Improved request handling

Also:
 * Shelve -> Pickle to avoid weird db problems
 * Serving schemas and contexts

.gitignore

@@ -2,5 +2,6 @@
 .*
 *egg-info
 dist
+build
 README.html
 __pycache__

senpy/blueprints.py

@@ -17,17 +17,18 @@
 """
 Blueprints for Senpy
 """
-from flask import Blueprint, request, current_app, render_template
+from flask import Blueprint, request, current_app, render_template, url_for, jsonify
-from .models import Error, Response, Plugins
+from .models import Error, Response, Plugins, read_schema
 from future.utils import iteritems
+from functools import wraps
 
 import json
 import logging
 
 logger = logging.getLogger(__name__)
 
-nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__)
+api_blueprint = Blueprint("api", __name__)
-demo_blueprint = Blueprint("Demo of the service. It includes an HTML+Javascript playground to test senpy", __name__)
+demo_blueprint = Blueprint("demo", __name__)
 
 API_PARAMS = {
     "algorithm": {
@@ -47,7 +48,7 @@ API_PARAMS = {
     },
 }
 
-BASIC_PARAMS = {
+NIF_PARAMS = {
     "algorithm": {
         "aliases": ["algorithm", "a", "algo"],
         "required": False,
@@ -104,7 +105,7 @@ BASIC_PARAMS = {
     },
 }
 
-def get_params(req, params=BASIC_PARAMS):
+def update_params(req, params=NIF_PARAMS):
     if req.method == 'POST':
         indict = req.form
     elif req.method == 'GET':
@@ -136,64 +137,73 @@ def get_params(req, params=BASIC_PARAMS):
                         parameters=outdict,
                         errors={param: error for param, error in
                                 iteritems(wrong_params)})
-        raise Error(message=message)
+        raise message
+    if hasattr(request, 'params'):
+        request.params.update(outdict)
+    else:
+        request.params = outdict
     return outdict
 
 
-def basic_analysis(params):
-    response = {"@context":
-                [("http://demos.gsi.dit.upm.es/"
-                  "eurosentiment/static/context.jsonld"),
-                 {
-                    "@base": "{}#".format(request.url.encode('utf-8'))
-                  }
-                 ],
-                "analysis": [{"@type": "marl:SentimentAnalysis"}],
-                "entries": []
-                }
-    if "language" in params:
-        response["language"] = params["language"]
-    for idx, sentence in enumerate(params["input"].split(".")):
-        response["entries"].append({
-            "@id": "Sentence{}".format(idx),
-            "nif:isString": sentence
-        })
-    return response
-
-
 @demo_blueprint.route('/')
 def index():
     return render_template("index.html")
 
+@api_blueprint.route('/contexts/<entity>.jsonld')
+def context(entity="context"):
+    return jsonify({"@context": Response.context})
 
-@nif_blueprint.route('/', methods=['POST', 'GET'])
+@api_blueprint.route('/schemas/<schema>')
-def api():
+def schema(schema="definitions"):
     try:
-        params = get_params(request)
+        return jsonify(read_schema(schema))
-        algo = params.get("algorithm", None)
+    except Exception: # Should be FileNotFoundError, but it's missing from py2
-        specific_params = current_app.senpy.parameters(algo)
+        return Error(message="Schema not found", status=404).flask()
-        logger.debug(
+
-            "Specific params: %s", json.dumps(specific_params, indent=4))
+def basic_api(f):
-        params.update(get_params(request, specific_params))
+    @wraps(f)
-        response = current_app.senpy.analyse(**params)
+    def decorated_function(*args, **kwargs):
-        in_headers = params["inHeaders"] != "0"
+        print('Getting request:')
-        prefix = params["prefix"]
+        print(request)
-        return response.flask(in_headers=in_headers, prefix=prefix)
+        update_params(request, params=API_PARAMS)
-    except Error as ex:
+        print('Params: %s' % request.params)
-        return ex.message.flask()
+        try:
+            response = f(*args, **kwargs)
+        except Error as ex:
+            response = ex
+        in_headers = request.params["inHeaders"] != "0"
+        prefix = request.params["prefix"]
+        headers = {'X-ORIGINAL-PARAMS': request.params}
+        return response.flask(in_headers=in_headers,
+                              prefix=prefix,
+                              headers=headers,
+                              context_uri=url_for('api.context', entity=type(response).__name__,
+                                                  _external=True))
+    return decorated_function
+    
+@api_blueprint.route('/', methods=['POST', 'GET'])
+@basic_api
+def api():
+    algo = request.params.get("algorithm", None)
+    specific_params = current_app.senpy.parameters(algo)
+    update_params(request, params=NIF_PARAMS)
+    logger.debug("Specific params: %s", json.dumps(specific_params, indent=4))
+    update_params(request, specific_params)
+    response = current_app.senpy.analyse(**request.params)
+    return response
 
 
-@nif_blueprint.route('/plugins/', methods=['POST', 'GET'])
+@api_blueprint.route('/plugins/', methods=['POST', 'GET'])
+@basic_api
 def plugins():
-    in_headers = get_params(request, API_PARAMS)["inHeaders"] != "0"
     sp = current_app.senpy
     dic = Plugins(plugins=list(sp.plugins.values()))
-    return dic.flask(in_headers=in_headers)
+    return dic
     
-@nif_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
+@api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
-@nif_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
+@api_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
+@basic_api
 def plugin(plugin=None, action="list"):
-    params = get_params(request, API_PARAMS)
     filt = {}
     sp = current_app.senpy
     plugs = sp.filter_plugins(name=plugin)
@@ -203,21 +213,19 @@ def plugin(plugin=None, action="list"):
     elif plugin in sp.plugins:
         response = sp.plugins[plugin]
     else:
-        return Error(message="Plugin not found", status=404).flask()
+        return Error(message="Plugin not found", status=404)
     if action == "list":
-        in_headers = params["inHeaders"] != "0"
+        return response
-        prefix = params['prefix']
-        return response.flask(in_headers=in_headers, prefix=prefix)
     method = "{}_plugin".format(action)
     if(hasattr(sp, method)):
         getattr(sp, method)(plugin)
-        return Response(message="Ok").flask()
+        return Response(message="Ok")
     else:
-        return Error(message="action '{}' not allowed".format(action)).flask()
+        return Error(message="action '{}' not allowed".format(action))
 
 if __name__ == '__main__':
     import config
 
-    app.register_blueprint(nif_blueprint)
+    app.register_blueprint(api_blueprint)
     app.debug = config.DEBUG
     app.run(host='0.0.0.0', port=5000)

senpy/extensions.py

@@ -8,7 +8,7 @@ monkey.patch_all()
 
 from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
 from .models import Error
-from .blueprints import nif_blueprint, demo_blueprint
+from .blueprints import api_blueprint, demo_blueprint
 
 from git import Repo, InvalidGitRepositoryError
 from functools import partial
@@ -58,7 +58,7 @@ class Senpy(object):
             app.teardown_appcontext(self.teardown)
         else:
             app.teardown_request(self.teardown)
-        app.register_blueprint(nif_blueprint, url_prefix="/api")
+        app.register_blueprint(api_blueprint, url_prefix="/api")
         app.register_blueprint(demo_blueprint, url_prefix="/")
 
     def add_folder(self, folder):
@@ -77,28 +77,30 @@ class Senpy(object):
         elif self.plugins:
             algo = self.default_plugin and self.default_plugin.name
         if not algo:
-            return Error(status=404,
+            raise Error(status=404,
-                         message=("No plugins found."
+                        message=("No plugins found."
-                                  " Please install one.").format(algo))
+                                 " Please install one.").format(algo))
-        if algo in self.plugins:
+        if algo not in self.plugins:
-            if self.plugins[algo].is_activated:
-                plug = self.plugins[algo]
-                resp = plug.analyse(**params)
-                resp.analysis.append(plug)
-                logger.debug("Returning analysis result: {}".format(resp))
-                return resp
-            else:
-                logger.debug("Plugin not activated: {}".format(algo))
-                return Error(status=400,
-                             message=("The algorithm '{}'"
-                                      " is not activated yet").format(algo))
-        else:
             logger.debug(("The algorithm '{}' is not valid\n"
                           "Valid algorithms: {}").format(algo,
                                                          self.plugins.keys()))
-            return Error(status=404,
+            raise Error(status=404,
-                         message="The algorithm '{}' is not valid"
+                        message="The algorithm '{}' is not valid"
-                                 .format(algo))
+                        .format(algo))
+
+        if not self.plugins[algo].is_activated:
+            logger.debug("Plugin not activated: {}".format(algo))
+            raise Error(status=400,
+                        message=("The algorithm '{}'"
+                                    " is not activated yet").format(algo))
+        plug = self.plugins[algo]
+        try:
+            resp = plug.analyse(**params)
+            resp.analysis.append(plug)
+            logger.debug("Returning analysis result: {}".format(resp))
+        except Exception as ex:
+            resp = Error(message=str(ex), status=500)
+        return resp
 
     @property
     def default_plugin(self):

senpy/models.py

@@ -63,28 +63,31 @@ class Context(dict):
 
 base_context = Context.load(CONTEXT_PATH)
 
-
 class SenpyMixin(object):
-    context = base_context
+    context = base_context["@context"]
 
     def flask(self,
               in_headers=False,
-              url="http://demos.gsi.dit.upm.es/senpy/senpy.jsonld",
+              headers=None,
-              prefix=None):
+              prefix=None,
+              **kwargs):
         """
         Return the values and error to be used in flask.
         So far, it returns a fixed context. We should store/generate different
         contexts if the plugin adds more aliases.
         """
-        headers = None
+        headers = headers or {}
+        kwargs["with_context"] = True
+        js = self.jsonld(**kwargs)
         if in_headers:
-            headers = {
+            url = js["@context"]
+            del js["@context"]
+            headers.update({
                 "Link": ('<%s>;'
                          'rel="http://www.w3.org/ns/json-ld#context";'
                          ' type="application/ld+json"' % url)
-            }
+            })
-        return FlaskResponse(self.to_JSON(with_context=not in_headers,
+        return FlaskResponse(json.dumps(js, indent=2, sort_keys=True),
-                                          prefix=prefix),
                              status=getattr(self, "status", 200),
                              headers=headers,
                              mimetype="application/json")
@@ -107,15 +110,27 @@ class SenpyMixin(object):
         return ser_or_down(self._plain_dict())
 
 
-    def jsonld(self, context=None, prefix=None, with_context=False):
+    def jsonld(self, prefix=None, with_context=True, context_uri=None):
         ser = self.serializable()
 
         if  with_context:
-            ser["@context"] = self.context.copy()
+            context = []
-
+            if context_uri:
+                context = context_uri
+            else:
+                context = self.context.copy()
             if prefix:
-                ser["@context"]["@base"] = prefix
+                # This sets @base for the document, which will be used in
+                # all relative URIs will. For example, if a uri is "Example" and
+                # prefix =s "http://example.com", the absolute URI after expanding
+                # with JSON-LD will be "http://example.com/Example"
 
+                prefix_context = {"@base": prefix}
+                if isinstance(context, list):
+                    context.append(prefix_context)
+                else:
+                    context = [context, prefix_context]
+            ser["@context"] = context
         return ser
 
 
@@ -184,11 +199,6 @@ class SenpyModel(SenpyMixin, dict):
         self.__delitem__(self._get_key(key))
         
     
-    @classmethod
-    def from_base(cls, name):
-        subschema = base_schema[name]
-        return warlock.model_factory(subschema, base_class=cls)
-
     def _plain_dict(self):
         d =  { k: v for (k,v) in self.items() if k[0] != "_"}
         d["@id"] = d.pop('id')

senpy/plugins.py

@@ -3,7 +3,7 @@ standard_library.install_aliases()
 
 import inspect
 import os.path
-import shelve
+import pickle
 import logging
 from .models import Response, PluginModel, Error
 
@@ -18,10 +18,7 @@ class SenpyPlugin(PluginModel):
         logger.debug("Initialising {}".format(info))
         super(SenpyPlugin, self).__init__(info)
         self.id = '{}_{}'.format(self.name, self.version)
-        self.params = info.get("extra_params", {})
         self._info = info
-        if "@id" not in self.params:
-            self.params["@id"] = "params_%s" % self.id
         self.is_activated = False
 
     def get_folder(self):
@@ -64,28 +61,34 @@ class ShelfMixin(object):
     @property
     def sh(self):
         if not hasattr(self, '_sh') or self._sh is None:
-            self._sh = shelve.open(self.shelf_file, writeback=True)
+            self.__dict__['_sh'] = {}
+            if os.path.isfile(self.shelf_file):
+                self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb'))
         return self._sh
 
     @sh.deleter
     def sh(self):
         if os.path.isfile(self.shelf_file):
             os.remove(self.shelf_file)
-        self.close()
+            del self.__dict__['_sh']
+        self.save()
 
     def __del__(self):
-        self.close()        
+        self.save()        
-        self.deactivate()
+        super(ShelfMixin, self).__del__()
 
     @property
     def shelf_file(self):
         if not hasattr(self, '_shelf_file') or not self._shelf_file:
             if hasattr(self, '_info') and 'shelf_file' in self._info:
-                self._shelf_file = self._info['shelf_file']
+                self.__dict__['_shelf_file'] = self._info['shelf_file']
             else:
-                self._shelf_file = os.path.join(self.get_folder(), self.name + '.db')
+                self._shelf_file = os.path.join(self.get_folder(), self.name + '.p')
         return self._shelf_file 
 
-    def close(self):
+    def save(self):
-        self.sh.close()
+        logger.debug('closing pickle')
-        del(self._sh)
+        if hasattr(self, '_sh') and self._sh is not None:
+            with open(self.shelf_file, 'wb') as f:
+                pickle.dump(self._sh, f)
+        del(self.__dict__['_sh'])

senpy/plugins/rand/rand.py

@@ -9,8 +9,7 @@ class Sentiment140Plugin(SentimentPlugin):
     def analyse(self, **params):
         lang = params.get("language", "auto")
 
-        p = params.get("prefix", None)
+        response = Results()
-        response = Results(prefix=p)
         polarity_value = max(-1, min(1, random.gauss(0.2, 0.2)))
         polarity = "marl:Neutral"
         if polarity_value > 0:

senpy/schemas/context.jsonld

@@ -1,33 +1,35 @@
 {
-  "@vocab": "http://www.gsi.dit.upm.es/ontologies/senpy#",
+  "@context": {
-  "dc": "http://dublincore.org/2012/06/14/dcelements#",
+    "@vocab": "http://www.gsi.dit.upm.es/ontologies/senpy#",
-  "me": "http://www.mixedemotions-project.eu/ns/model#",
+    "dc": "http://dublincore.org/2012/06/14/dcelements#",
-  "prov": "http://www.w3.org/ns/prov#",
+    "me": "http://www.mixedemotions-project.eu/ns/model#",
-  "nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#",
+    "prov": "http://www.w3.org/ns/prov#",
-  "marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#",
+    "nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#",
-  "onyx": "http://www.gsi.dit.upm.es/ontologies/onyx#",
+    "marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#",
-  "wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#",
+    "onyx": "http://www.gsi.dit.upm.es/ontologies/onyx#",
-  "xsd": "http://www.w3.org/2001/XMLSchema#",
+    "wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#",
-  "topics": {
+    "xsd": "http://www.w3.org/2001/XMLSchema#",
-    "@id": "dc:subject"
+    "topics": {
-  },
+        "@id": "dc:subject"
-  "entities": {
+    },
-  "@id": "me:hasEntities"
+    "entities": {
-  },
+    "@id": "me:hasEntities"
-  "suggestions": {
+    },
-    "@id": "me:hasSuggestions"
+    "suggestions": {
-  },
+        "@id": "me:hasSuggestions"
-  "emotions": {
+    },
-    "@id": "onyx:hasEmotionSet"
+    "emotions": {
-  },
+        "@id": "onyx:hasEmotionSet"
-  "sentiments": {
+    },
-    "@id": "marl:hasOpinion"
+    "sentiments": {
-  },
+        "@id": "marl:hasOpinion"
-  "entries": {
+    },
-    "@id": "prov:used"
+    "entries": {
-  },
+        "@id": "prov:used"
-  "analysis": {
+    },
-    "@id": "prov:wasGeneratedBy"
+    "analysis": {
+        "@id": "prov:wasGeneratedBy"
 
-}
+    }
+  }
 }

senpy/schemas/definitions.json

@@ -152,10 +152,14 @@
   },
   "Plugin": {
     "type": "object",
-    "required": ["@id"],
+    "required": ["@id", "extra_params"],
     "properties": {
       "@id": {
         "type": "string"
+      },
+      "extra_params": {
+        "type": "object",
+        "default": {}
       }
     }
   },

tests/test_blueprints.py

@@ -46,9 +46,6 @@ class BlueprintsTest(TestCase):
         self.assert200(resp)
         logging.debug("Got response: %s", resp.json)
         assert "@context" in resp.json
-        assert check_dict(
-            resp.json["@context"],
-            {"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"})
         assert "entries" in resp.json
 
     def test_list(self):
@@ -111,3 +108,16 @@ class BlueprintsTest(TestCase):
         sleep(0.5)
         resp = self.client.get("/api/plugins/default/")
         self.assert404(resp)
+
+    def test_context(self):
+        resp = self.client.get("/api/contexts/context.jsonld")
+        self.assert200(resp)
+        assert "@context" in resp.json
+        assert check_dict(
+            resp.json["@context"],
+            {"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"})
+
+    def test_schema(self):
+        resp = self.client.get("/api/schemas/definitions.json")
+        self.assert200(resp)
+        assert "$schema" in resp.json

tests/test_extensions.py

@@ -2,7 +2,9 @@ from __future__ import print_function
 import os
 import logging
 
+from functools import partial 
 from senpy.extensions import Senpy
+from senpy.models import Error
 from flask import Flask
 from flask.ext.testing import TestCase
 
@@ -54,9 +56,7 @@ class ExtensionsTest(TestCase):
     def test_noplugin(self):
         """ Don't analyse if there isn't any plugin installed """
         self.senpy.deactivate_all(sync=True)
-        resp = self.senpy.analyse(input="tupni")
+        self.assertRaises(Error, partial(self.senpy.analyse, input="tupni"))
-        logging.debug("Response: {}".format(resp))
-        assert resp.status == 404
 
     def test_analyse(self):
         """ Using a plugin """
@@ -67,12 +67,6 @@ class ExtensionsTest(TestCase):
         r2 = self.senpy.analyse(input="tupni", output="tuptuo")
         assert r1.analysis[0].id[:5] == "Dummy"
         assert r2.analysis[0].id[:5] == "Dummy"
-        for plug in self.senpy.plugins:
-            self.senpy.deactivate_plugin(plug, sync=True)
-        resp = self.senpy.analyse(input="tupni")
-        logging.debug("Response: {}".format(resp))
-        assert resp.status == 404
-
 
     def test_filtering(self):
         """ Filtering plugins """

tests/test_models.py

@@ -98,7 +98,7 @@ class ModelsTest(TestCase):
         c = p.jsonld()
         assert "info" not in c
         assert "repo" not in c
-        assert "params" in c
+        assert "extra_params" in c
         logging.debug("Framed:")
         logging.debug(c)
         p.validate()

tests/test_plugins.py

@@ -2,7 +2,7 @@
 
 import os
 import logging
-import shelve
+import pickle
 import shutil
 import tempfile
 
@@ -16,7 +16,6 @@ from senpy.plugins import SenpyPlugin, ShelfMixin
 class ShelfTest(ShelfMixin, SenpyPlugin):
 
     def test(self, key=None, value=None):
-        assert isinstance(self.sh, shelve.Shelf)
         assert key in self.sh
         print('Checking: sh[{}] == {}'.format(key, value))
         print('SH[{}]: {}'.format(key, self.sh[key]))
@@ -49,7 +48,9 @@ class ModelsTest(TestCase):
         a.sh['a'] = 'fromA'
         a.test(key='a', value='fromA')
 
-        sh = shelve.open(self.shelf_file)
+        a.save()
+
+        sh = pickle.load(open(self.shelf_file, 'rb'))
 
         assert sh['a'] == 'fromA'
 
@@ -61,7 +62,7 @@ class ModelsTest(TestCase):
                             'shelf_file': self.shelf_file})
         print('Shelf file: %s' % a.shelf_file)
         a.sh['a'] = 'fromA'
-        a.close()
+        a.save()
 
         b = ShelfTest(info={'name': 'shelve',
                             'version': 'test',