diff --git a/.gitignore b/.gitignore index d529d40..0299471 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,6 @@ .* *egg-info dist +build README.html __pycache__ \ No newline at end of file diff --git a/senpy/blueprints.py b/senpy/blueprints.py index e64b1c7..6635c55 100644 --- a/senpy/blueprints.py +++ b/senpy/blueprints.py @@ -17,17 +17,18 @@ """ Blueprints for Senpy """ -from flask import Blueprint, request, current_app, render_template -from .models import Error, Response, Plugins +from flask import Blueprint, request, current_app, render_template, url_for, jsonify +from .models import Error, Response, Plugins, read_schema from future.utils import iteritems +from functools import wraps import json import logging logger = logging.getLogger(__name__) -nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__) -demo_blueprint = Blueprint("Demo of the service. It includes an HTML+Javascript playground to test senpy", __name__) +api_blueprint = Blueprint("api", __name__) +demo_blueprint = Blueprint("demo", __name__) API_PARAMS = { "algorithm": { @@ -47,7 +48,7 @@ API_PARAMS = { }, } -BASIC_PARAMS = { +NIF_PARAMS = { "algorithm": { "aliases": ["algorithm", "a", "algo"], "required": False, @@ -104,7 +105,7 @@ BASIC_PARAMS = { }, } -def get_params(req, params=BASIC_PARAMS): +def update_params(req, params=NIF_PARAMS): if req.method == 'POST': indict = req.form elif req.method == 'GET': @@ -136,64 +137,73 @@ def get_params(req, params=BASIC_PARAMS): parameters=outdict, errors={param: error for param, error in iteritems(wrong_params)}) - raise Error(message=message) + raise message + if hasattr(request, 'params'): + request.params.update(outdict) + else: + request.params = outdict return outdict -def basic_analysis(params): - response = {"@context": - [("http://demos.gsi.dit.upm.es/" - "eurosentiment/static/context.jsonld"), - { - "@base": "{}#".format(request.url.encode('utf-8')) - } - ], - "analysis": [{"@type": "marl:SentimentAnalysis"}], - "entries": [] - } - if "language" in params: - response["language"] = params["language"] - for idx, sentence in enumerate(params["input"].split(".")): - response["entries"].append({ - "@id": "Sentence{}".format(idx), - "nif:isString": sentence - }) - return response - - @demo_blueprint.route('/') def index(): return render_template("index.html") +@api_blueprint.route('/contexts/.jsonld') +def context(entity="context"): + return jsonify({"@context": Response.context}) -@nif_blueprint.route('/', methods=['POST', 'GET']) -def api(): +@api_blueprint.route('/schemas/') +def schema(schema="definitions"): try: - params = get_params(request) - algo = params.get("algorithm", None) - specific_params = current_app.senpy.parameters(algo) - logger.debug( - "Specific params: %s", json.dumps(specific_params, indent=4)) - params.update(get_params(request, specific_params)) - response = current_app.senpy.analyse(**params) - in_headers = params["inHeaders"] != "0" - prefix = params["prefix"] - return response.flask(in_headers=in_headers, prefix=prefix) - except Error as ex: - return ex.message.flask() - - -@nif_blueprint.route('/plugins/', methods=['POST', 'GET']) + return jsonify(read_schema(schema)) + except Exception: # Should be FileNotFoundError, but it's missing from py2 + return Error(message="Schema not found", status=404).flask() + +def basic_api(f): + @wraps(f) + def decorated_function(*args, **kwargs): + print('Getting request:') + print(request) + update_params(request, params=API_PARAMS) + print('Params: %s' % request.params) + try: + response = f(*args, **kwargs) + except Error as ex: + response = ex + in_headers = request.params["inHeaders"] != "0" + prefix = request.params["prefix"] + headers = {'X-ORIGINAL-PARAMS': request.params} + return response.flask(in_headers=in_headers, + prefix=prefix, + headers=headers, + context_uri=url_for('api.context', entity=type(response).__name__, + _external=True)) + return decorated_function + +@api_blueprint.route('/', methods=['POST', 'GET']) +@basic_api +def api(): + algo = request.params.get("algorithm", None) + specific_params = current_app.senpy.parameters(algo) + update_params(request, params=NIF_PARAMS) + logger.debug("Specific params: %s", json.dumps(specific_params, indent=4)) + update_params(request, specific_params) + response = current_app.senpy.analyse(**request.params) + return response + + +@api_blueprint.route('/plugins/', methods=['POST', 'GET']) +@basic_api def plugins(): - in_headers = get_params(request, API_PARAMS)["inHeaders"] != "0" sp = current_app.senpy dic = Plugins(plugins=list(sp.plugins.values())) - return dic.flask(in_headers=in_headers) + return dic -@nif_blueprint.route('/plugins//', methods=['POST', 'GET']) -@nif_blueprint.route('/plugins//', methods=['POST', 'GET']) +@api_blueprint.route('/plugins//', methods=['POST', 'GET']) +@api_blueprint.route('/plugins//', methods=['POST', 'GET']) +@basic_api def plugin(plugin=None, action="list"): - params = get_params(request, API_PARAMS) filt = {} sp = current_app.senpy plugs = sp.filter_plugins(name=plugin) @@ -203,21 +213,19 @@ def plugin(plugin=None, action="list"): elif plugin in sp.plugins: response = sp.plugins[plugin] else: - return Error(message="Plugin not found", status=404).flask() + return Error(message="Plugin not found", status=404) if action == "list": - in_headers = params["inHeaders"] != "0" - prefix = params['prefix'] - return response.flask(in_headers=in_headers, prefix=prefix) + return response method = "{}_plugin".format(action) if(hasattr(sp, method)): getattr(sp, method)(plugin) - return Response(message="Ok").flask() + return Response(message="Ok") else: - return Error(message="action '{}' not allowed".format(action)).flask() + return Error(message="action '{}' not allowed".format(action)) if __name__ == '__main__': import config - app.register_blueprint(nif_blueprint) + app.register_blueprint(api_blueprint) app.debug = config.DEBUG app.run(host='0.0.0.0', port=5000) diff --git a/senpy/extensions.py b/senpy/extensions.py index 75d3fe8..b04980a 100644 --- a/senpy/extensions.py +++ b/senpy/extensions.py @@ -8,7 +8,7 @@ monkey.patch_all() from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin from .models import Error -from .blueprints import nif_blueprint, demo_blueprint +from .blueprints import api_blueprint, demo_blueprint from git import Repo, InvalidGitRepositoryError from functools import partial @@ -58,7 +58,7 @@ class Senpy(object): app.teardown_appcontext(self.teardown) else: app.teardown_request(self.teardown) - app.register_blueprint(nif_blueprint, url_prefix="/api") + app.register_blueprint(api_blueprint, url_prefix="/api") app.register_blueprint(demo_blueprint, url_prefix="/") def add_folder(self, folder): @@ -77,28 +77,30 @@ class Senpy(object): elif self.plugins: algo = self.default_plugin and self.default_plugin.name if not algo: - return Error(status=404, - message=("No plugins found." - " Please install one.").format(algo)) - if algo in self.plugins: - if self.plugins[algo].is_activated: - plug = self.plugins[algo] - resp = plug.analyse(**params) - resp.analysis.append(plug) - logger.debug("Returning analysis result: {}".format(resp)) - return resp - else: - logger.debug("Plugin not activated: {}".format(algo)) - return Error(status=400, - message=("The algorithm '{}'" - " is not activated yet").format(algo)) - else: + raise Error(status=404, + message=("No plugins found." + " Please install one.").format(algo)) + if algo not in self.plugins: logger.debug(("The algorithm '{}' is not valid\n" "Valid algorithms: {}").format(algo, self.plugins.keys())) - return Error(status=404, - message="The algorithm '{}' is not valid" - .format(algo)) + raise Error(status=404, + message="The algorithm '{}' is not valid" + .format(algo)) + + if not self.plugins[algo].is_activated: + logger.debug("Plugin not activated: {}".format(algo)) + raise Error(status=400, + message=("The algorithm '{}'" + " is not activated yet").format(algo)) + plug = self.plugins[algo] + try: + resp = plug.analyse(**params) + resp.analysis.append(plug) + logger.debug("Returning analysis result: {}".format(resp)) + except Exception as ex: + resp = Error(message=str(ex), status=500) + return resp @property def default_plugin(self): diff --git a/senpy/models.py b/senpy/models.py index faedfe1..5fd10ea 100644 --- a/senpy/models.py +++ b/senpy/models.py @@ -63,28 +63,31 @@ class Context(dict): base_context = Context.load(CONTEXT_PATH) - class SenpyMixin(object): - context = base_context + context = base_context["@context"] def flask(self, in_headers=False, - url="http://demos.gsi.dit.upm.es/senpy/senpy.jsonld", - prefix=None): + headers=None, + prefix=None, + **kwargs): """ Return the values and error to be used in flask. So far, it returns a fixed context. We should store/generate different contexts if the plugin adds more aliases. """ - headers = None + headers = headers or {} + kwargs["with_context"] = True + js = self.jsonld(**kwargs) if in_headers: - headers = { + url = js["@context"] + del js["@context"] + headers.update({ "Link": ('<%s>;' 'rel="http://www.w3.org/ns/json-ld#context";' ' type="application/ld+json"' % url) - } - return FlaskResponse(self.to_JSON(with_context=not in_headers, - prefix=prefix), + }) + return FlaskResponse(json.dumps(js, indent=2, sort_keys=True), status=getattr(self, "status", 200), headers=headers, mimetype="application/json") @@ -107,15 +110,27 @@ class SenpyMixin(object): return ser_or_down(self._plain_dict()) - def jsonld(self, context=None, prefix=None, with_context=False): + def jsonld(self, prefix=None, with_context=True, context_uri=None): ser = self.serializable() if with_context: - ser["@context"] = self.context.copy() - + context = [] + if context_uri: + context = context_uri + else: + context = self.context.copy() if prefix: - ser["@context"]["@base"] = prefix - + # This sets @base for the document, which will be used in + # all relative URIs will. For example, if a uri is "Example" and + # prefix =s "http://example.com", the absolute URI after expanding + # with JSON-LD will be "http://example.com/Example" + + prefix_context = {"@base": prefix} + if isinstance(context, list): + context.append(prefix_context) + else: + context = [context, prefix_context] + ser["@context"] = context return ser @@ -184,11 +199,6 @@ class SenpyModel(SenpyMixin, dict): self.__delitem__(self._get_key(key)) - @classmethod - def from_base(cls, name): - subschema = base_schema[name] - return warlock.model_factory(subschema, base_class=cls) - def _plain_dict(self): d = { k: v for (k,v) in self.items() if k[0] != "_"} d["@id"] = d.pop('id') diff --git a/senpy/plugins.py b/senpy/plugins.py index bf43d6d..65bb5f1 100644 --- a/senpy/plugins.py +++ b/senpy/plugins.py @@ -3,7 +3,7 @@ standard_library.install_aliases() import inspect import os.path -import shelve +import pickle import logging from .models import Response, PluginModel, Error @@ -18,10 +18,7 @@ class SenpyPlugin(PluginModel): logger.debug("Initialising {}".format(info)) super(SenpyPlugin, self).__init__(info) self.id = '{}_{}'.format(self.name, self.version) - self.params = info.get("extra_params", {}) self._info = info - if "@id" not in self.params: - self.params["@id"] = "params_%s" % self.id self.is_activated = False def get_folder(self): @@ -64,28 +61,34 @@ class ShelfMixin(object): @property def sh(self): if not hasattr(self, '_sh') or self._sh is None: - self._sh = shelve.open(self.shelf_file, writeback=True) + self.__dict__['_sh'] = {} + if os.path.isfile(self.shelf_file): + self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb')) return self._sh @sh.deleter def sh(self): if os.path.isfile(self.shelf_file): os.remove(self.shelf_file) - self.close() + del self.__dict__['_sh'] + self.save() def __del__(self): - self.close() - self.deactivate() + self.save() + super(ShelfMixin, self).__del__() @property def shelf_file(self): if not hasattr(self, '_shelf_file') or not self._shelf_file: if hasattr(self, '_info') and 'shelf_file' in self._info: - self._shelf_file = self._info['shelf_file'] + self.__dict__['_shelf_file'] = self._info['shelf_file'] else: - self._shelf_file = os.path.join(self.get_folder(), self.name + '.db') + self._shelf_file = os.path.join(self.get_folder(), self.name + '.p') return self._shelf_file - def close(self): - self.sh.close() - del(self._sh) + def save(self): + logger.debug('closing pickle') + if hasattr(self, '_sh') and self._sh is not None: + with open(self.shelf_file, 'wb') as f: + pickle.dump(self._sh, f) + del(self.__dict__['_sh']) diff --git a/senpy/plugins/rand/rand.py b/senpy/plugins/rand/rand.py index d7220ee..3c345e4 100644 --- a/senpy/plugins/rand/rand.py +++ b/senpy/plugins/rand/rand.py @@ -9,8 +9,7 @@ class Sentiment140Plugin(SentimentPlugin): def analyse(self, **params): lang = params.get("language", "auto") - p = params.get("prefix", None) - response = Results(prefix=p) + response = Results() polarity_value = max(-1, min(1, random.gauss(0.2, 0.2))) polarity = "marl:Neutral" if polarity_value > 0: diff --git a/senpy/schemas/context.jsonld b/senpy/schemas/context.jsonld index 220e6c6..939df0b 100644 --- a/senpy/schemas/context.jsonld +++ b/senpy/schemas/context.jsonld @@ -1,33 +1,35 @@ { - "@vocab": "http://www.gsi.dit.upm.es/ontologies/senpy#", - "dc": "http://dublincore.org/2012/06/14/dcelements#", - "me": "http://www.mixedemotions-project.eu/ns/model#", - "prov": "http://www.w3.org/ns/prov#", - "nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#", - "marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#", - "onyx": "http://www.gsi.dit.upm.es/ontologies/onyx#", - "wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#", - "xsd": "http://www.w3.org/2001/XMLSchema#", - "topics": { - "@id": "dc:subject" - }, - "entities": { - "@id": "me:hasEntities" - }, - "suggestions": { - "@id": "me:hasSuggestions" - }, - "emotions": { - "@id": "onyx:hasEmotionSet" - }, - "sentiments": { - "@id": "marl:hasOpinion" - }, - "entries": { - "@id": "prov:used" - }, - "analysis": { - "@id": "prov:wasGeneratedBy" + "@context": { + "@vocab": "http://www.gsi.dit.upm.es/ontologies/senpy#", + "dc": "http://dublincore.org/2012/06/14/dcelements#", + "me": "http://www.mixedemotions-project.eu/ns/model#", + "prov": "http://www.w3.org/ns/prov#", + "nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#", + "marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#", + "onyx": "http://www.gsi.dit.upm.es/ontologies/onyx#", + "wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "topics": { + "@id": "dc:subject" + }, + "entities": { + "@id": "me:hasEntities" + }, + "suggestions": { + "@id": "me:hasSuggestions" + }, + "emotions": { + "@id": "onyx:hasEmotionSet" + }, + "sentiments": { + "@id": "marl:hasOpinion" + }, + "entries": { + "@id": "prov:used" + }, + "analysis": { + "@id": "prov:wasGeneratedBy" -} + } + } } diff --git a/senpy/schemas/definitions.json b/senpy/schemas/definitions.json index 6f6d4f1..45eab88 100644 --- a/senpy/schemas/definitions.json +++ b/senpy/schemas/definitions.json @@ -152,10 +152,14 @@ }, "Plugin": { "type": "object", - "required": ["@id"], + "required": ["@id", "extra_params"], "properties": { "@id": { "type": "string" + }, + "extra_params": { + "type": "object", + "default": {} } } }, diff --git a/tests/test_blueprints.py b/tests/test_blueprints.py index 02862db..f596a57 100644 --- a/tests/test_blueprints.py +++ b/tests/test_blueprints.py @@ -46,9 +46,6 @@ class BlueprintsTest(TestCase): self.assert200(resp) logging.debug("Got response: %s", resp.json) assert "@context" in resp.json - assert check_dict( - resp.json["@context"], - {"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"}) assert "entries" in resp.json def test_list(self): @@ -111,3 +108,16 @@ class BlueprintsTest(TestCase): sleep(0.5) resp = self.client.get("/api/plugins/default/") self.assert404(resp) + + def test_context(self): + resp = self.client.get("/api/contexts/context.jsonld") + self.assert200(resp) + assert "@context" in resp.json + assert check_dict( + resp.json["@context"], + {"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"}) + + def test_schema(self): + resp = self.client.get("/api/schemas/definitions.json") + self.assert200(resp) + assert "$schema" in resp.json diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 5c3ca41..d95094e 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -2,7 +2,9 @@ from __future__ import print_function import os import logging +from functools import partial from senpy.extensions import Senpy +from senpy.models import Error from flask import Flask from flask.ext.testing import TestCase @@ -54,9 +56,7 @@ class ExtensionsTest(TestCase): def test_noplugin(self): """ Don't analyse if there isn't any plugin installed """ self.senpy.deactivate_all(sync=True) - resp = self.senpy.analyse(input="tupni") - logging.debug("Response: {}".format(resp)) - assert resp.status == 404 + self.assertRaises(Error, partial(self.senpy.analyse, input="tupni")) def test_analyse(self): """ Using a plugin """ @@ -67,12 +67,6 @@ class ExtensionsTest(TestCase): r2 = self.senpy.analyse(input="tupni", output="tuptuo") assert r1.analysis[0].id[:5] == "Dummy" assert r2.analysis[0].id[:5] == "Dummy" - for plug in self.senpy.plugins: - self.senpy.deactivate_plugin(plug, sync=True) - resp = self.senpy.analyse(input="tupni") - logging.debug("Response: {}".format(resp)) - assert resp.status == 404 - def test_filtering(self): """ Filtering plugins """ diff --git a/tests/test_models.py b/tests/test_models.py index e8cd5f7..f2408eb 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -98,7 +98,7 @@ class ModelsTest(TestCase): c = p.jsonld() assert "info" not in c assert "repo" not in c - assert "params" in c + assert "extra_params" in c logging.debug("Framed:") logging.debug(c) p.validate() diff --git a/tests/test_plugins.py b/tests/test_plugins.py index e66148c..a09326a 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -2,7 +2,7 @@ import os import logging -import shelve +import pickle import shutil import tempfile @@ -16,7 +16,6 @@ from senpy.plugins import SenpyPlugin, ShelfMixin class ShelfTest(ShelfMixin, SenpyPlugin): def test(self, key=None, value=None): - assert isinstance(self.sh, shelve.Shelf) assert key in self.sh print('Checking: sh[{}] == {}'.format(key, value)) print('SH[{}]: {}'.format(key, self.sh[key])) @@ -49,7 +48,9 @@ class ModelsTest(TestCase): a.sh['a'] = 'fromA' a.test(key='a', value='fromA') - sh = shelve.open(self.shelf_file) + a.save() + + sh = pickle.load(open(self.shelf_file, 'rb')) assert sh['a'] == 'fromA' @@ -61,7 +62,7 @@ class ModelsTest(TestCase): 'shelf_file': self.shelf_file}) print('Shelf file: %s' % a.shelf_file) a.sh['a'] = 'fromA' - a.close() + a.save() b = ShelfTest(info={'name': 'shelve', 'version': 'test',