From 14c9f618644bca32ee959f0c370bad3ed6b96021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Fri, 19 Feb 2016 19:24:09 +0100 Subject: [PATCH] Python 3 compatible There are also some slight changes to the JSON schemas and the use of JSON-LD. --- .gitignore | 1 + requirements.txt | 6 +- senpy/blueprints.py | 45 ++-- senpy/context.jsonld | 42 --- senpy/extensions.py | 5 +- senpy/models.py | 250 +++++++++++++----- senpy/plugins.py | 35 +-- senpy/plugins/rand/rand.py | 33 ++- senpy/plugins/sentiment140/sentiment140.py | 21 +- senpy/schemas/analysis.json | 4 + senpy/schemas/context.jsonld | 33 +++ senpy/schemas/definitions.json | 161 +++++++++++ senpy/schemas/emotion.json | 4 + senpy/schemas/emotionSet.json | 4 + senpy/schemas/entry.json | 4 + senpy/schemas/plugin.json | 3 + senpy/schemas/plugins.json | 3 + senpy/schemas/response.json | 4 + senpy/schemas/results.json | 4 + senpy/schemas/sentiment.json | 4 + senpy/schemas/suggestion.json | 4 + setup.cfg | 2 + setup.py | 4 +- test-requirements.txt | 3 +- tests/context.jsonld | 40 --- tests/dummy_plugin/dummy.py | 4 +- tests/models_test/__init__.py | 81 ------ tests/sleep_plugin/sleep.py | 4 +- .../__init__.py => test_blueprints.py} | 16 +- .../__init__.py => test_extensions.py} | 10 +- tests/test_models.py | 97 +++++++ .../__init__.py => test_plugins.py} | 25 +- 32 files changed, 614 insertions(+), 342 deletions(-) delete mode 100644 senpy/context.jsonld create mode 100644 senpy/schemas/analysis.json create mode 100644 senpy/schemas/context.jsonld create mode 100644 senpy/schemas/definitions.json create mode 100644 senpy/schemas/emotion.json create mode 100644 senpy/schemas/emotionSet.json create mode 100644 senpy/schemas/entry.json create mode 100644 senpy/schemas/plugin.json create mode 100644 senpy/schemas/plugins.json create mode 100644 senpy/schemas/response.json create mode 100644 senpy/schemas/results.json create mode 100644 senpy/schemas/sentiment.json create mode 100644 senpy/schemas/suggestion.json delete mode 100644 tests/context.jsonld delete mode 100644 tests/models_test/__init__.py rename tests/{blueprints_test/__init__.py => test_blueprints.py} (89%) rename tests/{extensions_test/__init__.py => test_extensions.py} (93%) create mode 100644 tests/test_models.py rename tests/{plugins_test/__init__.py => test_plugins.py} (84%) diff --git a/.gitignore b/.gitignore index 3568e02..d529d40 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ *egg-info dist README.html +__pycache__ \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index b33e090..3867eb5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,10 @@ Flask>=0.10.1 gunicorn>=19.0.0 requests>=2.4.1 GitPython>=0.3.2.RC1 -gevent>=1.0.1 +gevent>=1.1rc4 PyLD>=0.6.5 Flask-Testing>=0.4.2 +six +future +jsonschema +jsonref diff --git a/senpy/blueprints.py b/senpy/blueprints.py index f162d2a..937328d 100644 --- a/senpy/blueprints.py +++ b/senpy/blueprints.py @@ -17,8 +17,8 @@ """ Blueprints for Senpy """ -from flask import Blueprint, request, current_app, Flask, redirect, url_for, render_template -from .models import Error, Response, Leaf +from flask import Blueprint, request, current_app, render_template +from .models import Error, Response from future.utils import iteritems import json @@ -27,6 +27,7 @@ import logging logger = logging.getLogger(__name__) nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__) +demo_blueprint = Blueprint("Demo of the service. It includes an HTML+Javascript playground to test senpy", __name__) BASIC_PARAMS = { "algorithm": { @@ -40,15 +41,6 @@ BASIC_PARAMS = { } } -LIST_PARAMS = { - "params": { - "aliases": ["params", "with_params"], - "required": False, - "default": "0" - }, -} - - def get_params(req, params=BASIC_PARAMS): if req.method == 'POST': indict = req.form @@ -76,12 +68,11 @@ def get_params(req, params=BASIC_PARAMS): outdict[param] not in params[param]["options"]: wrong_params[param] = params[param] if wrong_params: - message = Error({"status": 404, - "message": "Missing or invalid parameters", - "parameters": outdict, - "errors": {param: error for param, error in - iteritems(wrong_params)} - }) + message = Error(status=404, + message="Missing or invalid parameters", + parameters=outdict, + errors={param: error for param, error in + iteritems(wrong_params)}) raise Error(message=message) return outdict @@ -107,12 +98,12 @@ def basic_analysis(params): return response -@nif_blueprint.route('/') +@demo_blueprint.route('/') def index(): return render_template("index.html") -@nif_blueprint.route('/api', methods=['POST', 'GET']) +@nif_blueprint.route('/', methods=['POST', 'GET']) def api(): try: params = get_params(request) @@ -128,7 +119,7 @@ def api(): return ex.message.flask() -@nif_blueprint.route("/api/default") +@nif_blueprint.route("/default") def default(): # return current_app.senpy.default_plugin plug = current_app.senpy.default_plugin @@ -139,9 +130,9 @@ def default(): return error.flask() -@nif_blueprint.route('/api/plugins/', methods=['POST', 'GET']) -@nif_blueprint.route('/api/plugins/', methods=['POST', 'GET']) -@nif_blueprint.route('/api/plugins//', methods=['POST', 'GET']) +@nif_blueprint.route('/plugins/', methods=['POST', 'GET']) +@nif_blueprint.route('/plugins//', methods=['POST', 'GET']) +@nif_blueprint.route('/plugins//', methods=['POST', 'GET']) def plugins(plugin=None, action="list"): filt = {} sp = current_app.senpy @@ -151,21 +142,19 @@ def plugins(plugin=None, action="list"): if plugin and not plugs: return "Plugin not found", 400 if action == "list": - with_params = get_params(request, LIST_PARAMS)["params"] == "1" in_headers = get_params(request, BASIC_PARAMS)["inHeaders"] != "0" if plugin: dic = plugs[plugin] else: dic = Response( - {plug: plugs[plug].jsonld(with_params) for plug in plugs}, - frame={}) + {plug: plugs[plug].serializable() for plug in plugs}) return dic.flask(in_headers=in_headers) method = "{}_plugin".format(action) if(hasattr(sp, method)): getattr(sp, method)(plugin) - return Leaf(message="Ok").flask() + return Response(message="Ok").flask() else: - return Error("action '{}' not allowed".format(action)).flask() + return Error(message="action '{}' not allowed".format(action)).flask() if __name__ == '__main__': diff --git a/senpy/context.jsonld b/senpy/context.jsonld deleted file mode 100644 index fa5f7e3..0000000 --- a/senpy/context.jsonld +++ /dev/null @@ -1,42 +0,0 @@ -{ - "dc": "http://purl.org/dc/terms/", - "dc:subject": { - "@type": "@id" - }, - "xsd": "http://www.w3.org/2001/XMLSchema#", - "marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#", - "nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#", - "onyx": "http://www.gsi.dit.upm.es/ontologies/onyx/ns#", - "emotions": { - "@container": "@set", - "@id": "onyx:hasEmotionSet" - }, - "opinions": { - "@container": "@set", - "@id": "marl:hasOpinion" - }, - "prov": "http://www.w3.org/ns/prov#", - "rdfs": "http://www.w3.org/2000/01/rdf-schema#", - "analysis": { - "@container": "@set", - "@id": "prov:wasInformedBy" - }, - "entries": { - "@container": "@set", - "@id": "prov:generated" - }, - "strings": { - "@container": "@set", - "@reverse": "nif:hasContext" - }, - "date": - { - "@id": "dc:date", - "@type": "xsd:dateTime" - }, - "text": { "@id": "nif:isString" }, - "wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#", - "xsd": "http://www.w3.org/2001/XMLSchema#", - "senpy": "http://www.gsi.dit.upm.es/ontologies/senpy/ns#", - "@vocab": "http://www.gsi.dit.upm.es/ontologies/senpy/ns#" -} diff --git a/senpy/extensions.py b/senpy/extensions.py index 50f6ace..bc4ae5f 100644 --- a/senpy/extensions.py +++ b/senpy/extensions.py @@ -8,7 +8,7 @@ monkey.patch_all() from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin from .models import Error -from .blueprints import nif_blueprint +from .blueprints import nif_blueprint, demo_blueprint from git import Repo, InvalidGitRepositoryError from functools import partial @@ -57,7 +57,8 @@ class Senpy(object): app.teardown_appcontext(self.teardown) else: app.teardown_request(self.teardown) - app.register_blueprint(nif_blueprint) + app.register_blueprint(nif_blueprint, url_prefix="/api") + app.register_blueprint(demo_blueprint, url_prefix="/") def add_folder(self, folder): logger.debug("Adding folder: %s", folder) diff --git a/senpy/models.py b/senpy/models.py index 474a79b..c54fc36 100644 --- a/senpy/models.py +++ b/senpy/models.py @@ -1,71 +1,71 @@ +''' +Senpy Models. + +This implementation should mirror the JSON schema definition. +For compatibility with Py3 and for easier debugging, this new version drops introspection +and adds all arguments to the models. +''' from __future__ import print_function from six import string_types +import time +import copy import json import os import logging +import jsonref +import jsonschema -from collections import defaultdict -from pyld import jsonld from flask import Response as FlaskResponse -class Response(object): +DEFINITIONS_FILE = 'definitions.json' +CONTEXT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld') - @property - def context(self): - if not hasattr(self, '_context'): - self._context = None - return self._context +def get_schema_path(schema_file, absolute=False): + if absolute: + return os.path.realpath(schema_file) + else: + return os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas', schema_file) + + +def read_schema(schema_file, absolute=False): + schema_path = get_schema_path(schema_file, absolute) + schema_uri = 'file://{}'.format(schema_path) + return jsonref.load(open(schema_path), base_uri=schema_uri) + + +base_schema = read_schema(DEFINITIONS_FILE) +logging.debug(base_schema) + +class Context(dict): @staticmethod - def get_context(context): - if isinstance(context, list): + def load(context): + logging.debug('Loading context: {}'.format(context)) + if not context: + return context + elif isinstance(context, list): contexts = [] for c in context: - contexts.append(Response.get_context(c)) + contexts.append(Context.load(c)) return contexts elif isinstance(context, dict): - return context + return Context(context) elif isinstance(context, string_types): try: with open(context) as f: - return json.loads(f.read()) + return Context(json.loads(f.read())) except IOError: return context else: raise AttributeError('Please, provide a valid context') - def jsonld(self, frame=None, options=None, - context=None, removeContext=None): - if removeContext is None: - removeContext = Response._context # Loop? - if frame is None: - frame = self._frame - if context is None: - context = self.context - else: - context = self.get_context(context) - # For some reason, this causes errors with pyld - # if options is None: - # options = {"expandContext": context.copy() } - js = self - if frame: - logging.debug("Framing: %s", json.dumps(self, indent=4)) - logging.debug("Framing with %s", json.dumps(frame, indent=4)) - js = jsonld.frame(js, frame, options) - logging.debug("Result: %s", json.dumps(js, indent=4)) - logging.debug("Compacting with %s", json.dumps(context, indent=4)) - js = jsonld.compact(js, context, options) - logging.debug("Result: %s", json.dumps(js, indent=4)) - if removeContext == context: - del js["@context"] - return js +base_context = Context.load(CONTEXT_PATH) + - def to_JSON(self, removeContext=None): - return json.dumps(self.jsonld(removeContext=removeContext), - default=lambda o: o.__dict__, - sort_keys=True, indent=4) +class SenpyMixin(object): + context = base_context def flask(self, in_headers=False, @@ -73,46 +73,166 @@ class Response(object): """ Return the values and error to be used in flask """ - js = self.jsonld() headers = None if in_headers: - ctx = js["@context"] headers = { "Link": ('<%s>;' 'rel="http://www.w3.org/ns/json-ld#context";' ' type="application/ld+json"' % url) } - del js["@context"] - return FlaskResponse(json.dumps(js, indent=4), - status=self.get("status", 200), + return FlaskResponse(self.to_JSON(with_context=not in_headers), + status=getattr(self, "status", 200), headers=headers, mimetype="application/json") - -class Entry(JSONLD): - pass -class Sentiment(JSONLD): - pass + def serializable(self): + def ser_or_down(item): + if hasattr(item, 'serializable'): + return item.serializable() + elif isinstance(item, dict): + temp = dict() + for kp in item: + vp = item[kp] + temp[kp] = ser_or_down(vp) + return temp + elif isinstance(item, list): + return list(ser_or_down(i) for i in item) + else: + return item + return ser_or_down(self._plain_dict()) -class EmotionSet(JSONLD): - pass + def jsonld(self, context=None, with_context=False): + ser = self.serializable() + if with_context: + ser["@context"] = self.context -class Emotion(JSONLD): - pass + return ser -class Suggestion(JSONLD): - pass + def to_JSON(self, *args, **kwargs): + js = json.dumps(self.jsonld(*args, **kwargs), indent=4, + sort_keys=True) + return js + + +class SenpyModel(SenpyMixin, dict): -class Error(BaseException, JSONLD): - # A better pattern would be this: - # htp://flask.pocoo.org/docs/0.10/patterns/apierrors/ - _frame = {} - _context = {} + schema = base_schema + prefix = None def __init__(self, *args, **kwargs): - self.message = kwargs.get('message', None) - super(Error, self).__init__(*args) + temp = dict(*args, **kwargs) + + reqs = self.schema.get('required', []) + for i in reqs: + if i not in temp: + prop = self.schema['properties'][i] + if 'default' in prop: + temp[i] = copy.deepcopy(prop['default']) + if 'context' in temp: + context = temp['context'] + del temp['context'] + self.__dict__['context'] = Context.load(context) + super(SenpyModel, self).__init__(temp) + + + def _get_key(self, key): + key = key.replace("__", ":", 1) + return key + + def __setitem__(self, key, value): + dict.__setitem__(self, key, value) + + + def __delitem__(self, key): + dict.__delitem__(self, key) + + def __getattr__(self, key): + try: + return self.__getitem__(self._get_key(key)) + except KeyError: + raise AttributeError(key) + + def __setattr__(self, key, value): + self.__setitem__(self._get_key(key), value) + + def __delattr__(self, key): + self.__delitem__(self._get_key(key)) + + + def validate(self, obj=None): + if not obj: + obj = self + if hasattr(obj, "jsonld"): + obj = obj.jsonld() + jsonschema.validate(obj, self.schema) + + + @classmethod + def from_base(cls, name): + subschema = base_schema[name] + return warlock.model_factory(subschema, base_class=cls) + + def _plain_dict(self): + d = { k: v for (k,v) in self.items() if k[0] != "_"} + if hasattr(self, "id"): + d["@id"] = self.id + return d + + @property + def id(self): + if not hasattr(self, '_id'): + self.__dict__["_id"] = '_:{}_{}'.format(type(self).__name__, time.time()) + return self._id + + @id.setter + def id(self, value): + self._id = value + + +class Response(SenpyModel): + schema = read_schema('response.json') + +class Results(SenpyModel): + schema = read_schema('results.json') + + def jsonld(self, context=None, with_context=True): + return super(Results, self).jsonld(context, with_context) + +class Entry(SenpyModel): + schema = read_schema('entry.json') + +class Sentiment(SenpyModel): + schema = read_schema('sentiment.json') + +class Analysis(SenpyModel): + schema = read_schema('analysis.json') + +class EmotionSet(SenpyModel): + schema = read_schema('emotionSet.json') + +class Suggestion(SenpyModel): + schema = read_schema('suggestion.json') + +class PluginModel(SenpyModel): + schema = read_schema('plugin.json') + +class Plugins(SenpyModel): + schema = read_schema('plugins.json') + +class Error(SenpyMixin, BaseException ): + + def __init__(self, message, status=500, params=None, errors=None, *args, **kwargs): + self.message = message + self.status = status + self.params = params or {} + self.errors = errors or "" + + def _plain_dict(self): + return self.__dict__ + + def __str__(self): + return str(self.jsonld()) diff --git a/senpy/plugins.py b/senpy/plugins.py index 735c93b..cd77ac2 100644 --- a/senpy/plugins.py +++ b/senpy/plugins.py @@ -5,7 +5,7 @@ import inspect import os.path import shelve import logging -from .models import Response, Leaf +from .models import Response, PluginModel, Error logger = logging.getLogger(__name__) @@ -58,36 +58,21 @@ PARAMS = { } -class SenpyPlugin(Leaf): - _context = Leaf.get_context(Response._context) - _frame = {"@context": _context, - "name": {}, - "extra_params": {"@container": "@index"}, - "@explicit": True, - "version": {}, - "repo": None, - "is_activated": {}, - "params": None, - } +class SenpyPlugin(PluginModel): def __init__(self, info=None): if not info: raise Error(message=("You need to provide configuration" "information for the plugin.")) logger.debug("Initialising {}".format(info)) - super(SenpyPlugin, self).__init__() self.name = info["name"] self.version = info["version"] - self.id = "{}_{}".format(self.name, self.version) self.params = info.get("params", PARAMS.copy()) if "@id" not in self.params: self.params["@id"] = "params_%s" % self.id - self.extra_params = info.get("extra_params", {}) - self.params.update(self.extra_params.copy()) - if "@id" not in self.extra_params: - self.extra_params["@id"] = "extra_params_%s" % self.id self.is_activated = False self._info = info + super(SenpyPlugin, self).__init__() def get_folder(self): return os.path.dirname(inspect.getfile(self.__class__)) @@ -102,13 +87,6 @@ class SenpyPlugin(Leaf): def deactivate(self): pass - def jsonld(self, parameters=False, *args, **kwargs): - nframe = kwargs.pop("frame", self._frame) - if parameters: - nframe = nframe.copy() - nframe["params"] = {} - return super(SenpyPlugin, self).jsonld(frame=nframe, *args, **kwargs) - @property def id(self): return "{}_{}".format(self.name, self.version) @@ -123,6 +101,7 @@ class SentimentPlugin(SenpyPlugin): super(SentimentPlugin, self).__init__(info, *args, **kwargs) self.minPolarityValue = float(info.get("minPolarityValue", 0)) self.maxPolarityValue = float(info.get("maxPolarityValue", 1)) + self["@type"] = "marl:SentimentAnalysis" class EmotionPlugin(SenpyPlugin): @@ -131,6 +110,7 @@ class EmotionPlugin(SenpyPlugin): resp = super(EmotionPlugin, self).__init__(info, *args, **kwargs) self.minEmotionValue = float(info.get("minEmotionValue", 0)) self.maxEmotionValue = float(info.get("maxEmotionValue", 0)) + self["@type"] = "onyx:EmotionAnalysis" class ShelfMixin(object): @@ -145,6 +125,11 @@ class ShelfMixin(object): def sh(self): if os.path.isfile(self.shelf_file): os.remove(self.shelf_file) + self.close() + + def __del__(self): + self.close() + self.deactivate() @property def shelf_file(self): diff --git a/senpy/plugins/rand/rand.py b/senpy/plugins/rand/rand.py index 1cda512..d7220ee 100644 --- a/senpy/plugins/rand/rand.py +++ b/senpy/plugins/rand/rand.py @@ -2,7 +2,7 @@ import json import random from senpy.plugins import SentimentPlugin -from senpy.models import Response, Opinion, Entry +from senpy.models import Results, Sentiment, Entry class Sentiment140Plugin(SentimentPlugin): @@ -10,22 +10,33 @@ class Sentiment140Plugin(SentimentPlugin): lang = params.get("language", "auto") p = params.get("prefix", None) - response = Response(prefix=p) + response = Results(prefix=p) polarity_value = max(-1, min(1, random.gauss(0.2, 0.2))) polarity = "marl:Neutral" if polarity_value > 0: polarity = "marl:Positive" elif polarity_value < 0: polarity = "marl:Negative" - entry = Entry(id="Entry0", - text=params["input"], - prefix=p) - opinion = Opinion(id="Opinion0", - prefix=p, - hasPolarity=polarity, - polarityValue=polarity_value) - opinion["prov:wasGeneratedBy"] = self.id - entry.opinions.append(opinion) + entry = Entry({"id":":Entry0", + "nif:isString": params["input"]}) + sentiment = Sentiment({"id": ":Sentiment0", + "marl:hasPolarity": polarity, + "marl:polarityValue": polarity_value}) + sentiment["prov:wasGeneratedBy"] = self.id + entry.sentiments = [] + entry.sentiments.append(sentiment) entry.language = lang response.entries.append(entry) return response + + + + + + + + + + + + diff --git a/senpy/plugins/sentiment140/sentiment140.py b/senpy/plugins/sentiment140/sentiment140.py index a0431fc..ad9704a 100644 --- a/senpy/plugins/sentiment140/sentiment140.py +++ b/senpy/plugins/sentiment140/sentiment140.py @@ -2,7 +2,7 @@ import requests import json from senpy.plugins import SentimentPlugin -from senpy.models import Response, Opinion, Entry +from senpy.models import Results, Sentiment, Entry class Sentiment140Plugin(SentimentPlugin): @@ -16,7 +16,7 @@ class Sentiment140Plugin(SentimentPlugin): ) p = params.get("prefix", None) - response = Response(prefix=p) + response = Results(prefix=p) polarity_value = self.maxPolarityValue*int(res.json()["data"][0] ["polarity"]) * 0.25 polarity = "marl:Neutral" @@ -25,15 +25,16 @@ class Sentiment140Plugin(SentimentPlugin): polarity = "marl:Positive" elif polarity_value < neutral_value: polarity = "marl:Negative" + entry = Entry(id="Entry0", - text=params["input"], - prefix=p) - opinion = Opinion(id="Opinion0", - prefix=p, - hasPolarity=polarity, - polarityValue=polarity_value) - opinion["prov:wasGeneratedBy"] = self.id - entry.opinions.append(opinion) + nif__isString=params["input"]) + sentiment = Sentiment(id="Sentiment0", + prefix=p, + marl__hasPolarity=polarity, + marl__polarityValue=polarity_value) + sentiment.prov__wasGeneratedBy = self.id + entry.sentiments = [] + entry.sentiments.append(sentiment) entry.language = lang response.entries.append(entry) return response diff --git a/senpy/schemas/analysis.json b/senpy/schemas/analysis.json new file mode 100644 index 0000000..602ae27 --- /dev/null +++ b/senpy/schemas/analysis.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "definitions.json#/Analysis" +} diff --git a/senpy/schemas/context.jsonld b/senpy/schemas/context.jsonld new file mode 100644 index 0000000..220e6c6 --- /dev/null +++ b/senpy/schemas/context.jsonld @@ -0,0 +1,33 @@ +{ + "@vocab": "http://www.gsi.dit.upm.es/ontologies/senpy#", + "dc": "http://dublincore.org/2012/06/14/dcelements#", + "me": "http://www.mixedemotions-project.eu/ns/model#", + "prov": "http://www.w3.org/ns/prov#", + "nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#", + "marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#", + "onyx": "http://www.gsi.dit.upm.es/ontologies/onyx#", + "wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "topics": { + "@id": "dc:subject" + }, + "entities": { + "@id": "me:hasEntities" + }, + "suggestions": { + "@id": "me:hasSuggestions" + }, + "emotions": { + "@id": "onyx:hasEmotionSet" + }, + "sentiments": { + "@id": "marl:hasOpinion" + }, + "entries": { + "@id": "prov:used" + }, + "analysis": { + "@id": "prov:wasGeneratedBy" + +} +} diff --git a/senpy/schemas/definitions.json b/senpy/schemas/definitions.json new file mode 100644 index 0000000..4442de4 --- /dev/null +++ b/senpy/schemas/definitions.json @@ -0,0 +1,161 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "Results": { + "title": "Results", + "description": "The results of an analysis", + "type": "object", + "properties": { + "@context": { + "$ref": "#/Context" + }, + "@id": { + "description": "ID of the analysis", + "type": "string" + }, + "analysis": { + "type": "array", + "default": [], + "items": { + "$ref": "#/Analysis" + } + }, + "entries": { + "type": "array", + "default": [], + "items": { + "$ref": "#/Entry" + } + } + + }, + "required": ["@id", "analysis", "entries"] + }, + "Context": { + "description": "JSON-LD Context", + "type": ["array", "string", "object"] + }, + "Analysis": { + "description": "Senpy analysis", + "type": "object", + "properties": { + "@id": { + "type": "string" + }, + "@type": { + "type": "string", + "description": "Type of the analysis. e.g. marl:SentimentAnalysis" + } + }, + "required": ["@id", "@type"] + }, + "Entry": { + "properties": { + "@id": { + "type": "string" + }, + "@type": { + "enum": [["nif:RFC5147String", "nif:Context"]] + }, + "nif:isString": { + "description": "String contained in this Context", + "type": "string" + }, + "sentiments": { + "type": "array", + "items": {"$ref": "#/Sentiment" } + }, + "emotions": { + "type": "array", + "items": {"$ref": "#/EmotionSet" } + }, + "entities": { + "type": "array", + "items": {"$ref": "#/Entity" } + }, + "topics": { + "type": "array", + "items": {"$ref": "#/Topic" } + }, + "suggestions": { + "type": "array", + "items": {"$ref": "#/Suggestion" } + } + }, + "required": ["@id", "nif:isString"] + }, + "Sentiment": { + "properties": { + "@id": {"type": "string"}, + "nif:beginIndex": {"type": "integer"}, + "nif:endIndex": {"type": "integer"}, + "nif:anchorOf": { + "description": "Piece of context that contains the Sentiment", + "type": "string" + }, + "marl:hasPolarity": { + "enum": ["marl:Positive", "marl:Negative", "marl:Neutral"] + }, + "marl:polarityValue": { + "type": "number" + }, + "prov:wasGeneratedBy": { + "type": "string", + "description": "The ID of the analysis that generated this Sentiment. The full object should be included in the \"analysis\" property of the root object" + } + }, + "required": ["@id", "prov:wasGeneratedBy"] + }, + "EmotionSet": { + "properties": { + "@id": {"type": "string"}, + "nif:beginIndex": {"type": "integer"}, + "nif:endIndex": {"type": "integer"}, + "nif:anchorOf": { + "description": "Piece of context that contains the Sentiment", + "type": "string" + }, + "onyx:hasEmotion": { + "$ref": "#/Emotion" + }, + "prov:wasGeneratedBy": { + "type": "string", + "description": "The ID of the analysis that generated this Emotion. The full object should be included in the \"analysis\" property of the root object" + } + }, + "required": ["@id", "prov:wasGeneratedBy", "onyx:hasEmotion"] + }, + "Emotion": { + "type": "object" + }, + "Entity": { + "type": "object" + }, + "Topic": { + "type": "object" + }, + "Suggestion": { + "type": "object" + }, + "Plugins": { + "properties": { + "plugins": { + "type": "array", + "items": { + "$ref": "#/Plugin" + } + } + } + }, + "Plugin": { + "type": "object", + "required": ["@id"], + "properties": { + "@id": { + "type": "string" + } + } + }, + "Response": { + "type": "object" + } +} diff --git a/senpy/schemas/emotion.json b/senpy/schemas/emotion.json new file mode 100644 index 0000000..a1aa294 --- /dev/null +++ b/senpy/schemas/emotion.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "definitions.json#/Emotion" +} diff --git a/senpy/schemas/emotionSet.json b/senpy/schemas/emotionSet.json new file mode 100644 index 0000000..ad06f7d --- /dev/null +++ b/senpy/schemas/emotionSet.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "definitions.json#/EmotionSet" +} diff --git a/senpy/schemas/entry.json b/senpy/schemas/entry.json new file mode 100644 index 0000000..68b2c2d --- /dev/null +++ b/senpy/schemas/entry.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "definitions.json#/Entry" +} diff --git a/senpy/schemas/plugin.json b/senpy/schemas/plugin.json new file mode 100644 index 0000000..6dc2e66 --- /dev/null +++ b/senpy/schemas/plugin.json @@ -0,0 +1,3 @@ +{ + "$ref": "definitions.json#/Plugin" +} diff --git a/senpy/schemas/plugins.json b/senpy/schemas/plugins.json new file mode 100644 index 0000000..1f6fefa --- /dev/null +++ b/senpy/schemas/plugins.json @@ -0,0 +1,3 @@ +{ + "$ref": "definitions.json#/Plugins" +} diff --git a/senpy/schemas/response.json b/senpy/schemas/response.json new file mode 100644 index 0000000..4e2d53c --- /dev/null +++ b/senpy/schemas/response.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "definitions.json#/Response" +} diff --git a/senpy/schemas/results.json b/senpy/schemas/results.json new file mode 100644 index 0000000..a470c16 --- /dev/null +++ b/senpy/schemas/results.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "definitions.json#/Results" +} diff --git a/senpy/schemas/sentiment.json b/senpy/schemas/sentiment.json new file mode 100644 index 0000000..957b509 --- /dev/null +++ b/senpy/schemas/sentiment.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "definitions.json#/Sentiment" +} diff --git a/senpy/schemas/suggestion.json b/senpy/schemas/suggestion.json new file mode 100644 index 0000000..251ba10 --- /dev/null +++ b/senpy/schemas/suggestion.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "definitions.json#/Suggestion" +} diff --git a/setup.cfg b/setup.cfg index 5aef279..91932ab 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,4 @@ [metadata] description-file = README.rst +[aliases] +test=pytest diff --git a/setup.py b/setup.py index ed022b8..11431d6 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ except AttributeError: install_reqs = [str(ir.req) for ir in install_reqs] test_reqs = [str(ir.req) for ir in test_reqs] -VERSION = "0.4.11" +VERSION = "0.5" setup( name='senpy', @@ -34,7 +34,7 @@ extendable, so new algorithms and sources can be used. classifiers=[], install_requires=install_reqs, tests_require=test_reqs, - test_suite="nose.collector", + setup_requires=['pytest-runner',], include_package_data=True, entry_points={ 'console_scripts': [ diff --git a/test-requirements.txt b/test-requirements.txt index c5c60b7..625bed6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,2 @@ -nose +pytest mock -pbr diff --git a/tests/context.jsonld b/tests/context.jsonld deleted file mode 100644 index 936db98..0000000 --- a/tests/context.jsonld +++ /dev/null @@ -1,40 +0,0 @@ -{ - "dc": "http://purl.org/dc/terms/", - "dc:subject": { - "@type": "@id" - }, - "xsd": "http://www.w3.org/2001/XMLSchema#", - "marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#", - "nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#", - "onyx": "http://www.gsi.dit.upm.es/ontologies/onyx/ns#", - "emotions": { - "@container": "@set", - "@id": "onyx:hasEmotionSet" - }, - "opinions": { - "@container": "@set", - "@id": "marl:hasOpinion" - }, - "prov": "http://www.w3.org/ns/prov#", - "rdfs": "http://www.w3.org/2000/01/rdf-schema#", - "analysis": { - "@container": "@set", - "@id": "prov:wasInformedBy" - }, - "entries": { - "@container": "@set", - "@id": "prov:generated" - }, - "strings": { - "@container": "@set", - "@reverse": "nif:hasContext" - }, - "date": - { - "@id": "dc:date", - "@type": "xsd:dateTime" - }, - "text": { "@id": "nif:isString" }, - "wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#", - "xsd": "http://www.w3.org/2001/XMLSchema#" -} diff --git a/tests/dummy_plugin/dummy.py b/tests/dummy_plugin/dummy.py index 60b6e1c..bb39102 100644 --- a/tests/dummy_plugin/dummy.py +++ b/tests/dummy_plugin/dummy.py @@ -1,8 +1,8 @@ from senpy.plugins import SentimentPlugin -from senpy.models import Response +from senpy.models import Results class DummyPlugin(SentimentPlugin): def analyse(self, *args, **kwargs): - return Response() + return Results() diff --git a/tests/models_test/__init__.py b/tests/models_test/__init__.py deleted file mode 100644 index 69d34c1..0000000 --- a/tests/models_test/__init__.py +++ /dev/null @@ -1,81 +0,0 @@ -import os -import logging - -try: - import unittest.mock as mock -except ImportError: - import mock -import json -import os -from unittest import TestCase -from senpy.models import Response, Entry -from senpy.plugins import SenpyPlugin - - -class ModelsTest(TestCase): - - def test_response(self): - r = Response(context=os.path.normpath( - os.path.join(__file__, "..", "..", "context.jsonld"))) - assert("@context" in r) - assert(r._frame) - logging.debug("Default frame: %s", r._frame) - assert("marl" in r.context) - assert("entries" in r.context) - - r2 = Response(context=json.loads('{"test": "roger"}')) - assert("test" in r2.context) - - r3 = Response(context=None) - del r3.context - assert("@context" not in r3) - assert("entries" in r3) - assert("analysis" in r3) - - r4 = Response() - assert("@context" in r4) - assert("entries" in r4) - assert("analysis" in r4) - - dummy = SenpyPlugin({"name": "dummy", "version": 0}) - r5 = Response({"dummy": dummy}, context=None, frame=None) - logging.debug("Response 5: %s", r5) - assert("dummy" in r5) - assert(r5["dummy"].name == "dummy") - js = r5.jsonld(context={}, frame={}) - logging.debug("jsonld 5: %s", js) - assert("dummy" in js) - assert(js["dummy"].name == "dummy") - - r6 = Response() - r6.entries.append(Entry(text="Just testing")) - logging.debug("Reponse 6: %s", r6) - assert("@context" in r6) - assert("marl" in r6.context) - assert("entries" in r6.context) - js = r6.jsonld() - logging.debug("jsonld: %s", js) - assert("entries" in js) - assert("entries" in js) - assert("analysis" in js) - resp = r6.flask() - received = json.loads(resp.data.decode()) - logging.debug("Response: %s", js) - assert(received["entries"]) - assert(received["entries"][0]["text"] == "Just testing") - assert(received["entries"][0]["text"] != "Not testing") - - def test_opinions(self): - pass - - def test_plugins(self): - p = SenpyPlugin({"name": "dummy", "version": 0}) - c = p.jsonld() - assert "info" not in c - assert "repo" not in c - assert "params" not in c - logging.debug("Framed: %s", c) - assert "extra_params" in c - - def test_frame_response(self): - pass diff --git a/tests/sleep_plugin/sleep.py b/tests/sleep_plugin/sleep.py index 6468794..d8e0783 100644 --- a/tests/sleep_plugin/sleep.py +++ b/tests/sleep_plugin/sleep.py @@ -1,5 +1,5 @@ from senpy.plugins import SenpyPlugin -from senpy.models import Response +from senpy.models import Results from time import sleep @@ -14,4 +14,4 @@ class SleepPlugin(SenpyPlugin): def analyse(self, *args, **kwargs): sleep(float(kwargs.get("timeout", self.timeout))) - return Response() + return Results() diff --git a/tests/blueprints_test/__init__.py b/tests/test_blueprints.py similarity index 89% rename from tests/blueprints_test/__init__.py rename to tests/test_blueprints.py index 886e51a..306a26b 100644 --- a/tests/blueprints_test/__init__.py +++ b/tests/test_blueprints.py @@ -1,10 +1,6 @@ import os import logging -try: - import unittest.mock as mock -except ImportError: - import mock from senpy.extensions import Senpy from flask import Flask from flask.ext.testing import TestCase @@ -31,7 +27,7 @@ class BlueprintsTest(TestCase): """ Calling with no arguments should ask the user for more arguments """ - resp = self.client.get("/api") + resp = self.client.get("/api/") self.assert404(resp) logging.debug(resp.json) assert resp.json["status"] == 404 @@ -46,7 +42,7 @@ class BlueprintsTest(TestCase): The dummy plugin returns an empty response,\ it should contain the context """ - resp = self.client.get("/api?i=My aloha mohame") + resp = self.client.get("/api/?i=My aloha mohame") self.assert200(resp) logging.debug("Got response: %s", resp.json) assert "@context" in resp.json @@ -64,7 +60,7 @@ class BlueprintsTest(TestCase): assert "@context" in resp.json def test_headers(self): - for i, j in product(["/api/plugins/?nothing=", "/api?i=test&"], + for i, j in product(["/api/plugins/?nothing=", "/api/?i=test&"], ["headers", "inHeaders"]): resp = self.client.get("%s" % (i)) assert "@context" in resp.json @@ -77,7 +73,7 @@ class BlueprintsTest(TestCase): def test_detail(self): """ Show only one plugin""" - resp = self.client.get("/api/plugins/Dummy") + resp = self.client.get("/api/plugins/Dummy/") self.assert200(resp) logging.debug(resp.json) assert "@id" in resp.json @@ -88,14 +84,14 @@ class BlueprintsTest(TestCase): resp = self.client.get("/api/plugins/Dummy/deactivate") self.assert200(resp) sleep(0.5) - resp = self.client.get("/api/plugins/Dummy") + resp = self.client.get("/api/plugins/Dummy/") self.assert200(resp) assert "is_activated" in resp.json assert resp.json["is_activated"] == False resp = self.client.get("/api/plugins/Dummy/activate") self.assert200(resp) sleep(0.5) - resp = self.client.get("/api/plugins/Dummy") + resp = self.client.get("/api/plugins/Dummy/") self.assert200(resp) assert "is_activated" in resp.json assert resp.json["is_activated"] == True diff --git a/tests/extensions_test/__init__.py b/tests/test_extensions.py similarity index 93% rename from tests/extensions_test/__init__.py rename to tests/test_extensions.py index b85226f..5c3ca41 100644 --- a/tests/extensions_test/__init__.py +++ b/tests/test_extensions.py @@ -2,10 +2,6 @@ from __future__ import print_function import os import logging -try: - import unittest.mock as mock -except ImportError: - import mock from senpy.extensions import Senpy from flask import Flask from flask.ext.testing import TestCase @@ -15,7 +11,7 @@ class ExtensionsTest(TestCase): def create_app(self): self.app = Flask("test_extensions") - self.dir = os.path.join(os.path.dirname(__file__), "..") + self.dir = os.path.join(os.path.dirname(__file__)) self.senpy = Senpy(plugin_folder=self.dir, default_plugins=False) self.senpy.init_app(self.app) self.senpy.activate_plugin("Dummy", sync=True) @@ -60,7 +56,7 @@ class ExtensionsTest(TestCase): self.senpy.deactivate_all(sync=True) resp = self.senpy.analyse(input="tupni") logging.debug("Response: {}".format(resp)) - assert resp["status"] == 404 + assert resp.status == 404 def test_analyse(self): """ Using a plugin """ @@ -75,7 +71,7 @@ class ExtensionsTest(TestCase): self.senpy.deactivate_plugin(plug, sync=True) resp = self.senpy.analyse(input="tupni") logging.debug("Response: {}".format(resp)) - assert resp["status"] == 404 + assert resp.status == 404 def test_filtering(self): diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..3d709b1 --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,97 @@ +import os +import logging + +import jsonschema + +import json +import os +from unittest import TestCase +from senpy.models import Response, Entry, Results, Sentiment, EmotionSet, Error +from senpy.plugins import SenpyPlugin +from pprint import pprint + + +class ModelsTest(TestCase): + + def test_jsonld(self): + ctx = os.path.normpath(os.path.join(__file__, "..", "..", "..", "senpy", "schemas", "context.jsonld")) + prueba = {"@id": "test", + "analysis": [], + "entries": []} + r = Results(**prueba) + + print("Response's context: ") + pprint(r.context) + + j = r.jsonld(with_context=True) + print("As JSON:") + pprint(j) + assert("@context" in j) + assert("marl" in j["@context"]) + assert("entries" in j["@context"]) + + r6 = Results(**prueba) + r6.entries.append(Entry({"@id":"ohno", "nif:isString":"Just testing"})) + logging.debug("Reponse 6: %s", r6) + assert("marl" in r6.context) + assert("entries" in r6.context) + j6 = r6.jsonld(with_context=True) + logging.debug("jsonld: %s", j6) + assert("@context" in j6) + assert("entries" in j6) + assert("analysis" in j6) + resp = r6.flask() + received = json.loads(resp.data.decode()) + logging.debug("Response: %s", j6) + assert(received["entries"]) + assert(received["entries"][0]["nif:isString"] == "Just testing") + assert(received["entries"][0]["nif:isString"] != "Not testing") + + def test_entries(self): + e = Entry() + self.assertRaises(jsonschema.ValidationError, e.validate) + e.nif__isString = "this is a test" + e.nif__beginIndex = 0 + e.nif__endIndex = 10 + e.validate() + + def test_sentiment(self): + s = Sentiment() + self.assertRaises(jsonschema.ValidationError, s.validate) + s.nif__anchorOf = "so much testing" + s.prov__wasGeneratedBy = "" + s.validate() + + def test_emotion_set(self): + e = EmotionSet() + self.assertRaises(jsonschema.ValidationError, e.validate) + e.nif__anchorOf = "so much testing" + e.prov__wasGeneratedBy = "" + self.assertRaises(jsonschema.ValidationError, e.validate) + e.onyx__hasEmotion = {} + e.validate() + + def test_results(self): + r = Results() + e = Entry() + e.nif__isString = "Results test" + r.entries.append(e) + r.id = ":test_results" + r.validate() + + def test_sentiments(self): + pass + + def test_plugins(self): + self.assertRaises(Error, SenpyPlugin) + p = SenpyPlugin({"name": "dummy", "version": 0}) + c = p.jsonld() + assert "info" not in c + assert "repo" not in c + assert "params" in c + logging.debug("Framed:") + logging.debug(c) + p.validate() + + def test_frame_response(self): + pass diff --git a/tests/plugins_test/__init__.py b/tests/test_plugins.py similarity index 84% rename from tests/plugins_test/__init__.py rename to tests/test_plugins.py index 2c179a3..e66148c 100644 --- a/tests/plugins_test/__init__.py +++ b/tests/test_plugins.py @@ -1,17 +1,15 @@ -#!/bin/env python2 -# -*- py-which-shell: "python2"; -*- +#!/bin/env python + import os import logging import shelve +import shutil +import tempfile -try: - import unittest.mock as mock -except ImportError: - import mock import json import os from unittest import TestCase -from senpy.models import Response, Entry +from senpy.models import Results, Entry from senpy.plugins import SenpyPlugin, ShelfMixin @@ -27,14 +25,18 @@ class ShelfTest(ShelfMixin, SenpyPlugin): class ModelsTest(TestCase): - shelf_file = 'shelf_test.db' def tearDown(self): + if os.path.exists(self.shelf_dir): + shutil.rmtree(self.shelf_dir) + if os.path.isfile(self.shelf_file): os.remove(self.shelf_file) - setUp = tearDown + def setUp(self): + self.shelf_dir = tempfile.mkdtemp() + self.shelf_file = os.path.join(self.shelf_dir, "shelf") def test_shelf(self): ''' A shelf is created and the value is stored ''' @@ -45,11 +47,10 @@ class ModelsTest(TestCase): assert a.shelf_file == self.shelf_file a.sh['a'] = 'fromA' - a.test(key='a', value='fromA') - del(a) - assert os.path.isfile(self.shelf_file) + sh = shelve.open(self.shelf_file) + assert sh['a'] == 'fromA'