From 79c83e34a3cf5cdf62388c6fa4fdb134a73bd5e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Mon, 23 Feb 2015 02:13:31 +0100 Subject: [PATCH] Added random plugin and other features --- plugins/rand/rand.py | 32 +++++ plugins/rand/rand.senpy | 17 +++ plugins/sentiment140/sentiment140.py | 10 +- plugins/sentiment140/sentiment140.senpy | 4 +- requirements.txt | 3 +- senpy/__main__.py | 3 +- senpy/blueprints.py | 9 +- senpy/extensions.py | 13 +- senpy/models.py | 179 +++++++++++++++++++----- senpy/plugins.py | 64 +++------ setup.py | 2 +- tests/blueprints_test/__init__.py | 1 - tests/context.jsonld | 40 ++++++ tests/dummy_plugin/dummy.py | 2 - tests/extensions_test/__init__.py | 11 +- tests/models_test/__init__.py | 36 +++++ tests/sleep_plugin/sleep.py | 4 + 17 files changed, 329 insertions(+), 101 deletions(-) create mode 100644 plugins/rand/rand.py create mode 100644 plugins/rand/rand.senpy create mode 100644 tests/context.jsonld create mode 100644 tests/models_test/__init__.py diff --git a/plugins/rand/rand.py b/plugins/rand/rand.py new file mode 100644 index 0000000..43341a8 --- /dev/null +++ b/plugins/rand/rand.py @@ -0,0 +1,32 @@ +import json +import random + +from senpy.plugins import SentimentPlugin +from senpy.models import Response, Opinion, Entry + + +class Sentiment140Plugin(SentimentPlugin): + def analyse(self, **params): + lang = params.get("language", "auto") + + p = params.get("prefix", None) + response = Response(prefix=p) + #polarity_value = self.maxPolarityValue*int(res.json()["data"][0]["polarity"]) * 0.25 + polarity_value = max(-1, min(1, random.gauss(0.2,0.2))) + polarity = "marl:Neutral" + if polarity_value > 0: + polarity = "marl:Positive" + elif polarity_value < 0: + polarity = "marl:Negative" + entry = Entry(id="Entry0", + text=params["input"], + prefix=p) + opinion = Opinion(id="Opinion0", + prefix=p, + hasPolarity=polarity, + polarityValue=polarity_value) + opinion["prov:wasGeneratedBy"] = self.id + entry.opinions.append(opinion) + entry.language = lang + response.entries.append(entry) + return response diff --git a/plugins/rand/rand.senpy b/plugins/rand/rand.senpy new file mode 100644 index 0000000..4125694 --- /dev/null +++ b/plugins/rand/rand.senpy @@ -0,0 +1,17 @@ +{ + "name": "rand", + "module": "rand", + "description": "What my plugin broadly does", + "author": "@balkian", + "version": "0.1", + "extra_params": { + "language": { + "aliases": ["language", "l"], + "required": false, + "options": ["es", "en", "auto"] + } + }, + "requirements": {}, + "marl:maxPolarityValue": "1", + "marl:minPolarityValue": "-1" +} diff --git a/plugins/sentiment140/sentiment140.py b/plugins/sentiment140/sentiment140.py index 6977eb1..237a804 100644 --- a/plugins/sentiment140/sentiment140.py +++ b/plugins/sentiment140/sentiment140.py @@ -15,15 +15,19 @@ class Sentiment140Plugin(SentimentPlugin): ) ) - response = Response(base=params.get("prefix", None)) - polarity_value = int(res.json()["data"][0]["polarity"]) * 25 + p = params.get("prefix", None) + response = Response(prefix=p) + polarity_value = self.maxPolarityValue*int(res.json()["data"][0]["polarity"]) * 0.25 polarity = "marl:Neutral" if polarity_value > 50: polarity = "marl:Positive" elif polarity_value < 50: polarity = "marl:Negative" - entry = Entry(id="Entry0", text=params["input"]) + entry = Entry(id="Entry0", + text=params["input"], + prefix=p) opinion = Opinion(id="Opinion0", + prefix=p, hasPolarity=polarity, polarityValue=polarity_value) opinion["prov:wasGeneratedBy"] = self.id diff --git a/plugins/sentiment140/sentiment140.senpy b/plugins/sentiment140/sentiment140.senpy index b0901fe..453c23b 100644 --- a/plugins/sentiment140/sentiment140.senpy +++ b/plugins/sentiment140/sentiment140.senpy @@ -11,5 +11,7 @@ "options": ["es", "en", "auto"] } }, - "requirements": {} + "requirements": {}, + "maxPolarityValue": "1", + "minPolarityValue": "0" } diff --git a/requirements.txt b/requirements.txt index 533341b..81f1b06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ gunicorn==19.0.0 requests==2.4.1 GitPython==0.3.2.RC1 Yapsy>=1.10.423 -gevent>=1.0.1 \ No newline at end of file +gevent>=1.0.1 +PyLD>=0.6.5 \ No newline at end of file diff --git a/senpy/__main__.py b/senpy/__main__.py index d3b3fb9..961d9f9 100644 --- a/senpy/__main__.py +++ b/senpy/__main__.py @@ -38,8 +38,7 @@ if __name__ == '__main__': help='Logging level') parser.add_argument('--debug', "-d", - metavar="debug", - type=bool, + action='store_true', default=False, help='Run the application in debug mode') parser.add_argument('--host', diff --git a/senpy/blueprints.py b/senpy/blueprints.py index 4fd086b..833d4c9 100644 --- a/senpy/blueprints.py +++ b/senpy/blueprints.py @@ -62,7 +62,7 @@ def get_params(req, params=BASIC_PARAMS): "parameters": outdict, "errors": {param: error for param, error in wrong_params.iteritems()} } - raise ValueError(json.dumps(message)) + raise ValueError(message) return outdict @@ -88,13 +88,14 @@ def basic_analysis(params): @nif_blueprint.route('/', methods=['POST', 'GET']) def home(): try: - algo = get_params(request).get("algorithm", None) + params = get_params(request) + algo = params.get("algorithm", None) specific_params = current_app.senpy.parameters(algo) - params = get_params(request, specific_params) + params.update(get_params(request, specific_params)) response = current_app.senpy.analyse(**params) return jsonify(response) except ValueError as ex: - return ex.message + return jsonify(ex.message) except Exception as ex: return jsonify(status="400", message=ex.message) diff --git a/senpy/extensions.py b/senpy/extensions.py index 78a2dd4..2798831 100644 --- a/senpy/extensions.py +++ b/senpy/extensions.py @@ -12,6 +12,7 @@ import json logger = logging.getLogger(__name__) from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin +from .models import Error from .blueprints import nif_blueprint from git import Repo, InvalidGitRepositoryError @@ -70,12 +71,14 @@ class Senpy(object): if self.plugins[algo].is_activated: plug = self.plugins[algo] resp = plug.analyse(**params) - resp.analysis.append(plug.jsonable()) + resp.analysis.append(plug) return resp - logger.debug("Plugin not activated: {}".format(algo)) + else: + logger.debug("Plugin not activated: {}".format(algo)) + return Error(status=400, message="The algorithm '{}' is not activated yet".format(algo)) else: logger.debug("The algorithm '{}' is not valid\nValid algorithms: {}".format(algo, self.plugins.keys())) - return {"status": 400, "message": "The algorithm '{}' is not valid".format(algo)} + return Error(status=400, message="The algorithm '{}' is not valid".format(algo)) @property def default_plugin(self): @@ -156,9 +159,9 @@ class Senpy(object): module = candidate(info=info) try: repo_path = root - module.repo = Repo(repo_path) + module._repo = Repo(repo_path) except InvalidGitRepositoryError: - module.repo = None + module._repo = None except Exception as ex: logger.debug("Exception importing {}: {}".format(filename, ex)) return None, None diff --git a/senpy/models.py b/senpy/models.py index e210c84..4f43226 100644 --- a/senpy/models.py +++ b/senpy/models.py @@ -1,21 +1,41 @@ import json import os from collections import defaultdict +from pyld import jsonld -class Leaf(defaultdict): + +class Leaf(dict): _prefix = None + _frame = {} + _context = {} - def __init__(self, id=None, context=None, prefix=None, ofclass=list): - super(Leaf, self).__init__(ofclass) - if context: + def __init__(self, + id=None, + context=None, + vocab=None, + prefix=None, + frame=None): + super(Leaf, self).__init__() + if context is not None: self.context = context - if id: - self.id = id + elif self._context: + self.context = self._context + else: + self.context = {} + if frame is not None: + self._frame = frame self._prefix = prefix + self.id = id def __getattr__(self, key): - return super(Leaf, self).__getitem__(self._get_key(key)) + try: + return object.__getattr__(self, key) + except AttributeError: + try: + return super(Leaf, self).__getitem__(self._get_key(key)) + except KeyError: + raise AttributeError() def __setattr__(self, key, value): try: @@ -23,20 +43,40 @@ class Leaf(defaultdict): object.__setattr__(self, key, value) except AttributeError: key = self._get_key(key) - value = self.get_context(value) if key == "@context" else value + if key == "@context": + value = self.get_context(value) + elif key == "@id": + value = self.get_id(value) if key[0] == "_": object.__setattr__(self, key, value) else: - super(Leaf, self).__setitem__(key, value) + if value is None: + try: + super(Leaf, self).__delitem__(key) + except KeyError: + pass + else: + super(Leaf, self).__setitem__(key, value) + + def get_id(self, id): + """ + This is not the most elegant solution to change the @id attribute, but it + is the quickest way to have it included in the dictionary without extra + boilerplate. + """ + if id and self._prefix and ":" not in id: + return "{}{}".format(self._prefix, id) + else: + return id def __delattr__(self, key): return super(Leaf, self).__delitem__(self._get_key(key)) def _get_key(self, key): - if key in ["context", "id"]: + if key[0] == "_": + return key + elif key in ["context", "id"]: return "@{}".format(key) - elif self._prefix: - return "{}:{}".format(self._prefix, key) else: return key @@ -56,55 +96,128 @@ class Leaf(defaultdict): except IOError: return context + def compact(self): + return jsonld.compact(self, self.context) + + def frame(self, frame=None, options=None): + if frame is None: + frame = self._frame + if options is None: + options = {} + return jsonld.frame(self, frame, options) + + def jsonable(self, parameters=False, frame=None, options=None, context=None): + if frame is None: + frame = self._frame + if options is None: + options = {} + if context is None: + context = self._context + return jsonld.compact(jsonld.frame(self, frame, options), context) + #if parameters: + #resp["parameters"] = self.params + #elif self.extra_params: + #resp["extra_parameters"] = self.extra_params + #return resp + + + def to_JSON(self): + return json.dumps(self, + default=lambda o: o.__dict__, + sort_keys=True, indent=4) + + class Response(Leaf): - def __init__(self, context=None, base=None, *args, **kwargs): + _frame = { "@context": { + "analysis": { + "@container": "@set", + "@id": "prov:wasInformedBy" + }, + "date": { + "@id": "dc:date", + "@type": "xsd:dateTime" + }, + "dc": "http://purl.org/dc/terms/", + "dc:subject": { + "@type": "@id" + }, + "emotions": { + "@container": "@set", + "@id": "onyx:hasEmotionSet" + }, + "entries": { + "@container": "@set", + "@id": "prov:generated" + }, + "marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#", + "nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#", + "onyx": "http://www.gsi.dit.upm.es/ontologies/onyx/ns#", + "opinions": { + "@container": "@set", + "@id": "marl:hasOpinion" + }, + "prov": "http://www.w3.org/ns/prov#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "strings": { + "@container": "@set", + "@reverse": "nif:hasContext" + }, + "wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#", + "xsd": "http://www.w3.org/2001/XMLSchema#" + }, + "analysis": {}, + "entries": {} + } + + def __init__(self, context=None, *args, **kwargs): if context is None: context = "{}/context.jsonld".format(os.path.dirname( os.path.realpath(__file__))) super(Response, self).__init__(*args, context=context, **kwargs) - if base: - self.context["@base"] = base - self["analysis"] = [] - self["entries"] = [] + self.analysis = [] + self.entries = [] class Entry(Leaf): + _context = { + "@vocab": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#" + + } def __init__(self, text=None, emotion_sets=None, opinions=None, **kwargs): super(Entry, self).__init__(**kwargs) if text: self.text = text - if emotion_sets: - self.emotionSets = emotion_sets - if opinions: - self.opinions = opinions - + self.emotionSets = emotion_sets if emotion_sets else [] + self.opinions = opinions if opinions else [] class Opinion(Leaf): - #opinionContext = {"@vocab": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"} + _context = { + "@vocab": "http://www.gsi.dit.upm.es/ontologies/marl/ns#" + } def __init__(self, polarityValue=None, hasPolarity=None, *args, **kwargs): - super(Opinion, self).__init__( prefix="marl", - *args, + super(Opinion, self).__init__(*args, **kwargs) if polarityValue is not None: - self.polarityValue = polarityValue + self.hasPolarityValue = polarityValue if hasPolarity is not None: self.hasPolarity = hasPolarity class EmotionSet(Leaf): - emotionContext = {} + _context = {} def __init__(self, emotions=None, *args, **kwargs): if not emotions: emotions = [] - super(EmotionSet, self).__init__(context=EmotionSet.emotionContext, + super(EmotionSet, self).__init__(context=EmotionSet._context, *args, **kwargs) self.emotions = emotions or [] class Emotion(Leaf): - emotionContext = {} - def __init__(self, emotions=None, *args, **kwargs): - super(EmotionSet, self).__init__(context=Emotion.emotionContext, - *args, - **kwargs) + _context = {} + +class Error(Leaf): + def __init__(self, *args, **kwargs): + super(Error, self).__init__(*args) + self.update(kwargs) diff --git a/senpy/plugins.py b/senpy/plugins.py index b9c3251..a30838a 100644 --- a/senpy/plugins.py +++ b/senpy/plugins.py @@ -1,5 +1,6 @@ import logging import ConfigParser +from .models import Leaf logger = logging.getLogger(__name__) @@ -38,13 +39,24 @@ PARAMS = {"input": {"aliases": ["i", "input"], } -class SenpyPlugin(object): +class SenpyPlugin(Leaf): + _context = {"@vocab": "http://www.gsi.dit.upm.es/ontologies/senpy/ns#", + "info": None} + _frame = { "@context": _context, + "name": {}, + "@explicit": False, + "version": {}, + "repo": None, + "info": None, + } def __init__(self, info=None): if not info: raise ValueError("You need to provide configuration information for the plugin.") logger.debug("Initialising {}".format(info)) + super(SenpyPlugin, self).__init__() self.name = info["name"] self.version = info["version"] + self.id="{}_{}".format(self.name, self.version) self.params = info.get("params", PARAMS.copy()) self.extra_params = info.get("extra_params", {}) self.params.update(self.extra_params) @@ -66,49 +78,15 @@ class SenpyPlugin(object): def id(self): return "{}_{}".format(self.name, self.version) - def jsonable(self, parameters=False): - resp = { - "@id": "{}_{}".format(self.name, self.version), - "is_activated": self.is_activated, - } - if hasattr(self, "repo") and self.repo: - resp["repo"] = self.repo.remotes[0].url - if parameters: - resp["parameters"] = self.params - elif self.extra_params: - resp["extra_parameters"] = self.extra_params - return resp - class SentimentPlugin(SenpyPlugin): - def __init__(self, - min_polarity_value=0, - max_polarity_value=1, - **kwargs): - super(SentimentPlugin, self).__init__(**kwargs) - self.minPolarityValue = min_polarity_value - self.maxPolarityValue = max_polarity_value - - def jsonable(self, *args, **kwargs): - resp = super(SentimentPlugin, self).jsonable(*args, **kwargs) - resp["marl:maxPolarityValue"] = self.maxPolarityValue - resp["marl:minPolarityValue"] = self.minPolarityValue - return resp - + def __init__(self, info, *args, **kwargs): + super(SentimentPlugin, self).__init__(info, *args, **kwargs) + self.minPolarityValue = float(info.get("minPolarityValue", 0)) + self.maxPolarityValue = float(info.get("maxPolarityValue", 1)) class EmotionPlugin(SenpyPlugin): - def __init__(self, - min_emotion_value=0, - max_emotion_value=1, - emotion_category=None, - **kwargs): - super(EmotionPlugin, self).__init__(**kwargs) - self.minEmotionValue = min_emotion_value - self.maxEmotionValue = max_emotion_value - self.emotionCategory = emotion_category - - def jsonable(self, *args, **kwargs): - resp = super(EmotionPlugin, self).jsonable(*args, **kwargs) - resp["onyx:minEmotionValue"] = self.minEmotionValue - resp["onyx:maxEmotionValue"] = self.maxEmotionValue - return resp + def __init__(self, info, *args, **kwargs): + resp = super(EmotionPlugin, self).__init__(info, *args, **kwargs) + self.minEmotionValue = float(info.get("minEmotionValue", 0)) + self.maxEmotionValue = float(info.get("maxEmotionValue", 0)) diff --git a/setup.py b/setup.py index 29f1f8d..f5af8bb 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ install_reqs = parse_requirements("requirements.txt") # e.g. ['django==1.5.1', 'mezzanine==1.4.6'] reqs = [str(ir.req) for ir in install_reqs] -VERSION = "0.3.1" +VERSION = "0.3.2" print(reqs) diff --git a/tests/blueprints_test/__init__.py b/tests/blueprints_test/__init__.py index 49d9689..3dcc383 100644 --- a/tests/blueprints_test/__init__.py +++ b/tests/blueprints_test/__init__.py @@ -1,4 +1,3 @@ - import os import logging diff --git a/tests/context.jsonld b/tests/context.jsonld new file mode 100644 index 0000000..936db98 --- /dev/null +++ b/tests/context.jsonld @@ -0,0 +1,40 @@ +{ + "dc": "http://purl.org/dc/terms/", + "dc:subject": { + "@type": "@id" + }, + "xsd": "http://www.w3.org/2001/XMLSchema#", + "marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#", + "nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#", + "onyx": "http://www.gsi.dit.upm.es/ontologies/onyx/ns#", + "emotions": { + "@container": "@set", + "@id": "onyx:hasEmotionSet" + }, + "opinions": { + "@container": "@set", + "@id": "marl:hasOpinion" + }, + "prov": "http://www.w3.org/ns/prov#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "analysis": { + "@container": "@set", + "@id": "prov:wasInformedBy" + }, + "entries": { + "@container": "@set", + "@id": "prov:generated" + }, + "strings": { + "@container": "@set", + "@reverse": "nif:hasContext" + }, + "date": + { + "@id": "dc:date", + "@type": "xsd:dateTime" + }, + "text": { "@id": "nif:isString" }, + "wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#", + "xsd": "http://www.w3.org/2001/XMLSchema#" +} diff --git a/tests/dummy_plugin/dummy.py b/tests/dummy_plugin/dummy.py index 4a7419e..49cba59 100644 --- a/tests/dummy_plugin/dummy.py +++ b/tests/dummy_plugin/dummy.py @@ -2,7 +2,5 @@ from senpy.plugins import SentimentPlugin from senpy.models import Response class DummyPlugin(SentimentPlugin): - pass - def analyse(self, *args, **kwargs): return Response() \ No newline at end of file diff --git a/tests/extensions_test/__init__.py b/tests/extensions_test/__init__.py index 305f61b..fdcd2df 100644 --- a/tests/extensions_test/__init__.py +++ b/tests/extensions_test/__init__.py @@ -52,11 +52,12 @@ class ExtensionsTest(TestCase): def test_analyse(self): """ Using a plugin """ - with mock.patch.object(self.senpy.plugins["Dummy"], "analyse") as mocked: - self.senpy.analyse(algorithm="Dummy", input="tupni", output="tuptuo") - self.senpy.analyse(input="tupni", output="tuptuo") - mocked.assert_any_call(input="tupni", output="tuptuo", algorithm="Dummy") - mocked.assert_any_call(input="tupni", output="tuptuo") + # I was using mock until plugin started inheriting Leaf (defaultdict with + # __setattr__ and __getattr__. + r1 = self.senpy.analyse(algorithm="Dummy", input="tupni", output="tuptuo") + r2 = self.senpy.analyse(input="tupni", output="tuptuo") + assert r1.analysis[0].id[:5] == "Dummy" + assert r2.analysis[0].id[:5] == "Dummy" for plug in self.senpy.plugins: self.senpy.deactivate_plugin(plug, sync=True) resp = self.senpy.analyse(input="tupni") diff --git a/tests/models_test/__init__.py b/tests/models_test/__init__.py new file mode 100644 index 0000000..d52c1b5 --- /dev/null +++ b/tests/models_test/__init__.py @@ -0,0 +1,36 @@ +import os +import logging + +try: + import unittest.mock as mock +except ImportError: + import mock +import json +import os +from unittest import TestCase +from senpy.models import Response +from senpy.plugins import SenpyPlugin + +class ModelsTest(TestCase): + def test_response(self): + r = Response(context=os.path.normpath(os.path.join(__file__, "..", "..", "context.jsonld"))) + assert("@context" in r) + assert("marl" in r.context) + r2 = Response(context=json.loads('{"test": "roger"}')) + assert("test" in r2.context) + r3 = Response(context=None) + del r3.context + assert("@context" not in r3) + assert("entries" in r3) + assert("analysis" in r3) + + def test_opinions(self): + pass + + def test_frame_plugin(self): + p = SenpyPlugin({"name": "dummy", "version": 0}) + c = p.frame() + assert "info" not in c + + def test_frame_response(self): + pass diff --git a/tests/sleep_plugin/sleep.py b/tests/sleep_plugin/sleep.py index 30a0157..a848e36 100644 --- a/tests/sleep_plugin/sleep.py +++ b/tests/sleep_plugin/sleep.py @@ -1,4 +1,5 @@ from senpy.plugins import SenpyPlugin +from senpy.models import Response from time import sleep class SleepPlugin(SenpyPlugin): @@ -8,3 +9,6 @@ class SleepPlugin(SenpyPlugin): def activate(self, *args, **kwargs): sleep(self.timeout) + + def analyse(self, *args, **kwargs): + return Response()