From 9f6a6f5ecdd8922ecbbea706cc6e9300f053fe5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Mon, 27 Feb 2017 11:37:43 +0100 Subject: [PATCH] Loads of changes! * Added conversion plugins (API might change!) * Added conversion to the analysis pipeline * Changed behaviour of --default-plugins (it adds conversion plugins regardless) * Added emotionModel [sic] and emotionConversion models //TODO add conversion tests //TODO add conversion to docs --- .gitlab-ci.yml | 3 + Dockerfile-3.4 | 9 +- Dockerfile-3.5 | 2 +- Makefile | 24 ++- docs/examples/results/example-complete.json | 3 +- docs/examples/results/example-suggestion.json | 3 +- docs/plugins.rst | 138 ++++++++++++---- requirements.txt | 5 +- senpy/__init__.py | 19 +-- senpy/__main__.py | 5 +- senpy/api.py | 41 +++-- senpy/blueprints.py | 65 +++++--- senpy/client.py | 14 +- senpy/extensions.py | 153 +++++++++++++----- senpy/models.py | 152 ++++++++++------- senpy/plugins.py | 58 ++++--- senpy/plugins/conversion/emotion/ekman2vad.py | 56 +++++++ .../conversion/emotion/ekman2vad.senpy | 35 ++++ senpy/plugins/example/emoRand/emoRand.py | 18 +++ senpy/plugins/example/emoRand/emoRand.senpy | 9 ++ senpy/plugins/{ => example}/rand/rand.py | 13 +- senpy/plugins/example/rand/rand.senpy | 10 ++ senpy/plugins/rand/rand.senpy | 18 --- .../sentiment140/sentiment140.py | 17 +- .../sentiment/sentiment140/sentiment140.senpy | 21 +++ senpy/plugins/sentiment140/sentiment140.senpy | 18 --- "senpy/schemas/\\" | 7 - senpy/schemas/context.jsonld | 45 ++++-- senpy/schemas/emotionAnalysis.json | 5 +- senpy/schemas/emotionConversion.json | 12 ++ senpy/schemas/emotionConversionPlugin.json | 19 +++ senpy/schemas/emotionPlugin.json | 2 +- senpy/schemas/entry.json | 15 +- senpy/schemas/error.json | 12 +- senpy/schemas/sentimentPlugin.json | 2 +- senpy/schemas/suggestion.json | 3 +- senpy/static/css/main.css | 30 ++-- senpy/static/js/main.js | 57 ++++--- senpy/templates/index.html | 89 ++++++---- senpy/version.py | 21 +-- setup.cfg | 3 + setup.py | 4 +- tests/dummy_plugin/dummy.py | 7 - tests/dummy_plugin/dummy.senpy | 7 - tests/plugins/dummy_plugin/dummy.py | 7 + tests/plugins/dummy_plugin/dummy.senpy | 15 ++ .../plugins/dummy_plugin/dummy_required.senpy | 14 ++ tests/{ => plugins}/sleep_plugin/sleep.py | 7 +- tests/{ => plugins}/sleep_plugin/sleep.senpy | 0 tests/test_blueprints.py | 34 +++- tests/test_client.py | 17 +- tests/test_extensions.py | 29 ++-- tests/test_models.py | 56 +++++-- tests/test_plugins.py | 17 ++ version.sh | 2 + 55 files changed, 986 insertions(+), 461 deletions(-) create mode 100644 senpy/plugins/conversion/emotion/ekman2vad.py create mode 100644 senpy/plugins/conversion/emotion/ekman2vad.senpy create mode 100644 senpy/plugins/example/emoRand/emoRand.py create mode 100644 senpy/plugins/example/emoRand/emoRand.senpy rename senpy/plugins/{ => example}/rand/rand.py (62%) create mode 100644 senpy/plugins/example/rand/rand.senpy delete mode 100644 senpy/plugins/rand/rand.senpy rename senpy/plugins/{ => sentiment}/sentiment140/sentiment140.py (69%) create mode 100644 senpy/plugins/sentiment/sentiment140/sentiment140.senpy delete mode 100644 senpy/plugins/sentiment140/sentiment140.senpy delete mode 100644 "senpy/schemas/\\" create mode 100644 senpy/schemas/emotionConversion.json create mode 100644 senpy/schemas/emotionConversionPlugin.json delete mode 100644 tests/dummy_plugin/dummy.py delete mode 100644 tests/dummy_plugin/dummy.senpy create mode 100644 tests/plugins/dummy_plugin/dummy.py create mode 100644 tests/plugins/dummy_plugin/dummy.senpy create mode 100644 tests/plugins/dummy_plugin/dummy_required.senpy rename tests/{ => plugins}/sleep_plugin/sleep.py (51%) rename tests/{ => plugins}/sleep_plugin/sleep.senpy (100%) create mode 100755 version.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2b14f37..7bfe306 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,6 +7,9 @@ variables: DOCKER_DRIVER: overlay DOCKERFILE: Dockerfile +before_script: + - sh version.sh > senpy/VERSION + stages: - test - images diff --git a/Dockerfile-3.4 b/Dockerfile-3.4 index c631112..118230d 100644 --- a/Dockerfile-3.4 +++ b/Dockerfile-3.4 @@ -1,10 +1,13 @@ -from python:3.4-slim +from python:3.4 + +RUN mkdir /cache/ +ENV PIP_CACHE_DIR=/cache/ WORKDIR /usr/src/app ADD requirements.txt /usr/src/app/ RUN pip install --use-wheel -r requirements.txt ADD . /usr/src/app/ -RUN pip install --use-wheel . +RUN pip install . VOLUME /data/ @@ -13,6 +16,6 @@ RUN mkdir /senpy-plugins/ WORKDIR /senpy-plugins/ ONBUILD ADD . /senpy-plugins/ -ONBUILD RUN python -m senpy -f /senpy-plugins +ONBUILD RUN python -m senpy --only-install -f /senpy-plugins ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"] \ No newline at end of file diff --git a/Dockerfile-3.5 b/Dockerfile-3.5 index 1899c5b..314cc0d 100644 --- a/Dockerfile-3.5 +++ b/Dockerfile-3.5 @@ -1,4 +1,4 @@ -from python:3.5 +FROM python:3.5 RUN mkdir /cache/ ENV PIP_CACHE_DIR=/cache/ diff --git a/Makefile b/Makefile index fb6e75d..7b6076b 100644 --- a/Makefile +++ b/Makefile @@ -2,13 +2,19 @@ PYVERSIONS=3.5 3.4 2.7 PYMAIN=$(firstword $(PYVERSIONS)) NAME=senpy REPO=gsiupm -VERSION=$(shell git describe --tags) -TARNAME=$(NAME)-$(subst -,.,$(VERSION)).tar.gz +VERSION=$(shell ./version.sh) +TARNAME=$(NAME)-$(VERSION).tar.gz IMAGENAME=$(REPO)/$(NAME):$(VERSION) TEST_COMMAND=gitlab-runner exec docker --cache-dir=/tmp/gitlabrunner --docker-volumes /tmp/gitlabrunner:/tmp/gitlabrunner --env CI_PROJECT_NAME=$(NAME) all: build run +FORCE: + +version: FORCE + @echo $(VERSION) > $(NAME)/VERSION + @echo $(NAME) $(VERSION) + yapf: yapf -i -r senpy yapf -i -r tests @@ -36,7 +42,13 @@ quick_test: $(addprefix test-,$(PYMAIN)) test: $(addprefix test-,$(PYVERSIONS)) debug-%: - (docker start $(NAME)-debug && docker attach $(NAME)-debug) || docker run -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti --name $(NAME)-debug '$(IMAGENAME)-python$* pip install -r test-requirements.txt' + @docker start $(NAME)-debug || (\ + $(MAKE) build-$*; \ + docker run -d -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -p 5000:5000 -ti --name $(NAME)-debug '$(IMAGENAME)-python$*'; \ + docker exec -ti $(NAME)-debug pip install -r test-requirements.txt; \ + )\ + + docker attach $(NAME)-debug debug: debug-$(PYMAIN) @@ -77,7 +89,9 @@ pip_upload: pip_test: $(addprefix pip_test-,$(PYVERSIONS)) -run: build - docker run --rm -p 5000:5000 -ti '$(IMAGENAME)-python$(PYMAIN)' +run-%: build-% + docker run --rm -p 5000:5000 -ti '$(IMAGENAME)-python$(PYMAIN)' --default-plugins + +run: run-$(PYMAIN) .PHONY: test test-% build-% build test pip_test run yapf dev diff --git a/docs/examples/results/example-complete.json b/docs/examples/results/example-complete.json index 53d105b..d735e6e 100644 --- a/docs/examples/results/example-complete.json +++ b/docs/examples/results/example-complete.json @@ -53,7 +53,8 @@ "@id": "http://micro.blog/status1#char=16,77", "nif:beginIndex": 16, "nif:endIndex": 77, - "nif:anchorOf": "put your Windows Phone on your newest #open technology program" + "nif:anchorOf": "put your Windows Phone on your newest #open technology program", + "prov:wasGeneratedBy": "me:SgAnalysis1" } ], "sentiments": [ diff --git a/docs/examples/results/example-suggestion.json b/docs/examples/results/example-suggestion.json index 2ba943a..be585bb 100644 --- a/docs/examples/results/example-suggestion.json +++ b/docs/examples/results/example-suggestion.json @@ -24,7 +24,8 @@ "@id": "http://micro.blog/status1#char=16,77", "nif:beginIndex": 16, "nif:endIndex": 77, - "nif:anchorOf": "put your Windows Phone on your newest #open technology program" + "nif:anchorOf": "put your Windows Phone on your newest #open technology program", + "prov:wasGeneratedBy": "me:SgAnalysis1" } ], "sentiments": [ diff --git a/docs/plugins.rst b/docs/plugins.rst index ec04a3b..91308f5 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -2,46 +2,127 @@ Developing new plugins ---------------------- Each plugin represents a different analysis process.There are two types of files that are needed by senpy for loading a plugin: -Plugins Interface -======= - Definition file, has the ".senpy" extension. - Code file, is a python file. +This separation will allow us to deploy plugins that use the same code but employ different parameters. +For instance, one could use the same classifier and processing in several plugins, but train with different datasets. +This scenario is particularly useful for evaluation purposes. + +The only limitation is that the name of each plugin needs to be unique. + Plugins Definitions =================== -The definition file can be written in JSON or YAML, where the data representation consists on attribute-value pairs. -The principal attributes are: +The definition file contains all the attributes of the plugin, and can be written in YAML or JSON. +The most important attributes are: -* name: plugin name used in senpy to call the plugin. -* module: indicates the module that will be loaded +* **name**: unique name that senpy will use internally to identify the plugin. +* **module**: indicates the module that contains the plugin code, which will be automatically loaded by senpy. +* **version** +* extra_params: used to specify parameters that the plugin accepts that are not already part of the senpy API. Those parameters may be required, and have aliased names. For instance: -.. code:: python + .. code:: yaml + + extra_params: + hello_param: + aliases: # required + - hello_param + - hello + required: true + default: Hi you + values: + - Hi you + - Hello y'all + - Howdy + + Parameter validation will fail if a required parameter without a default has not been provided, or if the definition includes a set of values and the provided one does not match one of them. + + +A complete example: + +.. code:: yaml + + name: + module: + version: 0.1 + +And the json equivalent: + +.. code:: json { - "name" : "senpyPlugin", - "module" : "{python code file}" + "name": "", + "module": "", + "version": "0.1" } -.. code:: python - - name: senpyPlugin - module: {python code file} Plugins Code -================= +============ The basic methods in a plugin are: * __init__ * activate: used to load memory-hungry resources * deactivate: used to free up resources -* analyse: called in every user requests. It takes in the parameters supplied by a user and should return a senpy Response. +* analyse_entry: called in every user requests. It takes in the parameters supplied by a user and should yield one or more ``Entry`` objects. Plugins are loaded asynchronously, so don't worry if the activate method takes too long. The plugin will be marked as activated once it is finished executing the method. + +Example plugin +============== + +In this section, we will implement a basic sentiment analysis plugin. +To determine the polarity of each entry, the plugin will compare the length of the string to a threshold. +This threshold will be included in the definition file. + +The definition file would look like this: + +.. code:: yaml + + name: helloworld + module: helloworld + version: 0.0 + threshold: 10 + + +Now, in a file named ``helloworld.py``: + +.. code:: python + + #!/bin/env python + #helloworld.py + + from senpy.plugins import SenpyPlugin + from senpy.models import Sentiment + + + class HelloWorld(SenpyPlugin): + + def analyse_entry(entry, params): + '''Basically do nothing with each entry''' + + sentiment = Sentiment() + if len(entry.text) < self.threshold: + sentiment['marl:hasPolarity'] = 'marl:Positive' + else: + sentiment['marl:hasPolarity'] = 'marl:Negative' + entry.sentiments.append(sentiment) + yield entry + + F.A.Q. ====== +Why does the analyse function yield instead of return? +?????????????????????????????????????????????????????? + +This is so that plugins may add new entries to the response or filter some of them. +For instance, a `context detection` plugin may add a new entry for each context in the original entry. +On the other hand, a conveersion plugin may leave out those entries that do not contain relevant information. + + If I'm using a classifier, where should I train it? ??????????????????????????????????????????????????? @@ -78,17 +159,17 @@ This example ilustrate how to implement the Sentiment140 service as a plugin in .. code:: python class Sentiment140Plugin(SentimentPlugin): - def analyse(self, **params): + def analyse_entry(self, entry, params): + text = entry.text lang = params.get("language", "auto") res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson", json.dumps({"language": lang, - "data": [{"text": params["input"]}] + "data": [{"text": text}] } ) ) p = params.get("prefix", None) - response = Results(prefix=p) polarity_value = self.maxPolarityValue*int(res.json()["data"][0] ["polarity"]) * 0.25 polarity = "marl:Neutral" @@ -98,18 +179,13 @@ This example ilustrate how to implement the Sentiment140 service as a plugin in elif polarity_value < neutral_value: polarity = "marl:Negative" - entry = Entry(id="Entry0", - nif__isString=params["input"]) sentiment = Sentiment(id="Sentiment0", prefix=p, marl__hasPolarity=polarity, marl__polarityValue=polarity_value) sentiment.prov__wasGeneratedBy = self.id - entry.sentiments = [] entry.sentiments.append(sentiment) - entry.language = lang - response.entries.append(entry) - return response + yield entry Where can I define extra parameters to be introduced in the request to my plugin? @@ -143,9 +219,9 @@ The extraction of this paremeter is used in the analyse method of the Plugin int Where can I set up variables for using them in my plugin? ????????????????????????????????????????????????????????? -You can add these variables in the definition file with the extracture of attribute-value pair. +You can add these variables in the definition file with the structure of attribute-value pairs. -Once you have added your variables, the next step is to extract them into the plugin. The plugin's __init__ method has a parameter called `info` where you can extract the values of the variables. This info parameter has the structure of a python dictionary. +Every field added to the definition file is available to the plugin instance. Can I activate a DEBUG mode for my plugin? ??????????????????????????????????????????? @@ -154,7 +230,15 @@ You can activate the DEBUG mode by the command-line tool using the option -d. .. code:: bash - python -m senpy -d + senpy -d + + +Additionally, with the ``--pdb`` option you will be dropped into a pdb post mortem shell if an exception is raised. + +.. code:: bash + + senpy --pdb + Where can I find more code examples? ???????????????????????????????????? diff --git a/requirements.txt b/requirements.txt index f8401eb..0dcc894 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,5 @@ Flask>=0.10.1 -gunicorn>=19.0.0 requests>=2.4.1 -GitPython>=0.3.2.RC1 gevent>=1.1rc4 PyLD>=0.6.5 six @@ -9,4 +7,5 @@ future jsonschema jsonref PyYAML -semver +rdflib +rdflib-jsonld diff --git a/senpy/__init__.py b/senpy/__init__.py index c2489c7..c9cfdf0 100644 --- a/senpy/__init__.py +++ b/senpy/__init__.py @@ -20,21 +20,10 @@ Sentiment analysis server in Python from __future__ import print_function from .version import __version__ -try: - import semver - __version_info__ = semver.parse_version_info(__version__) +import logging - if __version_info__.prerelease: - import logging - logger = logging.getLogger(__name__) - msg = 'WARNING: You are using a pre-release version of {} ({})'.format( - __name__, __version__) - if len(logging.root.handlers) > 0: - logger.info(msg) - else: - import sys - print(msg, file=sys.stderr) -except ImportError: - print('semver not installed. Not doing version checking') +logger = logging.getLogger(__name__) + +logger.info('Using senpy version: {}'.format(__version__)) __all__ = ['api', 'blueprints', 'cli', 'extensions', 'models', 'plugins'] diff --git a/senpy/__main__.py b/senpy/__main__.py index 72cec87..932f57c 100644 --- a/senpy/__main__.py +++ b/senpy/__main__.py @@ -74,7 +74,7 @@ def main(): parser.add_argument( '--host', type=str, - default="127.0.0.1", + default="0.0.0.0", help='Use 0.0.0.0 to accept requests from any host.') parser.add_argument( '--port', @@ -93,8 +93,7 @@ def main(): '-i', action='store_true', default=False, - help='Do not run a server, only install plugin dependencies' - ) + help='Do not run a server, only install plugin dependencies') args = parser.parse_args() logging.basicConfig() rl = logging.getLogger() diff --git a/senpy/api.py b/senpy/api.py index 0609e26..c90dec5 100644 --- a/senpy/api.py +++ b/senpy/api.py @@ -7,6 +7,31 @@ API_PARAMS = { "algorithm": { "aliases": ["algorithm", "a", "algo"], "required": False, + }, + "outformat": { + "@id": "outformat", + "aliases": ["outformat", "o"], + "default": "json-ld", + "required": True, + "options": ["json-ld", "turtle"], + }, + "expanded-jsonld": { + "@id": "expanded-jsonld", + "aliases": ["expanded", "expanded-jsonld"], + "required": True, + "default": 0 + }, + "emotionModel": { + "@id": "emotionModel", + "aliases": ["emotionModel", "emoModel"], + "required": False + }, + "conversion": { + "@id": "conversion", + "description": "How to show the elements that have (not) been converted", + "required": True, + "options": ["filtered", "nested", "full"], + "default": "full" } } @@ -47,13 +72,6 @@ NIF_PARAMS = { "default": "direct", "options": ["direct", "url", "file"], }, - "outformat": { - "@id": "outformat", - "aliases": ["outformat", "o"], - "default": "json-ld", - "required": False, - "options": ["json-ld"], - }, "language": { "@id": "language", "aliases": ["language", "l"], @@ -76,12 +94,12 @@ NIF_PARAMS = { def parse_params(indict, spec=NIF_PARAMS): - outdict = {} + logger.debug("Parsing: {}\n{}".format(indict, spec)) + outdict = indict.copy() wrong_params = {} for param, options in iteritems(spec): if param[0] != "@": # Exclude json-ld properties - logger.debug("Param: %s - Options: %s", param, options) - for alias in options["aliases"]: + for alias in options.get("aliases", []): if alias in indict: outdict[param] = indict[alias] if param not in outdict: @@ -95,8 +113,9 @@ def parse_params(indict, spec=NIF_PARAMS): outdict[param] not in spec[param]["options"]: wrong_params[param] = spec[param] if wrong_params: + logger.debug("Error parsing: %s", wrong_params) message = Error( - status=404, + status=400, message="Missing or invalid parameters", parameters=outdict, errors={param: error diff --git a/senpy/blueprints.py b/senpy/blueprints.py index bcc8b63..fd00955 100644 --- a/senpy/blueprints.py +++ b/senpy/blueprints.py @@ -17,10 +17,11 @@ """ Blueprints for Senpy """ -from flask import (Blueprint, request, current_app, - render_template, url_for, jsonify) +from flask import (Blueprint, request, current_app, render_template, url_for, + jsonify) from .models import Error, Response, Plugins, read_schema -from .api import WEB_PARAMS, parse_params +from .api import WEB_PARAMS, API_PARAMS, parse_params +from .version import __version__ from functools import wraps import logging @@ -29,6 +30,7 @@ logger = logging.getLogger(__name__) api_blueprint = Blueprint("api", __name__) demo_blueprint = Blueprint("demo", __name__) +ns_blueprint = Blueprint("ns", __name__) def get_params(req): @@ -43,12 +45,21 @@ def get_params(req): @demo_blueprint.route('/') def index(): - return render_template("index.html") + return render_template("index.html", version=__version__) @api_blueprint.route('/contexts/.jsonld') def context(entity="context"): - return jsonify({"@context": Response.context}) + context = Response._context + context['@vocab'] = url_for('ns.index', _external=True) + return jsonify({"@context": context}) + + +@ns_blueprint.route('/') # noqa: F811 +def index(): + context = Response._context + context['@vocab'] = url_for('.ns', _external=True) + return jsonify({"@context": context}) @api_blueprint.route('/schemas/') @@ -62,26 +73,39 @@ def schema(schema="definitions"): def basic_api(f): @wraps(f) def decorated_function(*args, **kwargs): - print('Getting request:') - print(request) raw_params = get_params(request) - web_params = parse_params(raw_params, spec=WEB_PARAMS) + headers = {'X-ORIGINAL-PARAMS': raw_params} + # Get defaults + web_params = parse_params({}, spec=WEB_PARAMS) + api_params = parse_params({}, spec=API_PARAMS) - if hasattr(request, 'params'): - request.params.update(raw_params) - else: - request.params = raw_params + outformat = 'json-ld' try: + print('Getting request:') + print(request) + web_params = parse_params(raw_params, spec=WEB_PARAMS) + api_params = parse_params(raw_params, spec=API_PARAMS) + if hasattr(request, 'params'): + request.params.update(api_params) + else: + request.params = api_params response = f(*args, **kwargs) except Error as ex: response = ex - in_headers = web_params["inHeaders"] != "0" - headers = {'X-ORIGINAL-PARAMS': raw_params} + + in_headers = web_params['inHeaders'] != "0" + expanded = api_params['expanded-jsonld'] + outformat = api_params['outformat'] + return response.flask( in_headers=in_headers, headers=headers, - context_uri=url_for( - 'api.context', entity=type(response).__name__, _external=True)) + prefix=url_for('.api', _external=True), + context_uri=url_for('api.context', + entity=type(response).__name__, + _external=True), + outformat=outformat, + expanded=expanded) return decorated_function @@ -106,10 +130,11 @@ def plugins(): def plugin(plugin=None): sp = current_app.senpy if plugin == 'default' and sp.default_plugin: - response = sp.default_plugin - plugin = response.name - elif plugin in sp.plugins: - response = sp.plugins[plugin] + return sp.default_plugin + plugins = sp.filter_plugins( + id='plugins/{}'.format(plugin)) or sp.filter_plugins(name=plugin) + if plugins: + response = list(plugins.values())[0] else: return Error(message="Plugin not found", status=404) return response diff --git a/senpy/client.py b/senpy/client.py index 53668ff..e64af35 100644 --- a/senpy/client.py +++ b/senpy/client.py @@ -6,7 +6,6 @@ logger = logging.getLogger(__name__) class Client(object): - def __init__(self, endpoint): self.endpoint = endpoint @@ -15,9 +14,7 @@ class Client(object): def request(self, path=None, method='GET', **params): url = '{}{}'.format(self.endpoint, path) - response = requests.request(method=method, - url=url, - params=params) + response = requests.request(method=method, url=url, params=params) try: resp = models.from_dict(response.json()) resp.validate(resp) @@ -30,8 +27,9 @@ class Client(object): '#### Response:\n' '\tCode: {code}' '\tContent: {content}' - '\n').format(error=ex, - url=url, - code=response.status_code, - content=response.content)) + '\n').format( + error=ex, + url=url, + code=response.status_code, + content=response.content)) raise ex diff --git a/senpy/extensions.py b/senpy/extensions.py index 2bc4e74..6231b80 100644 --- a/senpy/extensions.py +++ b/senpy/extensions.py @@ -1,15 +1,15 @@ """ +Main class for Senpy. +It orchestrates plugin (de)activation and analysis. """ from future import standard_library standard_library.install_aliases() -from .plugins import SentimentPlugin -from .models import Error -from .blueprints import api_blueprint, demo_blueprint +from .plugins import SentimentPlugin, SenpyPlugin +from .models import Error, Entry, Results +from .blueprints import api_blueprint, demo_blueprint, ns_blueprint from .api import API_PARAMS, NIF_PARAMS, parse_params -from git import Repo, InvalidGitRepositoryError - from threading import Thread import os @@ -30,18 +30,21 @@ class Senpy(object): def __init__(self, app=None, - plugin_folder="plugins", + plugin_folder=".", default_plugins=False): self.app = app - self._search_folders = set() self._plugin_list = [] self._outdated = True + self._default = None self.add_folder(plugin_folder) if default_plugins: - base_folder = os.path.join(os.path.dirname(__file__), "plugins") - self.add_folder(base_folder) + self.add_folder('plugins', from_root=True) + else: + # Add only conversion plugins + self.add_folder(os.path.join('plugins', 'conversion'), + from_root=True) if app is not None: self.init_app(app) @@ -60,9 +63,12 @@ class Senpy(object): else: app.teardown_request(self.teardown) app.register_blueprint(api_blueprint, url_prefix="/api") + app.register_blueprint(ns_blueprint, url_prefix="/ns") app.register_blueprint(demo_blueprint, url_prefix="/") - def add_folder(self, folder): + def add_folder(self, folder, from_root=False): + if from_root: + folder = os.path.join(os.path.dirname(__file__), folder) logger.debug("Adding folder: %s", folder) if os.path.isdir(folder): self._search_folders.add(folder) @@ -70,10 +76,9 @@ class Senpy(object): else: logger.debug("Not a folder: %s", folder) - def analyse(self, **params): - algo = None - logger.debug("analysing with params: {}".format(params)) + def _find_plugin(self, params): api_params = parse_params(params, spec=API_PARAMS) + algo = None if "algorithm" in api_params and api_params["algorithm"]: algo = api_params["algorithm"] elif self.plugins: @@ -97,32 +102,114 @@ class Senpy(object): status=400, message=("The algorithm '{}'" " is not activated yet").format(algo)) - plug = self.plugins[algo] + return self.plugins[algo] + + def _get_params(self, params, plugin): nif_params = parse_params(params, spec=NIF_PARAMS) - extra_params = plug.get('extra_params', {}) + extra_params = plugin.get('extra_params', {}) specific_params = parse_params(params, spec=extra_params) nif_params.update(specific_params) + return nif_params + + def _get_entries(self, params): + entry = None + if params['informat'] == 'text': + entry = Entry(text=params['input']) + else: + raise NotImplemented('Only text input format implemented') + yield entry + + def analyse(self, **api_params): + logger.debug("analysing with params: {}".format(api_params)) + plugin = self._find_plugin(api_params) + nif_params = self._get_params(api_params, plugin) + resp = Results() + if 'with_parameters' in api_params: + resp.parameters = nif_params try: - resp = plug.analyse(**nif_params) - resp.analysis.append(plug) + entries = [] + for i in self._get_entries(nif_params): + entries += list(plugin.analyse_entry(i, nif_params)) + resp.entries = entries + self.convert_emotions(resp, plugin, nif_params) + resp.analysis.append(plugin.id) logger.debug("Returning analysis result: {}".format(resp)) except Error as ex: logger.exception('Error returning analysis result') resp = ex except Exception as ex: - resp = Error(message=str(ex), status=500) logger.exception('Error returning analysis result') + resp = Error(message=str(ex), status=500) return resp + def _conversion_candidates(self, fromModel, toModel): + candidates = self.filter_plugins(**{'@type': 'emotionConversionPlugin'}) + for name, candidate in candidates.items(): + for pair in candidate.onyx__doesConversion: + logging.debug(pair) + + if pair['onyx:conversionFrom'] == fromModel \ + and pair['onyx:conversionTo'] == toModel: + # logging.debug('Found candidate: {}'.format(candidate)) + yield candidate + + def convert_emotions(self, resp, plugin, params): + """ + Conversion of all emotions in a response. + In addition to converting from one model to another, it has + to include the conversion plugin to the analysis list. + Needless to say, this is far from an elegant solution, but it works. + @todo refactor and clean up + """ + fromModel = plugin.get('onyx:usesEmotionModel', None) + toModel = params.get('emotionModel', None) + output = params.get('conversion', None) + logger.debug('Asked for model: {}'.format(toModel)) + logger.debug('Analysis plugin uses model: {}'.format(fromModel)) + + if not toModel: + return + try: + candidate = next(self._conversion_candidates(fromModel, toModel)) + except StopIteration: + e = Error(('No conversion plugin found for: ' + '{} -> {}'.format(fromModel, toModel))) + e.original_response = resp + e.parameters = params + raise e + newentries = [] + for i in resp.entries: + if output == "full": + newemotions = i.emotions.copy() + else: + newemotions = [] + for j in i.emotions: + for k in candidate.convert(j, fromModel, toModel, params): + k.prov__wasGeneratedBy = candidate.id + if output == 'nested': + k.prov__wasDerivedFrom = j + newemotions.append(k) + i.emotions = newemotions + newentries.append(i) + resp.entries = newentries + resp.analysis.append(candidate.id) + @property def default_plugin(self): - candidates = self.filter_plugins(is_activated=True) - if len(candidates) > 0: - candidate = list(candidates.values())[0] - logger.debug("Default: {}".format(candidate.name)) - return candidate + candidate = self._default + if not candidate: + candidates = self.filter_plugins(is_activated=True) + if len(candidates) > 0: + candidate = list(candidates.values())[0] + logger.debug("Default: {}".format(candidate)) + return candidate + + @default_plugin.setter + def default_plugin(self, value): + if isinstance(value, SenpyPlugin): + self._default = value else: - return None + self._default = self.plugins[value] def activate_all(self, sync=False): ps = [] @@ -164,6 +251,7 @@ class Senpy(object): plugin.name, ex, traceback.format_exc()) logger.error(msg) raise Error(msg) + if sync: act() else: @@ -184,8 +272,8 @@ class Senpy(object): plugin.deactivate() logger.info("Plugin deactivated: {}".format(plugin.name)) except Exception as ex: - logger.error("Error deactivating plugin {}: {}".format( - plugin.name, ex)) + logger.error( + "Error deactivating plugin {}: {}".format(plugin.name, ex)) logger.error("Trace: {}".format(traceback.format_exc())) if sync: @@ -237,13 +325,6 @@ class Senpy(object): logger.debug("No valid plugin for: {}".format(module)) return module = candidate(info=info) - repo_path = root - try: - module._repo = Repo(repo_path) - except InvalidGitRepositoryError: - logger.debug("The plugin {} is not in a Git repository".format( - module)) - module._repo = None return name, module @classmethod @@ -261,7 +342,7 @@ class Senpy(object): for root, dirnames, filenames in os.walk(search_folder): for filename in fnmatch.filter(filenames, '*.senpy'): name, plugin = self._load_plugin(root, filename) - if plugin and name not in self._plugin_list: + if plugin and name: plugins[name] = plugin self._outdated = False @@ -282,8 +363,8 @@ class Senpy(object): def matches(plug): res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items()) - logger.debug("matching {} with {}: {}".format(plug.name, kwargs, - res)) + logger.debug( + "matching {} with {}: {}".format(plug.name, kwargs, res)) return res if not kwargs: diff --git a/senpy/models.py b/senpy/models.py index ec6ef80..72c3d7f 100644 --- a/senpy/models.py +++ b/senpy/models.py @@ -16,6 +16,9 @@ import jsonref import jsonschema from flask import Response as FlaskResponse +from pyld import jsonld + +from rdflib import Graph import logging @@ -72,31 +75,60 @@ base_context = Context.load(CONTEXT_PATH) class SenpyMixin(object): - context = base_context["@context"] + _context = base_context["@context"] - def flask(self, in_headers=True, headers=None, **kwargs): + def flask(self, + in_headers=True, + headers=None, + outformat='json-ld', + **kwargs): """ Return the values and error to be used in flask. So far, it returns a fixed context. We should store/generate different contexts if the plugin adds more aliases. """ headers = headers or {} - kwargs["with_context"] = True - js = self.jsonld(**kwargs) - if in_headers: - url = js["@context"] - del js["@context"] + kwargs["with_context"] = not in_headers + content, mimetype = self.serialize(format=outformat, + with_mime=True, + **kwargs) + + if outformat == 'json-ld' and in_headers: headers.update({ - "Link": ('<%s>;' - 'rel="http://www.w3.org/ns/json-ld#context";' - ' type="application/ld+json"' % url) + "Link": + ('<%s>;' + 'rel="http://www.w3.org/ns/json-ld#context";' + ' type="application/ld+json"' % kwargs.get('context_uri')) }) return FlaskResponse( - json.dumps( - js, indent=2, sort_keys=True), + response=content, status=getattr(self, "status", 200), headers=headers, - mimetype="application/json") + mimetype=mimetype) + + def serialize(self, format='json-ld', with_mime=False, **kwargs): + js = self.jsonld(**kwargs) + if format == 'json-ld': + content = json.dumps(js, indent=2, sort_keys=True) + mimetype = "application/json" + elif format in ['turtle', ]: + logger.debug(js) + content = json.dumps(js, indent=2, sort_keys=True) + g = Graph().parse( + data=content, + format='json-ld', + base=kwargs.get('prefix'), + context=self._context) + logger.debug( + 'Parsing with prefix: {}'.format(kwargs.get('prefix'))) + content = g.serialize(format='turtle').decode('utf-8') + mimetype = 'text/{}'.format(format) + else: + raise Error('Unknown outformat: {}'.format(format)) + if with_mime: + return content, mimetype + else: + return content def serializable(self): def ser_or_down(item): @@ -115,28 +147,30 @@ class SenpyMixin(object): return ser_or_down(self._plain_dict()) - def jsonld(self, with_context=False, context_uri=None): + def jsonld(self, + with_context=True, + context_uri=None, + prefix=None, + expanded=False): ser = self.serializable() - if with_context: - context = [] - if context_uri: - context = context_uri - else: - context = self.context.copy() - if hasattr(self, 'prefix'): - # This sets @base for the document, which will be used in - # all relative URIs. For example, if a uri is "Example" and - # prefix =s "http://example.com", the absolute URI after - # expanding with JSON-LD will be "http://example.com/Example" - - prefix_context = {"@base": self.prefix} - if isinstance(context, list): - context.append(prefix_context) - else: - context = [context, prefix_context] - ser["@context"] = context - return ser + result = jsonld.compact( + ser, + self._context, + options={ + 'base': prefix, + 'expandContext': self._context, + 'senpy': prefix + }) + if context_uri: + result['@context'] = context_uri + if expanded: + result = jsonld.expand( + result, options={'base': prefix, + 'expandContext': self._context}) + if not with_context: + del result['@context'] + return result def to_JSON(self, *args, **kwargs): js = json.dumps(self.jsonld(*args, **kwargs), indent=4, sort_keys=True) @@ -161,13 +195,14 @@ class BaseModel(SenpyMixin, dict): if 'id' in kwargs: self.id = kwargs.pop('id') elif kwargs.pop('_auto_id', True): - self.id = '_:{}_{}'.format( - type(self).__name__, time.time()) + self.id = '_:{}_{}'.format(type(self).__name__, time.time()) temp = dict(*args, **kwargs) - for obj in [self.schema, ] + self.schema.get('allOf', []): + for obj in [ + self.schema, + ] + self.schema.get('allOf', []): for k, v in obj.get('properties', {}).items(): - if 'default' in v: + if 'default' in v and k not in temp: temp[k] = copy.deepcopy(v['default']) for i in temp: @@ -175,10 +210,6 @@ class BaseModel(SenpyMixin, dict): if nk != i: temp[nk] = temp[i] del temp[i] - if 'context' in temp: - context = temp['context'] - del temp['context'] - self.__dict__['context'] = Context.load(context) try: temp['@type'] = getattr(self, '@type') except AttributeError: @@ -239,10 +270,11 @@ def from_schema(name, schema_file=None, base_classes=None): base_classes = base_classes or [] base_classes.append(BaseModel) schema_file = schema_file or '{}.json'.format(name) - class_name = '{}{}'.format(i[0].upper(), i[1:]) + class_name = '{}{}'.format(name[0].upper(), name[1:]) newclass = type(class_name, tuple(base_classes), {}) setattr(newclass, '@type', name) setattr(newclass, 'schema', read_schema(schema_file)) + setattr(newclass, 'class_name', class_name) register(newclass, name) return newclass @@ -253,29 +285,31 @@ def _add_from_schema(*args, **kwargs): del generatedClass -for i in ['response', - 'results', - 'entry', - 'sentiment', - 'analysis', - 'emotionSet', - 'emotion', - 'emotionModel', - 'suggestion', - 'plugin', - 'emotionPlugin', - 'sentimentPlugin', - 'plugins']: +for i in [ + 'analysis', + 'emotion', + 'emotionConversion', + 'emotionConversionPlugin', + 'emotionAnalysis', + 'emotionModel', + 'emotionPlugin', + 'emotionSet', + 'entry', + 'plugin', + 'plugins', + 'response', + 'results', + 'sentiment', + 'sentimentPlugin', + 'suggestion', +]: _add_from_schema(i) _ErrorModel = from_schema('error') class Error(SenpyMixin, BaseException): - def __init__(self, - message, - *args, - **kwargs): + def __init__(self, message, *args, **kwargs): super(Error, self).__init__(self, message, message) self._error = _ErrorModel(message=message, *args, **kwargs) self.message = message diff --git a/senpy/plugins.py b/senpy/plugins.py index 512b125..693cb43 100644 --- a/senpy/plugins.py +++ b/senpy/plugins.py @@ -6,6 +6,7 @@ import os.path import pickle import logging import tempfile +import copy from . import models logger = logging.getLogger(__name__) @@ -13,21 +14,38 @@ logger = logging.getLogger(__name__) class SenpyPlugin(models.Plugin): def __init__(self, info=None): + """ + Provides a canonical name for plugins and serves as base for other + kinds of plugins. + """ if not info: raise models.Error(message=("You need to provide configuration" "information for the plugin.")) logger.debug("Initialising {}".format(info)) - super(SenpyPlugin, self).__init__(info) - self.id = '{}_{}'.format(self.name, self.version) - self._info = info + id = 'plugins/{}_{}'.format(info['name'], info['version']) + super(SenpyPlugin, self).__init__(id=id, **info) self.is_activated = False def get_folder(self): return os.path.dirname(inspect.getfile(self.__class__)) def analyse(self, *args, **kwargs): - logger.debug("Analysing with: {} {}".format(self.name, self.version)) - pass + raise NotImplemented( + 'Your method should implement either analyse or analyse_entry') + + def analyse_entry(self, entry, parameters): + """ An implemented plugin should override this method. + This base method is here to adapt old style plugins which only + implement the *analyse* function. + Note that this method may yield an annotated entry or a list of + entries (e.g. in a tokenizer) + """ + text = entry['text'] + params = copy.copy(parameters) + params['input'] = text + results = self.analyse(**params) + for i in results.entries: + yield i def activate(self): pass @@ -35,25 +53,24 @@ class SenpyPlugin(models.Plugin): def deactivate(self): pass - def __del__(self): - ''' Destructor, to make sure all the resources are freed ''' - self.deactivate() - -class SentimentPlugin(SenpyPlugin, models.SentimentPlugin): +class SentimentPlugin(models.SentimentPlugin, SenpyPlugin): def __init__(self, info, *args, **kwargs): super(SentimentPlugin, self).__init__(info, *args, **kwargs) self.minPolarityValue = float(info.get("minPolarityValue", 0)) self.maxPolarityValue = float(info.get("maxPolarityValue", 1)) - self["@type"] = "marl:SentimentAnalysis" -class EmotionPlugin(SentimentPlugin, models.EmotionPlugin): +class EmotionPlugin(models.EmotionPlugin, SenpyPlugin): def __init__(self, info, *args, **kwargs): super(EmotionPlugin, self).__init__(info, *args, **kwargs) - self.minEmotionValue = float(info.get("minEmotionValue", 0)) - self.maxEmotionValue = float(info.get("maxEmotionValue", 0)) - self["@type"] = "onyx:EmotionAnalysis" + self.minEmotionValue = float(info.get("minEmotionValue", -1)) + self.maxEmotionValue = float(info.get("maxEmotionValue", 1)) + + +class EmotionConversionPlugin(models.EmotionConversionPlugin, SenpyPlugin): + def __init__(self, info, *args, **kwargs): + super(EmotionConversionPlugin, self).__init__(info, *args, **kwargs) class ShelfMixin(object): @@ -74,13 +91,10 @@ class ShelfMixin(object): @property def shelf_file(self): - if not hasattr(self, '_shelf_file') or not self._shelf_file: - if hasattr(self, '_info') and 'shelf_file' in self._info: - self.__dict__['_shelf_file'] = self._info['shelf_file'] - else: - self._shelf_file = os.path.join(tempfile.gettempdir(), - self.name + '.p') - return self._shelf_file + if 'shelf_file' not in self or not self['shelf_file']: + self.shelf_file = os.path.join(tempfile.gettempdir(), + self.name + '.p') + return self['shelf_file'] def save(self): logger.debug('saving pickle') diff --git a/senpy/plugins/conversion/emotion/ekman2vad.py b/senpy/plugins/conversion/emotion/ekman2vad.py new file mode 100644 index 0000000..0fed98d --- /dev/null +++ b/senpy/plugins/conversion/emotion/ekman2vad.py @@ -0,0 +1,56 @@ +from senpy.plugins import EmotionConversionPlugin +from senpy.models import EmotionSet, Emotion, Error + +import logging +logger = logging.getLogger(__name__) + +import math + + +class WNA2VAD(EmotionConversionPlugin): + + def _ekman_to_vad(self, ekmanSet): + potency = 0 + arousal = 0 + dominance = 0 + for e in ekmanSet.onyx__hasEmotion: + category = e.onyx__hasEmotionCategory + centroid = self.centroids[category] + potency += centroid['V'] + arousal += centroid['A'] + dominance += centroid['D'] + e = Emotion({'emoml:potency': potency, + 'emoml:arousal': arousal, + 'emoml:dominance': dominance}) + return e + + def _vad_to_ekman(self, VADEmotion): + V = VADEmotion['emoml:valence'] + A = VADEmotion['emoml:potency'] + D = VADEmotion['emoml:dominance'] + emotion = '' + value = 10000000000000000000000.0 + for state in self.centroids: + valence = V - self.centroids[state]['V'] + arousal = A - self.centroids[state]['A'] + dominance = D - self.centroids[state]['D'] + new_value = math.sqrt((valence**2) + + (arousal**2) + + (dominance**2)) + if new_value < value: + value = new_value + emotion = state + result = Emotion(onyx__hasEmotionCategory=emotion) + return result + + def convert(self, emotionSet, fromModel, toModel, params): + logger.debug('{}\n{}\n{}\n{}'.format(emotionSet, fromModel, toModel, params)) + e = EmotionSet() + if fromModel == 'emoml:big6': + e.onyx__hasEmotion.append(self._ekman_to_vad(emotionSet)) + elif fromModel == 'emoml:fsre-dimensions': + for i in emotionSet.onyx__hasEmotion: + e.onyx__hasEmotion.append(self._vad_to_ekman(e)) + else: + raise Error('EMOTION MODEL NOT KNOWN') + yield e diff --git a/senpy/plugins/conversion/emotion/ekman2vad.senpy b/senpy/plugins/conversion/emotion/ekman2vad.senpy new file mode 100644 index 0000000..d50b13d --- /dev/null +++ b/senpy/plugins/conversion/emotion/ekman2vad.senpy @@ -0,0 +1,35 @@ +--- +name: Ekman2VAD +module: ekman2vad +description: Plugin to convert from Ekman to VAD +version: 0.1 +onyx:doesConversion: + - onyx:conversionFrom: emoml:big6 + onyx:conversionTo: emoml:fsre-dimensions + - onyx:conversionFrom: emoml:fsre-dimensions + onyx:conversionTo: wna:WNAModel +centroids: + emoml:big6anger: + A: 6.95 + D: 5.1 + V: 2.7 + emoml:big6disgust: + A: 5.3 + D: 8.05 + V: 2.7 + emoml:big6fear: + A: 6.5 + D: 3.6 + V: 3.2 + emoml:big6happiness: + A: 7.22 + D: 6.28 + V: 8.6 + emoml:big6sadness: + A: 5.21 + D: 2.82 + V: 2.21 +aliases: + A: emoml:arousal + V: emoml:potency + D: emoml:dominance \ No newline at end of file diff --git a/senpy/plugins/example/emoRand/emoRand.py b/senpy/plugins/example/emoRand/emoRand.py new file mode 100644 index 0000000..8de8e81 --- /dev/null +++ b/senpy/plugins/example/emoRand/emoRand.py @@ -0,0 +1,18 @@ +import random + +from senpy.plugins import EmotionPlugin +from senpy.models import EmotionSet, Emotion + + +class RmoRandPlugin(EmotionPlugin): + def analyse_entry(self, entry, params): + category = "emoml:big6happiness" + number = max(-1, min(1, random.gauss(0, 0.5))) + if number > 0: + category = "emoml:big6anger" + emotionSet = EmotionSet() + emotion = Emotion({"onyx:hasEmotionCategory": category}) + emotionSet.onyx__hasEmotion.append(emotion) + emotionSet.prov__wasGeneratedBy = self.id + entry.emotions.append(emotionSet) + yield entry diff --git a/senpy/plugins/example/emoRand/emoRand.senpy b/senpy/plugins/example/emoRand/emoRand.senpy new file mode 100644 index 0000000..a3ffae8 --- /dev/null +++ b/senpy/plugins/example/emoRand/emoRand.senpy @@ -0,0 +1,9 @@ +--- +name: emoRand +module: emoRand +description: A sample plugin that returns a random emotion annotation +author: "@balkian" +version: '0.1' +url: "https://github.com/gsi-upm/senpy-plugins-community" +requirements: {} +onyx:usesEmotionModel: "emoml:big6" \ No newline at end of file diff --git a/senpy/plugins/rand/rand.py b/senpy/plugins/example/rand/rand.py similarity index 62% rename from senpy/plugins/rand/rand.py rename to senpy/plugins/example/rand/rand.py index a6f7639..aa92c70 100644 --- a/senpy/plugins/rand/rand.py +++ b/senpy/plugins/example/rand/rand.py @@ -1,29 +1,24 @@ import random from senpy.plugins import SentimentPlugin -from senpy.models import Results, Sentiment, Entry +from senpy.models import Sentiment -class Sentiment140Plugin(SentimentPlugin): - def analyse(self, **params): +class RandPlugin(SentimentPlugin): + def analyse_entry(self, entry, params): lang = params.get("language", "auto") - response = Results() polarity_value = max(-1, min(1, random.gauss(0.2, 0.2))) polarity = "marl:Neutral" if polarity_value > 0: polarity = "marl:Positive" elif polarity_value < 0: polarity = "marl:Negative" - entry = Entry({"id": ":Entry0", "nif:isString": params["input"]}) sentiment = Sentiment({ - "id": ":Sentiment0", "marl:hasPolarity": polarity, "marl:polarityValue": polarity_value }) sentiment["prov:wasGeneratedBy"] = self.id - entry.sentiments = [] entry.sentiments.append(sentiment) entry.language = lang - response.entries.append(entry) - return response + yield entry diff --git a/senpy/plugins/example/rand/rand.senpy b/senpy/plugins/example/rand/rand.senpy new file mode 100644 index 0000000..b7ee693 --- /dev/null +++ b/senpy/plugins/example/rand/rand.senpy @@ -0,0 +1,10 @@ +--- +name: rand +module: rand +description: A sample plugin that returns a random sentiment annotation +author: "@balkian" +version: '0.1' +url: "https://github.com/gsi-upm/senpy-plugins-community" +requirements: {} +marl:maxPolarityValue: '1' +marl:minPolarityValue: "-1" diff --git a/senpy/plugins/rand/rand.senpy b/senpy/plugins/rand/rand.senpy deleted file mode 100644 index b7c8c57..0000000 --- a/senpy/plugins/rand/rand.senpy +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "rand", - "module": "rand", - "description": "What my plugin broadly does", - "author": "@balkian", - "version": "0.1", - "extra_params": { - "language": { - "@id": "lang_rand", - "aliases": ["language", "l"], - "required": false, - "options": ["es", "en", "auto"] - } - }, - "requirements": {}, - "marl:maxPolarityValue": "1", - "marl:minPolarityValue": "-1" -} diff --git a/senpy/plugins/sentiment140/sentiment140.py b/senpy/plugins/sentiment/sentiment140/sentiment140.py similarity index 69% rename from senpy/plugins/sentiment140/sentiment140.py rename to senpy/plugins/sentiment/sentiment140/sentiment140.py index 3090c02..b8e6d6f 100644 --- a/senpy/plugins/sentiment140/sentiment140.py +++ b/senpy/plugins/sentiment/sentiment140/sentiment140.py @@ -2,24 +2,22 @@ import requests import json from senpy.plugins import SentimentPlugin -from senpy.models import Results, Sentiment, Entry +from senpy.models import Sentiment class Sentiment140Plugin(SentimentPlugin): - def analyse(self, **params): + def analyse_entry(self, entry, params): lang = params.get("language", "auto") res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson", json.dumps({ "language": lang, "data": [{ - "text": params["input"] + "text": entry.text }] })) - p = params.get("prefix", None) - response = Results(prefix=p) - polarity_value = self.maxPolarityValue * int(res.json()["data"][0][ - "polarity"]) * 0.25 + polarity_value = self.maxPolarityValue * int( + res.json()["data"][0]["polarity"]) * 0.25 polarity = "marl:Neutral" neutral_value = self.maxPolarityValue / 2.0 if polarity_value > neutral_value: @@ -27,9 +25,7 @@ class Sentiment140Plugin(SentimentPlugin): elif polarity_value < neutral_value: polarity = "marl:Negative" - entry = Entry(id="Entry0", nif__isString=params["input"]) sentiment = Sentiment( - id="Sentiment0", prefix=p, marl__hasPolarity=polarity, marl__polarityValue=polarity_value) @@ -37,5 +33,4 @@ class Sentiment140Plugin(SentimentPlugin): entry.sentiments = [] entry.sentiments.append(sentiment) entry.language = lang - response.entries.append(entry) - return response + yield entry diff --git a/senpy/plugins/sentiment/sentiment140/sentiment140.senpy b/senpy/plugins/sentiment/sentiment140/sentiment140.senpy new file mode 100644 index 0000000..f2c92b3 --- /dev/null +++ b/senpy/plugins/sentiment/sentiment140/sentiment140.senpy @@ -0,0 +1,21 @@ +--- +name: sentiment140 +module: sentiment140 +description: "Connects to the sentiment140 free API: http://sentiment140.com" +author: "@balkian" +version: '0.2' +url: "https://github.com/gsi-upm/senpy-plugins-community" +extra_params: + language: + "@id": lang_sentiment140 + aliases: + - language + - l + required: false + options: + - es + - en + - auto +requirements: {} +maxPolarityValue: 1 +minPolarityValue: 0 \ No newline at end of file diff --git a/senpy/plugins/sentiment140/sentiment140.senpy b/senpy/plugins/sentiment140/sentiment140.senpy deleted file mode 100644 index 976ae4a..0000000 --- a/senpy/plugins/sentiment140/sentiment140.senpy +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "sentiment140", - "module": "sentiment140", - "description": "What my plugin broadly does", - "author": "@balkian", - "version": "0.1", - "extra_params": { - "language": { - "@id": "lang_sentiment140", - "aliases": ["language", "l"], - "required": false, - "options": ["es", "en", "auto"] - } - }, - "requirements": {}, - "maxPolarityValue": "1", - "minPolarityValue": "0" -} diff --git "a/senpy/schemas/\\" "b/senpy/schemas/\\" deleted file mode 100644 index dd76fa5..0000000 --- "a/senpy/schemas/\\" +++ /dev/null @@ -1,7 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "description": "Senpy analysis", - "allOf": [{ - "$ref": "atom.json" - }] -} diff --git a/senpy/schemas/context.jsonld b/senpy/schemas/context.jsonld index 136c683..795b0e6 100644 --- a/senpy/schemas/context.jsonld +++ b/senpy/schemas/context.jsonld @@ -6,34 +6,51 @@ "prov": "http://www.w3.org/ns/prov#", "nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#", "marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#", - "onyx": "http://www.gsi.dit.upm.es/ontologies/onyx#", - "wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#", + "onyx": "http://www.gsi.dit.upm.es/ontologies/onyx/ns#", + "wna": "http://www.gsi.dit.upm.es/ontologies/wnaffect/ns#", + "emoml": "http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/emotionml/ns#", "xsd": "http://www.w3.org/2001/XMLSchema#", "topics": { - "@id": "dc:subject" + "@id": "dc:subject" }, "entities": { - "@id": "me:hasEntities" + "@id": "me:hasEntities" }, "suggestions": { - "@id": "me:hasSuggestions", - "@container": "@set" + "@id": "me:hasSuggestions", + "@container": "@set" }, "emotions": { - "@id": "onyx:hasEmotionSet", - "@container": "@set" + "@id": "onyx:hasEmotionSet", + "@container": "@set" }, "sentiments": { - "@id": "marl:hasOpinion", - "@container": "@set" + "@id": "marl:hasOpinion", + "@container": "@set" }, "entries": { - "@id": "prov:used", - "@container": "@set" + "@id": "prov:used", + "@container": "@set" }, "analysis": { - "@id": "prov:wasGeneratedBy" - + "@id": "AnalysisInvolved", + "@type": "@id", + "@container": "@set" + }, + "prov:wasGeneratedBy": { + "@type": "@id" + }, + "onyx:usesEmotionModel": { + "@type": "@id" + }, + "onyx:hasEmotionCategory": { + "@type": "@id" + }, + "onyx:conversionFrom": { + "@type": "@id" + }, + "onyx:conversionTo": { + "@type": "@id" } } } diff --git a/senpy/schemas/emotionAnalysis.json b/senpy/schemas/emotionAnalysis.json index 41a9f60..6b64db7 100644 --- a/senpy/schemas/emotionAnalysis.json +++ b/senpy/schemas/emotionAnalysis.json @@ -6,13 +6,14 @@ {"$ref": "analysis.json"}, {"properties": { - "onyx:hasEmotionModel": { + "onyx:usesEmotionModel": { "anyOf": [ {"type": "string"}, {"$ref": "emotionModel.json"} ] } }, - "required": ["onyx:hasEmotionModel"] + "required": ["onyx:hasEmotionModel", + "@type"] }] } diff --git a/senpy/schemas/emotionConversion.json b/senpy/schemas/emotionConversion.json new file mode 100644 index 0000000..b330b39 --- /dev/null +++ b/senpy/schemas/emotionConversion.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "properties": { + "onyx:conversionFrom": { + "$ref": "emotionModel.json" + }, + "onyx:conversionTo": { + "$ref": "emotionModel.json" + } + }, + "required": ["onyx:conversionFrom", "onyx:conversionTo"] +} diff --git a/senpy/schemas/emotionConversionPlugin.json b/senpy/schemas/emotionConversionPlugin.json new file mode 100644 index 0000000..1c8b604 --- /dev/null +++ b/senpy/schemas/emotionConversionPlugin.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "allOf": [ + { + "$ref": "plugin.json" + }, + { + "properties": { + "onyx:doesConversion": { + "type": "array", + "items": { + "$ref": "emotionConversion.json" + } + } + } + } + ] +} diff --git a/senpy/schemas/emotionPlugin.json b/senpy/schemas/emotionPlugin.json index c36a859..cd010c4 100644 --- a/senpy/schemas/emotionPlugin.json +++ b/senpy/schemas/emotionPlugin.json @@ -1,7 +1,7 @@ { "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", - "$allOf": [ + "allOf": [ { "$ref": "plugin.json" }, diff --git a/senpy/schemas/entry.json b/senpy/schemas/entry.json index 9eef1e2..c1bb4fc 100644 --- a/senpy/schemas/entry.json +++ b/senpy/schemas/entry.json @@ -11,23 +11,28 @@ }, "sentiments": { "type": "array", - "items": {"$ref": "sentiment.json" } + "items": {"$ref": "sentiment.json" }, + "default": [] }, "emotions": { "type": "array", - "items": {"$ref": "emotionSet.json" } + "items": {"$ref": "emotionSet.json" }, + "default": [] }, "entities": { "type": "array", - "items": {"$ref": "entity.json" } + "items": {"$ref": "entity.json" }, + "default": [] }, "topics": { "type": "array", - "items": {"$ref": "topic.json" } + "items": {"$ref": "topic.json" }, + "default": [] }, "suggestions": { "type": "array", - "items": {"$ref": "suggestion.json" } + "items": {"$ref": "suggestion.json" }, + "default": [] } }, "required": ["@id", "nif:isString"] diff --git a/senpy/schemas/error.json b/senpy/schemas/error.json index 7cdf582..cbbd176 100644 --- a/senpy/schemas/error.json +++ b/senpy/schemas/error.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-04/schema#", "description": "Base schema for all Senpy objects", "type": "object", - "$allOf": [ + "allOf": [ {"$ref": "atom.json"}, { "properties": { @@ -10,14 +10,14 @@ "type": "string" }, "errors": { - "type": "list", + "type": "array", "items": {"type": "object"} }, "status": { - "type": "int" - }, - "required": ["message"] - } + "type": "number" + } + }, + "required": ["message"] } ] } diff --git a/senpy/schemas/sentimentPlugin.json b/senpy/schemas/sentimentPlugin.json index 26830cd..b20152e 100644 --- a/senpy/schemas/sentimentPlugin.json +++ b/senpy/schemas/sentimentPlugin.json @@ -1,7 +1,7 @@ { "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", - "$allOf": [ + "allOf": [ { "$ref": "plugin.json" }, diff --git a/senpy/schemas/suggestion.json b/senpy/schemas/suggestion.json index 70090b1..da1ecf3 100644 --- a/senpy/schemas/suggestion.json +++ b/senpy/schemas/suggestion.json @@ -1,4 +1,5 @@ { "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object" + "type": "object", + "required": ["@id", "prov:wasGeneratedBy"] } diff --git a/senpy/static/css/main.css b/senpy/static/css/main.css index 8365597..112be75 100644 --- a/senpy/static/css/main.css +++ b/senpy/static/css/main.css @@ -8,7 +8,7 @@ body { } #inputswrapper { min-height:100%; - background: white; + /* background: white; */ position:relative; min-width: 800px; height: 100%; @@ -50,25 +50,16 @@ body { #form { width: 100%; } -#results { +.results { overflow: auto; - padding: 20px; - background: lightgray; - -moz-border-radius: 20px; - -webkit-border-radius: 20px; - -khtml-border-radius: 20px; - border-radius: 20px; + /* padding: 20px; */ + background: white; + /* -moz-border-radius: 20px; */ + /* -webkit-border-radius: 20px; */ + /* -khtml-border-radius: 20px; */ + /* border-radius: 20px; */ } -#jsonraw { - overflow: auto; - padding: 20px; - background: lightgray; - -moz-border-radius: 20px; - -webkit-border-radius: 20px; - -khtml-border-radius: 20px; - border-radius: 20px; -} #input_request { margin-top: 5px; display:block; @@ -156,3 +147,8 @@ textarea{ #header { font-family: 'Architects Daughter', cursive; } + +#results-div { + /* background: white; */ + display: none; +} diff --git a/senpy/static/js/main.js b/senpy/static/js/main.js index b8dcbe9..cf9f2f1 100644 --- a/senpy/static/js/main.js +++ b/senpy/static/js/main.js @@ -32,39 +32,48 @@ $(document).ready(function() { var availablePlugins = document.getElementById('availablePlugins'); plugins = response.plugins; for (r in plugins){ - if (plugins[r]["name"]){ - if (plugins[r]["name"] == defaultPlugin["name"]){ - if (plugins[r]["is_activated"]){ - html+= "" + plugin = plugins[r] + if (plugin["name"]){ + if (plugin["name"] == defaultPlugin["name"]){ + if (plugin["is_activated"]){ + html+= "" }else{ - html+= "" + html+= "" } } else{ - if (plugins[r]["is_activated"]){ - html+= "" + if (plugin["is_activated"]){ + html+= "" } else{ - html+= "" + html+= "" } } } - if (plugins[r]["extra_params"]){ - plugins_params[plugins[r]["name"]]={}; - for (param in plugins[r]["extra_params"]){ - if (typeof plugins[r]["extra_params"][param] !="string"){ + if (plugin["extra_params"]){ + plugins_params[plugin["name"]]={}; + for (param in plugin["extra_params"]){ + if (typeof plugin["extra_params"][param] !="string"){ var params = new Array(); - var alias = plugins[r]["extra_params"][param]["aliases"][0]; + var alias = plugin["extra_params"][param]["aliases"][0]; params[alias]=new Array(); - for (option in plugins[r]["extra_params"][param]["options"]){ - params[alias].push(plugins[r]["extra_params"][param]["options"][option]) + for (option in plugin["extra_params"][param]["options"]){ + params[alias].push(plugin["extra_params"][param]["options"][option]) } - plugins_params[plugins[r]["name"]][alias] = (params[alias]) + plugins_params[plugin["name"]][alias] = (params[alias]) } } } var pluginList = document.createElement('li'); - pluginList.innerHTML = "" + plugins[r]["name"] + "" + ": " + plugins[r]["description"] + + newHtml = "" + if(plugin.url) { + newHtml= "" + plugin.name + ""; + }else { + newHtml= plugin["name"]; + } + newHtml += ": " + replaceURLWithHTMLLinks(plugin.description); + pluginList.innerHTML = newHtml; availablePlugins.appendChild(pluginList) } document.getElementById('plugins').innerHTML = html; @@ -96,6 +105,10 @@ function change_params(){ function load_JSON(){ url = "/api"; + var container = document.getElementById('results'); + var rawcontainer = document.getElementById("jsonraw"); + rawcontainer.innerHTML = ''; + container.innerHTML = ''; var plugin = document.getElementById("plugins").options[document.getElementById("plugins").selectedIndex].value; var input = encodeURIComponent(document.getElementById("input").value); url += "?algo="+plugin+"&i="+input @@ -108,18 +121,14 @@ function load_JSON(){ } } var response = JSON.parse($.ajax({type: "GET", url: url , async: false}).responseText); - var container = document.getElementById('results'); var options = { mode: 'view' }; - try { - container.removeChild(container.firstChild); - } - catch(err) { - } var editor = new JSONEditor(container, options, response); - document.getElementById("jsonraw").innerHTML = replaceURLWithHTMLLinks(JSON.stringify(response, undefined, 2)) + editor.expandAll(); + rawcontainer.innerHTML = replaceURLWithHTMLLinks(JSON.stringify(response, undefined, 2)) document.getElementById("input_request").innerHTML = ""+url+"" + document.getElementById("results-div").style.display = 'block'; } diff --git a/senpy/templates/index.html b/senpy/templates/index.html index d9fe0c6..9626de1 100755 --- a/senpy/templates/index.html +++ b/senpy/templates/index.html @@ -2,7 +2,7 @@ - Playground + Playground {{version}} @@ -25,49 +25,68 @@

Playground -

+

v{{ version}}

-
+
+
-
-
-

Test Senpy

-
-

- Test it ยป +

+

About Senpy

+

Senpy is a framework to build semantic sentiment and emotion analysis services. It does so by using a mix of web and semantic technologies, such as JSON-LD, RDFlib and Flask.

+

Senpy makes it easy to develop and publish your own analysis algorithms (plugins in senpy terms). +

+

+ This website is the senpy Playground, which allows you to test the instance of senpy in this server. It provides a user-friendly interface to the functions exposed by the senpy API. +

+

+ Once you get comfortable with the parameters and results, you are encouraged to issue your own requests to the API endpoint, which should be here. +

+

+ These are some of the things you can do with the API: +

+

- -
+
-
Follow us on GitHub
-
-
-
Enjoy.
+
+ Available Plugins +
+
    -
    -

    Available Plugins

    -
    -
      -
      - + +
      +
      If you are new to senpy, you might want to read senpy's documentation
      +
      +       + +
      +
      Feel free to follow us on GitHub
      +
      +       +
      +
      Enjoy.
      -
      +
      @@ -81,30 +100,32 @@ I cannot believe it!

      - Analyse! + Analyse! +
      - +
      + +
      -
      +
      -
      - 
      
-                    
      
+                  
      
+                    
      
+                  
      
                 
      
                 
                 
      
-                    
      
-                       
      
-                    
      
+                  
      
+                    
      
+                  
      
                 
      
               
      
             
      
-          
      
         
      
            
 
diff --git a/senpy/version.py b/senpy/version.py
index c527fd0..1d5ec0d 100644
--- a/senpy/version.py
+++ b/senpy/version.py
@@ -1,5 +1,4 @@
 import os
-import subprocess
 import logging
 
 logger = logging.getLogger(__name__)
@@ -8,27 +7,9 @@ ROOT = os.path.dirname(__file__)
 DEFAULT_FILE = os.path.join(ROOT, 'VERSION')
 
 
-def git_version():
-    try:
-        res = subprocess.check_output(['git', 'describe',
-                                       '--tags', '--dirty']).decode('utf-8')
-        return res.strip()
-    except subprocess.CalledProcessError:
-        return None
-
-
 def read_version(versionfile=DEFAULT_FILE):
     with open(versionfile) as f:
         return f.read().strip()
 
 
-def write_version(version, versionfile=DEFAULT_FILE):
-    version = version or git_version()
-    if not version:
-        raise ValueError('You need to provide a valid version')
-    with open(versionfile, 'w') as f:
-        f.write(version)
-
-
-__version__ = git_version() or read_version()
-write_version(__version__)
+__version__ = read_version()
diff --git a/setup.cfg b/setup.cfg
index 178ba5f..c1b329b 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -7,3 +7,6 @@ test=pytest
 # finishing the imports. flake8 thinks that we're doing the imports too late,
 # but it's actually ok
 ignore = E402
+max-line-length = 100
+[bdist_wheel]
+universal=1
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 9044fe3..7dee40a 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,8 @@
 import pip
 from setuptools import setup
-from pip.req import parse_requirements
 # parse_requirements() returns generator of pip.req.InstallRequirement objects
+from pip.req import parse_requirements
+from senpy import __version__
 
 try:
     install_reqs = parse_requirements(
@@ -15,7 +16,6 @@ except AttributeError:
 install_reqs = [str(ir.req) for ir in install_reqs]
 test_reqs = [str(ir.req) for ir in test_reqs]
 
-from senpy import __version__
 
 setup(
     name='senpy',
diff --git a/tests/dummy_plugin/dummy.py b/tests/dummy_plugin/dummy.py
deleted file mode 100644
index 7ab4a2d..0000000
--- a/tests/dummy_plugin/dummy.py
+++ /dev/null
@@ -1,7 +0,0 @@
-from senpy.plugins import SentimentPlugin
-from senpy.models import Results
-
-
-class DummyPlugin(SentimentPlugin):
-    def analyse(self, *args, **kwargs):
-        return Results()
diff --git a/tests/dummy_plugin/dummy.senpy b/tests/dummy_plugin/dummy.senpy
deleted file mode 100644
index 996e614..0000000
--- a/tests/dummy_plugin/dummy.senpy
+++ /dev/null
@@ -1,7 +0,0 @@
-{
-    "name": "Dummy",
-    "module": "dummy",
-    "description": "I am dummy",
-    "author": "@balkian",
-    "version": "0.1"
-}
diff --git a/tests/plugins/dummy_plugin/dummy.py b/tests/plugins/dummy_plugin/dummy.py
new file mode 100644
index 0000000..6f3f091
--- /dev/null
+++ b/tests/plugins/dummy_plugin/dummy.py
@@ -0,0 +1,7 @@
+from senpy.plugins import SentimentPlugin
+
+
+class DummyPlugin(SentimentPlugin):
+    def analyse_entry(self, entry, params):
+        entry.text = entry.text[::-1]
+        yield entry
diff --git a/tests/plugins/dummy_plugin/dummy.senpy b/tests/plugins/dummy_plugin/dummy.senpy
new file mode 100644
index 0000000..ea0c405
--- /dev/null
+++ b/tests/plugins/dummy_plugin/dummy.senpy
@@ -0,0 +1,15 @@
+{
+    "name": "Dummy",
+    "module": "dummy",
+    "description": "I am dummy",
+    "author": "@balkian",
+    "version": "0.1",
+    "extra_params": {
+        "example": {
+            "@id": "example_parameter",
+            "aliases": ["example", "ex"],
+            "required": false,
+            "default": 0
+        }
+    }
+}
diff --git a/tests/plugins/dummy_plugin/dummy_required.senpy b/tests/plugins/dummy_plugin/dummy_required.senpy
new file mode 100644
index 0000000..3e361f6
--- /dev/null
+++ b/tests/plugins/dummy_plugin/dummy_required.senpy
@@ -0,0 +1,14 @@
+{
+    "name": "DummyRequired",
+    "module": "dummy",
+    "description": "I am dummy",
+    "author": "@balkian",
+    "version": "0.1",
+    "extra_params": {
+        "example": {
+            "@id": "example_parameter",
+            "aliases": ["example", "ex"],
+            "required": true
+        }
+    }
+}
diff --git a/tests/sleep_plugin/sleep.py b/tests/plugins/sleep_plugin/sleep.py
similarity index 51%
rename from tests/sleep_plugin/sleep.py
rename to tests/plugins/sleep_plugin/sleep.py
index bcc1cbe..510c2a9 100644
--- a/tests/sleep_plugin/sleep.py
+++ b/tests/plugins/sleep_plugin/sleep.py
@@ -1,5 +1,4 @@
 from senpy.plugins import SenpyPlugin
-from senpy.models import Results
 from time import sleep
 
 
@@ -7,6 +6,6 @@ class SleepPlugin(SenpyPlugin):
     def activate(self, *args, **kwargs):
         sleep(self.timeout)
 
-    def analyse(self, *args, **kwargs):
-        sleep(float(kwargs.get("timeout", self.timeout)))
-        return Results()
+    def analyse_entry(self, entry, params):
+        sleep(float(params.get("timeout", self.timeout)))
+        yield entry
diff --git a/tests/sleep_plugin/sleep.senpy b/tests/plugins/sleep_plugin/sleep.senpy
similarity index 100%
rename from tests/sleep_plugin/sleep.senpy
rename to tests/plugins/sleep_plugin/sleep.senpy
diff --git a/tests/test_blueprints.py b/tests/test_blueprints.py
index 6674992..2eaae64 100644
--- a/tests/test_blueprints.py
+++ b/tests/test_blueprints.py
@@ -25,6 +25,8 @@ class BlueprintsTest(TestCase):
         self.dir = os.path.join(os.path.dirname(__file__), "..")
         self.senpy.add_folder(self.dir)
         self.senpy.activate_plugin("Dummy", sync=True)
+        self.senpy.activate_plugin("DummyRequired", sync=True)
+        self.senpy.default_plugin = 'Dummy'
 
     def assertCode(self, resp, code):
         self.assertEqual(resp.status_code, code)
@@ -34,12 +36,12 @@ class BlueprintsTest(TestCase):
         Calling with no arguments should ask the user for more arguments
         """
         resp = self.client.get("/api/")
-        self.assertCode(resp, 404)
+        self.assertCode(resp, 400)
         js = parse_resp(resp)
         logging.debug(js)
-        assert js["status"] == 404
+        assert js["status"] == 400
         atleast = {
-            "status": 404,
+            "status": 400,
             "message": "Missing or invalid parameters",
         }
         assert check_dict(js, atleast)
@@ -56,6 +58,28 @@ class BlueprintsTest(TestCase):
         assert "@context" in js
         assert "entries" in js
 
+    def test_analysis_extra(self):
+        """
+        Extra params that have a default should
+        """
+        resp = self.client.get("/api/?i=My aloha mohame&algo=Dummy")
+        self.assertCode(resp, 200)
+        js = parse_resp(resp)
+        logging.debug("Got response: %s", js)
+        assert "@context" in js
+        assert "entries" in js
+
+    def test_analysis_extra_required(self):
+        """
+        Extra params that have a required argument that does not
+        have a default should raise an error.
+        """
+        resp = self.client.get("/api/?i=My aloha mohame&algo=DummyRequired")
+        self.assertCode(resp, 400)
+        js = parse_resp(resp)
+        logging.debug("Got response: %s", js)
+        assert isinstance(js, models.Error)
+
     def test_error(self):
         """
         The dummy plugin returns an empty response,\
@@ -102,7 +126,7 @@ class BlueprintsTest(TestCase):
         js = parse_resp(resp)
         logging.debug(js)
         assert "@id" in js
-        assert js["@id"] == "Dummy_0.1"
+        assert js["@id"] == "plugins/Dummy_0.1"
 
     def test_default(self):
         """ Show only one plugin"""
@@ -111,7 +135,7 @@ class BlueprintsTest(TestCase):
         js = parse_resp(resp)
         logging.debug(js)
         assert "@id" in js
-        assert js["@id"] == "Dummy_0.1"
+        assert js["@id"] == "plugins/Dummy_0.1"
 
     def test_context(self):
         resp = self.client.get("/api/contexts/context.jsonld")
diff --git a/tests/test_client.py b/tests/test_client.py
index 2690d88..2597258 100644
--- a/tests/test_client.py
+++ b/tests/test_client.py
@@ -9,9 +9,10 @@ from senpy.models import Results, Error
 
 
 class Call(dict):
-
     def __init__(self, obj):
         self.obj = obj.jsonld()
+        self.status_code = 200
+        self.content = self.json()
 
     def json(self):
         return self.obj
@@ -29,14 +30,14 @@ class ModelsTest(TestCase):
         with patch('requests.request', return_value=success) as patched:
             resp = client.analyse('hello')
             assert isinstance(resp, Results)
-        patched.assert_called_with(url=endpoint + '/',
-                                   method='GET',
-                                   params={'input': 'hello'})
+        patched.assert_called_with(
+            url=endpoint + '/', method='GET', params={'input': 'hello'})
         error = Call(Error('Nothing'))
         with patch('requests.request', return_value=error) as patched:
             resp = client.analyse(input='hello', algorithm='NONEXISTENT')
             assert isinstance(resp, Error)
-        patched.assert_called_with(url=endpoint + '/',
-                                   method='GET',
-                                   params={'input': 'hello',
-                                           'algorithm': 'NONEXISTENT'})
+        patched.assert_called_with(
+            url=endpoint + '/',
+            method='GET',
+            params={'input': 'hello',
+                    'algorithm': 'NONEXISTENT'})
diff --git a/tests/test_extensions.py b/tests/test_extensions.py
index 5e1faa0..7c89aca 100644
--- a/tests/test_extensions.py
+++ b/tests/test_extensions.py
@@ -16,7 +16,7 @@ from unittest import TestCase
 
 class ExtensionsTest(TestCase):
     def setUp(self):
-        self.app = Flask("test_extensions")
+        self.app = Flask('test_extensions')
         self.dir = os.path.join(os.path.dirname(__file__))
         self.senpy = Senpy(plugin_folder=self.dir,
                            app=self.app,
@@ -45,7 +45,7 @@ class ExtensionsTest(TestCase):
             'requirements': ['noop'],
             'version': 0
         }
-        root = os.path.join(self.dir, 'dummy_plugin')
+        root = os.path.join(self.dir, 'plugins', 'dummy_plugin')
         name, module = self.senpy._load_plugin_from_info(info, root=root)
         assert name == 'TestPip'
         assert module
@@ -55,7 +55,7 @@ class ExtensionsTest(TestCase):
     def test_installing(self):
         """ Enabling a plugin """
         self.senpy.activate_all(sync=True)
-        assert len(self.senpy.plugins) == 2
+        assert len(self.senpy.plugins) >= 3
         assert self.senpy.plugins["Sleep"].is_activated
 
     def test_disabling(self):
@@ -75,11 +75,12 @@ class ExtensionsTest(TestCase):
     def test_noplugin(self):
         """ Don't analyse if there isn't any plugin installed """
         self.senpy.deactivate_all(sync=True)
-        self.assertRaises(Error, partial(self.senpy.analyse,
-                                         input="tupni"))
-        self.assertRaises(Error, partial(self.senpy.analyse,
-                                         input="tupni",
-                                         algorithm='Dummy'))
+        self.assertRaises(Error, partial(self.senpy.analyse, input="tupni"))
+        self.assertRaises(Error,
+                          partial(
+                              self.senpy.analyse,
+                              input="tupni",
+                              algorithm='Dummy'))
 
     def test_analyse(self):
         """ Using a plugin """
@@ -88,17 +89,20 @@ class ExtensionsTest(TestCase):
         r1 = self.senpy.analyse(
             algorithm="Dummy", input="tupni", output="tuptuo")
         r2 = self.senpy.analyse(input="tupni", output="tuptuo")
-        assert r1.analysis[0].id[:5] == "Dummy"
-        assert r2.analysis[0].id[:5] == "Dummy"
+        assert r1.analysis[0] == "plugins/Dummy_0.1"
+        assert r2.analysis[0] == "plugins/Dummy_0.1"
+        assert r1.entries[0].text == 'input'
 
     def test_analyse_error(self):
         mm = mock.MagicMock()
-        mm.analyse.side_effect = Error('error on analysis', status=900)
+        mm.analyse_entry.side_effect = Error('error on analysis', status=900)
         self.senpy.plugins['MOCK'] = mm
         resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
         assert resp['message'] == 'error on analysis'
         assert resp['status'] == 900
         mm.analyse.side_effect = Exception('generic exception on analysis')
+        mm.analyse_entry.side_effect = Exception(
+            'generic exception on analysis')
         resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
         assert resp['message'] == 'generic exception on analysis'
         assert resp['status'] == 500
@@ -110,8 +114,7 @@ class ExtensionsTest(TestCase):
         assert self.senpy.filter_plugins(name="Dummy", is_activated=True)
         self.senpy.deactivate_plugin("Dummy", sync=True)
         assert not len(
-            self.senpy.filter_plugins(
-                name="Dummy", is_activated=True))
+            self.senpy.filter_plugins(name="Dummy", is_activated=True))
 
     def test_load_default_plugins(self):
         senpy = Senpy(plugin_folder=self.dir, default_plugins=True)
diff --git a/tests/test_models.py b/tests/test_models.py
index 5ced7f3..df48c79 100644
--- a/tests/test_models.py
+++ b/tests/test_models.py
@@ -3,20 +3,25 @@ import logging
 import jsonschema
 
 import json
+import rdflib
 from unittest import TestCase
-from senpy.models import Entry, Results, Sentiment, EmotionSet, Error
+from senpy.models import (Emotion,
+                          EmotionAnalysis,
+                          EmotionSet,
+                          Entry,
+                          Error,
+                          Results,
+                          Sentiment)
 from senpy.plugins import SenpyPlugin
 from pprint import pprint
 
 
 class ModelsTest(TestCase):
     def test_jsonld(self):
-        prueba = {"id": "test",
-                  "analysis": [],
-                  "entries": []}
+        prueba = {"id": "test", "analysis": [], "entries": []}
         r = Results(**prueba)
         print("Response's context: ")
-        pprint(r.context)
+        pprint(r._context)
 
         assert r.id == "test"
 
@@ -30,14 +35,11 @@ class ModelsTest(TestCase):
         assert "id" not in j
 
         r6 = Results(**prueba)
-        e = Entry({
-            "@id": "ohno",
-            "nif:isString": "Just testing"
-        })
+        e = Entry({"@id": "ohno", "nif:isString": "Just testing"})
         r6.entries.append(e)
         logging.debug("Reponse 6: %s", r6)
-        assert ("marl" in r6.context)
-        assert ("entries" in r6.context)
+        assert ("marl" in r6._context)
+        assert ("entries" in r6._context)
         j6 = r6.jsonld(with_context=True)
         logging.debug("jsonld: %s", j6)
         assert ("@context" in j6)
@@ -113,5 +115,35 @@ class ModelsTest(TestCase):
         s = str(r)
         assert "_testing" not in s
 
-    def test_frame_response(self):
+    def test_turtle(self):
+        """Any model should be serializable as a turtle file"""
+        ana = EmotionAnalysis()
+        res = Results()
+        res.analysis.append(ana)
+        entry = Entry(text='Just testing')
+        eSet = EmotionSet()
+        emotion = Emotion()
+        entry.emotions.append(eSet)
+        res.entries.append(entry)
+        eSet.onyx__hasEmotion.append(emotion)
+        eSet.prov__wasGeneratedBy = ana.id
+        triples = ('ana a :Analysis',
+                   'entry a :entry',
+                   '      nif:isString "Just testing"',
+                   '      onyx:hasEmotionSet eSet',
+                   'eSet a onyx:EmotionSet',
+                   '     prov:wasGeneratedBy ana',
+                   '     onyx:hasEmotion emotion',
+                   'emotion a onyx:Emotion',
+                   'res a :results',
+                   '    me:AnalysisInvoloved ana',
+                   '    prov:used entry')
+
+        t = res.serialize(format='turtle')
+        print(t)
+        g = rdflib.Graph().parse(data=t, format='turtle')
+        assert len(g) == len(triples)
+
+    def test_convert_emotions(self):
+        """It should be possible to convert between different emotion models"""
         pass
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 734e643..946f77e 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -77,6 +77,7 @@ class PluginsTest(TestCase):
         })
         a.activate()
 
+        assert a.shelf_file == self.shelf_file
         res1 = a.analyse(input=1)
         assert res1.entries[0].nif__isString == 1
         res2 = a.analyse(input=1)
@@ -103,3 +104,19 @@ class PluginsTest(TestCase):
         assert b.sh['a'] == 'fromA'
         b.sh['a'] = 'fromB'
         assert b.sh['a'] == 'fromB'
+
+    def test_extra_params(self):
+        ''' Should be able to set extra parameters'''
+        a = ShelfDummyPlugin(info={
+            'name': 'shelve',
+            'version': 'test',
+            'shelf_file': self.shelf_file,
+            'extra_params': {
+                'example': {
+                    'aliases': ['example', 'ex'],
+                    'required': True,
+                    'default': 'nonsense'
+                }
+            }
+        })
+        assert 'example' in a.extra_params
diff --git a/version.sh b/version.sh
new file mode 100755
index 0000000..c8d1f03
--- /dev/null
+++ b/version.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+VERSION=$(git describe --long --tags --dirty)