Filter conversion plugins

Closes #12

* Shows only analysis plugins by default on /api/plugins
* Adds a plugin_type parameter to get other types of plugins
* default_plugin chosen from analysis plugins

Dockerfile.template

@@ -6,7 +6,7 @@ RUN mkdir /cache/ /senpy-plugins /data/
 
 VOLUME /data/
 
-ENV PIP_CACHE_DIR=/cache/
+ENV PIP_CACHE_DIR=/cache/ SENPY_DATA=/data
 
 ONBUILD COPY . /senpy-plugins/
 ONBUILD RUN python -m senpy --only-install -f /senpy-plugins

Makefile

@@ -85,8 +85,6 @@ git_push:
 pip_upload:
 	python setup.py sdist upload ;
 
-pip_test: $(addprefix pip_test-,$(PYVERSIONS))
-
 run-%: build-%
 	docker run --rm -p 5000:5000 -ti '$(IMAGEWTAG)-python$(PYMAIN)' --default-plugins
 
@@ -95,12 +93,16 @@ run: run-$(PYMAIN)
 push-latest: build-$(PYMAIN)
 	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
 	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGENAME)'
-	docker push '$(IMAGENAME)'
+	docker push '$(IMAGENAME):latest'
 	docker push '$(IMAGEWTAG)'
 
 push-%: build-%
 	docker push $(IMAGENAME):$(VERSION)-python$*
 
+push: $(addprefix push-,$(PYVERSIONS))
+	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
+	docker push  $(IMAGENAME):$(VERSION)
+
 ci:
 	gitlab-runner exec docker --docker-volumes /var/run/docker.sock:/var/run/docker.sock --env CI_PROJECT_NAME=$(NAME) ${action}
 

docs/api.rst

@@ -62,6 +62,7 @@ NIF API
    :query o outformat: one of `turtle` (default), `text`, `json-ld`
    :query p prefix: prefix for the URIs
    :query algo algorithm: algorithm/plugin to use for the analysis. For a list of options, see :http:get:`/api/plugins`. If not provided, the default plugin will be used (:http:get:`/api/plugins/default`).
+   :query algo emotionModel: desired emotion model in the results. If the requested algorithm does not use that emotion model, there are conversion plugins specifically for this. If none of the plugins match, an error will be returned, which includes the results *as is*.
 
    :reqheader Accept: the response content type depends on
                       :mailheader:`Accept` header
@@ -69,6 +70,7 @@ NIF API
                             header of request
    :statuscode 200: no error
    :statuscode 404: service not found
+   :statuscode 400: error while processing the request
 
 .. http:post:: /api
 
@@ -94,50 +96,52 @@ NIF API
             "@context": {
                    ...
             },
-            "sentiment140": {
+            "@type": "plugins",
-                "name": "sentiment140", 
+            "plugins": [
-                "is_activated": true, 
+                  {
-                "version": "0.1", 
+                    "name": "sentiment140", 
-                "extra_params": {
+                    "is_activated": true, 
-                    "@id": "extra_params_sentiment140_0.1", 
+                    "version": "0.1", 
-                    "language": {
+                    "extra_params": {
-                        "required": false, 
+                        "@id": "extra_params_sentiment140_0.1", 
-                        "@id": "lang_sentiment140", 
+                        "language": {
-                        "options": [
+                            "required": false, 
-                            "es", 
+                            "@id": "lang_sentiment140", 
-                            "en", 
+                            "options": [
-                            "auto"
+                                "es", 
-                        ], 
+                                "en", 
-                        "aliases": [
+                                "auto"
-                            "language", 
+                            ], 
-                            "l"
+                            "aliases": [
-                        ]
+                                "language", 
-                    }
+                                "l"
-                }, 
+                            ]
-                "@id": "sentiment140_0.1"
+                        }
-            }, 
+                    }, 
-            "rand": {
+                    "@id": "sentiment140_0.1"
-                "name": "rand", 
+                }, {
-                "is_activated": true, 
+                    "name": "rand", 
-                "version": "0.1", 
+                    "is_activated": true, 
-                "extra_params": {
+                    "version": "0.1", 
-                    "@id": "extra_params_rand_0.1", 
+                    "extra_params": {
-                    "language": {
+                        "@id": "extra_params_rand_0.1", 
-                        "required": false, 
+                        "language": {
-                        "@id": "lang_rand", 
+                            "required": false, 
-                        "options": [
+                            "@id": "lang_rand", 
-                            "es", 
+                            "options": [
-                            "en", 
+                                "es", 
-                            "auto"
+                                "en", 
-                        ], 
+                                "auto"
-                        "aliases": [
+                            ], 
-                            "language", 
+                            "aliases": [
-                            "l"
+                                "language", 
-                        ]
+                                "l"
-                    }
+                            ]
-                }, 
+                        }
-                "@id": "rand_0.1"
+                    }, 
-            }
+                    "@id": "rand_0.1"
+                }
+            ]
         }
 
 
@@ -148,7 +152,7 @@ NIF API
 
    .. sourcecode:: http
 
-      GET /api/plugins/rand HTTP/1.1
+      GET /api/plugins/rand/ HTTP/1.1
       Host: localhost
       Accept: application/json, text/javascript
 
@@ -159,6 +163,7 @@ NIF API
 
       {
           "@id": "rand_0.1",
+          "@type": "sentimentPlugin",
           "extra_params": {
               "@id": "extra_params_rand_0.1",
               "language": {
@@ -185,24 +190,3 @@ NIF API
 
    Return the information about the default plugin.
 
-.. http:get:: /api/plugins/<pluginname>/{de}activate
-
-   {De}activate a plugin.
-
-   **Example request**:
-
-   .. sourcecode:: http
-
-      GET /api/plugins/rand/deactivate HTTP/1.1
-      Host: localhost
-      Accept: application/json, text/javascript
-
-
-   **Example response**:
-
-   .. sourcecode:: http
-
-       {
-          "@context": {}, 
-          "message": "Ok"
-       }

docs/conversion.rst

@@ -0,0 +1,116 @@
+Conversion
+----------
+
+Senpy includes experimental support for emotion/sentiment conversion plugins.
+
+
+Use
+===
+
+Consider the original query: http://127.0.0.1:5000/api/?i=hello&algo=emoRand
+
+The requested plugin (emoRand) returns emotions using Ekman's model (or big6 in EmotionML):
+
+.. code:: json
+
+
+          ... rest of the document ...
+          {
+            "@type": "emotionSet",
+            "onyx:hasEmotion": {
+                "@type": "emotion",
+                "onyx:hasEmotionCategory": "emoml:big6anger"
+            },
+            "prov:wasGeneratedBy": "plugins/emoRand_0.1"
+          }
+
+          
+
+To get these emotions in VAD space (FSRE dimensions in EmotionML), we'd do this:
+
+http://127.0.0.1:5000/api/?i=hello&algo=emoRand&emotionModel=emoml:fsre-dimensions
+
+This call, provided there is a valid conversion plugin from Ekman's to VAD, would return something like this:
+
+.. code:: json
+
+
+          ... rest of the document ...
+          {
+            "@type": "emotionSet",
+            "onyx:hasEmotion": {
+                "@type": "emotion",
+                "onyx:hasEmotionCategory": "emoml:big6anger"
+                },
+            "prov:wasGeneratedBy": "plugins/emoRand_0.1"
+          }, {
+            "@type": "emotionSet",
+            "onyx:hasEmotion": {
+                "@type": "emotion",
+                "A": 7.22,
+                "D": 6.28,
+                "V": 8.6
+            },
+            "prov:wasGeneratedBy": "plugins/Ekman2VAD_0.1"
+
+          }
+
+
+That is called a *full* response, as it simply adds the converted emotion alongside.
+It is also possible to get the original emotion nested within the new converted emotion, using the `conversion=nested` parameter:
+
+.. code:: json
+
+
+          ... rest of the document ...
+          {
+            "@type": "emotionSet",
+            "onyx:hasEmotion": {
+                "@type": "emotion",
+                "onyx:hasEmotionCategory": "emoml:big6anger"
+                },
+            "prov:wasGeneratedBy": "plugins/emoRand_0.1"
+            "onyx:wasDerivedFrom": {
+                "@type": "emotionSet",
+                "onyx:hasEmotion": {
+                    "@type": "emotion",
+                    "A": 7.22,
+                    "D": 6.28,
+                    "V": 8.6
+                },
+                "prov:wasGeneratedBy": "plugins/Ekman2VAD_0.1"
+             }
+
+          }
+
+
+Lastly, `conversion=filtered` would only return the converted emotions.
+
+Developing a conversion plugin
+================================
+
+Conversion plugins are discovered by the server just like any other plugin.
+The difference is the slightly different API, and the need to specify the `source` and `target` of the conversion.
+For instance, an emotion conversion plugin needs the following:
+
+
+.. code:: yaml
+          
+
+          ---
+          onyx:doesConversion:
+            - onyx:conversionFrom: emoml:big6
+              onyx:conversionTo: emoml:fsre-dimensions
+            - onyx:conversionFrom: emoml:fsre-dimensions
+              onyx:conversionTo: emoml:big6
+
+
+
+
+.. code:: python
+
+
+          class MyConversion(EmotionConversionPlugin):
+
+              def convert(self, emotionSet, fromModel, toModel, params):
+                  pass

docs/index.rst

@@ -1,8 +1,3 @@
-.. Senpy documentation master file, created by
-   sphinx-quickstart on Tue Feb 24 08:57:32 2015.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
-
 Welcome to Senpy's documentation!
 =================================
 
@@ -15,5 +10,6 @@ Contents:
    api
    schema
    plugins
+   conversion
    demo
    :maxdepth: 2

docs/plugins.rst

@@ -1,5 +1,7 @@
 Developing new plugins
 ----------------------
+This document describes how to develop a new analysis plugin. For an example of conversion plugins, see :doc:`conversion`.
+
 Each plugin represents a different analysis process.There are two types of files that are needed by senpy for loading a plugin:
 
 - Definition file, has the ".senpy" extension.

docs/usage.rst

@@ -48,8 +48,8 @@ Once the server is launched, there is a basic endpoint in the server, which prov
 
 In case you want to know the different endpoints of the server, there is more information available in the NIF API section_.
 
-Video example
+CLI
-=============
+===
 
 This video shows how to use senpy through command-line tool.
 
@@ -58,18 +58,23 @@ https://asciinema.org/a/9uwef1ghkjk062cw2t4mhzpyk
 Request example in python
 =========================
 
-This example shows how to make a request to a plugin.
+This example shows how to make a request to the default plugin:
 
 .. code:: python
 
-    import requests
+    from senpy.client import Client
-    import json
 
-    r = requests.get('http://127.0.0.1:5000/api/?algo=rand&i=Testing')
+    c = Client('http://127.0.0.1:5000/api/')
-    response = r.content.decode('utf-8')
+    r = c.analyse('hello world')
-    response_json = json.loads(response)
+
+    for entry in r.entries:
+          print('{} -> {}'.format(entry.text, entry.emotions))
 
 
 
 .. _section: http://senpy.readthedocs.org/en/latest/api.html
 
+
+Conversion
+==========
+See :doc:`conversion`

senpy/__init__.py

@@ -17,7 +17,6 @@
 """
 Sentiment analysis server in Python
 """
-from __future__ import print_function
 from .version import __version__
 
 import logging

senpy/api.py

@@ -26,6 +26,13 @@ API_PARAMS = {
         "aliases": ["emotionModel", "emoModel"],
         "required": False
     },
+    "plugin_type": {
+        "@id": "pluginType",
+        "description": 'What kind of plugins to list',
+        "aliases": ["pluginType", "plugin_type"],
+        "required": True,
+        "default": "analysisPlugin"
+    },
     "conversion": {
         "@id": "conversion",
         "description": "How to show the elements that have (not) been converted",

senpy/blueprints.py

@@ -121,7 +121,9 @@ def api():
 @basic_api
 def plugins():
     sp = current_app.senpy
-    dic = Plugins(plugins=list(sp.plugins.values()))
+    ptype = request.params.get('plugin_type')
+    plugins = sp.filter_plugins(plugin_type=ptype)
+    dic = Plugins(plugins=list(plugins.values()))
     return dic
 
 

senpy/client.py

@@ -18,7 +18,6 @@ class Client(object):
         try:
             resp = models.from_dict(response.json())
             resp.validate(resp)
-            return resp
         except Exception as ex:
             logger.error(('There seems to be a problem with the response:\n'
                           '\tURL: {url}\n'
@@ -33,3 +32,6 @@ class Client(object):
                               code=response.status_code,
                               content=response.content))
             raise ex
+        if isinstance(resp, models.Error):
+            raise resp
+        return resp

senpy/extensions.py

@@ -5,7 +5,8 @@ It orchestrates plugin (de)activation and analysis.
 from future import standard_library
 standard_library.install_aliases()
 
-from .plugins import SentimentPlugin, SenpyPlugin
+from . import plugins
+from .plugins import SenpyPlugin
 from .models import Error, Entry, Results
 from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
 from .api import API_PARAMS, NIF_PARAMS, parse_params
@@ -253,7 +254,7 @@ class Senpy(object):
                 logger.error(msg)
                 raise Error(msg)
 
-        if sync:
+        if sync or 'async' in plugin and not plugin.async:
             act()
         else:
             th = Thread(target=act)
@@ -277,7 +278,7 @@ class Senpy(object):
                     "Error deactivating plugin {}: {}".format(plugin.name, ex))
                 logger.error("Trace: {}".format(traceback.format_exc()))
 
-        if sync:
+        if sync or 'async' in plugin and not plugin.async:
             deact()
         else:
             th = Thread(target=deact)
@@ -367,6 +368,22 @@ class Senpy(object):
 
     def filter_plugins(self, **kwargs):
         """ Filter plugins by different criteria """
+        ptype = kwargs.pop('plugin_type', None)
+        logger.debug('#' * 100)
+        logger.debug('ptype {}'.format(ptype))
+        if ptype:
+            try:
+                ptype = ptype[0].upper() + ptype[1:]
+                pclass = getattr(plugins, ptype)
+                logger.debug('Class: {}'.format(pclass))
+                candidates = filter(lambda x: isinstance(x, pclass),
+                                    self.plugins.values())
+            except AttributeError:
+                raise Error('{} is not a valid type'.format(ptype))
+        else:
+            candidates = self.plugins.values()
+
+        logger.debug(candidates)
 
         def matches(plug):
             res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
@@ -374,15 +391,11 @@ class Senpy(object):
                 "matching {} with {}: {}".format(plug.name, kwargs, res))
             return res
 
-        if not kwargs:
+        if kwargs:
-            return self.plugins
+            candidates = filter(matches, candidates)
-        else:
+        return {p.name: p for p in candidates}
-            return {n: p for n, p in self.plugins.items() if matches(p)}
 
-    def sentiment_plugins(self):
+    @property
-        """ Return only the sentiment plugins """
+    def analysis_plugins(self):
-        return {
+        """ Return only the analysis plugins """
-            p: plugin
+        return self.filter_plugins(plugin_type='analysisPlugin')
-            for p, plugin in self.plugins.items()
-            if isinstance(plugin, SentimentPlugin)
-        }

senpy/plugins/__init__.py

@@ -3,6 +3,7 @@ standard_library.install_aliases()
 
 import inspect
 import os.path
+import os
 import pickle
 import logging
 import tempfile
@@ -29,6 +30,15 @@ class SenpyPlugin(models.Plugin):
     def get_folder(self):
         return os.path.dirname(inspect.getfile(self.__class__))
 
+    def activate(self):
+        pass
+
+    def deactivate(self):
+        pass
+
+
+class AnalysisPlugin(SenpyPlugin):
+
     def analyse(self, *args, **kwargs):
         raise NotImplemented(
             'Your method should implement either analyse or analyse_entry')
@@ -47,30 +57,27 @@ class SenpyPlugin(models.Plugin):
         for i in results.entries:
             yield i
 
-    def activate(self):
-        pass
 
-    def deactivate(self):
+class ConversionPlugin(SenpyPlugin):
-        pass
+    pass
 
 
-class SentimentPlugin(models.SentimentPlugin, SenpyPlugin):
+class SentimentPlugin(models.SentimentPlugin, AnalysisPlugin):
     def __init__(self, info, *args, **kwargs):
         super(SentimentPlugin, self).__init__(info, *args, **kwargs)
         self.minPolarityValue = float(info.get("minPolarityValue", 0))
         self.maxPolarityValue = float(info.get("maxPolarityValue", 1))
 
 
-class EmotionPlugin(models.EmotionPlugin, SenpyPlugin):
+class EmotionPlugin(models.EmotionPlugin, AnalysisPlugin):
     def __init__(self, info, *args, **kwargs):
         super(EmotionPlugin, self).__init__(info, *args, **kwargs)
         self.minEmotionValue = float(info.get("minEmotionValue", -1))
         self.maxEmotionValue = float(info.get("maxEmotionValue", 1))
 
 
-class EmotionConversionPlugin(models.EmotionConversionPlugin, SenpyPlugin):
+class EmotionConversionPlugin(models.EmotionConversionPlugin, ConversionPlugin):
-    def __init__(self, info, *args, **kwargs):
+    pass
-        super(EmotionConversionPlugin, self).__init__(info, *args, **kwargs)
 
 
 class ShelfMixin(object):
@@ -92,8 +99,8 @@ class ShelfMixin(object):
     @property
     def shelf_file(self):
         if 'shelf_file' not in self or not self['shelf_file']:
-            self.shelf_file = os.path.join(tempfile.gettempdir(),
+            sd = os.environ.get('SENPY_DATA', tempfile.gettempdir())
-                                           self.name + '.p')
+            self.shelf_file = os.path.join(sd, self.name + '.p')
         return self['shelf_file']
 
     def save(self):

senpy/plugins/conversion/centroids.py

@@ -13,7 +13,7 @@ class CentroidConversion(EmotionConversionPlugin):
         for e in original.onyx__hasEmotion:
             category = e.onyx__hasEmotionCategory
             if category in self.centroids:
-                for dim, value in self.centroids[category].iteritems():
+                for dim, value in self.centroids[category].items():
                     try:
                         res[dim] += value
                     except Exception:

senpy/version.py

@@ -8,8 +8,12 @@ DEFAULT_FILE = os.path.join(ROOT, 'VERSION')
 
 
 def read_version(versionfile=DEFAULT_FILE):
-    with open(versionfile) as f:
+    try:
-        return f.read().strip()
+        with open(versionfile) as f:
+            return f.read().strip()
+    except IOError:
+        logger.error('Running an unknown version of senpy. Be careful!.')
+        return '0.0'
 
 
 __version__ = read_version()

tests/test_client.py

@@ -34,8 +34,11 @@ class ModelsTest(TestCase):
             url=endpoint + '/', method='GET', params={'input': 'hello'})
         error = Call(Error('Nothing'))
         with patch('requests.request', return_value=error) as patched:
-            resp = client.analyse(input='hello', algorithm='NONEXISTENT')
+            try:
-            assert isinstance(resp, Error)
+                client.analyse(input='hello', algorithm='NONEXISTENT')
+                raise Exception('Exceptions should be raised. This is not golang')
+            except Error:
+                pass
         patched.assert_called_with(
             url=endpoint + '/',
             method='GET',