Filter conversion plugins

Closes #12

* Shows only analysis plugins by default on /api/plugins
* Adds a plugin_type parameter to get other types of plugins
* default_plugin chosen from analysis plugins

README.rst

@@ -23,7 +23,7 @@ Through PIP
 
 .. code:: bash
 
-   pip install -U --user senpy
+   pip install --user senpy
 
    
 Alternatively, you can use the development version:
@@ -42,53 +42,6 @@ Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/s
 
 To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --default-plugins -f /plugins``
 
-
-Developing
-----------
-
-Developing/debugging
-********************
-This command will run the senpy container using the latest image available, mounting your current folder so you get your latest code:
-
-.. code:: bash
-
-
-    # Python 3.5
-    make dev
-    # Python 2.7
-    make dev-2.7
-
-Building a docker image
-***********************
-
-.. code:: bash
-
-
-    # Python 3.5
-    make build-3.5
-    # Python 2.7
-    make build-2.7
-
-Testing
-*******
-
-.. code:: bash
-
-
-    make test
-
-Running
-*******
-This command will run the senpy server listening on localhost:5000
-
-.. code:: bash
-
-
-    # Python 3.5
-    make run-3.5
-    # Python 2.7
-    make run-2.7
-
 Usage
 -----
 
@@ -96,14 +49,12 @@ However, the easiest and recommended way is to just use the command-line tool to
 
 .. code:: bash
 
-
    senpy
 
 or, alternatively:
 
 .. code:: bash
 
-
     python -m senpy
 
 

senpy/blueprints.py

@@ -113,11 +113,8 @@ def basic_api(f):
 @api_blueprint.route('/', methods=['POST', 'GET'])
 @basic_api
 def api():
-    try:
-        response = current_app.senpy.analyse(**request.params)
-        return response
-    except Error as ex:
-        return ex
+    response = current_app.senpy.analyse(**request.params)
+    return response
 
 
 @api_blueprint.route('/plugins/', methods=['POST', 'GET'])

senpy/extensions.py

@@ -7,7 +7,7 @@ standard_library.install_aliases()
 
 from . import plugins
 from .plugins import SenpyPlugin
-from .models import Error, Entry, Results, from_dict
+from .models import Error, Entry, Results
 from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
 from .api import API_PARAMS, NIF_PARAMS, parse_params
 
@@ -78,101 +78,70 @@ class Senpy(object):
         else:
             logger.debug("Not a folder: %s", folder)
 
-    def _find_plugins(self, params):
-        if not self.analysis_plugins:
+    def _find_plugin(self, params):
+        api_params = parse_params(params, spec=API_PARAMS)
+        algo = None
+        if "algorithm" in api_params and api_params["algorithm"]:
+            algo = api_params["algorithm"]
+        elif self.plugins:
+            algo = self.default_plugin and self.default_plugin.name
+        if not algo:
             raise Error(
                 status=404,
                 message=("No plugins found."
-                         " Please install one."))
-        api_params = parse_params(params, spec=API_PARAMS)
-        algos = None
-        if "algorithm" in api_params and api_params["algorithm"]:
-            algos = api_params["algorithm"].split(',')
-        elif self.default_plugin:
-            algos = [self.default_plugin.name, ]
-        else:
+                         " Please install one.").format(algo))
+        if algo not in self.plugins:
+            logger.debug(("The algorithm '{}' is not valid\n"
+                          "Valid algorithms: {}").format(algo,
+                                                         self.plugins.keys()))
             raise Error(
                 status=404,
-                message="No default plugin found, and None provided")
+                message="The algorithm '{}' is not valid".format(algo))
 
-        plugins = list()
-        for algo in algos:
-            if algo not in self.plugins:
-                logger.debug(("The algorithm '{}' is not valid\n"
-                              "Valid algorithms: {}").format(algo,
-                                                             self.plugins.keys()))
-                raise Error(
-                    status=404,
-                    message="The algorithm '{}' is not valid".format(algo))
+        if not self.plugins[algo].is_activated:
+            logger.debug("Plugin not activated: {}".format(algo))
+            raise Error(
+                status=400,
+                message=("The algorithm '{}'"
+                         " is not activated yet").format(algo))
+        return self.plugins[algo]
 
-            if not self.plugins[algo].is_activated:
-                logger.debug("Plugin not activated: {}".format(algo))
-                raise Error(
-                    status=400,
-                    message=("The algorithm '{}'"
-                             " is not activated yet").format(algo))
-            plugins.append(self.plugins[algo])
-        return plugins
-
-    def _get_params(self, params, plugin=None):
+    def _get_params(self, params, plugin):
         nif_params = parse_params(params, spec=NIF_PARAMS)
-        if plugin:
-            extra_params = plugin.get('extra_params', {})
-            specific_params = parse_params(params, spec=extra_params)
-            nif_params.update(specific_params)
+        extra_params = plugin.get('extra_params', {})
+        specific_params = parse_params(params, spec=extra_params)
+        nif_params.update(specific_params)
         return nif_params
 
     def _get_entries(self, params):
+        entry = None
         if params['informat'] == 'text':
-            results = Results()
             entry = Entry(text=params['input'])
-            results.entries.append(entry)
-        elif params['informat'] == 'json-ld':
-            results = from_dict(params['input'])
         else:
-            raise NotImplemented('Informat {} is not implemented'.format(params['informat']))
-        return results
-
-    def _process_entries(self, entries, plugins, nif_params):
-        if not plugins:
-            for i in entries:
-                yield i
-            return
-        plugin = plugins[0]
-        specific_params = self._get_params(nif_params, plugin)
-        results = plugin.analyse_entries(entries, specific_params)
-        for i in self._process_entries(results, plugins[1:], nif_params):
-            yield i
-
-    def _process_response(self, resp, plugins, nif_params):
-        entries = resp.entries
-        resp.entries = []
-        for plug in plugins:
-            resp.analysis.append(plug.id)
-        for i in self._process_entries(entries, plugins, nif_params):
-            resp.entries.append(i)
-        return resp
+            raise NotImplemented('Only text input format implemented')
+        yield entry
 
     def analyse(self, **api_params):
-        """
-        Main method that analyses a request, either from CLI or HTTP.
-        It uses a dictionary of parameters, provided by the user.
-        """
         logger.debug("analysing with params: {}".format(api_params))
-        plugins = self._find_plugins(api_params)
-        nif_params = self._get_params(api_params)
-        resp = self._get_entries(nif_params)
+        plugin = self._find_plugin(api_params)
+        nif_params = self._get_params(api_params, plugin)
+        resp = Results()
         if 'with_parameters' in api_params:
             resp.parameters = nif_params
         try:
-            resp = self._process_response(resp, plugins, nif_params)
-            self.convert_emotions(resp, plugins, nif_params)
+            entries = []
+            for i in self._get_entries(nif_params):
+                entries += list(plugin.analyse_entry(i, nif_params))
+            resp.entries = entries
+            self.convert_emotions(resp, plugin, nif_params)
+            resp.analysis.append(plugin.id)
             logger.debug("Returning analysis result: {}".format(resp))
-        except (Error, Exception) as ex:
-            if not isinstance(ex, Error):
-                ex = Error(message=str(ex), status=500)
+        except Error as ex:
             logger.exception('Error returning analysis result')
-            raise ex
+            resp = ex
+        except Exception as ex:
+            logger.exception('Error returning analysis result')
+            resp = Error(message=str(ex), status=500)
         return resp
 
     def _conversion_candidates(self, fromModel, toModel):
@@ -186,7 +155,7 @@ class Senpy(object):
                     # logging.debug('Found candidate: {}'.format(candidate))
                     yield candidate
 
-    def convert_emotions(self, resp, plugins, params):
+    def convert_emotions(self, resp, plugin, params):
         """
         Conversion of all emotions in a response.
         In addition to converting from one model to another, it has
@@ -194,35 +163,29 @@ class Senpy(object):
         Needless to say, this is far from an elegant solution, but it works.
         @todo refactor and clean up
         """
+        fromModel = plugin.get('onyx:usesEmotionModel', None)
         toModel = params.get('emotionModel', None)
+        output = params.get('conversion', None)
+        logger.debug('Asked for model: {}'.format(toModel))
+        logger.debug('Analysis plugin uses model: {}'.format(fromModel))
+
         if not toModel:
             return
-
-        logger.debug('Asked for model: {}'.format(toModel))
-        output = params.get('conversion', None)
-        candidates = {}
-        for plugin in plugins:
-            try:
-                fromModel = plugin.get('onyx:usesEmotionModel', None)
-                candidates[plugin.id] = next(self._conversion_candidates(fromModel, toModel))
-                logger.debug('Analysis plugin {} uses model: {}'.format(plugin.id, fromModel))
-            except StopIteration:
-                e = Error(('No conversion plugin found for: '
-                           '{} -> {}'.format(fromModel, toModel)))
-                e.original_response = resp
-                e.parameters = params
-                raise e
+        try:
+            candidate = next(self._conversion_candidates(fromModel, toModel))
+        except StopIteration:
+            e = Error(('No conversion plugin found for: '
+                       '{} -> {}'.format(fromModel, toModel)))
+            e.original_response = resp
+            e.parameters = params
+            raise e
         newentries = []
-        resp.analysis = set(resp.analysis)
         for i in resp.entries:
             if output == "full":
                 newemotions = copy.deepcopy(i.emotions)
             else:
                 newemotions = []
             for j in i.emotions:
-                plugname = j['prov:wasGeneratedBy']
-                candidate = candidates[plugname]
-                resp.analysis.add(candidate.id)
                 for k in candidate.convert(j, fromModel, toModel, params):
                     k.prov__wasGeneratedBy = candidate.id
                     if output == 'nested':
@@ -231,6 +194,7 @@ class Senpy(object):
             i.emotions = newemotions
             newentries.append(i)
         resp.entries = newentries
+        resp.analysis.append(candidate.id)
 
     @property
     def default_plugin(self):

senpy/plugins/__init__.py

@@ -13,7 +13,7 @@ from .. import models
 logger = logging.getLogger(__name__)
 
 
-class Plugin(models.Plugin):
+class SenpyPlugin(models.Plugin):
     def __init__(self, info=None):
         """
         Provides a canonical name for plugins and serves as base for other
@@ -24,7 +24,7 @@ class Plugin(models.Plugin):
                                         "information for the plugin."))
         logger.debug("Initialising {}".format(info))
         id = 'plugins/{}_{}'.format(info['name'], info['version'])
-        super(Plugin, self).__init__(id=id, **info)
+        super(SenpyPlugin, self).__init__(id=id, **info)
         self.is_activated = False
 
     def get_folder(self):
@@ -37,10 +37,7 @@ class Plugin(models.Plugin):
         pass
 
 
-SenpyPlugin = Plugin
-
-
-class AnalysisPlugin(Plugin):
+class AnalysisPlugin(SenpyPlugin):
 
     def analyse(self, *args, **kwargs):
         raise NotImplemented(
@@ -60,14 +57,8 @@ class AnalysisPlugin(Plugin):
         for i in results.entries:
             yield i
 
-    def analyse_entries(self, entries, parameters):
-        for entry in entries:
-            logger.debug('Analysing entry with plugin {}: {}'.format(self, entry))
-            for result in self.analyse_entry(entry, parameters):
-                yield result
 
-
-class ConversionPlugin(Plugin):
+class ConversionPlugin(SenpyPlugin):
     pass
 
 

senpy/plugins/conversion/centroids.py

@@ -6,33 +6,6 @@ logger = logging.getLogger(__name__)
 
 
 class CentroidConversion(EmotionConversionPlugin):
-    def __init__(self, info):
-        if 'centroids' not in info:
-            raise Error('Centroid conversion plugins should provide '
-                        'the centroids in their senpy file')
-        if 'onyx:doesConversion' not in info:
-            if 'centroids_direction' not in info:
-                raise Error('Please, provide centroids direction')
-
-            cf, ct = info['centroids_direction']
-            info['onyx:doesConversion'] = [{
-                'onyx:conversionFrom': cf,
-                'onyx:conversionTo': ct
-            }, {
-                'onyx:conversionFrom': ct,
-                'onyx:conversionTo': cf
-            }]
-
-        if 'aliases' in info:
-            aliases = info['aliases']
-            ncentroids = {}
-            for k1, v1 in info['centroids'].items():
-                nv1 = {}
-                for k2, v2 in v1.items():
-                    nv1[aliases.get(k2, k2)] = v2
-                ncentroids[aliases.get(k1, k1)] = nv1
-            info['centroids'] = ncentroids
-        super(CentroidConversion, self).__init__(info)
 
     def _forward_conversion(self, original):
         """Sum the VAD value of all categories found."""
@@ -52,7 +25,7 @@ class CentroidConversion(EmotionConversionPlugin):
         dimensions = list(self.centroids.values())[0]
 
         def distance(e1, e2):
-            return sum((e1[k] - e2.get(k, 0)) for k in dimensions)
+            return sum((e1[k] - e2.get(self.aliases[k], 0)) for k in dimensions)
 
         emotion = ''
         mindistance = 10000000000000000000000.0
@@ -67,12 +40,11 @@ class CentroidConversion(EmotionConversionPlugin):
     def convert(self, emotionSet, fromModel, toModel, params):
 
         cf, ct = self.centroids_direction
-        logger.debug(
-            '{}\n{}\n{}\n{}'.format(emotionSet, fromModel, toModel, params))
+        logger.debug('{}\n{}\n{}\n{}'.format(emotionSet, fromModel, toModel, params))
         e = EmotionSet()
-        if fromModel == cf and toModel == ct:
+        if fromModel == cf:
             e.onyx__hasEmotion.append(self._forward_conversion(emotionSet))
-        elif fromModel == ct and toModel == cf:
+        elif fromModel == ct:
             for i in emotionSet.onyx__hasEmotion:
                 e.onyx__hasEmotion.append(self._backwards_conversion(i))
         else:

senpy/plugins/conversion/emotion/ekman2fsre.senpy

@@ -1,39 +0,0 @@
----
-name: Ekman2FSRE
-module: senpy.plugins.conversion.centroids
-description: Plugin to convert emotion sets from Ekman to VAD
-version: 0.1
-# No need to specify onyx:doesConversion because centroids.py adds it automatically from centroids_direction
-centroids:
-  anger:
-    A: 6.95
-    D: 5.1
-    V: 2.7
-  disgust:
-    A: 5.3
-    D: 8.05
-    V: 2.7
-  fear:
-    A: 6.5
-    D: 3.6
-    V: 3.2
-  happiness:
-    A: 7.22
-    D: 6.28
-    V: 8.6
-  sadness:
-    A: 5.21
-    D: 2.82
-    V: 2.21
-centroids_direction:
-  - emoml:big6
-  - emoml:fsre-dimensions
-aliases: # These are aliases for any key in the centroid, to avoid repeating a long name several times
-  A: emoml:arousal
-  V: emoml:valence
-  D: emoml:dominance
-  anger: emoml:big6anger
-  disgust: emoml:big6disgust
-  fear: emoml:big6fear
-  happiness: emoml:big6happiness
-  sadness: emoml:big6sadness

senpy/plugins/conversion/emotion/ekman2vad.senpy

@@ -1,39 +1,38 @@
 ---
-name: Ekman2PAD
+name: Ekman2VAD
 module: senpy.plugins.conversion.centroids
 description: Plugin to convert emotion sets from Ekman to VAD
 version: 0.1
-# No need to specify onyx:doesConversion because centroids.py adds it automatically from centroids_direction
+onyx:doesConversion:
+  - onyx:conversionFrom: emoml:big6
+    onyx:conversionTo: emoml:fsre-dimensions
+  - onyx:conversionFrom: emoml:fsre-dimensions
+    onyx:conversionTo: emoml:big6
 centroids:
-  anger:
+  emoml:big6anger:
     A: 6.95
     D: 5.1
     V: 2.7
-  disgust:
+  emoml:big6disgust:
     A: 5.3
     D: 8.05
     V: 2.7
-  fear:
+  emoml:big6fear:
     A: 6.5
     D: 3.6
     V: 3.2
-  happiness:
+  emoml:big6happiness:
     A: 7.22
     D: 6.28
     V: 8.6
-  sadness:
+  emoml:big6sadness:
     A: 5.21
     D: 2.82
     V: 2.21
 centroids_direction:
   - emoml:big6
-  - emoml:pad
-aliases: # These are aliases for any key in the centroid, to avoid repeating a long name several times
+  - emoml:fsre-dimensions
+aliases:
   A: emoml:arousal
   V: emoml:valence
-  D: emoml:dominance
-  anger: emoml:big6anger
-  disgust: emoml:big6disgust
-  fear: emoml:big6fear
-  happiness: emoml:big6happiness
-  sadness: emoml:big6sadness
+  D: emoml:dominance

senpy/schemas/context.jsonld

@@ -37,12 +37,6 @@
       "@type": "@id",
       "@container": "@set"
     },
-    "plugins": {
-      "@container": "@list"
-    },
-    "options": {
-      "@container": "@set"
-    },
     "prov:wasGeneratedBy": {
       "@type": "@id"
     },

senpy/schemas/plugins.json

@@ -6,7 +6,6 @@
       "properties": {
         "plugins": {
           "type": "array",
-          "default": [],
           "items": {
             "$ref": "plugin.json"
           }

tests/plugins/dummy_plugin/dummy.py

@@ -4,5 +4,4 @@ from senpy.plugins import SentimentPlugin
 class DummyPlugin(SentimentPlugin):
     def analyse_entry(self, entry, params):
         entry.text = entry.text[::-1]
-        entry.reversed = entry.get('reversed', 0) + 1
         yield entry

tests/plugins/sleep_plugin/sleep.py

@@ -1,8 +1,8 @@
-from senpy.plugins import AnalysisPlugin
+from senpy.plugins import SenpyPlugin
 from time import sleep
 
 
-class SleepPlugin(AnalysisPlugin):
+class SleepPlugin(SenpyPlugin):
     def activate(self, *args, **kwargs):
         sleep(self.timeout)
 

tests/test_extensions.py

@@ -10,7 +10,7 @@ except ImportError:
 
 from functools import partial
 from senpy.extensions import Senpy
-from senpy.models import Error, Results, Entry, EmotionSet, Emotion, Plugin
+from senpy.models import Error, Results, Entry, EmotionSet, Emotion
 from flask import Flask
 from unittest import TestCase
 
@@ -98,26 +98,17 @@ class ExtensionsTest(TestCase):
 
     def test_analyse_error(self):
         mm = mock.MagicMock()
-        mm.id = 'magic_mock'
-        mm.analyse_entries.side_effect = Error('error on analysis', status=500)
+        mm.analyse_entry.side_effect = Error('error on analysis', status=900)
         self.senpy.plugins['MOCK'] = mm
-        try:
-            self.senpy.analyse(input='nothing', algorithm='MOCK')
-            assert False
-        except Error as ex:
-            assert ex['message'] == 'error on analysis'
-            assert ex['status'] == 500
-
+        resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
+        assert resp['message'] == 'error on analysis'
+        assert resp['status'] == 900
         mm.analyse.side_effect = Exception('generic exception on analysis')
-        mm.analyse_entries.side_effect = Exception(
+        mm.analyse_entry.side_effect = Exception(
             'generic exception on analysis')
-
-        try:
-            self.senpy.analyse(input='nothing', algorithm='MOCK')
-            assert False
-        except Error as ex:
-            assert ex['message'] == 'generic exception on analysis'
-            assert ex['status'] == 500
+        resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
+        assert resp['message'] == 'generic exception on analysis'
+        assert resp['status'] == 500
 
     def test_filtering(self):
         """ Filtering plugins """
@@ -134,12 +125,11 @@ class ExtensionsTest(TestCase):
 
     def test_convert_emotions(self):
         self.senpy.activate_all()
-        plugin = Plugin({
+        plugin = {
             'id': 'imaginary',
             'onyx:usesEmotionModel': 'emoml:fsre-dimensions'
-        })
+        }
         eSet1 = EmotionSet()
-        eSet1.prov__wasGeneratedBy = plugin['id']
         eSet1['onyx:hasEmotion'].append(Emotion({
             'emoml:arousal': 1,
             'emoml:potency': 0,
@@ -155,19 +145,19 @@ class ExtensionsTest(TestCase):
                   'conversion': 'full'}
         r1 = deepcopy(response)
         self.senpy.convert_emotions(r1,
-                                    [plugin, ],
+                                    plugin,
                                     params)
         assert len(r1.entries[0].emotions) == 2
         params['conversion'] = 'nested'
         r2 = deepcopy(response)
         self.senpy.convert_emotions(r2,
-                                    [plugin, ],
+                                    plugin,
                                     params)
         assert len(r2.entries[0].emotions) == 1
         assert r2.entries[0].emotions[0]['prov:wasDerivedFrom'] == eSet1
         params['conversion'] = 'filtered'
         r3 = deepcopy(response)
         self.senpy.convert_emotions(r3,
-                                    [plugin, ],
+                                    plugin,
                                     params)
         assert len(r3.entries[0].emotions) == 1

tests/test_models.py

@@ -11,10 +11,8 @@ from senpy.models import (Emotion,
                           Entry,
                           Error,
                           Results,
-                          Sentiment,
-                          Plugins,
-                          Plugin)
-from senpy import plugins
+                          Sentiment)
+from senpy.plugins import SenpyPlugin
 from pprint import pprint
 
 
@@ -55,8 +53,8 @@ class ModelsTest(TestCase):
         assert (received["entries"][0]["nif:isString"] != "Not testing")
 
     def test_id(self):
-        """ Adding the id after creation should overwrite the automatic ID
-        """
+        ''' Adding the id after creation should overwrite the automatic ID
+        '''
         r = Entry()
         j = r.jsonld()
         assert '@id' in j
@@ -96,16 +94,8 @@ class ModelsTest(TestCase):
         r.validate()
 
     def test_plugins(self):
-        self.assertRaises(Error, plugins.Plugin)
-        p = plugins.Plugin({"name": "dummy",
-                            "version": 0,
-                            "extra_params": {
-                                "none": {
-                                    "options": ["es", ],
-                                    "required": False,
-                                    "default": "0"
-                                }
-                            }})
+        self.assertRaises(Error, SenpyPlugin)
+        p = SenpyPlugin({"name": "dummy", "version": 0})
         c = p.jsonld()
         assert "info" not in c
         assert "repo" not in c
@@ -113,13 +103,11 @@ class ModelsTest(TestCase):
         logging.debug("Framed:")
         logging.debug(c)
         p.validate()
-        assert "es" in c['extra_params']['none']['options']
-        assert isinstance(c['extra_params']['none']['options'], list)
 
     def test_str(self):
         """The string representation shouldn't include private variables"""
         r = Results()
-        p = plugins.Plugin({"name": "STR test", "version": 0})
+        p = SenpyPlugin({"name": "STR test", "version": 0})
         p._testing = 0
         s = str(p)
         assert "_testing" not in s
@@ -155,15 +143,3 @@ class ModelsTest(TestCase):
         print(t)
         g = rdflib.Graph().parse(data=t, format='turtle')
         assert len(g) == len(triples)
-
-    def test_single_plugin(self):
-        """A response with a single plugin should still return a list"""
-        plugs = Plugins()
-        for i in range(10):
-            p = Plugin({'id': str(i),
-                        'version': 0,
-                        'description': 'dummy'})
-            plugs.plugins.append(p)
-        assert isinstance(plugs.plugins, list)
-        js = plugs.jsonld()
-        assert isinstance(js['plugins'], list)