Merge branch '17-...' into 0.8.x

Closes #18

README.rst

@@ -23,7 +23,7 @@ Through PIP
 
 .. code:: bash
 
-   pip install --user senpy
+   pip install -U --user senpy
 
    
 Alternatively, you can use the development version:
@@ -42,6 +42,53 @@ Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/s
 
 To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --default-plugins -f /plugins``
 
+
+Developing
+----------
+
+Developing/debugging
+********************
+This command will run the senpy container using the latest image available, mounting your current folder so you get your latest code:
+
+.. code:: bash
+
+
+    # Python 3.5
+    make dev
+    # Python 2.7
+    make dev-2.7
+
+Building a docker image
+***********************
+
+.. code:: bash
+
+
+    # Python 3.5
+    make build-3.5
+    # Python 2.7
+    make build-2.7
+
+Testing
+*******
+
+.. code:: bash
+
+
+    make test
+
+Running
+*******
+This command will run the senpy server listening on localhost:5000
+
+.. code:: bash
+
+
+    # Python 3.5
+    make run-3.5
+    # Python 2.7
+    make run-2.7
+
 Usage
 -----
 
@@ -49,12 +96,14 @@ However, the easiest and recommended way is to just use the command-line tool to
 
 .. code:: bash
 
+
    senpy
 
 or, alternatively:
 
 .. code:: bash
 
+
     python -m senpy
 
 

senpy/blueprints.py

@@ -113,8 +113,11 @@ def basic_api(f):
 @api_blueprint.route('/', methods=['POST', 'GET'])
 @basic_api
 def api():
-    response = current_app.senpy.analyse(**request.params)
+    try:
-    return response
+        response = current_app.senpy.analyse(**request.params)
+        return response
+    except Error as ex:
+        return ex
 
 
 @api_blueprint.route('/plugins/', methods=['POST', 'GET'])

senpy/extensions.py

@@ -7,7 +7,7 @@ standard_library.install_aliases()
 
 from . import plugins
 from .plugins import SenpyPlugin
-from .models import Error, Entry, Results
+from .models import Error, Entry, Results, from_dict
 from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
 from .api import API_PARAMS, NIF_PARAMS, parse_params
 
@@ -78,70 +78,101 @@ class Senpy(object):
         else:
             logger.debug("Not a folder: %s", folder)
 
-    def _find_plugin(self, params):
+    def _find_plugins(self, params):
-        api_params = parse_params(params, spec=API_PARAMS)
+        if not self.analysis_plugins:
-        algo = None
-        if "algorithm" in api_params and api_params["algorithm"]:
-            algo = api_params["algorithm"]
-        elif self.plugins:
-            algo = self.default_plugin and self.default_plugin.name
-        if not algo:
             raise Error(
                 status=404,
                 message=("No plugins found."
-                         " Please install one.").format(algo))
+                         " Please install one."))
-        if algo not in self.plugins:
+        api_params = parse_params(params, spec=API_PARAMS)
-            logger.debug(("The algorithm '{}' is not valid\n"
+        algos = None
-                          "Valid algorithms: {}").format(algo,
+        if "algorithm" in api_params and api_params["algorithm"]:
-                                                         self.plugins.keys()))
+            algos = api_params["algorithm"].split(',')
+        elif self.default_plugin:
+            algos = [self.default_plugin.name, ]
+        else:
             raise Error(
                 status=404,
-                message="The algorithm '{}' is not valid".format(algo))
+                message="No default plugin found, and None provided")
 
-        if not self.plugins[algo].is_activated:
+        plugins = list()
-            logger.debug("Plugin not activated: {}".format(algo))
+        for algo in algos:
-            raise Error(
+            if algo not in self.plugins:
-                status=400,
+                logger.debug(("The algorithm '{}' is not valid\n"
-                message=("The algorithm '{}'"
+                              "Valid algorithms: {}").format(algo,
-                         " is not activated yet").format(algo))
+                                                             self.plugins.keys()))
-        return self.plugins[algo]
+                raise Error(
+                    status=404,
+                    message="The algorithm '{}' is not valid".format(algo))
 
-    def _get_params(self, params, plugin):
+            if not self.plugins[algo].is_activated:
+                logger.debug("Plugin not activated: {}".format(algo))
+                raise Error(
+                    status=400,
+                    message=("The algorithm '{}'"
+                             " is not activated yet").format(algo))
+            plugins.append(self.plugins[algo])
+        return plugins
+
+    def _get_params(self, params, plugin=None):
         nif_params = parse_params(params, spec=NIF_PARAMS)
-        extra_params = plugin.get('extra_params', {})
+        if plugin:
-        specific_params = parse_params(params, spec=extra_params)
+            extra_params = plugin.get('extra_params', {})
-        nif_params.update(specific_params)
+            specific_params = parse_params(params, spec=extra_params)
+            nif_params.update(specific_params)
         return nif_params
 
     def _get_entries(self, params):
-        entry = None
         if params['informat'] == 'text':
+            results = Results()
             entry = Entry(text=params['input'])
+            results.entries.append(entry)
+        elif params['informat'] == 'json-ld':
+            results = from_dict(params['input'])
         else:
-            raise NotImplemented('Only text input format implemented')
+            raise NotImplemented('Informat {} is not implemented'.format(params['informat']))
-        yield entry
+        return results
+
+    def _process_entries(self, entries, plugins, nif_params):
+        if not plugins:
+            for i in entries:
+                yield i
+            return
+        plugin = plugins[0]
+        specific_params = self._get_params(nif_params, plugin)
+        results = plugin.analyse_entries(entries, specific_params)
+        for i in self._process_entries(results, plugins[1:], nif_params):
+            yield i
+
+    def _process_response(self, resp, plugins, nif_params):
+        entries = resp.entries
+        resp.entries = []
+        for plug in plugins:
+            resp.analysis.append(plug.id)
+        for i in self._process_entries(entries, plugins, nif_params):
+            resp.entries.append(i)
+        return resp
 
     def analyse(self, **api_params):
+        """
+        Main method that analyses a request, either from CLI or HTTP.
+        It uses a dictionary of parameters, provided by the user.
+        """
         logger.debug("analysing with params: {}".format(api_params))
-        plugin = self._find_plugin(api_params)
+        plugins = self._find_plugins(api_params)
-        nif_params = self._get_params(api_params, plugin)
+        nif_params = self._get_params(api_params)
-        resp = Results()
+        resp = self._get_entries(nif_params)
         if 'with_parameters' in api_params:
             resp.parameters = nif_params
         try:
-            entries = []
+            resp = self._process_response(resp, plugins, nif_params)
-            for i in self._get_entries(nif_params):
+            self.convert_emotions(resp, plugins, nif_params)
-                entries += list(plugin.analyse_entry(i, nif_params))
-            resp.entries = entries
-            self.convert_emotions(resp, plugin, nif_params)
-            resp.analysis.append(plugin.id)
             logger.debug("Returning analysis result: {}".format(resp))
-        except Error as ex:
+        except (Error, Exception) as ex:
+            if not isinstance(ex, Error):
+                ex = Error(message=str(ex), status=500)
             logger.exception('Error returning analysis result')
-            resp = ex
+            raise ex
-        except Exception as ex:
-            logger.exception('Error returning analysis result')
-            resp = Error(message=str(ex), status=500)
         return resp
 
     def _conversion_candidates(self, fromModel, toModel):
@@ -155,7 +186,7 @@ class Senpy(object):
                     # logging.debug('Found candidate: {}'.format(candidate))
                     yield candidate
 
-    def convert_emotions(self, resp, plugin, params):
+    def convert_emotions(self, resp, plugins, params):
         """
         Conversion of all emotions in a response.
         In addition to converting from one model to another, it has
@@ -163,29 +194,35 @@ class Senpy(object):
         Needless to say, this is far from an elegant solution, but it works.
         @todo refactor and clean up
         """
-        fromModel = plugin.get('onyx:usesEmotionModel', None)
         toModel = params.get('emotionModel', None)
-        output = params.get('conversion', None)
-        logger.debug('Asked for model: {}'.format(toModel))
-        logger.debug('Analysis plugin uses model: {}'.format(fromModel))
-
         if not toModel:
             return
-        try:
+
-            candidate = next(self._conversion_candidates(fromModel, toModel))
+        logger.debug('Asked for model: {}'.format(toModel))
-        except StopIteration:
+        output = params.get('conversion', None)
-            e = Error(('No conversion plugin found for: '
+        candidates = {}
-                       '{} -> {}'.format(fromModel, toModel)))
+        for plugin in plugins:
-            e.original_response = resp
+            try:
-            e.parameters = params
+                fromModel = plugin.get('onyx:usesEmotionModel', None)
-            raise e
+                candidates[plugin.id] = next(self._conversion_candidates(fromModel, toModel))
+                logger.debug('Analysis plugin {} uses model: {}'.format(plugin.id, fromModel))
+            except StopIteration:
+                e = Error(('No conversion plugin found for: '
+                           '{} -> {}'.format(fromModel, toModel)))
+                e.original_response = resp
+                e.parameters = params
+                raise e
         newentries = []
+        resp.analysis = set(resp.analysis)
         for i in resp.entries:
             if output == "full":
                 newemotions = copy.deepcopy(i.emotions)
             else:
                 newemotions = []
             for j in i.emotions:
+                plugname = j['prov:wasGeneratedBy']
+                candidate = candidates[plugname]
+                resp.analysis.add(candidate.id)
                 for k in candidate.convert(j, fromModel, toModel, params):
                     k.prov__wasGeneratedBy = candidate.id
                     if output == 'nested':
@@ -194,7 +231,6 @@ class Senpy(object):
             i.emotions = newemotions
             newentries.append(i)
         resp.entries = newentries
-        resp.analysis.append(candidate.id)
 
     @property
     def default_plugin(self):

senpy/plugins/__init__.py

@@ -13,7 +13,7 @@ from .. import models
 logger = logging.getLogger(__name__)
 
 
-class SenpyPlugin(models.Plugin):
+class Plugin(models.Plugin):
     def __init__(self, info=None):
         """
         Provides a canonical name for plugins and serves as base for other
@@ -24,7 +24,7 @@ class SenpyPlugin(models.Plugin):
                                         "information for the plugin."))
         logger.debug("Initialising {}".format(info))
         id = 'plugins/{}_{}'.format(info['name'], info['version'])
-        super(SenpyPlugin, self).__init__(id=id, **info)
+        super(Plugin, self).__init__(id=id, **info)
         self.is_activated = False
 
     def get_folder(self):
@@ -37,7 +37,10 @@ class SenpyPlugin(models.Plugin):
         pass
 
 
-class AnalysisPlugin(SenpyPlugin):
+SenpyPlugin = Plugin
+
+
+class AnalysisPlugin(Plugin):
 
     def analyse(self, *args, **kwargs):
         raise NotImplemented(
@@ -57,8 +60,14 @@ class AnalysisPlugin(SenpyPlugin):
         for i in results.entries:
             yield i
 
+    def analyse_entries(self, entries, parameters):
+        for entry in entries:
+            logger.debug('Analysing entry with plugin {}: {}'.format(self, entry))
+            for result in self.analyse_entry(entry, parameters):
+                yield result
 
-class ConversionPlugin(SenpyPlugin):
+
+class ConversionPlugin(Plugin):
     pass
 
 

senpy/plugins/conversion/centroids.py

@@ -6,6 +6,33 @@ logger = logging.getLogger(__name__)
 
 
 class CentroidConversion(EmotionConversionPlugin):
+    def __init__(self, info):
+        if 'centroids' not in info:
+            raise Error('Centroid conversion plugins should provide '
+                        'the centroids in their senpy file')
+        if 'onyx:doesConversion' not in info:
+            if 'centroids_direction' not in info:
+                raise Error('Please, provide centroids direction')
+
+            cf, ct = info['centroids_direction']
+            info['onyx:doesConversion'] = [{
+                'onyx:conversionFrom': cf,
+                'onyx:conversionTo': ct
+            }, {
+                'onyx:conversionFrom': ct,
+                'onyx:conversionTo': cf
+            }]
+
+        if 'aliases' in info:
+            aliases = info['aliases']
+            ncentroids = {}
+            for k1, v1 in info['centroids'].items():
+                nv1 = {}
+                for k2, v2 in v1.items():
+                    nv1[aliases.get(k2, k2)] = v2
+                ncentroids[aliases.get(k1, k1)] = nv1
+            info['centroids'] = ncentroids
+        super(CentroidConversion, self).__init__(info)
 
     def _forward_conversion(self, original):
         """Sum the VAD value of all categories found."""
@@ -25,7 +52,7 @@ class CentroidConversion(EmotionConversionPlugin):
         dimensions = list(self.centroids.values())[0]
 
         def distance(e1, e2):
-            return sum((e1[k] - e2.get(self.aliases[k], 0)) for k in dimensions)
+            return sum((e1[k] - e2.get(k, 0)) for k in dimensions)
 
         emotion = ''
         mindistance = 10000000000000000000000.0
@@ -40,11 +67,12 @@ class CentroidConversion(EmotionConversionPlugin):
     def convert(self, emotionSet, fromModel, toModel, params):
 
         cf, ct = self.centroids_direction
-        logger.debug('{}\n{}\n{}\n{}'.format(emotionSet, fromModel, toModel, params))
+        logger.debug(
+            '{}\n{}\n{}\n{}'.format(emotionSet, fromModel, toModel, params))
         e = EmotionSet()
-        if fromModel == cf:
+        if fromModel == cf and toModel == ct:
             e.onyx__hasEmotion.append(self._forward_conversion(emotionSet))
-        elif fromModel == ct:
+        elif fromModel == ct and toModel == cf:
             for i in emotionSet.onyx__hasEmotion:
                 e.onyx__hasEmotion.append(self._backwards_conversion(i))
         else:

senpy/plugins/conversion/emotion/ekman2fsre.senpy

@@ -0,0 +1,39 @@
+---
+name: Ekman2FSRE
+module: senpy.plugins.conversion.centroids
+description: Plugin to convert emotion sets from Ekman to VAD
+version: 0.1
+# No need to specify onyx:doesConversion because centroids.py adds it automatically from centroids_direction
+centroids:
+  anger:
+    A: 6.95
+    D: 5.1
+    V: 2.7
+  disgust:
+    A: 5.3
+    D: 8.05
+    V: 2.7
+  fear:
+    A: 6.5
+    D: 3.6
+    V: 3.2
+  happiness:
+    A: 7.22
+    D: 6.28
+    V: 8.6
+  sadness:
+    A: 5.21
+    D: 2.82
+    V: 2.21
+centroids_direction:
+  - emoml:big6
+  - emoml:fsre-dimensions
+aliases: # These are aliases for any key in the centroid, to avoid repeating a long name several times
+  A: emoml:arousal
+  V: emoml:valence
+  D: emoml:dominance
+  anger: emoml:big6anger
+  disgust: emoml:big6disgust
+  fear: emoml:big6fear
+  happiness: emoml:big6happiness
+  sadness: emoml:big6sadness

senpy/plugins/conversion/emotion/ekman2vad.senpy

@@ -1,38 +1,39 @@
 ---
-name: Ekman2VAD
+name: Ekman2PAD
 module: senpy.plugins.conversion.centroids
 description: Plugin to convert emotion sets from Ekman to VAD
 version: 0.1
-onyx:doesConversion:
+# No need to specify onyx:doesConversion because centroids.py adds it automatically from centroids_direction
-  - onyx:conversionFrom: emoml:big6
-    onyx:conversionTo: emoml:fsre-dimensions
-  - onyx:conversionFrom: emoml:fsre-dimensions
-    onyx:conversionTo: emoml:big6
 centroids:
-  emoml:big6anger:
+  anger:
     A: 6.95
     D: 5.1
     V: 2.7
-  emoml:big6disgust:
+  disgust:
     A: 5.3
     D: 8.05
     V: 2.7
-  emoml:big6fear:
+  fear:
     A: 6.5
     D: 3.6
     V: 3.2
-  emoml:big6happiness:
+  happiness:
     A: 7.22
     D: 6.28
     V: 8.6
-  emoml:big6sadness:
+  sadness:
     A: 5.21
     D: 2.82
     V: 2.21
 centroids_direction:
   - emoml:big6
-  - emoml:fsre-dimensions
+  - emoml:pad
-aliases:
+aliases: # These are aliases for any key in the centroid, to avoid repeating a long name several times
   A: emoml:arousal
   V: emoml:valence
-  D: emoml:dominance
+  D: emoml:dominance
+  anger: emoml:big6anger
+  disgust: emoml:big6disgust
+  fear: emoml:big6fear
+  happiness: emoml:big6happiness
+  sadness: emoml:big6sadness

senpy/schemas/context.jsonld

@@ -37,6 +37,12 @@
       "@type": "@id",
       "@container": "@set"
     },
+    "plugins": {
+      "@container": "@list"
+    },
+    "options": {
+      "@container": "@set"
+    },
     "prov:wasGeneratedBy": {
       "@type": "@id"
     },

senpy/schemas/plugins.json

@@ -6,6 +6,7 @@
       "properties": {
         "plugins": {
           "type": "array",
+          "default": [],
           "items": {
             "$ref": "plugin.json"
           }

tests/plugins/dummy_plugin/dummy.py

@@ -4,4 +4,5 @@ from senpy.plugins import SentimentPlugin
 class DummyPlugin(SentimentPlugin):
     def analyse_entry(self, entry, params):
         entry.text = entry.text[::-1]
+        entry.reversed = entry.get('reversed', 0) + 1
         yield entry

tests/plugins/sleep_plugin/sleep.py

@@ -1,8 +1,8 @@
-from senpy.plugins import SenpyPlugin
+from senpy.plugins import AnalysisPlugin
 from time import sleep
 
 
-class SleepPlugin(SenpyPlugin):
+class SleepPlugin(AnalysisPlugin):
     def activate(self, *args, **kwargs):
         sleep(self.timeout)
 

tests/test_extensions.py

@@ -10,7 +10,7 @@ except ImportError:
 
 from functools import partial
 from senpy.extensions import Senpy
-from senpy.models import Error, Results, Entry, EmotionSet, Emotion
+from senpy.models import Error, Results, Entry, EmotionSet, Emotion, Plugin
 from flask import Flask
 from unittest import TestCase
 
@@ -98,17 +98,26 @@ class ExtensionsTest(TestCase):
 
     def test_analyse_error(self):
         mm = mock.MagicMock()
-        mm.analyse_entry.side_effect = Error('error on analysis', status=900)
+        mm.id = 'magic_mock'
+        mm.analyse_entries.side_effect = Error('error on analysis', status=500)
         self.senpy.plugins['MOCK'] = mm
-        resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
+        try:
-        assert resp['message'] == 'error on analysis'
+            self.senpy.analyse(input='nothing', algorithm='MOCK')
-        assert resp['status'] == 900
+            assert False
+        except Error as ex:
+            assert ex['message'] == 'error on analysis'
+            assert ex['status'] == 500
+
         mm.analyse.side_effect = Exception('generic exception on analysis')
-        mm.analyse_entry.side_effect = Exception(
+        mm.analyse_entries.side_effect = Exception(
             'generic exception on analysis')
-        resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
+
-        assert resp['message'] == 'generic exception on analysis'
+        try:
-        assert resp['status'] == 500
+            self.senpy.analyse(input='nothing', algorithm='MOCK')
+            assert False
+        except Error as ex:
+            assert ex['message'] == 'generic exception on analysis'
+            assert ex['status'] == 500
 
     def test_filtering(self):
         """ Filtering plugins """
@@ -125,11 +134,12 @@ class ExtensionsTest(TestCase):
 
     def test_convert_emotions(self):
         self.senpy.activate_all()
-        plugin = {
+        plugin = Plugin({
             'id': 'imaginary',
             'onyx:usesEmotionModel': 'emoml:fsre-dimensions'
-        }
+        })
         eSet1 = EmotionSet()
+        eSet1.prov__wasGeneratedBy = plugin['id']
         eSet1['onyx:hasEmotion'].append(Emotion({
             'emoml:arousal': 1,
             'emoml:potency': 0,
@@ -145,19 +155,19 @@ class ExtensionsTest(TestCase):
                   'conversion': 'full'}
         r1 = deepcopy(response)
         self.senpy.convert_emotions(r1,
-                                    plugin,
+                                    [plugin, ],
                                     params)
         assert len(r1.entries[0].emotions) == 2
         params['conversion'] = 'nested'
         r2 = deepcopy(response)
         self.senpy.convert_emotions(r2,
-                                    plugin,
+                                    [plugin, ],
                                     params)
         assert len(r2.entries[0].emotions) == 1
         assert r2.entries[0].emotions[0]['prov:wasDerivedFrom'] == eSet1
         params['conversion'] = 'filtered'
         r3 = deepcopy(response)
         self.senpy.convert_emotions(r3,
-                                    plugin,
+                                    [plugin, ],
                                     params)
         assert len(r3.entries[0].emotions) == 1

tests/test_models.py

@@ -11,8 +11,10 @@ from senpy.models import (Emotion,
                           Entry,
                           Error,
                           Results,
-                          Sentiment)
+                          Sentiment,
-from senpy.plugins import SenpyPlugin
+                          Plugins,
+                          Plugin)
+from senpy import plugins
 from pprint import pprint
 
 
@@ -53,8 +55,8 @@ class ModelsTest(TestCase):
         assert (received["entries"][0]["nif:isString"] != "Not testing")
 
     def test_id(self):
-        ''' Adding the id after creation should overwrite the automatic ID
+        """ Adding the id after creation should overwrite the automatic ID
-        '''
+        """
         r = Entry()
         j = r.jsonld()
         assert '@id' in j
@@ -94,8 +96,16 @@ class ModelsTest(TestCase):
         r.validate()
 
     def test_plugins(self):
-        self.assertRaises(Error, SenpyPlugin)
+        self.assertRaises(Error, plugins.Plugin)
-        p = SenpyPlugin({"name": "dummy", "version": 0})
+        p = plugins.Plugin({"name": "dummy",
+                            "version": 0,
+                            "extra_params": {
+                                "none": {
+                                    "options": ["es", ],
+                                    "required": False,
+                                    "default": "0"
+                                }
+                            }})
         c = p.jsonld()
         assert "info" not in c
         assert "repo" not in c
@@ -103,11 +113,13 @@ class ModelsTest(TestCase):
         logging.debug("Framed:")
         logging.debug(c)
         p.validate()
+        assert "es" in c['extra_params']['none']['options']
+        assert isinstance(c['extra_params']['none']['options'], list)
 
     def test_str(self):
         """The string representation shouldn't include private variables"""
         r = Results()
-        p = SenpyPlugin({"name": "STR test", "version": 0})
+        p = plugins.Plugin({"name": "STR test", "version": 0})
         p._testing = 0
         s = str(p)
         assert "_testing" not in s
@@ -143,3 +155,15 @@ class ModelsTest(TestCase):
         print(t)
         g = rdflib.Graph().parse(data=t, format='turtle')
         assert len(g) == len(triples)
+
+    def test_single_plugin(self):
+        """A response with a single plugin should still return a list"""
+        plugs = Plugins()
+        for i in range(10):
+            p = Plugin({'id': str(i),
+                        'version': 0,
+                        'description': 'dummy'})
+            plugs.plugins.append(p)
+        assert isinstance(plugs.plugins, list)
+        js = plugs.jsonld()
+        assert isinstance(js['plugins'], list)