Refactored conversion and postprocessing

Replace algorithm list with a tuple

Closes #56

See merge request senpy/senpy!25

.gitlab-ci.yml

@@ -31,29 +31,19 @@ test-2.7:
   variables:
     PYTHON_VERSION: "2.7"
 
-.image: &image_definition
+push:
   stage: push
   script:
-    - make -e push-$PYTHON_VERSION
+    - make -e push
   only:
     - tags
     - triggers
     - fix-makefiles
 
-push-3.5:
-  <<: *image_definition
-  variables:
-    PYTHON_VERSION: "3.5"
-
-push-2.7:
-  <<: *image_definition
-  variables:
-    PYTHON_VERSION: "2.7"
-
 push-latest:
-  <<: *image_definition
+  stage: push
-  variables:
+  script:
-    PYTHON_VERSION: latest
+    - make -e push-latest
   only:
     - master
     - triggers

.makefiles/git.mk

@@ -22,7 +22,4 @@ else
 	rm $(KEY_FILE)
 endif
 
-push:: git-push
+.PHONY:: commit tag git-push git-pull push-github 
-pull:: git-pull
-
-.PHONY:: commit tag push git-push git-pull push-github 

.makefiles/makefiles.mk

@@ -1,17 +1,15 @@
 makefiles-remote:
-	@git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git 2>/dev/null || true
+	git ls-remote --exit-code  makefiles 2> /dev/null || git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git
 
 makefiles-commit: makefiles-remote
 	git add -f .makefiles
 	git commit -em "Updated makefiles from ${NAME}"
 
 makefiles-push:
+	git fetch makefiles $(NAME)
 	git subtree push --prefix=.makefiles/ makefiles $(NAME)
 
 makefiles-pull: makefiles-remote
 	git subtree pull --prefix=.makefiles/ makefiles master --squash
 
-pull:: makefiles-pull
+.PHONY:: makefiles-remote makefiles-commit makefiles-push makefiles-pull
-push:: makefiles-push
-
-.PHONY:: makefiles-remote makefiles-commit makefiles-push makefiles-pull pull push

example-plugins/async_plugin.py

@@ -11,7 +11,7 @@ class Async(AnalysisPlugin):
     '''An example of an asynchronous module'''
     author = '@balkian'
     version = '0.2'
-    async = True
+    sync = False
 
     def _do_async(self, num_processes):
         pool = multiprocessing.Pool(processes=num_processes)

requirements.txt

@@ -12,3 +12,4 @@ rdflib-jsonld
 numpy
 scipy
 scikit-learn
+responses

senpy/api.py

@@ -3,10 +3,8 @@ from .models import Error, Results, Entry, from_string
 import logging
 logger = logging.getLogger(__name__)
 
-
 boolean = [True, False]
 
-
 API_PARAMS = {
     "algorithm": {
         "aliases": ["algorithms", "a", "algo"],
@@ -140,6 +138,15 @@ NIF_PARAMS = {
     }
 }
 
+BUILTIN_PARAMS = {}
+
+for d in [
+        NIF_PARAMS, CLI_PARAMS, WEB_PARAMS, PLUGINS_PARAMS, EVAL_PARAMS,
+        API_PARAMS
+]:
+    for k, v in d.items():
+        BUILTIN_PARAMS[k] = v
+
 
 def parse_params(indict, *specs):
     if not specs:
@@ -164,7 +171,7 @@ def parse_params(indict, *specs):
                 continue
             if "options" in options:
                 if options["options"] == boolean:
-                    outdict[param] = outdict[param] in [None, True, 'true', '1']
+                    outdict[param] = str(outdict[param]).lower() in ['true', '1']
                 elif outdict[param] not in options["options"]:
                     wrong_params[param] = spec[param]
     if wrong_params:
@@ -175,16 +182,24 @@ def parse_params(indict, *specs):
             parameters=outdict,
             errors=wrong_params)
         raise message
-    if 'algorithm' in outdict and not isinstance(outdict['algorithm'], list):
+    if 'algorithm' in outdict and not isinstance(outdict['algorithm'], tuple):
-        outdict['algorithm'] = list(outdict['algorithm'].split(','))
+        outdict['algorithm'] = tuple(outdict['algorithm'].split(','))
     return outdict
 
 
-def parse_extra_params(request, plugin=None):
+def parse_extra_params(request, plugins=None):
+    plugins = plugins or []
     params = request.parameters.copy()
-    if plugin:
+    for plugin in plugins:
-        extra_params = parse_params(params, plugin.get('extra_params', {}))
+        if plugin:
-        params.update(extra_params)
+            extra_params = parse_params(params, plugin.get('extra_params', {}))
+            for k, v in extra_params.items():
+                if k not in BUILTIN_PARAMS:
+                    if k in params:  # Set by another plugin
+                        del params[k]
+                    else:
+                        params[k] = v
+                params['{}.{}'.format(plugin.name, k)] = v
     return params
 
 
@@ -194,12 +209,12 @@ def parse_call(params):
     params = parse_params(params, NIF_PARAMS)
     if params['informat'] == 'text':
         results = Results()
-        entry = Entry(nif__isString=params['input'],
+        entry = Entry(nif__isString=params['input'], id='#')  # Use @base
-                      id='#')  # Use @base
         results.entries.append(entry)
     elif params['informat'] == 'json-ld':
         results = from_string(params['input'], cls=Results)
     else:  # pragma: no cover
-        raise NotImplementedError('Informat {} is not implemented'.format(params['informat']))
+        raise NotImplementedError('Informat {} is not implemented'.format(
+            params['informat']))
     results.parameters = params
     return results

senpy/blueprints.py

@@ -69,7 +69,7 @@ def encoded_url(url=None, base=None):
         if request.method == 'GET':
             url = request.full_path[1:]  # Remove the first slash
         else:
-            hash(frozenset(request.form.params().items()))
+            hash(frozenset(tuple(request.parameters.items())))
             code = 'hash:{}'.format(hash)
 
     code = code or base64.urlsafe_b64encode(url.encode()).decode()
@@ -184,15 +184,22 @@ def basic_api(f):
     return decorated_function
 
 
-@api_blueprint.route('/', methods=['POST', 'GET'])
+@api_blueprint.route('/', defaults={'plugin': None}, methods=['POST', 'GET'])
+@api_blueprint.route('/<path:plugin>', methods=['POST', 'GET'])
 @basic_api
-def api_root():
+def api_root(plugin):
     if request.parameters['help']:
         dic = dict(api.API_PARAMS, **api.NIF_PARAMS)
         response = Help(valid_parameters=dic)
         return response
     req = api.parse_call(request.parameters)
-    return current_app.senpy.analyse(req)
+    if plugin:
+        plugin = plugin.replace('+', '/')
+        plugin = plugin.split('/')
+        req.parameters['algorithm'] = tuple(plugin)
+    results = current_app.senpy.analyse(req)
+    results.analysis = set(i.id for i in results.analysis)
+    return results
 
 
 @api_blueprint.route('/evaluate/', methods=['POST', 'GET'])
@@ -221,7 +228,7 @@ def plugins():
 
 @api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
 @basic_api
-def plugin(plugin=None):
+def plugin(plugin):
     sp = current_app.senpy
     return sp.get_plugin(plugin)
 

senpy/client.py

@@ -13,7 +13,7 @@ class Client(object):
         return self.request('/', method=method, input=input, **kwargs)
 
     def evaluate(self, input, method='GET', **kwargs):
-        return self.request('/evaluate', method = method, input=input, **kwargs)
+        return self.request('/evaluate', method=method, input=input, **kwargs)
 
     def plugins(self, *args, **kwargs):
         resp = self.request(path='/plugins').plugins
@@ -24,8 +24,12 @@ class Client(object):
         return {d.name: d for d in resp}
 
     def request(self, path=None, method='GET', **params):
-        url = '{}{}'.format(self.endpoint, path)
+        url = '{}{}'.format(self.endpoint.rstrip('/'), path)
-        response = requests.request(method=method, url=url, params=params)
+        if method == 'POST':
+            response = requests.post(url=url, data=params)
+        else:
+            response = requests.request(method=method, url=url, params=params)
+
         try:
             resp = models.from_dict(response.json())
         except Exception as ex:

senpy/extensions.py

@@ -6,7 +6,6 @@ from future import standard_library
 standard_library.install_aliases()
 
 from . import plugins, api
-from .plugins import Plugin, evaluate
 from .models import Error, AggregatedEvaluation
 from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
 
@@ -17,7 +16,6 @@ import copy
 import errno
 import logging
 
-
 from . import gsitk_compat
 
 logger = logging.getLogger(__name__)
@@ -25,6 +23,7 @@ logger = logging.getLogger(__name__)
 
 class Senpy(object):
     """ Default Senpy extension for Flask """
+
     def __init__(self,
                  app=None,
                  plugin_folder=".",
@@ -50,7 +49,7 @@ class Senpy(object):
             self.add_folder('plugins', from_root=True)
         else:
             # Add only conversion plugins
-            self.add_folder(os.path.join('plugins', 'conversion'),
+            self.add_folder(os.path.join('plugins', 'postprocessing'),
                             from_root=True)
         self.app = app
         if app is not None:
@@ -115,6 +114,7 @@ class Senpy(object):
             raise AttributeError("Not a folder or does not exist: %s", folder)
 
     def _get_plugins(self, request):
+        '''Get a list of plugins that should be run for a specific request'''
         if not self.analysis_plugins:
             raise Error(
                 status=404,
@@ -132,34 +132,32 @@ class Senpy(object):
         plugins = list()
         for algo in algos:
             algo = algo.lower()
+            if algo == 'conversion':
+                continue  # Allow 'conversion' as a virtual plugin, which does nothing
             if algo not in self._plugins:
                 msg = ("The algorithm '{}' is not valid\n"
                        "Valid algorithms: {}").format(algo,
                                                       self._plugins.keys())
                 logger.debug(msg)
-                raise Error(
+                raise Error(status=404, message=msg)
-                    status=404,
-                    message=msg)
             plugins.append(self._plugins[algo])
+
         return plugins
 
-    def _process_entries(self, entries, req, plugins):
+    def _process(self, req, pending, done=None):
         """
         Recursively process the entries with the first plugin in the list, and pass the results
         to the rest of the plugins.
         """
-        if not plugins:
+        done = done or []
-            for i in entries:
+        if not pending:
-                yield i
+            return req
-            return
+
-        plugin = plugins[0]
+        plugin = pending[0]
-        self._activate(plugin)  # Make sure the plugin is activated
+        results = plugin.process(req, conversions_applied=done)
-        specific_params = api.parse_extra_params(req, plugin)
+        if plugin not in results.analysis:
-        req.analysis.append({'plugin': plugin,
+            results.analysis.append(plugin)
-                             'parameters': specific_params})
+        return self._process(results, pending[1:], done)
-        results = plugin.analyse_entries(entries, specific_params)
-        for i in self._process_entries(results, req, plugins[1:]):
-            yield i
 
     def install_deps(self):
         plugins.install_deps(*self.plugins())
@@ -171,17 +169,88 @@ class Senpy(object):
         by api.parse_call().
         """
         logger.debug("analysing request: {}".format(request))
-        entries = request.entries
-        request.entries = []
         plugins = self._get_plugins(request)
-        results = request
+        request.parameters = api.parse_extra_params(request, plugins)
-        for i in self._process_entries(entries, results, plugins):
+        results = self._process(request, plugins)
-            results.entries.append(i)
+        logger.debug("Got analysis result: {}".format(results))
-        self.convert_emotions(results)
+        results = self.postprocess(results)
-        logger.debug("Returning analysis result: {}".format(results))
+        logger.debug("Returning post-processed result: {}".format(results))
-        results.analysis = [i['plugin'].id for i in results.analysis]
         return results
 
+    def convert_emotions(self, resp):
+        """
+        Conversion of all emotions in a response **in place**.
+        In addition to converting from one model to another, it has
+        to include the conversion plugin to the analysis list.
+        Needless to say, this is far from an elegant solution, but it works.
+        @todo refactor and clean up
+        """
+        plugins = resp.analysis
+
+        params = resp.parameters
+        toModel = params.get('emotionModel', None)
+        if not toModel:
+            return resp
+
+        logger.debug('Asked for model: {}'.format(toModel))
+        output = params.get('conversion', None)
+        candidates = {}
+        for plugin in plugins:
+            try:
+                fromModel = plugin.get('onyx:usesEmotionModel', None)
+                candidates[plugin.id] = next(self._conversion_candidates(fromModel, toModel))
+                logger.debug('Analysis plugin {} uses model: {}'.format(
+                    plugin.id, fromModel))
+            except StopIteration:
+                e = Error(('No conversion plugin found for: '
+                           '{} -> {}'.format(fromModel, toModel)),
+                          status=404)
+                e.original_response = resp
+                e.parameters = params
+                raise e
+        newentries = []
+        done = []
+        for i in resp.entries:
+            if output == "full":
+                newemotions = copy.deepcopy(i.emotions)
+            else:
+                newemotions = []
+            for j in i.emotions:
+                plugname = j['prov:wasGeneratedBy']
+                candidate = candidates[plugname]
+                done.append({'plugin': candidate, 'parameters': params})
+                for k in candidate.convert(j, fromModel, toModel, params):
+                    k.prov__wasGeneratedBy = candidate.id
+                    if output == 'nested':
+                        k.prov__wasDerivedFrom = j
+                    newemotions.append(k)
+            i.emotions = newemotions
+            newentries.append(i)
+        resp.entries = newentries
+        return resp
+
+    def _conversion_candidates(self, fromModel, toModel):
+        candidates = self.plugins(plugin_type=plugins.EmotionConversion)
+        for candidate in candidates:
+            for pair in candidate.onyx__doesConversion:
+                logging.debug(pair)
+                if candidate.can_convert(fromModel, toModel):
+                    yield candidate
+
+    def postprocess(self, response):
+        '''
+        Transform the results from the analysis plugins.
+        It has some pre-defined post-processing like emotion conversion,
+        and it also allows plugins to auto-select themselves.
+        '''
+
+        response = self.convert_emotions(response)
+
+        for plug in self.plugins(plugin_type=plugins.PostProcessing):
+            if plug.check(response, response.analysis):
+                response = plug.process(response)
+        return response
+
     def _get_datasets(self, request):
         if not self.datasets:
             raise Error(
@@ -192,8 +261,8 @@ class Senpy(object):
         for dataset in datasets_name:
             if dataset not in self.datasets:
                 logger.debug(("The dataset '{}' is not valid\n"
-                              "Valid datasets: {}").format(dataset,
+                              "Valid datasets: {}").format(
-                                                           self.datasets.keys()))
+                                  dataset, self.datasets.keys()))
                 raise Error(
                     status=404,
                     message="The dataset '{}' is not valid".format(dataset))
@@ -220,77 +289,18 @@ class Senpy(object):
         results.parameters = params
         datasets = self._get_datasets(results)
         plugins = self._get_plugins(results)
-        for eval in evaluate(plugins, datasets):
+        for eval in plugins.evaluate(plugins, datasets):
             results.evaluations.append(eval)
         if 'with_parameters' not in results.parameters:
             del results.parameters
         logger.debug("Returning evaluation result: {}".format(results))
         return results
 
-    def _conversion_candidates(self, fromModel, toModel):
-        candidates = self.plugins(plugin_type='emotionConversionPlugin')
-        for candidate in candidates:
-            for pair in candidate.onyx__doesConversion:
-                logging.debug(pair)
-
-                if pair['onyx:conversionFrom'] == fromModel \
-                   and pair['onyx:conversionTo'] == toModel:
-                    yield candidate
-
-    def convert_emotions(self, resp):
-        """
-        Conversion of all emotions in a response **in place**.
-        In addition to converting from one model to another, it has
-        to include the conversion plugin to the analysis list.
-        Needless to say, this is far from an elegant solution, but it works.
-        @todo refactor and clean up
-        """
-        plugins = [i['plugin'] for i in resp.analysis]
-        params = resp.parameters
-        toModel = params.get('emotionModel', None)
-        if not toModel:
-            return
-
-        logger.debug('Asked for model: {}'.format(toModel))
-        output = params.get('conversion', None)
-        candidates = {}
-        for plugin in plugins:
-            try:
-                fromModel = plugin.get('onyx:usesEmotionModel', None)
-                candidates[plugin.id] = next(self._conversion_candidates(fromModel, toModel))
-                logger.debug('Analysis plugin {} uses model: {}'.format(plugin.id, fromModel))
-            except StopIteration:
-                e = Error(('No conversion plugin found for: '
-                           '{} -> {}'.format(fromModel, toModel)),
-                          status=404)
-                e.original_response = resp
-                e.parameters = params
-                raise e
-        newentries = []
-        for i in resp.entries:
-            if output == "full":
-                newemotions = copy.deepcopy(i.emotions)
-            else:
-                newemotions = []
-            for j in i.emotions:
-                plugname = j['prov:wasGeneratedBy']
-                candidate = candidates[plugname]
-                resp.analysis.append({'plugin': candidate,
-                                      'parameters': params})
-                for k in candidate.convert(j, fromModel, toModel, params):
-                    k.prov__wasGeneratedBy = candidate.id
-                    if output == 'nested':
-                        k.prov__wasDerivedFrom = j
-                    newemotions.append(k)
-            i.emotions = newemotions
-            newentries.append(i)
-        resp.entries = newentries
-
     @property
     def default_plugin(self):
         if not self._default or not self._default.is_activated:
-            candidates = self.plugins(plugin_type='analysisPlugin',
+            candidates = self.plugins(
-                                      is_activated=True)
+                plugin_type='analysisPlugin', is_activated=True)
             if len(candidates) > 0:
                 self._default = candidates[0]
             else:
@@ -300,7 +310,7 @@ class Senpy(object):
 
     @default_plugin.setter
     def default_plugin(self, value):
-        if isinstance(value, Plugin):
+        if isinstance(value, plugins.Plugin):
             if not value.is_activated:
                 raise AttributeError('The default plugin has to be activated.')
             self._default = value
@@ -352,7 +362,8 @@ class Senpy(object):
 
         logger.info("Activating plugin: {}".format(plugin.name))
 
-        if sync or 'async' in plugin and not plugin.async:
+        if sync or not getattr(plugin, 'async', True) or getattr(
+                plugin, 'sync', False):
             return self._activate(plugin)
         else:
             th = Thread(target=partial(self._activate, plugin))
@@ -375,7 +386,8 @@ class Senpy(object):
 
         self._set_active(plugin, False)
 
-        if sync or 'async' in plugin and not plugin.async:
+        if sync or not getattr(plugin, 'async', True) or not getattr(
+                plugin, 'sync', False):
             self._deactivate(plugin)
         else:
             th = Thread(target=partial(self._deactivate, plugin))

senpy/gsitk_compat.py

@@ -1,5 +1,7 @@
 import logging
 
+from pkg_resources import parse_version, get_distribution, DistributionNotFound
+
 logger = logging.getLogger(__name__)
 
 MSG = 'GSITK is not (properly) installed.'
@@ -12,12 +14,18 @@ def raise_exception(*args, **kwargs):
 
 
 try:
+    gsitk_distro = get_distribution("gsitk")
+    GSITK_VERSION = parse_version(gsitk_distro.version)
+    GSITK_AVAILABLE = GSITK_VERSION > parse_version("0.1.9.1")  # Earlier versions have a bug
+except DistributionNotFound:
+    GSITK_AVAILABLE = False
+    GSITK_VERSION = ()
+
+if GSITK_AVAILABLE:
     from gsitk.datasets.datasets import DatasetManager
     from gsitk.evaluation.evaluation import Evaluation as Eval
     from sklearn.pipeline import Pipeline
-    GSITK_AVAILABLE = True
     modules = locals()
-except ImportError:
+else:
-    logger.warn(IMPORTMSG)
+    logger.warning(IMPORTMSG)
-    GSITK_AVAILABLE = False
     DatasetManager = Eval = Pipeline = raise_exception

senpy/plugins/__init__.py

@@ -1,7 +1,6 @@
 from future import standard_library
 standard_library.install_aliases()
 
-
 from future.utils import with_metaclass
 from functools import partial
 
@@ -10,7 +9,6 @@ import os
 import re
 import pickle
 import logging
-import copy
 import pprint
 
 import inspect
@@ -24,7 +22,7 @@ import nltk
 from .. import models, utils
 from .. import api
 from .. import gsitk_compat
-
+from .. import testing
 
 logger = logging.getLogger(__name__)
 
@@ -45,16 +43,19 @@ class PluginMeta(models.BaseMeta):
             if doc:
                 attrs['description'] = doc
             else:
-                logger.warn(('Plugin {} does not have a description. '
+                logger.warning(
-                             'Please, add a short summary to help other developers').format(name))
+                    ('Plugin {} does not have a description. '
+                     'Please, add a short summary to help other developers'
+                     ).format(name))
         cls = super(PluginMeta, mcs).__new__(mcs, name, bases, attrs)
 
         if alias in mcs._classes:
             if os.environ.get('SENPY_TESTING', ""):
-                raise Exception(('The type of plugin {} already exists. '
+                raise Exception(
-                                'Please, choose a different name').format(name))
+                    ('The type of plugin {} already exists. '
+                     'Please, choose a different name').format(name))
             else:
-                logger.warn('Overloading plugin class: {}'.format(alias))
+                logger.warning('Overloading plugin class: {}'.format(alias))
         mcs._classes[alias] = cls
         return cls
 
@@ -86,10 +87,12 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)):
         if info:
             self.update(info)
         self.validate()
-        self.id = 'endpoint:plugins/{}_{}'.format(self['name'], self['version'])
+        self.id = 'endpoint:plugins/{}_{}'.format(self['name'],
+                                                  self['version'])
         self.is_activated = False
         self._lock = threading.Lock()
-        self._directory = os.path.abspath(os.path.dirname(inspect.getfile(self.__class__)))
+        self._directory = os.path.abspath(
+            os.path.dirname(inspect.getfile(self.__class__)))
 
         data_folder = data_folder or os.getcwd()
         subdir = os.path.join(data_folder, self.name)
@@ -117,7 +120,8 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)):
             if x not in self:
                 missing.append(x)
         if missing:
-            raise models.Error('Missing configuration parameters: {}'.format(missing))
+            raise models.Error(
+                'Missing configuration parameters: {}'.format(missing))
 
     def get_folder(self):
         return os.path.dirname(inspect.getfile(self.__class__))
@@ -128,35 +132,86 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)):
     def deactivate(self):
         pass
 
+    def process(self, request, **kwargs):
+        """
+        An implemented plugin should override this method.
+        Here, we assume that a process_entries method exists."""
+        newentries = list(
+            self.process_entries(request.entries, request.parameters))
+        request.entries = newentries
+        return request
+
+    def process_entries(self, entries, parameters):
+        for entry in entries:
+            self.log.debug('Processing entry with plugin {}: {}'.format(
+                self, entry))
+            results = self.process_entry(entry, parameters)
+            if inspect.isgenerator(results):
+                for result in results:
+                    yield result
+            else:
+                yield results
+
+    def process_entry(self, entry, parameters):
+        """
+        This base method is here to adapt plugins which only
+        implement the *process* function.
+        Note that this method may yield an annotated entry or a list of
+        entries (e.g. in a tokenizer)
+        """
+        raise NotImplementedError(
+            'You need to implement process, process_entries or process_entry in your plugin'
+        )
+
     def test(self, test_cases=None):
         if not test_cases:
             if not hasattr(self, 'test_cases'):
-                raise AttributeError(('Plugin {} [{}] does not have any defined '
+                raise AttributeError(
-                                      'test cases').format(self.id,
+                    ('Plugin {} [{}] does not have any defined '
-                                                           inspect.getfile(self.__class__)))
+                     'test cases').format(self.id,
+                                          inspect.getfile(self.__class__)))
             test_cases = self.test_cases
         for case in test_cases:
             try:
                 self.test_case(case)
-                self.log.debug('Test case passed:\n{}'.format(pprint.pformat(case)))
+                self.log.debug('Test case passed:\n{}'.format(
+                    pprint.pformat(case)))
             except Exception as ex:
-                self.log.warn('Test case failed:\n{}'.format(pprint.pformat(case)))
+                self.log.warning('Test case failed:\n{}'.format(
+                    pprint.pformat(case)))
                 raise
 
-    def test_case(self, case):
+    def test_case(self, case, mock=testing.MOCK_REQUESTS):
+        if 'entry' not in case and 'input' in case:
+            entry = models.Entry(_auto_id=False)
+            entry.nif__isString = case['input']
+            case['entry'] = entry
         entry = models.Entry(case['entry'])
         given_parameters = case.get('params', case.get('parameters', {}))
         expected = case.get('expected', None)
         should_fail = case.get('should_fail', False)
+        responses = case.get('responses', [])
+
         try:
-            params = api.parse_params(given_parameters, self.extra_params)
+            request = models.Response()
-            res = list(self.analyse_entries([entry, ], params))
+            request.parameters = api.parse_params(given_parameters,
+                                                  self.extra_params)
+            request.entries = [
+                entry,
+            ]
+
+            method = partial(self.process, request)
+
+            if mock:
+                res = method()
+            else:
+                with testing.patch_all_requests(responses):
+                    res = method()
 
             if not isinstance(expected, list):
                 expected = [expected]
-            utils.check_template(res, expected)
+            utils.check_template(res.entries, expected)
-            for r in res:
+            res.validate()
-                r.validate()
         except models.Error:
             if should_fail:
                 return
@@ -193,40 +248,26 @@ class Analysis(Plugin):
     A subclass of Plugin that analyses text and provides an annotation.
     '''
 
-    def analyse(self, *args, **kwargs):
+    def analyse(self, request, parameters):
-        raise NotImplementedError(
+        return super(Analysis, self).process(request)
-            'Your plugin should implement either analyse or analyse_entry')
-
-    def analyse_entry(self, entry, parameters):
-        """ An implemented plugin should override this method.
-        This base method is here to adapt old style plugins which only
-        implement the *analyse* function.
-        Note that this method may yield an annotated entry or a list of
-        entries (e.g. in a tokenizer)
-        """
-        text = entry['nif:isString']
-        params = copy.copy(parameters)
-        params['input'] = text
-        results = self.analyse(**params)
-        for i in results.entries:
-            yield i
 
     def analyse_entries(self, entries, parameters):
-        for entry in entries:
+        for i in super(Analysis, self).process_entries(entries, parameters):
-            self.log.debug('Analysing entry with plugin {}: {}'.format(self, entry))
+            yield i
-            results = self.analyse_entry(entry, parameters)
-            if inspect.isgenerator(results):
-                for result in results:
-                    yield result
-            else:
-                yield results
 
-    def test_case(self, case):
+    def process(self, request, **kwargs):
-        if 'entry' not in case and 'input' in case:
+        return self.analyse(request, request.parameters)
-            entry = models.Entry(_auto_id=False)
+
-            entry.nif__isString = case['input']
+    def process_entries(self, entries, parameters):
-            case['entry'] = entry
+        for i in self.analyse_entries(entries, parameters):
-        super(Analysis, self).test_case(case)
+            yield i
+
+    def process_entry(self, entry, parameters, **kwargs):
+        if hasattr(self, 'analyse_entry'):
+            for i in self.analyse_entry(entry, parameters):
+                yield i
+        else:
+            super(Analysis, self).process_entry(entry, parameters, **kwargs)
 
 
 AnalysisPlugin = Analysis
@@ -237,7 +278,20 @@ class Conversion(Plugin):
     A subclass of Plugins that convert between different annotation models.
     e.g. a conversion of emotion models, or normalization of sentiment values.
     '''
-    pass
+
+    def process(self, response, plugins=None, **kwargs):
+        plugins = plugins or []
+        newentries = []
+        for entry in response.entries:
+            newentries.append(
+                self.convert_entry(entry, response.parameters, plugins))
+        response.entries = newentries
+        return response
+
+    def convert_entry(self, entry, parameters, conversions_applied):
+        raise NotImplementedError(
+            'You should implement a way to convert each entry, or a custom process method'
+        )
 
 
 ConversionPlugin = Conversion
@@ -274,12 +328,28 @@ class EmotionConversion(Conversion):
     '''
     A subclass of Conversion that converts emotion annotations using different models
     '''
-    pass
+
+    def can_convert(self, fromModel, toModel):
+        '''
+        Whether this plugin can convert from fromModel to toModel.
+        If fromModel is None, it is interpreted as "any Model"
+        '''
+        for pair in self.onyx__doesConversion:
+            if (pair['onyx:conversionTo'] == toModel) and \
+               ((fromModel is None) or (pair['onyx:conversionFrom'] == fromModel)):
+                return True
+        return False
 
 
 EmotionConversionPlugin = EmotionConversion
 
 
+class PostProcessing(Plugin):
+    def check(self, request, plugins):
+        '''Should this plugin be run for this request?'''
+        return False
+
+
 class Box(AnalysisPlugin):
     '''
     Black box plugins delegate analysis to a function.
@@ -304,9 +374,10 @@ class Box(AnalysisPlugin):
         return output
 
     def predict_one(self, input):
-        raise NotImplementedError('You should define the behavior of this plugin')
+        raise NotImplementedError(
+            'You should define the behavior of this plugin')
 
-    def analyse_entries(self, entries, params):
+    def process_entries(self, entries, params):
         for entry in entries:
             input = self.input(entry=entry, params=params)
             results = self.predict_one(input=input)
@@ -375,7 +446,6 @@ class EmotionBox(TextBox, EmotionPlugin):
 
 
 class MappingMixin(object):
-
     @property
     def mappings(self):
         return self._mappings
@@ -385,11 +455,10 @@ class MappingMixin(object):
         self._mappings = value
 
     def output(self, output, entry, params):
-        output = self.mappings.get(output,
+        output = self.mappings.get(output, self.mappings.get(
-                                   self.mappings.get('default', output))
+            'default', output))
-        return super(MappingMixin, self).output(output=output,
+        return super(MappingMixin, self).output(
-                                                entry=entry,
+            output=output, entry=entry, params=params)
-                                                params=params)
 
 
 class ShelfMixin(object):
@@ -402,7 +471,8 @@ class ShelfMixin(object):
                     with self.open(self.shelf_file, 'rb') as p:
                         self._sh = pickle.load(p)
                 except (IndexError, EOFError, pickle.UnpicklingError):
-                    self.log.warning('Corrupted shelf file: {}'.format(self.shelf_file))
+                    self.log.warning('Corrupted shelf file: {}'.format(
+                        self.shelf_file))
                     if not self.get('force_shelf', False):
                         raise
         return self._sh
@@ -450,8 +520,7 @@ def pfilter(plugins, plugin_type=Analysis, **kwargs):
             plugin_type = plugin_type[0].upper() + plugin_type[1:]
             pclass = globals()[plugin_type]
             logger.debug('Class: {}'.format(pclass))
-            candidates = filter(lambda x: isinstance(x, pclass),
+            candidates = filter(lambda x: isinstance(x, pclass), plugins)
-                                plugins)
         except KeyError:
             raise models.Error('{} is not a valid type'.format(plugin_type))
     else:
@@ -461,8 +530,7 @@ def pfilter(plugins, plugin_type=Analysis, **kwargs):
 
     def matches(plug):
         res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
-        logger.debug(
+        logger.debug("matching {} with {}: {}".format(plug.name, kwargs, res))
-            "matching {} with {}: {}".format(plug.name, kwargs, res))
         return res
 
     if kwargs:
@@ -496,14 +564,14 @@ def install_deps(*plugins):
             for req in requirements:
                 pip_args.append(req)
             logger.info('Installing requirements: ' + str(requirements))
-            process = subprocess.Popen(pip_args,
+            process = subprocess.Popen(
-                                       stdout=subprocess.PIPE,
+                pip_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-                                       stderr=subprocess.PIPE)
             _log_subprocess_output(process)
             exitcode = process.wait()
             installed = True
             if exitcode != 0:
-                raise models.Error("Dependencies not properly installed: {}".format(pip_args))
+                raise models.Error(
+                    "Dependencies not properly installed: {}".format(pip_args))
         nltk_resources |= set(info.get('nltk_resources', []))
 
     installed |= nltk.download(list(nltk_resources))
@@ -546,7 +614,7 @@ def from_folder(folders, loader=from_path, **kwargs):
 
 
 def from_info(info, root=None, install_on_fail=True, **kwargs):
-    if any(x not in info for x in ('module',)):
+    if any(x not in info for x in ('module', )):
         raise ValueError('Plugin info is not valid: {}'.format(info))
     module = info["module"]
 
@@ -583,7 +651,8 @@ def one_from_module(module, root, info, **kwargs):
     if '@type' in info:
         cls = PluginMeta.from_type(info['@type'])
         return cls(info=info, **kwargs)
-    instance = next(from_module(module=module, root=root, info=info, **kwargs), None)
+    instance = next(
+        from_module(module=module, root=root, info=info, **kwargs), None)
     if not instance:
         raise Exception("No valid plugin for: {}".format(module))
     return instance
@@ -607,7 +676,8 @@ def _instances_in_module(module):
 
 def _from_module_name(module, root, info=None, **kwargs):
     module = load_module(module, root)
-    for plugin in _from_loaded_module(module=module, root=root, info=info, **kwargs):
+    for plugin in _from_loaded_module(
+            module=module, root=root, info=info, **kwargs):
         yield plugin
 
 
@@ -619,9 +689,10 @@ def _from_loaded_module(module, info=None, **kwargs):
 
 
 def evaluate(plugins, datasets, **kwargs):
-    ev = gsitk_compat.Eval(tuples=None,
+    ev = gsitk_compat.Eval(
-                           datasets=datasets,
+        tuples=None,
-                           pipelines=[plugin.as_pipe() for plugin in plugins])
+        datasets=datasets,
+        pipelines=[plugin.as_pipe() for plugin in plugins])
     ev.evaluate()
     results = ev.results
     evaluations = evaluations_to_JSONLD(results, **kwargs)

senpy/plugins/postprocessing/emotion/ekman2fsre.senpy

@@ -1,6 +1,6 @@
 ---
 name: Ekman2FSRE
-module: senpy.plugins.conversion.emotion.centroids
+module: senpy.plugins.postprocessing.emotion.centroids
 description: Plugin to convert emotion sets from Ekman to VAD
 version: 0.2
 # No need to specify onyx:doesConversion because centroids.py adds it automatically from centroids_direction

senpy/plugins/postprocessing/emotion/ekman2vad.senpy

@@ -1,6 +1,6 @@
 ---
 name: Ekman2PAD
-module: senpy.plugins.conversion.emotion.centroids
+module: senpy.plugins.postprocessing.emotion.centroids
 description: Plugin to convert emotion sets from Ekman to VAD
 version: 0.2
 # No need to specify onyx:doesConversion because centroids.py adds it automatically from centroids_direction

senpy/plugins/postprocessing/emotion/maxEmotion_plugin.py

@@ -0,0 +1,196 @@
+from senpy import PostProcessing, easy_test
+
+
+class MaxEmotion(PostProcessing):
+    '''Plugin to extract the emotion with highest value from an EmotionSet'''
+    author = '@dsuarezsouto'
+    version = '0.1'
+
+    def process_entry(self, entry, params):
+        if len(entry.emotions) < 1:
+            yield entry
+            return
+
+        set_emotions = entry.emotions[0]['onyx:hasEmotion']
+
+        # If there is only one emotion, do not modify it
+        if len(set_emotions) < 2:
+            yield entry
+            return
+
+        max_emotion = set_emotions[0]
+
+        # Extract max emotion from the set emotions (emotion with highest intensity)
+        for tmp_emotion in set_emotions:
+            if tmp_emotion['onyx:hasEmotionIntensity'] > max_emotion[
+                    'onyx:hasEmotionIntensity']:
+                max_emotion = tmp_emotion
+
+        if max_emotion['onyx:hasEmotionIntensity'] == 0:
+            max_emotion['onyx:hasEmotionCategory'] = "neutral"
+            max_emotion['onyx:hasEmotionIntensity'] = 1.0
+
+        entry.emotions[0]['onyx:hasEmotion'] = [max_emotion]
+
+        entry.emotions[0]['prov:wasGeneratedBy'] = "maxSentiment"
+        yield entry
+
+    def check(self, request, plugins):
+        return 'maxemotion' in request.parameters and self not in plugins
+
+    # Test Cases:
+    #   1 Normal Situation.
+    #   2 Case to return a Neutral Emotion.
+    test_cases = [
+        {
+            "name":
+            "If there are several emotions within an emotion set, reduce it to one.",
+            "entry": {
+                "@type":
+                "entry",
+                "emotions": [
+                    {
+                        "@id":
+                        "Emotions0",
+                        "@type":
+                        "emotionSet",
+                        "onyx:hasEmotion": [
+                            {
+                                "@id": "_:Emotion_1538121033.74",
+                                "@type": "emotion",
+                                "onyx:hasEmotionCategory": "anger",
+                                "onyx:hasEmotionIntensity": 0
+                            },
+                            {
+                                "@id": "_:Emotion_1538121033.74",
+                                "@type": "emotion",
+                                "onyx:hasEmotionCategory": "joy",
+                                "onyx:hasEmotionIntensity": 0.3333333333333333
+                            },
+                            {
+                                "@id": "_:Emotion_1538121033.74",
+                                "@type": "emotion",
+                                "onyx:hasEmotionCategory": "negative-fear",
+                                "onyx:hasEmotionIntensity": 0
+                            },
+                            {
+                                "@id": "_:Emotion_1538121033.74",
+                                "@type": "emotion",
+                                "onyx:hasEmotionCategory": "sadness",
+                                "onyx:hasEmotionIntensity": 0
+                            },
+                            {
+                                "@id": "_:Emotion_1538121033.74",
+                                "@type": "emotion",
+                                "onyx:hasEmotionCategory": "disgust",
+                                "onyx:hasEmotionIntensity": 0
+                            }
+                        ]
+                    }
+                ],
+                "nif:isString":
+                "Test"
+            },
+            'expected': {
+                "@type":
+                "entry",
+                "emotions": [
+                    {
+                        "@id":
+                        "Emotions0",
+                        "@type":
+                        "emotionSet",
+                        "onyx:hasEmotion": [
+                            {
+                                "@id": "_:Emotion_1538121033.74",
+                                "@type": "emotion",
+                                "onyx:hasEmotionCategory": "joy",
+                                "onyx:hasEmotionIntensity": 0.3333333333333333
+                            }
+                        ],
+                        "prov:wasGeneratedBy":
+                        'maxSentiment'
+                    }
+                ],
+                "nif:isString":
+                "Test"
+            }
+        },
+        {
+            "name":
+            "If the maximum emotion has an intensity of 0, return a neutral emotion.",
+            "entry": {
+                "@type":
+                "entry",
+                "emotions": [{
+                    "@id":
+                    "Emotions0",
+                    "@type":
+                    "emotionSet",
+                    "onyx:hasEmotion": [
+                        {
+                            "@id": "_:Emotion_1538121033.74",
+                            "@type": "emotion",
+                            "onyx:hasEmotionCategory": "anger",
+                            "onyx:hasEmotionIntensity": 0
+                        },
+                        {
+                            "@id": "_:Emotion_1538121033.74",
+                            "@type": "emotion",
+                            "onyx:hasEmotionCategory": "joy",
+                            "onyx:hasEmotionIntensity": 0
+                        },
+                        {
+                            "@id":
+                            "_:Emotion_1538121033.74",
+                            "@type":
+                            "emotion",
+                            "onyx:hasEmotionCategory":
+                            "negative-fear",
+                            "onyx:hasEmotionIntensity":
+                            0
+                        },
+                        {
+                            "@id": "_:Emotion_1538121033.74",
+                            "@type": "emotion",
+                            "onyx:hasEmotionCategory":
+                            "sadness",
+                            "onyx:hasEmotionIntensity": 0
+                        },
+                        {
+                            "@id": "_:Emotion_1538121033.74",
+                            "@type": "emotion",
+                            "onyx:hasEmotionCategory":
+                            "disgust",
+                            "onyx:hasEmotionIntensity": 0
+                        }]
+                }],
+                "nif:isString":
+                "Test"
+            },
+            'expected': {
+                "@type":
+                "entry",
+                "emotions": [{
+                    "@id":
+                    "Emotions0",
+                    "@type":
+                    "emotionSet",
+                    "onyx:hasEmotion": [{
+                        "@id": "_:Emotion_1538121033.74",
+                        "@type": "emotion",
+                        "onyx:hasEmotionCategory": "neutral",
+                        "onyx:hasEmotionIntensity": 1
+                    }],
+                    "prov:wasGeneratedBy":
+                    'maxSentiment'
+                }],
+                "nif:isString":
+                "Test"
+            }
+        }
+    ]
+
+
+if __name__ == '__main__':
+    easy_test()

senpy/plugins/sentiment/sentiment140/sentiment140_plugin.py

@@ -4,6 +4,8 @@ import json
 from senpy.plugins import SentimentPlugin
 from senpy.models import Sentiment
 
+ENDPOINT = 'http://www.sentiment140.com/api/bulkClassifyJson'
+
 
 class Sentiment140(SentimentPlugin):
     '''Connects to the sentiment140 free API: http://sentiment140.com'''
@@ -26,7 +28,7 @@ class Sentiment140(SentimentPlugin):
 
     def analyse_entry(self, entry, params):
         lang = params["language"]
-        res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
+        res = requests.post(ENDPOINT,
                             json.dumps({
                                 "language": lang,
                                 "data": [{
@@ -52,18 +54,6 @@ class Sentiment140(SentimentPlugin):
         entry.language = lang
         yield entry
 
-    def test(self, *args, **kwargs):
-        '''
-        To avoid calling the sentiment140 API, we will mock the results
-        from requests.
-        '''
-        from senpy.testing import patch_requests
-        expected = {"data": [{"polarity": 4}]}
-        with patch_requests(expected) as (request, response):
-            super(Sentiment140, self).test(*args, **kwargs)
-            assert request.called
-            assert response.json.called
-
     test_cases = [
         {
             'entry': {
@@ -77,6 +67,9 @@ class Sentiment140(SentimentPlugin):
                         'marl:hasPolarity': 'marl:Positive',
                     }
                 ]
-            }
+            },
+            'responses': [{'url': ENDPOINT,
+                           'method': 'POST',
+                           'json': {'data': [{'polarity': 4}]}}]
         }
     ]

senpy/testing.py

@@ -1,36 +1,31 @@
-try:
-    from unittest.mock import patch, MagicMock
-except ImportError:
-    from mock import patch, MagicMock
-
 from past.builtins import basestring
 
-
+import os
-import json
+import responses as requestmock
-from contextlib import contextmanager
 
 from .models import BaseModel
 
 
-@contextmanager
+MOCK_REQUESTS = os.environ.get('MOCK_REQUESTS', '').lower() in ['no', 'false']
-def patch_requests(value, code=200):
+
-    success = MagicMock()
+
-    if isinstance(value, BaseModel):
+def patch_all_requests(responses):
-        value = value.jsonld()
+
-    if not isinstance(value, basestring):
+    patched = requestmock.RequestsMock()
-        data = json.dumps(value)
+
+    for response in responses or []:
+        args = response.copy()
+        if 'json' in args and isinstance(args['json'], BaseModel):
+            args['json'] = args['json'].jsonld()
+        args['method'] = getattr(requestmock, args.get('method', 'GET'))
+        patched.add(**args)
+    return patched
+
+
+def patch_requests(url, response, method='GET', status=200):
+    args = {'url': url, 'method': method, 'status': status}
+    if isinstance(response, basestring):
+        args['body'] = response
     else:
-        data = value
+        args['json'] = response
-
+    return patch_all_requests([args])
-    success.json.return_value = value
-
-    success.status_code = code
-    success.content = data
-    success.text = data
-
-    method_mocker = MagicMock()
-    method_mocker.return_value = success
-    with patch.multiple('requests', request=method_mocker,
-                        get=method_mocker, post=method_mocker):
-        yield method_mocker, success
-        assert method_mocker.called

tests/test_blueprints.py

@@ -67,10 +67,25 @@ class BlueprintsTest(TestCase):
         logging.debug("Got response: %s", js)
         assert "@context" in js
         assert "entries" in js
+        assert len(js['analysis']) == 1
+
+    def test_analysis_post(self):
+        """
+        The results for a POST request should be the same as for a GET request.
+        """
+        resp = self.client.post("/api/", data={'i': 'My aloha mohame',
+                                               'algorithm': 'rand',
+                                               'with_parameters': True})
+        self.assertCode(resp, 200)
+        js = parse_resp(resp)
+        logging.debug("Got response: %s", js)
+        assert "@context" in js
+        assert "entries" in js
+        assert len(js['analysis']) == 1
 
     def test_analysis_extra(self):
         """
-        Extra params that have a default should
+        Extra params that have a default should use it
         """
         resp = self.client.get("/api/?i=My aloha mohame&algo=Dummy&with_parameters=true")
         self.assertCode(resp, 200)
@@ -95,6 +110,44 @@ class BlueprintsTest(TestCase):
         resp = self.client.get("/api/?i=My aloha mohame&algo=DummyRequired&example=a")
         self.assertCode(resp, 200)
 
+    def test_analysis_url(self):
+        """
+        The algorithm can also be specified as part of the URL
+        """
+        self.app.config['TESTING'] = False  # Errors are expected in this case
+        resp = self.client.get("/api/DummyRequired?i=My aloha mohame")
+        self.assertCode(resp, 400)
+        js = parse_resp(resp)
+        logging.debug("Got response: %s", js)
+        assert isinstance(js, models.Error)
+        resp = self.client.get("/api/DummyRequired?i=My aloha mohame&example=notvalid")
+        self.assertCode(resp, 400)
+        resp = self.client.get("/api/DummyRequired?i=My aloha mohame&example=a")
+        self.assertCode(resp, 200)
+
+    def test_analysis_chain(self):
+        """
+        More than one algorithm can be specified. Plugins will then be chained
+        """
+        resp = self.client.get("/api/Dummy?i=My aloha mohame")
+        js = parse_resp(resp)
+        assert len(js['analysis']) == 1
+        assert js['entries'][0]['nif:isString'] == 'My aloha mohame'[::-1]
+
+        resp = self.client.get("/api/Dummy/Dummy?i=My aloha mohame")
+        # Calling dummy twice, should return the same string
+        self.assertCode(resp, 200)
+        js = parse_resp(resp)
+        assert len(js['analysis']) == 1
+        assert js['entries'][0]['nif:isString'] == 'My aloha mohame'
+
+        resp = self.client.get("/api/Dummy+Dummy?i=My aloha mohame")
+        # Same with pluses instead of slashes
+        self.assertCode(resp, 200)
+        js = parse_resp(resp)
+        assert len(js['analysis']) == 1
+        assert js['entries'][0]['nif:isString'] == 'My aloha mohame'
+
     def test_error(self):
         """
         The dummy plugin returns an empty response,\

tests/test_client.py

@@ -14,22 +14,28 @@ class ModelsTest(TestCase):
     def test_client(self):
         endpoint = 'http://dummy/'
         client = Client(endpoint)
-        with patch_requests(Results()) as (request, response):
+        with patch_requests('http://dummy/', Results()):
             resp = client.analyse('hello')
             assert isinstance(resp, Results)
-        request.assert_called_with(
+        with patch_requests('http://dummy/', Error('Nothing')):
-            url=endpoint + '/', method='GET', params={'input': 'hello'})
-        with patch_requests(Error('Nothing')) as (request, response):
             try:
                 client.analyse(input='hello', algorithm='NONEXISTENT')
                 raise Exception('Exceptions should be raised. This is not golang')
             except Error:
                 pass
-        request.assert_called_with(
+
-            url=endpoint + '/',
+    def test_client_post(self):
-            method='GET',
+        endpoint = 'http://dummy/'
-            params={'input': 'hello',
+        client = Client(endpoint)
-                    'algorithm': 'NONEXISTENT'})
+        with patch_requests('http://dummy/', Results()):
+            resp = client.analyse('hello')
+            assert isinstance(resp, Results)
+        with patch_requests('http://dummy/', Error('Nothing'), method='POST'):
+            try:
+                client.analyse(input='hello', method='POST', algorithm='NONEXISTENT')
+                raise Exception('Exceptions should be raised. This is not golang')
+            except Error:
+                pass
 
     def test_plugins(self):
         endpoint = 'http://dummy/'
@@ -37,11 +43,8 @@ class ModelsTest(TestCase):
         plugins = Plugins()
         p1 = AnalysisPlugin({'name': 'AnalysisP1', 'version': 0, 'description': 'No'})
         plugins.plugins = [p1, ]
-        with patch_requests(plugins) as (request, response):
+        with patch_requests('http://dummy/plugins', plugins):
             response = client.plugins()
             assert isinstance(response, dict)
             assert len(response) == 1
             assert "AnalysisP1" in response
-        request.assert_called_with(
-            url=endpoint + '/plugins', method='GET',
-            params={})

tests/test_extensions.py

@@ -121,8 +121,8 @@ class ExtensionsTest(TestCase):
         # Leaf (defaultdict with  __setattr__ and __getattr__.
         r1 = analyse(self.senpy, algorithm="Dummy", input="tupni", output="tuptuo")
         r2 = analyse(self.senpy, input="tupni", output="tuptuo")
-        assert r1.analysis[0] == "endpoint:plugins/Dummy_0.1"
+        assert r1.analysis[0].id == "endpoint:plugins/Dummy_0.1"
-        assert r2.analysis[0] == "endpoint:plugins/Dummy_0.1"
+        assert r2.analysis[0].id == "endpoint:plugins/Dummy_0.1"
         assert r1.entries[0]['nif:isString'] == 'input'
 
     def test_analyse_empty(self):
@@ -156,8 +156,8 @@ class ExtensionsTest(TestCase):
         r2 = analyse(self.senpy,
                      input="tupni",
                      output="tuptuo")
-        assert r1.analysis[0] == "endpoint:plugins/Dummy_0.1"
+        assert r1.analysis[0].id == "endpoint:plugins/Dummy_0.1"
-        assert r2.analysis[0] == "endpoint:plugins/Dummy_0.1"
+        assert r2.analysis[0].id == "endpoint:plugins/Dummy_0.1"
         assert r1.entries[0]['nif:isString'] == 'input'
 
     def test_analyse_error(self):
@@ -165,7 +165,7 @@ class ExtensionsTest(TestCase):
         mm.id = 'magic_mock'
         mm.name = 'mock'
         mm.is_activated = True
-        mm.analyse_entries.side_effect = Error('error in analysis', status=500)
+        mm.process.side_effect = Error('error in analysis', status=500)
         self.senpy.add_plugin(mm)
         try:
             analyse(self.senpy, input='nothing', algorithm='MOCK')
@@ -175,8 +175,7 @@ class ExtensionsTest(TestCase):
             assert ex['status'] == 500
 
         ex = Exception('generic exception on analysis')
-        mm.analyse.side_effect = ex
+        mm.process.side_effect = ex
-        mm.analyse_entries.side_effect = ex
 
         try:
             analyse(self.senpy, input='nothing', algorithm='MOCK')
@@ -211,27 +210,28 @@ class ExtensionsTest(TestCase):
             'emoml:valence': 0
         }))
         response = Results({
-            'analysis': [{'plugin': plugin}],
+            'analysis': [plugin],
             'entries': [Entry({
                 'nif:isString': 'much ado about nothing',
                 'emotions': [eSet1]
             })]
         })
         params = {'emotionModel': 'emoml:big6',
+                  'algorithm': ['conversion'],
                   'conversion': 'full'}
         r1 = deepcopy(response)
         r1.parameters = params
-        self.senpy.convert_emotions(r1)
+        self.senpy.analyse(r1)
         assert len(r1.entries[0].emotions) == 2
         params['conversion'] = 'nested'
         r2 = deepcopy(response)
         r2.parameters = params
-        self.senpy.convert_emotions(r2)
+        self.senpy.analyse(r2)
         assert len(r2.entries[0].emotions) == 1
         assert r2.entries[0].emotions[0]['prov:wasDerivedFrom'] == eSet1
         params['conversion'] = 'filtered'
         r3 = deepcopy(response)
         r3.parameters = params
-        self.senpy.convert_emotions(r3)
+        self.senpy.analyse(r3)
         assert len(r3.entries[0].emotions) == 1
         r3.jsonld()

tests/test_plugins.py

@@ -1,15 +1,15 @@
 #!/bin/env python
 
 import os
-import sys
 import pickle
 import shutil
 import tempfile
 
-from unittest import TestCase
+from unittest import TestCase, skipIf
 from senpy.models import Results, Entry, EmotionSet, Emotion, Plugins
 from senpy import plugins
-from senpy.plugins.conversion.emotion.centroids import CentroidConversion
+from senpy.plugins.postprocessing.emotion.centroids import CentroidConversion
+from senpy.gsitk_compat import GSITK_AVAILABLE
 
 import pandas as pd
 
@@ -346,13 +346,15 @@ class PluginsTest(TestCase):
         smart_metrics = results[0].metrics[0]
         assert abs(smart_metrics['accuracy'] - 1) < 0.01
 
+    @skipIf(not GSITK_AVAILABLE, "GSITK is not available")
     def test_evaluation(self):
-        if sys.version_info < (3, 0):
+        self._test_evaluation()
-            with self.assertRaises(Exception) as context:
+
-                self._test_evaluation()
+    @skipIf(GSITK_AVAILABLE, "GSITK is available")
-            self.assertTrue('GSITK ' in str(context.exception))
+    def test_evaluation_unavailable(self):
-        else:
+        with self.assertRaises(Exception) as context:
             self._test_evaluation()
+        self.assertTrue('GSITK ' in str(context.exception))
 
 
 def make_mini_test(fpath):

tests/test_test.py

@@ -5,28 +5,29 @@ import json
 from senpy.testing import patch_requests
 from senpy.models import Results
 
+ENDPOINT = 'http://example.com'
+
 
 class TestTest(TestCase):
     def test_patch_text(self):
-        with patch_requests('hello'):
+        with patch_requests(ENDPOINT, 'hello'):
-            r = requests.get('http://example.com')
+            r = requests.get(ENDPOINT)
             assert r.text == 'hello'
-            assert r.content == 'hello'
 
     def test_patch_json(self):
         r = Results()
-        with patch_requests(r):
+        with patch_requests(ENDPOINT, r):
-            res = requests.get('http://example.com')
+            res = requests.get(ENDPOINT)
-            assert res.content == json.dumps(r.jsonld())
+            assert res.text == json.dumps(r.jsonld())
             js = res.json()
             assert js
             assert js['@type'] == r['@type']
 
     def test_patch_dict(self):
         r = {'nothing': 'new'}
-        with patch_requests(r):
+        with patch_requests(ENDPOINT, r):
-            res = requests.get('http://example.com')
+            res = requests.get(ENDPOINT)
-            assert res.content == json.dumps(r)
+            assert res.text == json.dumps(r)
             js = res.json()
             assert js
             assert js['nothing'] == 'new'