Merge branches into 0.8.x

'25-validation-errors'
'27-add-method-to-get-list-of-plugins'
'28-fix-multiprocessing-issues'

requirements.txt

@@ -1,6 +1,6 @@
 Flask>=0.10.1
 requests>=2.4.1
-gevent>=1.1rc4
+tornado>=4.4.3
 PyLD>=0.6.5
 six
 future

senpy/__main__.py

@@ -22,15 +22,16 @@ the server.
 
 from flask import Flask
 from senpy.extensions import Senpy
-from gevent.wsgi import WSGIServer
+from tornado.wsgi import WSGIContainer
-from gevent.monkey import patch_all
+from tornado.httpserver import HTTPServer
+from tornado.ioloop import IOLoop
+
+
 import logging
 import os
 import argparse
 import senpy
 
-patch_all(thread=False)
-
 SERVER_PORT = os.environ.get("PORT", 5000)
 
 
@@ -92,9 +93,10 @@ def main():
     print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,
                                                             args.port))
     if not app.debug:
-        http_server = WSGIServer((args.host, args.port), app)
+        http_server = HTTPServer(WSGIContainer(app))
+        http_server.listen(args.port, address=args.host)
         try:
-            http_server.serve_forever()
+            IOLoop.instance().start()
         except KeyboardInterrupt:
             print('Bye!')
         http_server.stop()

senpy/client.py

@@ -1,6 +1,7 @@
 import requests
 import logging
 from . import models
+from .plugins import default_plugin_type
 
 logger = logging.getLogger(__name__)
 
@@ -12,6 +13,10 @@ class Client(object):
     def analyse(self, input, method='GET', **kwargs):
         return self.request('/', method=method, input=input, **kwargs)
 
+    def plugins(self, ptype=default_plugin_type):
+        resp = self.request(path='/plugins', plugin_type=ptype).plugins
+        return {p.name: p for p in resp}
+
     def request(self, path=None, method='GET', **params):
         url = '{}{}'.format(self.endpoint, path)
         response = requests.request(method=method, url=url, params=params)

senpy/extensions.py

@@ -183,7 +183,7 @@ class Senpy(object):
         return resp
 
     def _conversion_candidates(self, fromModel, toModel):
-        candidates = self.filter_plugins(**{'@type': 'emotionConversionPlugin'})
+        candidates = self.filter_plugins(plugin_type='emotionConversionPlugin')
         for name, candidate in candidates.items():
             for pair in candidate.onyx__doesConversion:
                 logging.debug(pair)
@@ -303,6 +303,7 @@ class Senpy(object):
         else:
             th = Thread(target=act)
             th.start()
+            return th
 
     def deactivate_plugin(self, plugin_name, sync=False):
         try:
@@ -327,6 +328,7 @@ class Senpy(object):
         else:
             th = Thread(target=deact)
             th.start()
+            return th
 
     @classmethod
     def validate_info(cls, info):
@@ -417,33 +419,7 @@ class Senpy(object):
         return self._plugin_list
 
     def filter_plugins(self, **kwargs):
-        """ Filter plugins by different criteria """
+        return plugins.pfilter(self.plugins, **kwargs)
-        ptype = kwargs.pop('plugin_type', None)
-        logger.debug('#' * 100)
-        logger.debug('ptype {}'.format(ptype))
-        if ptype:
-            try:
-                ptype = ptype[0].upper() + ptype[1:]
-                pclass = getattr(plugins, ptype)
-                logger.debug('Class: {}'.format(pclass))
-                candidates = filter(lambda x: isinstance(x, pclass),
-                                    self.plugins.values())
-            except AttributeError:
-                raise Error('{} is not a valid type'.format(ptype))
-        else:
-            candidates = self.plugins.values()
-
-        logger.debug(candidates)
-
-        def matches(plug):
-            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
-            logger.debug(
-                "matching {} with {}: {}".format(plug.name, kwargs, res))
-            return res
-
-        if kwargs:
-            candidates = filter(matches, candidates)
-        return {p.name: p for p in candidates}
 
     @property
     def analysis_plugins(self):

senpy/plugins/__init__.py

@@ -9,6 +9,7 @@ import logging
 import tempfile
 import copy
 from .. import models
+from ..api import API_PARAMS
 
 logger = logging.getLogger(__name__)
 
@@ -117,3 +118,40 @@ class ShelfMixin(object):
         if hasattr(self, '_sh') and self._sh is not None:
             with open(self.shelf_file, 'wb') as f:
                 pickle.dump(self._sh, f)
+
+
+default_plugin_type = API_PARAMS['plugin_type']['default']
+
+
+def pfilter(plugins, **kwargs):
+    """ Filter plugins by different criteria """
+    if isinstance(plugins, models.Plugins):
+        plugins = plugins.plugins
+    elif isinstance(plugins, dict):
+        plugins = plugins.values()
+    ptype = kwargs.pop('plugin_type', default_plugin_type)
+    logger.debug('#' * 100)
+    logger.debug('ptype {}'.format(ptype))
+    if ptype:
+        try:
+            ptype = ptype[0].upper() + ptype[1:]
+            pclass = globals()[ptype]
+            logger.debug('Class: {}'.format(pclass))
+            candidates = filter(lambda x: isinstance(x, pclass),
+                                plugins)
+        except KeyError:
+            raise models.Error('{} is not a valid type'.format(ptype))
+    else:
+        candidates = plugins
+
+    logger.debug(candidates)
+
+    def matches(plug):
+        res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
+        logger.debug(
+            "matching {} with {}: {}".format(plug.name, kwargs, res))
+        return res
+
+    if kwargs:
+        candidates = filter(matches, candidates)
+    return {p.name: p for p in candidates}

senpy/schemas/context.jsonld

@@ -37,12 +37,12 @@
       "@type": "@id",
       "@container": "@set"
     },
-    "plugins": {
-      "@container": "@list"
-    },
     "options": {
       "@container": "@set"
     },
+    "plugins": {
+      "@container": "@set"
+    },
     "prov:wasGeneratedBy": {
       "@type": "@id"
     },

senpy/schemas/plugins.json

@@ -10,8 +10,6 @@
           "items": {
             "$ref": "plugin.json"
           }
-        },
-        "@type": {
         }
       }
     }

tests/plugins/async_plugin/asyncplugin.py

@@ -0,0 +1,23 @@
+from senpy.plugins import AnalysisPlugin
+
+import multiprocessing
+
+
+def _train(process_number):
+    return process_number
+
+
+class AsyncPlugin(AnalysisPlugin):
+    def _do_async(self, num_processes):
+        pool = multiprocessing.Pool(processes=num_processes)
+        values = pool.map(_train, range(num_processes))
+
+        return values
+
+    def activate(self):
+        self.value = self._do_async(4)
+
+    def analyse_entry(self, entry, params):
+        values = self._do_async(2)
+        entry.async_values = values
+        yield entry

tests/plugins/async_plugin/asyncplugin.senpy

@@ -0,0 +1,8 @@
+---
+name: Async
+module: asyncplugin
+description: I am async
+author: "@balkian"
+version: '0.1'
+async: true
+extra_params: {}

tests/test_client.py

@@ -4,18 +4,21 @@ try:
 except ImportError:
     from mock import patch
 
+import json
+
 from senpy.client import Client
-from senpy.models import Results, Error
+from senpy.models import Results, Plugins, Error
+from senpy.plugins import AnalysisPlugin, default_plugin_type
 
 
 class Call(dict):
     def __init__(self, obj):
-        self.obj = obj.jsonld()
+        self.obj = obj.serialize()
         self.status_code = 200
         self.content = self.json()
 
     def json(self):
-        return self.obj
+        return json.loads(self.obj)
 
 
 class ModelsTest(TestCase):
@@ -44,3 +47,19 @@ class ModelsTest(TestCase):
             method='GET',
             params={'input': 'hello',
                     'algorithm': 'NONEXISTENT'})
+
+    def test_plugins(self):
+        endpoint = 'http://dummy/'
+        client = Client(endpoint)
+        plugins = Plugins()
+        p1 = AnalysisPlugin({'name': 'AnalysisP1', 'version': 0, 'description': 'No'})
+        plugins.plugins = [p1, ]
+        success = Call(plugins)
+        with patch('requests.request', return_value=success) as patched:
+            response = client.plugins()
+            assert isinstance(response, dict)
+            assert len(response) == 1
+            assert "AnalysisP1" in response
+        patched.assert_called_with(
+            url=endpoint + '/plugins', method='GET',
+            params={'plugin_type': default_plugin_type})

tests/test_extensions.py

@@ -167,7 +167,7 @@ class ExtensionsTest(TestCase):
         assert len(senpy.plugins) > 1
 
     def test_convert_emotions(self):
-        self.senpy.activate_all()
+        self.senpy.activate_all(sync=True)
         plugin = Plugin({
             'id': 'imaginary',
             'onyx:usesEmotionModel': 'emoml:fsre-dimensions'
@@ -205,3 +205,14 @@ class ExtensionsTest(TestCase):
                                     [plugin, ],
                                     params)
         assert len(r3.entries[0].emotions) == 1
+
+    # def test_async_plugin(self):
+    #     """ We should accept multiprocessing plugins with async=False"""
+    #     thread1 = self.senpy.activate_plugin("Async", sync=False)
+    #     thread1.join(timeout=1)
+    #     assert len(self.senpy.plugins['Async'].value) == 4
+
+    #     resp = self.senpy.analyse(input='nothing', algorithm='Async')
+
+    #     assert len(resp.entries[0].async_values) == 2
+    #     self.senpy.activate_plugin("Async", sync=True)

tests/test_models.py

@@ -109,13 +109,15 @@ class ModelsTest(TestCase):
                                 }
                             }})
         c = p.jsonld()
-        assert "info" not in c
+        assert '@type' in c
-        assert "repo" not in c
+        assert c['@type'] == 'plugin'
-        assert "extra_params" in c
+        assert 'info' not in c
-        logging.debug("Framed:")
+        assert 'repo' not in c
+        assert 'extra_params' in c
+        logging.debug('Framed:')
         logging.debug(c)
         p.validate()
-        assert "es" in c['extra_params']['none']['options']
+        assert 'es' in c['extra_params']['none']['options']
         assert isinstance(c['extra_params']['none']['options'], list)
 
     def test_str(self):
@@ -158,14 +160,20 @@ class ModelsTest(TestCase):
         g = rdflib.Graph().parse(data=t, format='turtle')
         assert len(g) == len(triples)
 
+    def test_plugin_list(self):
+        """The plugin list should be of type \"plugins\""""
+        plugs = Plugins()
+        c = plugs.jsonld()
+        assert '@type' in c
+        assert c['@type'] == 'plugins'
+
     def test_single_plugin(self):
         """A response with a single plugin should still return a list"""
         plugs = Plugins()
-        for i in range(10):
+        p = Plugin({'id': str(1),
-            p = Plugin({'id': str(i),
+                    'version': 0,
-                        'version': 0,
+                    'description': 'dummy'})
-                        'description': 'dummy'})
+        plugs.plugins.append(p)
-            plugs.plugins.append(p)
         assert isinstance(plugs.plugins, list)
         js = plugs.jsonld()
         assert isinstance(js['plugins'], list)