Several fixes

* Refactored BaseModel for efficiency
* Added plugin metaclass to keep track of plugin types
* Moved plugins to examples dir (in a previous commit)
* Simplified validation in parse_params
* Added convenience methods to mock requests in tests
* Changed help schema to use `.valid_parameters` instead of `.parameters`,
which was used in results to show parameters provided by the user.
* Improved UI
    * Added basic parameters
    * Fixed bugs in parameter handling
    * Refactored and cleaned code

example-plugins/async_plugin/asyncplugin.py

@@ -0,0 +1,26 @@
+from senpy.plugins import AnalysisPlugin
+
+import multiprocessing
+
+
+def _train(process_number):
+    return process_number
+
+
+class AsyncPlugin(AnalysisPlugin):
+    def _do_async(self, num_processes):
+        pool = multiprocessing.Pool(processes=num_processes)
+        values = pool.map(_train, range(num_processes))
+
+        return values
+
+    def activate(self):
+        self.value = self._do_async(4)
+
+    def analyse_entry(self, entry, params):
+        values = self._do_async(2)
+        entry.async_values = values
+        yield entry
+
+    def test(self):
+        pass

example-plugins/async_plugin/asyncplugin.senpy

@@ -0,0 +1,8 @@
+---
+name: Async
+module: asyncplugin
+description: I am async
+author: "@balkian"
+version: '0.1'
+async: true
+extra_params: {}

example-plugins/dummy_plugin/dummy.py

@@ -0,0 +1,11 @@
+from senpy.plugins import SentimentPlugin
+
+
+class DummyPlugin(SentimentPlugin):
+    def analyse_entry(self, entry, params):
+        entry['nif:isString'] = entry['nif:isString'][::-1]
+        entry.reversed = entry.get('reversed', 0) + 1
+        yield entry
+
+    def test(self):
+        pass

example-plugins/dummy_plugin/dummy.senpy

@@ -0,0 +1,15 @@
+{
+    "name": "Dummy",
+    "module": "dummy",
+    "description": "I am dummy",
+    "author": "@balkian",
+    "version": "0.1",
+    "extra_params": {
+        "example": {
+            "@id": "example_parameter",
+            "aliases": ["example", "ex"],
+            "required": false,
+            "default": 0
+        }
+    }
+}

example-plugins/dummy_plugin/dummy_noinfo.py

@@ -0,0 +1,27 @@
+from senpy.plugins import SentimentPlugin
+
+
+class DummyPlugin(SentimentPlugin):
+
+    description = 'This is a dummy self-contained plugin'
+    author = '@balkian'
+    version = '0.1'
+
+    def analyse_entry(self, entry, params):
+        entry['nif:isString'] = entry['nif:isString'][::-1]
+        entry.reversed = entry.get('reversed', 0) + 1
+        yield entry
+
+    test_cases = [{
+        "entry": {
+            "nif:isString": "Hello world!"
+        },
+        "expected": [{
+            "nif:isString": "!dlrow olleH"
+        }]
+    }]
+
+
+if __name__ == '__main__':
+    d = DummyPlugin()
+    d.test()

example-plugins/dummy_plugin/dummy_noinfo.senpy

@@ -0,0 +1,2 @@
+name: DummyNoInfo
+module: dummy_noinfo

example-plugins/dummy_plugin/dummy_required.senpy

@@ -0,0 +1,14 @@
+{
+    "name": "DummyRequired",
+    "module": "dummy",
+    "description": "I am dummy",
+    "author": "@balkian",
+    "version": "0.1",
+    "extra_params": {
+        "example": {
+            "@id": "example_parameter",
+            "aliases": ["example", "ex"],
+            "required": true
+        }
+    }
+}

example-plugins/noop/noop_plugin.py

@@ -0,0 +1,5 @@
+from senpy.plugins import SentimentPlugin
+
+
+class DummyPlugin(SentimentPlugin):
+    import noop

example-plugins/sleep_plugin/sleep.py

@@ -0,0 +1,14 @@
+from senpy.plugins import AnalysisPlugin
+from time import sleep
+
+
+class SleepPlugin(AnalysisPlugin):
+    def activate(self, *args, **kwargs):
+        sleep(self.timeout)
+
+    def analyse_entry(self, entry, params):
+        sleep(float(params.get("timeout", self.timeout)))
+        yield entry
+
+    def test(self):
+        pass

example-plugins/sleep_plugin/sleep.senpy

@@ -0,0 +1,16 @@
+{
+    "name": "Sleep",
+    "module": "sleep",
+    "description": "I am dummy",
+    "author": "@balkian",
+    "version": "0.1",
+    "timeout": 0.05,
+    "extra_params": {
+        "timeout": {
+            "@id": "timeout_sleep",
+            "aliases": ["timeout", "to"],
+            "required": false,
+            "default": 0
+        }
+     }
+}

requirements.txt

@@ -3,7 +3,6 @@ requests>=2.4.1
 tornado>=4.4.3
 PyLD>=0.6.5
 nltk
-six
 future
 jsonschema
 jsonref

senpy/__init__.py

@@ -19,6 +19,9 @@ Sentiment analysis server in Python
 """
 from .version import __version__
 
+from future.standard_library import install_aliases
+install_aliases()
+
 import logging
 
 logger = logging.getLogger(__name__)

senpy/__main__.py

@@ -67,7 +67,7 @@ def main():
         '--plugins-folder',
         '-f',
         type=str,
-        default='plugins',
+        default='.',
         help='Where to look for plugins.')
     parser.add_argument(
         '--only-install',

senpy/api.py

@@ -13,8 +13,9 @@ API_PARAMS = {
     "expanded-jsonld": {
         "@id": "expanded-jsonld",
         "aliases": ["expanded"],
+        "options": "boolean",
         "required": True,
-        "default": 0
+        "default": False
     },
     "with_parameters": {
         "aliases": ['withparameters',
@@ -23,13 +24,6 @@ API_PARAMS = {
         "default": False,
         "required": True
     },
-    "plugin_type": {
-        "@id": "pluginType",
-        "description": 'What kind of plugins to list',
-        "aliases": ["pluginType"],
-        "required": True,
-        "default": "analysisPlugin"
-    },
     "outformat": {
         "@id": "outformat",
         "aliases": ["o"],
@@ -59,6 +53,16 @@ API_PARAMS = {
     }
 }
 
+PLUGINS_PARAMS = {
+    "plugin_type": {
+        "@id": "pluginType",
+        "description": 'What kind of plugins to list',
+        "aliases": ["pluginType"],
+        "required": True,
+        "default": 'analysisPlugin'
+    }
+}
+
 WEB_PARAMS = {
     "inHeaders": {
         "aliases": ["headers"],
@@ -126,7 +130,8 @@ def parse_params(indict, *specs):
     wrong_params = {}
     for spec in specs:
         for param, options in iteritems(spec):
-            if param[0] != "@":  # Exclude json-ld properties
+            if param[0] == "@":  # Exclude json-ld properties
+                continue
             for alias in options.get("aliases", []):
                 # Replace each alias with the correct name of the parameter
                 if alias in indict and alias is not param:
@@ -134,15 +139,16 @@ def parse_params(indict, *specs):
                     del indict[alias]
                     continue
             if param not in outdict:
-                    if options.get("required", False) and "default" not in options:
-                        wrong_params[param] = spec[param]
-                    else:
                 if "default" in options:
+                    # We assume the default is correct
                     outdict[param] = options["default"]
-                elif "options" in spec[param]:
+                elif options.get("required", False):
-                    if spec[param]["options"] == "boolean":
+                    wrong_params[param] = spec[param]
+                continue
+            if "options" in options:
+                if options["options"] == "boolean":
                     outdict[param] = outdict[param] in [None, True, 'true', '1']
-                    elif outdict[param] not in spec[param]["options"]:
+                elif outdict[param] not in options["options"]:
                     wrong_params[param] = spec[param]
     if wrong_params:
         logger.debug("Error parsing: %s", wrong_params)
@@ -158,7 +164,7 @@ def parse_params(indict, *specs):
     return outdict
 
 
-def get_extra_params(request, plugin=None):
+def parse_extra_params(request, plugin=None):
     params = request.parameters.copy()
     if plugin:
         extra_params = parse_params(params, plugin.get('extra_params', {}))
@@ -177,6 +183,6 @@ def parse_call(params):
     elif params['informat'] == 'json-ld':
         results = from_string(params['input'], cls=Results)
     else:
-        raise NotImplemented('Informat {} is not implemented'.format(params['informat']))
+        raise NotImplementedError('Informat {} is not implemented'.format(params['informat']))
     results.parameters = params
     return results

senpy/blueprints.py

@@ -25,6 +25,7 @@ from .version import __version__
 from functools import wraps
 
 import logging
+import traceback
 import json
 
 logger = logging.getLogger(__name__)
@@ -72,12 +73,19 @@ def schema(schema="definitions"):
 
 
 def basic_api(f):
+    default_params = {
+        'inHeaders': False,
+        'expanded-jsonld': False,
+        'outformat': 'json-ld',
+        'with_parameters': True,
+    }
+
     @wraps(f)
     def decorated_function(*args, **kwargs):
         raw_params = get_params(request)
         headers = {'X-ORIGINAL-PARAMS': json.dumps(raw_params)}
+        params = default_params
 
-        outformat = 'json-ld'
         try:
             print('Getting request:')
             print(request)
@@ -87,26 +95,32 @@ def basic_api(f):
             else:
                 request.parameters = params
             response = f(*args, **kwargs)
-        except Error as ex:
+        except (Exception) as ex:
-            response = ex
-            response.parameters = params
-            logger.error(ex)
             if current_app.debug:
                 raise
+            if not isinstance(ex, Error):
+                msg = "{}:\n\t{}".format(ex,
+                                         traceback.format_exc())
+                ex = Error(message=msg, status=500)
+            logger.exception('Error returning analysis result')
+            response = ex
+            response.parameters = raw_params
+            logger.error(ex)
 
-        in_headers = params['inHeaders']
+        if 'parameters' in response and not params['with_parameters']:
-        expanded = params['expanded-jsonld']
+            print(response)
-        outformat = params['outformat']
+            print(response.data)
+            del response.parameters
 
         return response.flask(
-            in_headers=in_headers,
+            in_headers=params['inHeaders'],
             headers=headers,
             prefix=url_for('.api_root', _external=True),
             context_uri=url_for('api.context',
                                 entity=type(response).__name__,
                                 _external=True),
-            outformat=outformat,
+            outformat=params['outformat'],
-            expanded=expanded)
+            expanded=params['expanded-jsonld'])
 
     return decorated_function
 
@@ -116,19 +130,18 @@ def basic_api(f):
 def api_root():
     if request.parameters['help']:
         dic = dict(api.API_PARAMS, **api.NIF_PARAMS)
-        response = Help(parameters=dic)
+        response = Help(valid_parameters=dic)
         return response
-    else:
     req = api.parse_call(request.parameters)
-        response = current_app.senpy.analyse(req)
+    return current_app.senpy.analyse(req)
-        return response
 
 
 @api_blueprint.route('/plugins/', methods=['POST', 'GET'])
 @basic_api
 def plugins():
     sp = current_app.senpy
-    ptype = request.parameters.get('plugin_type')
+    params = api.parse_params(request.parameters, api.PLUGINS_PARAMS)
+    ptype = params.get('plugin_type')
     plugins = sp.filter_plugins(plugin_type=ptype)
     dic = Plugins(plugins=list(plugins.values()))
     return dic

senpy/client.py

@@ -1,7 +1,6 @@
 import requests
 import logging
 from . import models
-from .plugins import default_plugin_type
 
 logger = logging.getLogger(__name__)
 
@@ -13,8 +12,8 @@ class Client(object):
     def analyse(self, input, method='GET', **kwargs):
         return self.request('/', method=method, input=input, **kwargs)
 
-    def plugins(self, ptype=default_plugin_type):
+    def plugins(self, *args, **kwargs):
-        resp = self.request(path='/plugins', plugin_type=ptype).plugins
+        resp = self.request(path='/plugins').plugins
         return {p.name: p for p in resp}
 
     def request(self, path=None, method='GET', **params):

senpy/extensions.py

@@ -123,7 +123,7 @@ class Senpy(object):
             return
         plugin = plugins[0]
         self._activate(plugin)  # Make sure the plugin is activated
-        specific_params = api.get_extra_params(req, plugin)
+        specific_params = api.parse_extra_params(req, plugin)
         req.analysis.append({'plugin': plugin,
                              'parameters': specific_params})
         results = plugin.analyse_entries(entries, specific_params)
@@ -262,17 +262,11 @@ class Senpy(object):
         with plugin._lock:
             if plugin.is_activated:
                 return
-            try:
             plugin.activate()
             msg = "Plugin activated: {}".format(plugin.name)
             logger.info(msg)
             success = True
             self._set_active(plugin, success)
-            except Exception as ex:
-                msg = "Error activating plugin {} - {} : \n\t{}".format(
-                    plugin.name, ex, traceback.format_exc())
-                logger.error(msg)
-                raise Error(msg)
 
     def activate_plugin(self, plugin_name, sync=True):
         try:
@@ -294,13 +288,8 @@ class Senpy(object):
         with plugin._lock:
             if not plugin.is_activated:
                 return
-            try:
             plugin.deactivate()
             logger.info("Plugin deactivated: {}".format(plugin.name))
-            except Exception as ex:
-                logger.error(
-                    "Error deactivating plugin {}: {}".format(plugin.name, ex))
-                logger.error("Trace: {}".format(traceback.format_exc()))
 
     def deactivate_plugin(self, plugin_name, sync=True):
         try:

senpy/models.py

@@ -6,7 +6,11 @@ For compatibility with Py3 and for easier debugging, this new version drops
 introspection and adds all arguments to the models.
 '''
 from __future__ import print_function
-from six import string_types
+from future import standard_library
+standard_library.install_aliases()
+
+from future.utils import with_metaclass
+from past.builtins import basestring
 
 import time
 import copy
@@ -15,6 +19,8 @@ import os
 import jsonref
 import jsonschema
 import inspect
+from collections import UserDict
+from abc import ABCMeta
 
 from flask import Response as FlaskResponse
 from pyld import jsonld
@@ -62,7 +68,7 @@ class Context(dict):
             return contexts
         elif isinstance(context, dict):
             return Context(context)
-        elif isinstance(context, string_types):
+        elif isinstance(context, basestring):
             try:
                 with open(context) as f:
                     return Context(json.loads(f.read()))
@@ -75,9 +81,154 @@ class Context(dict):
 base_context = Context.load(CONTEXT_PATH)
 
 
-class SenpyMixin(object):
+class BaseMeta(ABCMeta):
+    '''
+    Metaclass for models. It extracts the default values for the fields in
+    the model.
+
+    For instance, instances of the following class wouldn't need to mark
+    their version or description on initialization:
+
+    .. code-block:: python
+
+       class MyPlugin(Plugin):
+           version=0.3
+           description='A dull plugin'
+
+
+    Note that these operations could be included in the __init__ of the
+    class, but it would be very inefficient.
+    '''
+    def __new__(mcs, name, bases, attrs, **kwargs):
+        defaults = {}
+        if 'schema' in attrs:
+            defaults = mcs.get_defaults(attrs['schema'])
+        for b in bases:
+            if hasattr(b, 'defaults'):
+                defaults.update(b.defaults)
+        info = mcs.attrs_to_dict(attrs)
+        defaults.update(info)
+        attrs['defaults'] = defaults
+        return super(BaseMeta, mcs).__new__(mcs, name, bases, attrs)
+
+    @staticmethod
+    def attrs_to_dict(attrs):
+        '''
+        Extract the attributes of the class.
+
+        This allows adding default values in the class definition.
+        e.g.:
+        '''
+        def is_attr(k, v):
+            return (not(inspect.isroutine(v) or
+                        inspect.ismethod(v) or
+                        inspect.ismodule(v) or
+                        isinstance(v, property)) and
+                    k[0] != '_' and
+                    k != 'schema' and
+                    k != 'data')
+
+        return {key: copy.deepcopy(value) for key, value in attrs.items() if is_attr(key, value)}
+
+    @staticmethod
+    def get_defaults(schema):
+        temp = {}
+        for obj in [
+                schema,
+        ] + schema.get('allOf', []):
+            for k, v in obj.get('properties', {}).items():
+                if 'default' in v and k not in temp:
+                    temp[k] = copy.deepcopy(v['default'])
+        return temp
+
+
+class CustomDict(UserDict, object):
+    '''
+    A dictionary whose elements can also be accessed as attributes. Since some
+    characters are not valid in the dot-notation, the attribute names also
+    converted. e.g.:
+
+    > d = CustomDict()
+    > d.key = d['ns:name'] = 1
+    > d.key == d['key']
+    True
+    > d.ns__name == d['ns:name']
+    '''
+
+    defaults = []
+
+    def __init__(self, *args, **kwargs):
+        temp = copy.deepcopy(self.defaults)
+        for arg in args:
+            temp.update(copy.deepcopy(arg))
+        for k, v in kwargs.items():
+            temp[self._get_key(k)] = v
+
+        super(CustomDict, self).__init__(temp)
+
+    @staticmethod
+    def _get_key(key):
+        if key is 'id':
+            key = '@id'
+        key = key.replace("__", ":", 1)
+        return key
+
+    @staticmethod
+    def _internal_key(key):
+        return key[0] == '_' or key == 'data'
+
+    def __getattr__(self, key):
+        '''
+        __getattr__ only gets called when the attribute could not be found
+        in the __dict__. So we only need to look for the the element in the
+        dictionary, or raise an Exception.
+        '''
+        mkey = self._get_key(key)
+        if not self._internal_key(key) and mkey in self:
+            return self[mkey]
+        raise AttributeError(key)
+
+    def __setattr__(self, key, value):
+        # Work as usual for internal properties or already existing
+        # properties
+        if self._internal_key(key) or key in self.__dict__:
+            return super(CustomDict, self).__setattr__(key, value)
+        key = self._get_key(key)
+        return self.__setitem__(self._get_key(key), value)
+
+    def __delattr__(self, key):
+        if self._internal_key(key):
+            return object.__delattr__(self, key)
+        key = self._get_key(key)
+        self.__delitem__(self._get_key(key))
+
+
+class BaseModel(with_metaclass(BaseMeta, CustomDict)):
+    '''
+    Entities of the base model are a special kind of dictionary that emulates
+    a JSON-LD object. The structure of the dictionary is checked via JSON-schema.
+    For convenience, the values can also be accessed as attributes
+    (a la Javascript). e.g.:
+
+    > myobject.key == myobject['key']
+    True
+    > myobject.ns__name == myobject['ns:name']
+    True
+    '''
+
+    schema = base_schema
     _context = base_context["@context"]
 
+    def __init__(self, *args, **kwargs):
+        auto_id = kwargs.pop('_auto_id', True)
+        super(BaseModel, self).__init__(*args, **kwargs)
+
+        if '@id' not in self and auto_id:
+            self.id = ':{}_{}'.format(type(self).__name__, time.time())
+
+        if '@type' not in self:
+            logger.warn('Created an instance of an unknown model')
+
     def flask(self,
               in_headers=True,
               headers=None,
@@ -146,7 +297,7 @@ class SenpyMixin(object):
             else:
                 return item
 
-        return ser_or_down(self._plain_dict())
+        return ser_or_down(self.data)
 
     def jsonld(self,
                with_context=True,
@@ -188,150 +339,21 @@ class SenpyMixin(object):
         return str(self.serialize())
 
 
-class BaseModel(SenpyMixin, dict):
+_subtypes = {}
-    '''
-    Entities of the base model are a special kind of dictionary that emulates
-    a JSON-LD object. For convenience, the values can also be accessed as attributes
-    (a la Javascript). e.g.:
-
-    > myobject.key == myobject['key']
-    True
-    > myobject.ns__name == myobject['ns:name']
-    True
-    '''
-
-    schema = base_schema
-
-    def __init__(self, *args, **kwargs):
-        self.attrs_to_dict()
-        if 'id' in kwargs:
-            self.id = kwargs.pop('id')
-        elif kwargs.pop('_auto_id', True):
-            self.id = '_:{}_{}'.format(type(self).__name__, time.time())
-
-        temp = self.get_defaults()
-        temp.update(dict(*args))
-        for k, v in kwargs.items():
-            temp[self._get_key(k)] = v
-        super(BaseModel, self).__init__(temp)
-
-        if '@type' not in self:
-            logger.warn('Created an instance of an unknown model')
-
-    def get_defaults(self):
-        temp = {}
-        for obj in [
-                self.schema,
-        ] + self.schema.get('allOf', []):
-            for k, v in obj.get('properties', {}).items():
-                if 'default' in v and k not in temp:
-                    temp[k] = copy.deepcopy(v['default'])
-        return temp
-
-    def attrs_to_dict(self):
-        '''
-        Copy the attributes of the class to the instance.
-
-        This allows adding default values in the class definition.
-        e.g.:
-
-        class MyPlugin(Plugin):
-            version=0.3
-            description='A dull plugin'
-        '''
-        def is_attr(x):
-            return not(inspect.isroutine(x) or inspect.ismethod(x) or isinstance(x, property))
-        for key, value in inspect.getmembers(self.__class__, is_attr):
-            if key[0] != '_' and key != 'schema':
-                self[key] = value
-
-    def _get_key(self, key):
-        if key is 'id':
-            key = '@id'
-        key = key.replace("__", ":", 1)
-        return key
-
-    def __delitem__(self, key):
-        key = self._get_key(key)
-        dict.__delitem__(self, key)
-
-    def _internal_key(self, key):
-        return key[0] == '_' or key in self.__dict__
-
-    def _plain_dict(self):
-        d = {k: v for (k, v) in self.items() if k[0] != "_"}
-        return d
-
-    def __getattr__(self, key):
-        '''
-        __getattr__ only gets called when the attribute could not
-        be found in the __dict__. So we only need to look for the
-        the element in the dictionary, or raise an Exception.
-        '''
-        if self._internal_key(key):
-            raise AttributeError(key)
-        return self.__getitem__(self._get_key(key))
-
-    def __setattr__(self, key, value):
-        if self._internal_key(key):
-            return super(BaseModel, self).__setattr__(key, value)
-        key = self._get_key(key)
-        return self.__setitem__(self._get_key(key), value)
-
-    def __delattr__(self, key):
-        if self._internal_key(key):
-            return object.__delattr__(self, key)
-        key = self._get_key(key)
-        self.__delitem__(self._get_key(key))
 
 
 def register(rsubclass, rtype=None):
     _subtypes[rtype or rsubclass.__name__] = rsubclass
 
 
-_subtypes = {}
-
-
-def from_dict(indict, cls=None):
-    if not cls:
-        target = indict.get('@type', None)
-        try:
-            if target and target in _subtypes:
-                cls = _subtypes[target]
-            else:
-                cls = BaseModel
-        except Exception:
-            cls = BaseModel
-    outdict = dict()
-    for k, v in indict.items():
-        if k == '@context':
-            pass
-        elif isinstance(v, dict):
-            v = from_dict(indict[k])
-        elif isinstance(v, list):
-            for ix, v2 in enumerate(v):
-                if isinstance(v2, dict):
-                    v[ix] = from_dict(v2)
-        outdict[k] = v
-    return cls(**outdict)
-
-
-def from_string(string, **kwargs):
-    return from_dict(json.loads(string), **kwargs)
-
-
-def from_json(injson):
-    indict = json.loads(injson)
-    return from_dict(indict)
-
-
 def from_schema(name, schema=None, schema_file=None, base_classes=None):
     base_classes = base_classes or []
     base_classes.append(BaseModel)
     schema_file = schema_file or '{}.json'.format(name)
     class_name = '{}{}'.format(name[0].upper(), name[1:])
     if '/' not in 'schema_file':
-        schema_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+        thisdir = os.path.dirname(os.path.realpath(__file__))
+        schema_file = os.path.join(thisdir,
                                    'schemas',
                                    schema_file)
 
@@ -354,6 +376,40 @@ def from_schema(name, schema=None, schema_file=None, base_classes=None):
     return newclass
 
 
+def from_dict(indict, cls=None):
+    if not cls:
+        target = indict.get('@type', None)
+        try:
+            if target and target in _subtypes:
+                cls = _subtypes[target]
+            else:
+                cls = BaseModel
+        except Exception:
+            cls = BaseModel
+    outdict = dict()
+    for k, v in indict.items():
+        if k == '@context':
+            pass
+        elif isinstance(v, dict):
+            v = from_dict(indict[k])
+        elif isinstance(v, list):
+            v = v[:]
+            for ix, v2 in enumerate(v):
+                if isinstance(v2, dict):
+                    v[ix] = from_dict(v2)
+        outdict[k] = copy.deepcopy(v)
+    return cls(**outdict)
+
+
+def from_string(string, **kwargs):
+    return from_dict(json.loads(string), **kwargs)
+
+
+def from_json(injson):
+    indict = json.loads(injson)
+    return from_dict(indict)
+
+
 def _add_from_schema(*args, **kwargs):
     generatedClass = from_schema(*args, **kwargs)
     globals()[generatedClass.__name__] = generatedClass
@@ -384,40 +440,14 @@ for i in [
 _ErrorModel = from_schema('error')
 
 
-class Error(SenpyMixin, Exception):
+class Error(_ErrorModel, Exception):
     def __init__(self, message, *args, **kwargs):
-        super(Error, self).__init__(self, message, message)
+        Exception.__init__(self, message)
-        self._error = _ErrorModel(message=message, *args, **kwargs)
+        super(Error, self).__init__(*args, **kwargs)
         self.message = message
 
-    def validate(self, obj=None):
+    def __hash__(self):
-        self._error.validate()
+        return Exception.__hash__(self)
-
-    def __getitem__(self, key):
-        return self._error[key]
-
-    def __setitem__(self, key, value):
-        self._error[key] = value
-
-    def __delitem__(self, key):
-        del self._error[key]
-
-    def __getattr__(self, key):
-        if key != '_error' and hasattr(self._error, key):
-            return getattr(self._error, key)
-        raise AttributeError(key)
-
-    def __setattr__(self, key, value):
-        if key != '_error':
-            return setattr(self._error, key, value)
-        else:
-            super(Error, self).__setattr__(key, value)
-
-    def __delattr__(self, key):
-        delattr(self._error, key)
-
-    def __str__(self):
-        return str(self.to_JSON(with_context=False))
 
 
 register(Error, 'error')

senpy/plugins/__init__.py

@@ -1,5 +1,6 @@
 from future import standard_library
 standard_library.install_aliases()
+from future.utils import with_metaclass
 
 import os.path
 import os
@@ -16,21 +17,33 @@ import yaml
 import threading
 
 from .. import models, utils
-from ..api import API_PARAMS
+from .. import api
 
 logger = logging.getLogger(__name__)
 
 
-class Plugin(models.Plugin):
+class PluginMeta(models.BaseMeta):
-    def __init__(self, info=None, data_folder=None):
+
+    def __new__(mcs, name, bases, attrs, **kwargs):
+        plugin_type = []
+        if hasattr(bases[0], 'plugin_type'):
+            plugin_type += bases[0].plugin_type
+        plugin_type.append(name)
+        attrs['plugin_type'] = plugin_type
+        return super(PluginMeta, mcs).__new__(mcs, name, bases, attrs)
+
+
+class Plugin(with_metaclass(PluginMeta, models.Plugin)):
+
+    def __init__(self, info=None, data_folder=None, **kwargs):
         """
         Provides a canonical name for plugins and serves as base for other
         kinds of plugins.
         """
         logger.debug("Initialising {}".format(info))
+        super(Plugin, self).__init__(**kwargs)
         if info:
             self.update(info)
-        super(Plugin, self).__init__(**self)
         if not self.validate():
             raise models.Error(message=("You need to provide configuration"
                                         "information for the plugin."))
@@ -57,7 +70,8 @@ class Plugin(models.Plugin):
                                   'test cases').format(self.id, inspect.getfile(self.__class__)))
         for case in self.test_cases:
             entry = models.Entry(case['entry'])
-            params = case.get('params', {})
+            given_parameters = case.get('params', {})
+            params = api.parse_params(given_parameters, self.extra_params)
             fails = case.get('fails', False)
             try:
                 res = list(self.analyse_entry(entry, params))
@@ -90,7 +104,7 @@ SenpyPlugin = Plugin
 class AnalysisPlugin(Plugin):
 
     def analyse(self, *args, **kwargs):
-        raise NotImplemented(
+        raise NotImplementedError(
             'Your method should implement either analyse or analyse_entry')
 
     def analyse_entry(self, entry, parameters):
@@ -118,17 +132,17 @@ class ConversionPlugin(Plugin):
     pass
 
 
-class SentimentPlugin(models.SentimentPlugin, AnalysisPlugin):
+class SentimentPlugin(AnalysisPlugin, models.SentimentPlugin):
     minPolarityValue = 0
     maxPolarityValue = 1
 
 
-class EmotionPlugin(models.EmotionPlugin, AnalysisPlugin):
+class EmotionPlugin(AnalysisPlugin, models.EmotionPlugin):
     minEmotionValue = 0
     maxEmotionValue = 1
 
 
-class EmotionConversionPlugin(models.EmotionConversionPlugin, ConversionPlugin):
+class EmotionConversionPlugin(ConversionPlugin):
     pass
 
 
@@ -171,19 +185,18 @@ class ShelfMixin(object):
                 pickle.dump(self._sh, f)
 
 
-default_plugin_type = API_PARAMS['plugin_type']['default']
-
-
 def pfilter(plugins, **kwargs):
     """ Filter plugins by different criteria """
     if isinstance(plugins, models.Plugins):
         plugins = plugins.plugins
     elif isinstance(plugins, dict):
         plugins = plugins.values()
-    ptype = kwargs.pop('plugin_type', default_plugin_type)
+    ptype = kwargs.pop('plugin_type', AnalysisPlugin)
     logger.debug('#' * 100)
     logger.debug('ptype {}'.format(ptype))
     if ptype:
+        if isinstance(ptype, PluginMeta):
+            ptype = ptype.__name__
         try:
             ptype = ptype[0].upper() + ptype[1:]
             pclass = globals()[ptype]

senpy/plugins/example/emoRand/emoRand.py

@@ -4,7 +4,7 @@ from senpy.plugins import EmotionPlugin
 from senpy.models import EmotionSet, Emotion, Entry
 
 
-class RmoRandPlugin(EmotionPlugin):
+class EmoRandPlugin(EmotionPlugin):
     def analyse_entry(self, entry, params):
         category = "emoml:big6happiness"
         number = max(-1, min(1, random.gauss(0, 0.5)))

senpy/plugins/misc/split.py

@@ -11,7 +11,7 @@ class SplitPlugin(AnalysisPlugin):
         nltk.download('punkt')
 
     def analyse_entry(self, entry, params):
-        chunker_type = params.get("delimiter", "sentence")
+        chunker_type = params["delimiter"]
         original_text = entry['nif:isString']
         if chunker_type == "sentence":
             tokenizer = PunktSentenceTokenizer()

senpy/plugins/sentiment/sentiment140/sentiment140.py

@@ -7,7 +7,7 @@ from senpy.models import Sentiment
 
 class Sentiment140Plugin(SentimentPlugin):
     def analyse_entry(self, entry, params):
-        lang = params.get("language", "auto")
+        lang = params["language"]
         res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
                             json.dumps({
                                 "language": lang,
@@ -35,6 +35,18 @@ class Sentiment140Plugin(SentimentPlugin):
         entry.language = lang
         yield entry
 
+    def test(self, *args, **kwargs):
+        '''
+        To avoid calling the sentiment140 API, we will mock the results
+        from requests.
+        '''
+        from senpy.test import patch_requests
+        expected = {"data": [{"polarity": 10}]}
+        with patch_requests(expected) as (request, response):
+            super(Sentiment140Plugin, self).test(*args, **kwargs)
+            assert request.called
+            assert response.json.called
+
     test_cases = [
         {
             'entry': {

senpy/plugins/sentiment/sentiment140/sentiment140.senpy

@@ -16,6 +16,7 @@ extra_params:
     - es
     - en
     - auto
+    default: auto
 requirements: {}
 maxPolarityValue: 1
 minPolarityValue: 0

senpy/schemas/help.json

@@ -7,11 +7,11 @@
       "description": "Help containing accepted parameters",
       "type": "object",
       "properties": {
-        "parameters": {
+        "valid_parameters": {
           "type": "object"
           }
         },
-        "required": "parameters"
+        "required": "valid_parameters"
       }
     ]
 }

senpy/schemas/plugin.json

@@ -1,7 +1,7 @@
 {
   "$schema": "http://json-schema.org/draft-04/schema#",
   "type": "object",
-  "required": ["@id", "extra_params"],
+  "required": ["@id", "name", "description", "version", "plugin_type"],
   "properties": {
     "@id": {
       "type": "string",
@@ -9,7 +9,19 @@
     },
     "name": {
       "type": "string",
-      "description": "The name of the plugin, which will be used in the algorithm detection phase"
+      "description": "The name of the plugin, which will be used in the algorithm detection phase."
+    },
+    "description": {
+      "type": "string",
+      "description": "A summary of what the plugin does, and pointers to further information."
+    },
+    "version": {
+      "type": "string",
+      "description": "The version of the plugin."
+    },
+    "plugin_type": {
+      "type": "string",
+      "description": "Sub-type of plugin. e.g. sentimentPlugin"
     },
     "extra_params": {
       "type": "object",

senpy/static/css/main.css

@@ -152,3 +152,18 @@ textarea{
     /* background: white; */
     display: none;
 }
+
+.deco-none {
+    color: inherit;
+    text-decoration: inherit;
+}
+
+.deco-none:link {
+    color: inherit;
+    text-decoration: inherit;
+}
+
+.deco-none:hover {
+    color: inherit;
+    text-decoration: inherit;
+}

senpy/static/js/main.js

@@ -1,7 +1,10 @@
 var ONYX = "http://www.gsi.dit.upm.es/ontologies/onyx/ns#";
 var RDF_TYPE =  "http://www.w3.org/1999/02/22-rdf-syntax-ns#type";
-var plugins_params={};
+var plugins_params = default_params = {};
-var default_params = JSON.parse($.ajax({type: "GET", url: "/api?help=true" , async: false}).responseText);
+var plugins = [];
+var defaultPlugin = {};
+var gplugins = {};
+
 function replaceURLWithHTMLLinks(text) {
     console.log('Text: ' + text);
     var exp = /(\b(https?|ftp|file):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])/ig;
@@ -25,39 +28,25 @@ function hashchanged(){
     }
 }
 
-$(document).ready(function() {
+
-    var response = JSON.parse($.ajax({type: "GET", url: "/api/plugins/" , async: false}).responseText);
+function get_plugins(response){
-    var defaultPlugin= JSON.parse($.ajax({type: "GET", url: "/api/plugins/default" , async: false}).responseText);
-    html="";
-    var availablePlugins = document.getElementById('availablePlugins');
     plugins = response.plugins;
-    gplugins = {};
+}
+
+function group_plugins(){
     for (r in plugins){
         ptype = plugins[r]['@type'];
         if(gplugins[ptype] == undefined){
-            gplugins[ptype] = [r]
+            gplugins[ptype] = [r];
         }else{
-            gplugins[ptype].push(r)
+            gplugins[ptype].push(r);
 	      }
     }
-    for (g in gplugins){	
+}
-        html += "<optgroup label=\""+g+"\">"
-        for (r in gplugins[g]){
-            plugin = plugins[gplugins[g][r]]
-            if (!plugin["name"]){
-                console.log("No name for plugin ", plugin);
-                continue;
-
-            }
-            html+= "<option value=\""+plugin["name"]+"\" "
-            if (plugin["name"] == defaultPlugin["name"]){
-                html+= " selected=\"selected\""
-            }
-            if (!plugin["is_activated"]){
-                html+= " disabled=\"disabled\" "
-            }
-            html+=">"+plugin["name"]+"</option>"
 
+function get_parameters(){
+    for (p in plugins){	
+        plugin = plugins[p];
         if (plugin["extra_params"]){
             plugins_params[plugin["name"]]={};
             for (param in plugin["extra_params"]){
@@ -73,8 +62,41 @@ $(document).ready(function() {
             }
         }
     }
-        var pluginEntry = document.createElement('li');
+}
 
+function draw_plugins_selection(){
+    html="";
+    group_plugins();
+    for (g in gplugins){	
+        html += "<optgroup label=\""+g+"\">"
+        for (r in gplugins[g]){
+            plugin = plugins[gplugins[g][r]]
+            if (!plugin["name"]){
+                console.log("No name for plugin ", plugin);
+                continue;
+
+            }
+            html+= "<option value=\""+plugin.name+"\" "
+            if (plugin["name"] == defaultPlugin["name"]){
+                html+= " selected=\"selected\""
+            }
+            if (!plugin["is_activated"]){
+                html+= " disabled=\"disabled\" "
+            }
+            html+=">"+plugin["name"]+"</option>"
+
+        }
+    }
+    html += "</optgroup>"
+    document.getElementById('plugins').innerHTML = html;
+}
+
+function draw_plugins_list(){
+    var availablePlugins = document.getElementById('availablePlugins');
+
+    for(p in plugins){
+        var pluginEntry = document.createElement('li');
+        plugin = plugins[p];
         newHtml = ""
         if(plugin.url) {
             newHtml= "<a href="+plugin.url+">" + plugin.name + "</a>";
@@ -85,9 +107,17 @@ $(document).ready(function() {
         pluginEntry.innerHTML = newHtml;
         availablePlugins.appendChild(pluginEntry)
     }
-    html += "</optgroup>"
+}
-    document.getElementById('plugins').innerHTML = html;
+
-    change_params();
+$(document).ready(function() {
+    var response = JSON.parse($.ajax({type: "GET", url: "/api/plugins/" , async: false}).responseText);
+    defaultPlugin= JSON.parse($.ajax({type: "GET", url: "/api/plugins/default" , async: false}).responseText);
+    get_plugins(response);
+    get_default_parameters();
+
+    draw_plugins_list();
+    draw_plugins_selection();
+    draw_parameters();
 
     $(window).on('hashchange', hashchanged);
     hashchanged();
@@ -95,49 +125,131 @@ $(document).ready(function() {
 
 });
 
+function get_default_parameters(){
+    default_params = JSON.parse($.ajax({type: "GET", url: "/api?help=true" , async: false}).responseText).valid_parameters;
+    // Remove the parameters that are always added
+    delete default_params["input"];
+    delete default_params["algorithm"];
+    delete default_params["help"];
 
-function change_params(){
+}
+
+function draw_default_parameters(){
+    var basic_params = document.getElementById("basic_params");
+    basic_params.innerHTML = params_div(default_params);
+}
+
+function draw_extra_parameters(){
     var plugin = document.getElementById("plugins").options[document.getElementById("plugins").selectedIndex].value;
-        html=""
+    get_parameters();
-		for (param in default_params){
-		  if ((default_params[param]['options']) && (['help','conversion'].indexOf(param) < 0)){  
-          html+= "<label> "+param+"</label>"
-          if (default_params[param]['options'].length < 1) {
-              html +="<input></input>";
-          }
-          else {
-              html+= "<select id=\""+param+"\" name=\""+param+"\">"
-              for (option in default_params[param]['options']){
-                  if (default_params[param]['options'][option] == default_params[param]['default']){  
-                      html+="<option value \""+default_params[param]['options'][option]+"\" selected >"+default_params[param]['options'][option]+"</option>"
-                  }
-                  else{
-                      html+="<option value \""+default_params[param]['options'][option]+"\">"+default_params[param]['options'][option]+"</option>"
 
+    var extra_params = document.getElementById("extra_params");
+    extra_params.innerHTML = params_div(plugins_params[plugin]);
+}
+
+function draw_parameters(){
+    draw_default_parameters();
+    draw_extra_parameters();
+}
+
+
+function add_default_params(){
+    var html = "";
+    // html += '<a href="#basic_params" class="btn btn-info" data-toggle="collapse">Basic API parameters</a>';
+    html += '<span id="basic_params" class="panel-collapse collapse">';
+    html += '<ul class="list-group">'
+    html += params_div(default_params);
+    html += '</span>';
+    return html;
+}
+
+function params_div(params){
+    var html = '<div class="container-fluid">';
+    if (Object.keys(params).length === 0) {
+        html += '<p class="text text-muted text-center">This plugin does not take any extra parameters</p>';
     }
+    // Iterate over the keys in order
+    pnames = Object.keys(params).sort()
+    for (ix in pnames){
+        pname = pnames[ix];
+        param = params[pname];
+        html+='<div class="form-group">';
+        html += '<div class="row">'
+        html+= '<label class="col-sm-2" for="'+pname+'">'+pname+'</label>'
+        if (param.options){
+            opts = param.options;
+            if(param.options.length == 1 && param.options[0] == 'boolean') {
+                opts = [true, false];
             }
+            html+= '<select class="col-sm-10" id="'+pname+"\" name=\""+pname+"\">"
+            var defaultopt = param.default;
+            for (option in opts){
+                isselected = "";
+                if (defaultopt != undefined && opts[option] == defaultopt ){
+                    isselected = ' selected="selected"'
                 }
-      html+="</select><br>"
+                html+="<option value=\""+opts[option]+'"' + isselected +
-      }
+                '>'+opts[option]+"</option>"
-		}
-        for (param in plugins_params[plugin]){
-          if (param || plugins_params[plugin][param].length > 1){
-              html+= "<label> Parameter "+param+"</label>"
-              param_opts = plugins_params[plugin][param]
-              if (param_opts.length > 0) {
-                  html+= "<select id=\""+param+"\" name=\""+param+"\">"
-                  for (option in param_opts){
-                      html+="<option value \""+param_opts[option]+"\">"+param_opts[option]+"</option>"
             }
             html+="</select>"
         }
         else {
-                  html +="<input id=\""+param+"\" name=\""+param+"\"></input>";
+            default_value = "";
+            if(param.default != undefined){
+                default_value = param.default;
+            };
+            html +='<input class="col-sm-10" id="'+pname+'" name="'+pname+'" value="' + default_value + '"></input>';
+        }
+        html+='</div>';
+        html+='<div class="row">';
+        if ('description' in param){
+            html += '<p class="form-text sm-sm-12 text-muted text-center">' + param.description + '</p>';
+
+        }
+        html+='</div>';
+        html+='</div>';
+    }
+    html+='</div>';
+    return html;
+}
+
+function _get_form_parameters(id){
+    var element = document.getElementById(id);
+    params = {};
+    var selects = element.getElementsByTagName('select');
+    var inputs = element.getElementsByTagName('input');
+
+    Array.prototype.forEach.call(selects, function (sel) {
+        key = sel.name;
+        value = sel.options[sel.selectedIndex].value
+        params[key] = value;
+    });
+
+    Array.prototype.forEach.call(inputs, function (el) {
+        params[el.name] = el.value;
+    });
+
+    for (k in params){
+        value = params[k];
+        if (value == "" || value === "undefined"){
+            delete params[k];
+
         }
     }
-        }
+
-        document.getElementById("params").innerHTML = html
+    return params;
-};
+}
+
+function get_form_parameters(){
+    var p1 = _get_form_parameters("basic_params");
+    var p2 = _get_form_parameters("extra_params");
+    return Object.assign(p1, p2);
+}
+
+function add_param(key, value){
+    value = encodeURIComponent(value);
+    return "&"+key+"="+value;
+}
 
 function load_JSON(){
     url = "/api";
@@ -148,31 +260,15 @@ function load_JSON(){
     var plugin = document.getElementById("plugins").options[document.getElementById("plugins").selectedIndex].value;
     var input = encodeURIComponent(document.getElementById("input").value);
     url += "?algo="+plugin+"&i="+input
-      for (param in plugins_params[plugin]){
-        if (param != null){
-            field = document.getElementById(param);
-            if (plugins_params[plugin][param].length > 0){
-                var param_value = encodeURIComponent(field.options[field.selectedIndex].text);
-            } else {
-                var param_value = encodeURIComponent(field.text);
-            }
-            if (param_value !== "undefined" && param_value.length > 0){
-                url+="&"+param+"="+param_value
-            }
-        }
-      }
 
-      for (param in default_params){
+    params = get_form_parameters();
-        if ((param != null) && (default_params[param]['options']) && (['help','conversion'].indexOf(param) < 0)){
+
-            var param_value = encodeURIComponent(document.getElementById(param).options[document.getElementById(param).selectedIndex].value);
+    for (key in params){
-            if (param_value){
+        url += add_param(key, params[key]);
-              url+="&"+param+"="+param_value
-            }
-        }
     }
 
     var response =  $.ajax({type: "GET", url: url , async: false}).responseText;
-  rawcontainer.innerHTML = replaceURLWithHTMLLinks(response)
+    rawcontainer.innerHTML = replaceURLWithHTMLLinks(response);
 
     document.getElementById("input_request").innerHTML = "<a href='"+url+"'>"+url+"</a>"
     document.getElementById("results-div").style.display = 'block';
@@ -183,12 +279,13 @@ function load_JSON(){
         };
         var editor = new JSONEditor(container, options, response);
         editor.expandAll();
+        // $('#results-div a[href="#viewer"]').tab('show');
+        $('#results-div a[href="#viewer"]').click();
+        // location.hash = 'raw';
     }
     catch(err){
         console.log("Error decoding JSON (got turtle?)");
+        $('#results-div a[href="#raw"]').click();
+        // location.hash = 'raw';
     }
-      
-
 }
-
-

senpy/templates/index.html

@@ -88,28 +88,52 @@
 
         <div class="tab-pane active" id="test">
           <div class="well">
-            <form id="form" onsubmit="return getPlugins();" accept-charset="utf-8">
+            <form id="form" class="container" onsubmit="return getPlugins();" accept-charset="utf-8">
-              <div id="inputswrapper">
                 <div><textarea id="input" class="boxsizingBorder" rows="5" name="i">This text makes me sad.
 whilst this text makes me happy and surprised at the same time.
-I cannot believe it!</textarea></div>
+I cannot believe it!</textarea>
-                <label>Select the plugin:</label>
-                <select id="plugins" name="plugins" onchange="change_params()">
-                </select>
-                        </br>
-                        <div id ="params">
                 </div>
-                </br>
+                <div>
+                    <label>Select the plugin:</label>
+                    <select id="plugins" name="plugins" onchange="draw_extra_parameters()">
+                    </select>
+                </div>
+                <!-- PARAMETERS -->
+                <div class="panel-group" id="parameters">
+                    <div class="panel panel-default">
+                        <a data-toggle="collapse" class="deco-none" href="#basic_params">
+                            <div class="panel-heading">
+                                <h4 class="panel-title">
+                                    Basic API parameters
+                                </h4>
+                            </div>
+                        </a>
+                        <div id="basic_params" class="panel-collapse collapse panel-body">
+                        </div>
+                    </div>
+                    <div class="panel panel-default">
+                        <a data-toggle="collapse" class="deco-none" href="#extra_params">
+                            <div class="panel-heading">
+                                <h4 class="panel-title">
+                                    Plugin extra parameters
+                                </h4>
+                            </div>
+                        </a>
+                        <div id="extra_params" class="panel-collapse collapse in panel-body">
+                        </div>
+                    </div>
+                </div> 
+                <!-- END PARAMETERS -->
+ 
                 <a id="preview" class="btn btn-lg btn-primary" onclick="load_JSON()">Analyse!</a>
                 <!--<button id="visualise" name="type" type="button">Visualise!</button>-->
-              </div>
             </form>
           </div>
             <span id="input_request"></span>
             <div id="results-div">
               <ul class="nav nav-tabs" role="tablist">
-                <li role="presentation" class="active"><a class="active" href="#viewer">Viewer</a></li>
+                <li role="presentation" class="active"><a data-toggle="tab" class="active" href="#viewer">Viewer</a></li>
-                <li role="presentation"><a class="active" href="#raw">Raw</a></li>
+                <li role="presentation"><a data-toggle="tab" class="active" href="#raw">Raw</a></li>
               </ul>
               <div class="tab-content" id="results-container">
                 

senpy/test.py

@@ -0,0 +1,43 @@
+try:
+    from unittest.mock import patch, MagicMock
+except ImportError:
+    from mock import patch, MagicMock
+
+
+import json
+from contextlib import contextmanager
+
+from .models import BaseModel
+
+
+class Call(dict):
+    def __init__(self, obj):
+        self.obj = obj.serialize()
+        self.status_code = 200
+        self.content = self.json()
+
+    def json(self):
+        return json.loads(self.obj)
+
+
+@contextmanager
+def patch_requests(value, code=200):
+    success = MagicMock()
+    if isinstance(value, BaseModel):
+        value = value.jsonld()
+    data = json.dumps(value)
+
+    success.json.return_value = value
+    success.data.return_value = data
+    success.status_code = code
+
+    if hasattr(value, 'jsonld'):
+        success.content = value.jsonld()
+    else:
+        success.content = json.dumps(value)
+    method_mocker = MagicMock()
+    method_mocker.return_value = success
+    with patch.multiple('requests', request=method_mocker,
+                        get=method_mocker, post=method_mocker):
+        yield method_mocker, success
+        assert method_mocker.called

senpy/utils.py

@@ -1,15 +1,17 @@
 from . import models
+from collections import MutableMapping
+
+# MutableMapping should be enough, but it causes problems with py2
+DICTCLASSES = (MutableMapping, dict, models.BaseModel)
 
 
 def check_template(indict, template):
-    if isinstance(template, dict) and isinstance(indict, dict):
+    if isinstance(template, DICTCLASSES) and isinstance(indict, DICTCLASSES):
         for k, v in template.items():
             if k not in indict:
                 return '{} not in {}'.format(k, indict)
             check_template(indict[k], v)
     elif isinstance(template, list) and isinstance(indict, list):
-        if len(indict) != len(template):
-            raise models.Error('Different size for {} and {}'.format(indict, template))
         for e in template:
             found = False
             for i in indict:
@@ -17,6 +19,7 @@ def check_template(indict, template):
                     check_template(i, e)
                     found = True
                 except models.Error as ex:
+                    # raise
                     continue
             if not found:
                 raise models.Error('{} not found in {}'.format(e, indict))

tests/test_blueprints.py

@@ -162,5 +162,5 @@ class BlueprintsTest(TestCase):
         resp = self.client.get("/api/?help=true")
         self.assertCode(resp, 200)
         js = parse_resp(resp)
-        assert "parameters" in js
+        assert "valid_parameters" in js
-        assert "help" in js["parameters"]
+        assert "help" in js["valid_parameters"]

tests/test_client.py

@@ -1,24 +1,9 @@
 from unittest import TestCase
-try:
-    from unittest.mock import patch
-except ImportError:
-    from mock import patch
-
-import json
 
+from senpy.test import patch_requests
 from senpy.client import Client
 from senpy.models import Results, Plugins, Error
-from senpy.plugins import AnalysisPlugin, default_plugin_type
+from senpy.plugins import AnalysisPlugin
-
-
-class Call(dict):
-    def __init__(self, obj):
-        self.obj = obj.serialize()
-        self.status_code = 200
-        self.content = self.json()
-
-    def json(self):
-        return json.loads(self.obj)
 
 
 class ModelsTest(TestCase):
@@ -29,20 +14,18 @@ class ModelsTest(TestCase):
     def test_client(self):
         endpoint = 'http://dummy/'
         client = Client(endpoint)
-        success = Call(Results())
+        with patch_requests(Results()) as (request, response):
-        with patch('requests.request', return_value=success) as patched:
             resp = client.analyse('hello')
             assert isinstance(resp, Results)
-        patched.assert_called_with(
+        request.assert_called_with(
             url=endpoint + '/', method='GET', params={'input': 'hello'})
-        error = Call(Error('Nothing'))
+        with patch_requests(Error('Nothing')) as (request, response):
-        with patch('requests.request', return_value=error) as patched:
             try:
                 client.analyse(input='hello', algorithm='NONEXISTENT')
                 raise Exception('Exceptions should be raised. This is not golang')
             except Error:
                 pass
-        patched.assert_called_with(
+        request.assert_called_with(
             url=endpoint + '/',
             method='GET',
             params={'input': 'hello',
@@ -54,12 +37,11 @@ class ModelsTest(TestCase):
         plugins = Plugins()
         p1 = AnalysisPlugin({'name': 'AnalysisP1', 'version': 0, 'description': 'No'})
         plugins.plugins = [p1, ]
-        success = Call(plugins)
+        with patch_requests(plugins) as (request, response):
-        with patch('requests.request', return_value=success) as patched:
             response = client.plugins()
             assert isinstance(response, dict)
             assert len(response) == 1
             assert "AnalysisP1" in response
-        patched.assert_called_with(
+        request.assert_called_with(
             url=endpoint + '/plugins', method='GET',
-            params={'plugin_type': default_plugin_type})
+            params={})

tests/test_extensions.py

@@ -106,7 +106,7 @@ class ExtensionsTest(TestCase):
         r2 = analyse(self.senpy, input="tupni", output="tuptuo")
         assert r1.analysis[0] == "plugins/Dummy_0.1"
         assert r2.analysis[0] == "plugins/Dummy_0.1"
-        assert r1.entries[0]['nif:iString'] == 'input'
+        assert r1.entries[0]['nif:isString'] == 'input'
 
     def test_analyse_jsonld(self):
         """ Using a plugin with JSON-LD input"""
@@ -130,7 +130,7 @@ class ExtensionsTest(TestCase):
                      output="tuptuo")
         assert r1.analysis[0] == "plugins/Dummy_0.1"
         assert r2.analysis[0] == "plugins/Dummy_0.1"
-        assert r1.entries[0]['nif:iString'] == 'input'
+        assert r1.entries[0]['nif:isString'] == 'input'
 
     def test_analyse_error(self):
         mm = mock.MagicMock()
@@ -185,7 +185,7 @@ class ExtensionsTest(TestCase):
         response = Results({
             'analysis': [{'plugin': plugin}],
             'entries': [Entry({
-                'nif:iString': 'much ado about nothing',
+                'nif:isString': 'much ado about nothing',
                 'emotions': [eSet1]
             })]
         })

tests/test_models.py

@@ -12,8 +12,8 @@ from senpy.models import (Emotion,
                           Error,
                           Results,
                           Sentiment,
+                          SentimentPlugin,
                           Plugins,
-                          Plugin,
                           from_string,
                           from_dict)
 from senpy import plugins
@@ -99,7 +99,7 @@ class ModelsTest(TestCase):
 
     def test_plugins(self):
         self.assertRaises(Error, plugins.Plugin)
-        p = plugins.Plugin({"name": "dummy",
+        p = plugins.SentimentPlugin({"name": "dummy",
                                      "description": "I do nothing",
                                      "version": 0,
                                      "extra_params": {
@@ -111,7 +111,7 @@ class ModelsTest(TestCase):
                                      }})
         c = p.jsonld()
         assert '@type' in c
-        assert c['@type'] == 'plugin'
+        assert c['@type'] == 'sentimentPlugin'
         assert 'info' not in c
         assert 'repo' not in c
         assert 'extra_params' in c
@@ -173,13 +173,14 @@ class ModelsTest(TestCase):
     def test_single_plugin(self):
         """A response with a single plugin should still return a list"""
         plugs = Plugins()
-        p = Plugin({'id': str(1),
+        p = SentimentPlugin({'id': str(1),
                              'version': 0,
                              'description': 'dummy'})
         plugs.plugins.append(p)
         assert isinstance(plugs.plugins, list)
         js = plugs.jsonld()
         assert isinstance(js['plugins'], list)
+        assert js['plugins'][0]['@type'] == 'sentimentPlugin'
 
     def test_from_string(self):
         results = {
@@ -192,6 +193,7 @@ class ModelsTest(TestCase):
             }]
         }
         recovered = from_dict(results)
+        assert recovered.id == results['@id']
         assert isinstance(recovered, Results)
         assert isinstance(recovered.entries[0], Entry)
 

tests/test_plugins.py

@@ -6,7 +6,7 @@ import shutil
 import tempfile
 
 from unittest import TestCase
-from senpy.models import Results, Entry, EmotionSet, Emotion
+from senpy.models import Results, Entry, EmotionSet, Emotion, Plugins
 from senpy import plugins
 from senpy.plugins.conversion.emotion.centroids import CentroidConversion
 
@@ -49,6 +49,25 @@ class PluginsTest(TestCase):
         assert os.path.isfile(a.shelf_file)
         os.remove(a.shelf_file)
 
+    def test_plugin_filter(self):
+        ps = Plugins()
+        for i in (plugins.SentimentPlugin,
+                  plugins.EmotionPlugin,
+                  plugins.AnalysisPlugin):
+            p = i(name='Plugin_{}'.format(i.__name__),
+                  description='TEST',
+                  version=0,
+                  author='NOBODY')
+            ps.plugins.append(p)
+        assert len(ps.plugins) == 3
+        cases = [('AnalysisPlugin', 3),
+                 ('SentimentPlugin', 1),
+                 ('EmotionPlugin', 1)]
+
+        for name, num in cases:
+            res = plugins.pfilter(ps.plugins, plugin_type=name)
+            assert len(res) == num
+
     def test_shelf(self):
         ''' A shelf is created and the value is stored '''
         newfile = self.shelf_file + "new"