From bfc588a9155658ac50a57b40be85b017f5b4f629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Mon, 1 Jan 2018 13:13:17 +0100 Subject: [PATCH] Several fixes * Refactored BaseModel for efficiency * Added plugin metaclass to keep track of plugin types * Moved plugins to examples dir (in a previous commit) * Simplified validation in parse_params * Added convenience methods to mock requests in tests * Changed help schema to use `.valid_parameters` instead of `.parameters`, which was used in results to show parameters provided by the user. * Improved UI * Added basic parameters * Fixed bugs in parameter handling * Refactored and cleaned code --- example-plugins/async_plugin/asyncplugin.py | 26 ++ .../async_plugin/asyncplugin.senpy | 8 + example-plugins/dummy_plugin/dummy.py | 11 + example-plugins/dummy_plugin/dummy.senpy | 15 + example-plugins/dummy_plugin/dummy_noinfo.py | 27 ++ .../dummy_plugin/dummy_noinfo.senpy | 2 + .../dummy_plugin/dummy_required.senpy | 14 + example-plugins/noop/noop_plugin.py | 5 + example-plugins/sleep_plugin/sleep.py | 14 + example-plugins/sleep_plugin/sleep.senpy | 16 + requirements.txt | 1 - senpy/__init__.py | 3 + senpy/__main__.py | 2 +- senpy/api.py | 62 ++-- senpy/blueprints.py | 47 ++- senpy/client.py | 5 +- senpy/extensions.py | 27 +- senpy/models.py | 344 ++++++++++-------- senpy/plugins/__init__.py | 39 +- senpy/plugins/example/emoRand/emoRand.py | 2 +- senpy/plugins/misc/split.py | 2 +- .../sentiment/sentiment140/sentiment140.py | 14 +- .../sentiment/sentiment140/sentiment140.senpy | 1 + senpy/schemas/help.json | 6 +- senpy/schemas/plugin.json | 16 +- senpy/static/css/main.css | 15 + senpy/static/js/main.js | 327 +++++++++++------ senpy/templates/index.html | 50 ++- senpy/test.py | 43 +++ senpy/utils.py | 9 +- tests/test_blueprints.py | 4 +- tests/test_client.py | 36 +- tests/test_extensions.py | 6 +- tests/test_models.py | 32 +- tests/test_plugins.py | 21 +- 35 files changed, 826 insertions(+), 426 deletions(-) create mode 100644 example-plugins/async_plugin/asyncplugin.py create mode 100644 example-plugins/async_plugin/asyncplugin.senpy create mode 100644 example-plugins/dummy_plugin/dummy.py create mode 100644 example-plugins/dummy_plugin/dummy.senpy create mode 100644 example-plugins/dummy_plugin/dummy_noinfo.py create mode 100644 example-plugins/dummy_plugin/dummy_noinfo.senpy create mode 100644 example-plugins/dummy_plugin/dummy_required.senpy create mode 100644 example-plugins/noop/noop_plugin.py create mode 100644 example-plugins/sleep_plugin/sleep.py create mode 100644 example-plugins/sleep_plugin/sleep.senpy create mode 100644 senpy/test.py diff --git a/example-plugins/async_plugin/asyncplugin.py b/example-plugins/async_plugin/asyncplugin.py new file mode 100644 index 0000000..a37f2cb --- /dev/null +++ b/example-plugins/async_plugin/asyncplugin.py @@ -0,0 +1,26 @@ +from senpy.plugins import AnalysisPlugin + +import multiprocessing + + +def _train(process_number): + return process_number + + +class AsyncPlugin(AnalysisPlugin): + def _do_async(self, num_processes): + pool = multiprocessing.Pool(processes=num_processes) + values = pool.map(_train, range(num_processes)) + + return values + + def activate(self): + self.value = self._do_async(4) + + def analyse_entry(self, entry, params): + values = self._do_async(2) + entry.async_values = values + yield entry + + def test(self): + pass diff --git a/example-plugins/async_plugin/asyncplugin.senpy b/example-plugins/async_plugin/asyncplugin.senpy new file mode 100644 index 0000000..8c71849 --- /dev/null +++ b/example-plugins/async_plugin/asyncplugin.senpy @@ -0,0 +1,8 @@ +--- +name: Async +module: asyncplugin +description: I am async +author: "@balkian" +version: '0.1' +async: true +extra_params: {} \ No newline at end of file diff --git a/example-plugins/dummy_plugin/dummy.py b/example-plugins/dummy_plugin/dummy.py new file mode 100644 index 0000000..8dd987f --- /dev/null +++ b/example-plugins/dummy_plugin/dummy.py @@ -0,0 +1,11 @@ +from senpy.plugins import SentimentPlugin + + +class DummyPlugin(SentimentPlugin): + def analyse_entry(self, entry, params): + entry['nif:isString'] = entry['nif:isString'][::-1] + entry.reversed = entry.get('reversed', 0) + 1 + yield entry + + def test(self): + pass diff --git a/example-plugins/dummy_plugin/dummy.senpy b/example-plugins/dummy_plugin/dummy.senpy new file mode 100644 index 0000000..ea0c405 --- /dev/null +++ b/example-plugins/dummy_plugin/dummy.senpy @@ -0,0 +1,15 @@ +{ + "name": "Dummy", + "module": "dummy", + "description": "I am dummy", + "author": "@balkian", + "version": "0.1", + "extra_params": { + "example": { + "@id": "example_parameter", + "aliases": ["example", "ex"], + "required": false, + "default": 0 + } + } +} diff --git a/example-plugins/dummy_plugin/dummy_noinfo.py b/example-plugins/dummy_plugin/dummy_noinfo.py new file mode 100644 index 0000000..0a653e2 --- /dev/null +++ b/example-plugins/dummy_plugin/dummy_noinfo.py @@ -0,0 +1,27 @@ +from senpy.plugins import SentimentPlugin + + +class DummyPlugin(SentimentPlugin): + + description = 'This is a dummy self-contained plugin' + author = '@balkian' + version = '0.1' + + def analyse_entry(self, entry, params): + entry['nif:isString'] = entry['nif:isString'][::-1] + entry.reversed = entry.get('reversed', 0) + 1 + yield entry + + test_cases = [{ + "entry": { + "nif:isString": "Hello world!" + }, + "expected": [{ + "nif:isString": "!dlrow olleH" + }] + }] + + +if __name__ == '__main__': + d = DummyPlugin() + d.test() diff --git a/example-plugins/dummy_plugin/dummy_noinfo.senpy b/example-plugins/dummy_plugin/dummy_noinfo.senpy new file mode 100644 index 0000000..da4e83e --- /dev/null +++ b/example-plugins/dummy_plugin/dummy_noinfo.senpy @@ -0,0 +1,2 @@ +name: DummyNoInfo +module: dummy_noinfo diff --git a/example-plugins/dummy_plugin/dummy_required.senpy b/example-plugins/dummy_plugin/dummy_required.senpy new file mode 100644 index 0000000..3e361f6 --- /dev/null +++ b/example-plugins/dummy_plugin/dummy_required.senpy @@ -0,0 +1,14 @@ +{ + "name": "DummyRequired", + "module": "dummy", + "description": "I am dummy", + "author": "@balkian", + "version": "0.1", + "extra_params": { + "example": { + "@id": "example_parameter", + "aliases": ["example", "ex"], + "required": true + } + } +} diff --git a/example-plugins/noop/noop_plugin.py b/example-plugins/noop/noop_plugin.py new file mode 100644 index 0000000..ba851b5 --- /dev/null +++ b/example-plugins/noop/noop_plugin.py @@ -0,0 +1,5 @@ +from senpy.plugins import SentimentPlugin + + +class DummyPlugin(SentimentPlugin): + import noop diff --git a/example-plugins/sleep_plugin/sleep.py b/example-plugins/sleep_plugin/sleep.py new file mode 100644 index 0000000..770dd3b --- /dev/null +++ b/example-plugins/sleep_plugin/sleep.py @@ -0,0 +1,14 @@ +from senpy.plugins import AnalysisPlugin +from time import sleep + + +class SleepPlugin(AnalysisPlugin): + def activate(self, *args, **kwargs): + sleep(self.timeout) + + def analyse_entry(self, entry, params): + sleep(float(params.get("timeout", self.timeout))) + yield entry + + def test(self): + pass diff --git a/example-plugins/sleep_plugin/sleep.senpy b/example-plugins/sleep_plugin/sleep.senpy new file mode 100644 index 0000000..166f234 --- /dev/null +++ b/example-plugins/sleep_plugin/sleep.senpy @@ -0,0 +1,16 @@ +{ + "name": "Sleep", + "module": "sleep", + "description": "I am dummy", + "author": "@balkian", + "version": "0.1", + "timeout": 0.05, + "extra_params": { + "timeout": { + "@id": "timeout_sleep", + "aliases": ["timeout", "to"], + "required": false, + "default": 0 + } + } +} diff --git a/requirements.txt b/requirements.txt index d145317..80cf572 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ requests>=2.4.1 tornado>=4.4.3 PyLD>=0.6.5 nltk -six future jsonschema jsonref diff --git a/senpy/__init__.py b/senpy/__init__.py index 4c757cc..49ea183 100644 --- a/senpy/__init__.py +++ b/senpy/__init__.py @@ -19,6 +19,9 @@ Sentiment analysis server in Python """ from .version import __version__ +from future.standard_library import install_aliases +install_aliases() + import logging logger = logging.getLogger(__name__) diff --git a/senpy/__main__.py b/senpy/__main__.py index 51998c9..4c05d1a 100644 --- a/senpy/__main__.py +++ b/senpy/__main__.py @@ -67,7 +67,7 @@ def main(): '--plugins-folder', '-f', type=str, - default='plugins', + default='.', help='Where to look for plugins.') parser.add_argument( '--only-install', diff --git a/senpy/api.py b/senpy/api.py index c21d3d2..dee9856 100644 --- a/senpy/api.py +++ b/senpy/api.py @@ -13,8 +13,9 @@ API_PARAMS = { "expanded-jsonld": { "@id": "expanded-jsonld", "aliases": ["expanded"], + "options": "boolean", "required": True, - "default": 0 + "default": False }, "with_parameters": { "aliases": ['withparameters', @@ -23,13 +24,6 @@ API_PARAMS = { "default": False, "required": True }, - "plugin_type": { - "@id": "pluginType", - "description": 'What kind of plugins to list', - "aliases": ["pluginType"], - "required": True, - "default": "analysisPlugin" - }, "outformat": { "@id": "outformat", "aliases": ["o"], @@ -59,6 +53,16 @@ API_PARAMS = { } } +PLUGINS_PARAMS = { + "plugin_type": { + "@id": "pluginType", + "description": 'What kind of plugins to list', + "aliases": ["pluginType"], + "required": True, + "default": 'analysisPlugin' + } +} + WEB_PARAMS = { "inHeaders": { "aliases": ["headers"], @@ -126,24 +130,26 @@ def parse_params(indict, *specs): wrong_params = {} for spec in specs: for param, options in iteritems(spec): - if param[0] != "@": # Exclude json-ld properties - for alias in options.get("aliases", []): - # Replace each alias with the correct name of the parameter - if alias in indict and alias is not param: - outdict[param] = indict[alias] - del indict[alias] - continue - if param not in outdict: - if options.get("required", False) and "default" not in options: - wrong_params[param] = spec[param] - else: - if "default" in options: - outdict[param] = options["default"] - elif "options" in spec[param]: - if spec[param]["options"] == "boolean": - outdict[param] = outdict[param] in [None, True, 'true', '1'] - elif outdict[param] not in spec[param]["options"]: - wrong_params[param] = spec[param] + if param[0] == "@": # Exclude json-ld properties + continue + for alias in options.get("aliases", []): + # Replace each alias with the correct name of the parameter + if alias in indict and alias is not param: + outdict[param] = indict[alias] + del indict[alias] + continue + if param not in outdict: + if "default" in options: + # We assume the default is correct + outdict[param] = options["default"] + elif options.get("required", False): + wrong_params[param] = spec[param] + continue + if "options" in options: + if options["options"] == "boolean": + outdict[param] = outdict[param] in [None, True, 'true', '1'] + elif outdict[param] not in options["options"]: + wrong_params[param] = spec[param] if wrong_params: logger.debug("Error parsing: %s", wrong_params) message = Error( @@ -158,7 +164,7 @@ def parse_params(indict, *specs): return outdict -def get_extra_params(request, plugin=None): +def parse_extra_params(request, plugin=None): params = request.parameters.copy() if plugin: extra_params = parse_params(params, plugin.get('extra_params', {})) @@ -177,6 +183,6 @@ def parse_call(params): elif params['informat'] == 'json-ld': results = from_string(params['input'], cls=Results) else: - raise NotImplemented('Informat {} is not implemented'.format(params['informat'])) + raise NotImplementedError('Informat {} is not implemented'.format(params['informat'])) results.parameters = params return results diff --git a/senpy/blueprints.py b/senpy/blueprints.py index 024af1a..7943e16 100644 --- a/senpy/blueprints.py +++ b/senpy/blueprints.py @@ -25,6 +25,7 @@ from .version import __version__ from functools import wraps import logging +import traceback import json logger = logging.getLogger(__name__) @@ -72,12 +73,19 @@ def schema(schema="definitions"): def basic_api(f): + default_params = { + 'inHeaders': False, + 'expanded-jsonld': False, + 'outformat': 'json-ld', + 'with_parameters': True, + } + @wraps(f) def decorated_function(*args, **kwargs): raw_params = get_params(request) headers = {'X-ORIGINAL-PARAMS': json.dumps(raw_params)} + params = default_params - outformat = 'json-ld' try: print('Getting request:') print(request) @@ -87,26 +95,32 @@ def basic_api(f): else: request.parameters = params response = f(*args, **kwargs) - except Error as ex: - response = ex - response.parameters = params - logger.error(ex) + except (Exception) as ex: if current_app.debug: raise + if not isinstance(ex, Error): + msg = "{}:\n\t{}".format(ex, + traceback.format_exc()) + ex = Error(message=msg, status=500) + logger.exception('Error returning analysis result') + response = ex + response.parameters = raw_params + logger.error(ex) - in_headers = params['inHeaders'] - expanded = params['expanded-jsonld'] - outformat = params['outformat'] + if 'parameters' in response and not params['with_parameters']: + print(response) + print(response.data) + del response.parameters return response.flask( - in_headers=in_headers, + in_headers=params['inHeaders'], headers=headers, prefix=url_for('.api_root', _external=True), context_uri=url_for('api.context', entity=type(response).__name__, _external=True), - outformat=outformat, - expanded=expanded) + outformat=params['outformat'], + expanded=params['expanded-jsonld']) return decorated_function @@ -116,19 +130,18 @@ def basic_api(f): def api_root(): if request.parameters['help']: dic = dict(api.API_PARAMS, **api.NIF_PARAMS) - response = Help(parameters=dic) - return response - else: - req = api.parse_call(request.parameters) - response = current_app.senpy.analyse(req) + response = Help(valid_parameters=dic) return response + req = api.parse_call(request.parameters) + return current_app.senpy.analyse(req) @api_blueprint.route('/plugins/', methods=['POST', 'GET']) @basic_api def plugins(): sp = current_app.senpy - ptype = request.parameters.get('plugin_type') + params = api.parse_params(request.parameters, api.PLUGINS_PARAMS) + ptype = params.get('plugin_type') plugins = sp.filter_plugins(plugin_type=ptype) dic = Plugins(plugins=list(plugins.values())) return dic diff --git a/senpy/client.py b/senpy/client.py index 48c1238..ae1e375 100644 --- a/senpy/client.py +++ b/senpy/client.py @@ -1,7 +1,6 @@ import requests import logging from . import models -from .plugins import default_plugin_type logger = logging.getLogger(__name__) @@ -13,8 +12,8 @@ class Client(object): def analyse(self, input, method='GET', **kwargs): return self.request('/', method=method, input=input, **kwargs) - def plugins(self, ptype=default_plugin_type): - resp = self.request(path='/plugins', plugin_type=ptype).plugins + def plugins(self, *args, **kwargs): + resp = self.request(path='/plugins').plugins return {p.name: p for p in resp} def request(self, path=None, method='GET', **params): diff --git a/senpy/extensions.py b/senpy/extensions.py index fc9068c..61a0c3a 100644 --- a/senpy/extensions.py +++ b/senpy/extensions.py @@ -123,7 +123,7 @@ class Senpy(object): return plugin = plugins[0] self._activate(plugin) # Make sure the plugin is activated - specific_params = api.get_extra_params(req, plugin) + specific_params = api.parse_extra_params(req, plugin) req.analysis.append({'plugin': plugin, 'parameters': specific_params}) results = plugin.analyse_entries(entries, specific_params) @@ -262,17 +262,11 @@ class Senpy(object): with plugin._lock: if plugin.is_activated: return - try: - plugin.activate() - msg = "Plugin activated: {}".format(plugin.name) - logger.info(msg) - success = True - self._set_active(plugin, success) - except Exception as ex: - msg = "Error activating plugin {} - {} : \n\t{}".format( - plugin.name, ex, traceback.format_exc()) - logger.error(msg) - raise Error(msg) + plugin.activate() + msg = "Plugin activated: {}".format(plugin.name) + logger.info(msg) + success = True + self._set_active(plugin, success) def activate_plugin(self, plugin_name, sync=True): try: @@ -294,13 +288,8 @@ class Senpy(object): with plugin._lock: if not plugin.is_activated: return - try: - plugin.deactivate() - logger.info("Plugin deactivated: {}".format(plugin.name)) - except Exception as ex: - logger.error( - "Error deactivating plugin {}: {}".format(plugin.name, ex)) - logger.error("Trace: {}".format(traceback.format_exc())) + plugin.deactivate() + logger.info("Plugin deactivated: {}".format(plugin.name)) def deactivate_plugin(self, plugin_name, sync=True): try: diff --git a/senpy/models.py b/senpy/models.py index 884493d..44f4d45 100644 --- a/senpy/models.py +++ b/senpy/models.py @@ -6,7 +6,11 @@ For compatibility with Py3 and for easier debugging, this new version drops introspection and adds all arguments to the models. ''' from __future__ import print_function -from six import string_types +from future import standard_library +standard_library.install_aliases() + +from future.utils import with_metaclass +from past.builtins import basestring import time import copy @@ -15,6 +19,8 @@ import os import jsonref import jsonschema import inspect +from collections import UserDict +from abc import ABCMeta from flask import Response as FlaskResponse from pyld import jsonld @@ -62,7 +68,7 @@ class Context(dict): return contexts elif isinstance(context, dict): return Context(context) - elif isinstance(context, string_types): + elif isinstance(context, basestring): try: with open(context) as f: return Context(json.loads(f.read())) @@ -75,9 +81,154 @@ class Context(dict): base_context = Context.load(CONTEXT_PATH) -class SenpyMixin(object): +class BaseMeta(ABCMeta): + ''' + Metaclass for models. It extracts the default values for the fields in + the model. + + For instance, instances of the following class wouldn't need to mark + their version or description on initialization: + + .. code-block:: python + + class MyPlugin(Plugin): + version=0.3 + description='A dull plugin' + + + Note that these operations could be included in the __init__ of the + class, but it would be very inefficient. + ''' + def __new__(mcs, name, bases, attrs, **kwargs): + defaults = {} + if 'schema' in attrs: + defaults = mcs.get_defaults(attrs['schema']) + for b in bases: + if hasattr(b, 'defaults'): + defaults.update(b.defaults) + info = mcs.attrs_to_dict(attrs) + defaults.update(info) + attrs['defaults'] = defaults + return super(BaseMeta, mcs).__new__(mcs, name, bases, attrs) + + @staticmethod + def attrs_to_dict(attrs): + ''' + Extract the attributes of the class. + + This allows adding default values in the class definition. + e.g.: + ''' + def is_attr(k, v): + return (not(inspect.isroutine(v) or + inspect.ismethod(v) or + inspect.ismodule(v) or + isinstance(v, property)) and + k[0] != '_' and + k != 'schema' and + k != 'data') + + return {key: copy.deepcopy(value) for key, value in attrs.items() if is_attr(key, value)} + + @staticmethod + def get_defaults(schema): + temp = {} + for obj in [ + schema, + ] + schema.get('allOf', []): + for k, v in obj.get('properties', {}).items(): + if 'default' in v and k not in temp: + temp[k] = copy.deepcopy(v['default']) + return temp + + +class CustomDict(UserDict, object): + ''' + A dictionary whose elements can also be accessed as attributes. Since some + characters are not valid in the dot-notation, the attribute names also + converted. e.g.: + + > d = CustomDict() + > d.key = d['ns:name'] = 1 + > d.key == d['key'] + True + > d.ns__name == d['ns:name'] + ''' + + defaults = [] + + def __init__(self, *args, **kwargs): + temp = copy.deepcopy(self.defaults) + for arg in args: + temp.update(copy.deepcopy(arg)) + for k, v in kwargs.items(): + temp[self._get_key(k)] = v + + super(CustomDict, self).__init__(temp) + + @staticmethod + def _get_key(key): + if key is 'id': + key = '@id' + key = key.replace("__", ":", 1) + return key + + @staticmethod + def _internal_key(key): + return key[0] == '_' or key == 'data' + + def __getattr__(self, key): + ''' + __getattr__ only gets called when the attribute could not be found + in the __dict__. So we only need to look for the the element in the + dictionary, or raise an Exception. + ''' + mkey = self._get_key(key) + if not self._internal_key(key) and mkey in self: + return self[mkey] + raise AttributeError(key) + + def __setattr__(self, key, value): + # Work as usual for internal properties or already existing + # properties + if self._internal_key(key) or key in self.__dict__: + return super(CustomDict, self).__setattr__(key, value) + key = self._get_key(key) + return self.__setitem__(self._get_key(key), value) + + def __delattr__(self, key): + if self._internal_key(key): + return object.__delattr__(self, key) + key = self._get_key(key) + self.__delitem__(self._get_key(key)) + + +class BaseModel(with_metaclass(BaseMeta, CustomDict)): + ''' + Entities of the base model are a special kind of dictionary that emulates + a JSON-LD object. The structure of the dictionary is checked via JSON-schema. + For convenience, the values can also be accessed as attributes + (a la Javascript). e.g.: + + > myobject.key == myobject['key'] + True + > myobject.ns__name == myobject['ns:name'] + True + ''' + + schema = base_schema _context = base_context["@context"] + def __init__(self, *args, **kwargs): + auto_id = kwargs.pop('_auto_id', True) + super(BaseModel, self).__init__(*args, **kwargs) + + if '@id' not in self and auto_id: + self.id = ':{}_{}'.format(type(self).__name__, time.time()) + + if '@type' not in self: + logger.warn('Created an instance of an unknown model') + def flask(self, in_headers=True, headers=None, @@ -146,7 +297,7 @@ class SenpyMixin(object): else: return item - return ser_or_down(self._plain_dict()) + return ser_or_down(self.data) def jsonld(self, with_context=True, @@ -188,108 +339,41 @@ class SenpyMixin(object): return str(self.serialize()) -class BaseModel(SenpyMixin, dict): - ''' - Entities of the base model are a special kind of dictionary that emulates - a JSON-LD object. For convenience, the values can also be accessed as attributes - (a la Javascript). e.g.: - - > myobject.key == myobject['key'] - True - > myobject.ns__name == myobject['ns:name'] - True - ''' - - schema = base_schema - - def __init__(self, *args, **kwargs): - self.attrs_to_dict() - if 'id' in kwargs: - self.id = kwargs.pop('id') - elif kwargs.pop('_auto_id', True): - self.id = '_:{}_{}'.format(type(self).__name__, time.time()) - - temp = self.get_defaults() - temp.update(dict(*args)) - for k, v in kwargs.items(): - temp[self._get_key(k)] = v - super(BaseModel, self).__init__(temp) - - if '@type' not in self: - logger.warn('Created an instance of an unknown model') - - def get_defaults(self): - temp = {} - for obj in [ - self.schema, - ] + self.schema.get('allOf', []): - for k, v in obj.get('properties', {}).items(): - if 'default' in v and k not in temp: - temp[k] = copy.deepcopy(v['default']) - return temp - - def attrs_to_dict(self): - ''' - Copy the attributes of the class to the instance. - - This allows adding default values in the class definition. - e.g.: - - class MyPlugin(Plugin): - version=0.3 - description='A dull plugin' - ''' - def is_attr(x): - return not(inspect.isroutine(x) or inspect.ismethod(x) or isinstance(x, property)) - for key, value in inspect.getmembers(self.__class__, is_attr): - if key[0] != '_' and key != 'schema': - self[key] = value - - def _get_key(self, key): - if key is 'id': - key = '@id' - key = key.replace("__", ":", 1) - return key +_subtypes = {} - def __delitem__(self, key): - key = self._get_key(key) - dict.__delitem__(self, key) - def _internal_key(self, key): - return key[0] == '_' or key in self.__dict__ +def register(rsubclass, rtype=None): + _subtypes[rtype or rsubclass.__name__] = rsubclass - def _plain_dict(self): - d = {k: v for (k, v) in self.items() if k[0] != "_"} - return d - def __getattr__(self, key): - ''' - __getattr__ only gets called when the attribute could not - be found in the __dict__. So we only need to look for the - the element in the dictionary, or raise an Exception. - ''' - if self._internal_key(key): - raise AttributeError(key) - return self.__getitem__(self._get_key(key)) +def from_schema(name, schema=None, schema_file=None, base_classes=None): + base_classes = base_classes or [] + base_classes.append(BaseModel) + schema_file = schema_file or '{}.json'.format(name) + class_name = '{}{}'.format(name[0].upper(), name[1:]) + if '/' not in 'schema_file': + thisdir = os.path.dirname(os.path.realpath(__file__)) + schema_file = os.path.join(thisdir, + 'schemas', + schema_file) - def __setattr__(self, key, value): - if self._internal_key(key): - return super(BaseModel, self).__setattr__(key, value) - key = self._get_key(key) - return self.__setitem__(self._get_key(key), value) + schema_path = 'file://' + schema_file - def __delattr__(self, key): - if self._internal_key(key): - return object.__delattr__(self, key) - key = self._get_key(key) - self.__delitem__(self._get_key(key)) + with open(schema_file) as f: + schema = json.load(f) + dct = {} -def register(rsubclass, rtype=None): - _subtypes[rtype or rsubclass.__name__] = rsubclass + resolver = jsonschema.RefResolver(schema_path, schema) + dct['@type'] = name + dct['_schema_file'] = schema_file + dct['schema'] = schema + dct['_validator'] = jsonschema.Draft4Validator(schema, resolver=resolver) + newclass = type(class_name, tuple(base_classes), dct) -_subtypes = {} + register(newclass, name) + return newclass def from_dict(indict, cls=None): @@ -309,10 +393,11 @@ def from_dict(indict, cls=None): elif isinstance(v, dict): v = from_dict(indict[k]) elif isinstance(v, list): + v = v[:] for ix, v2 in enumerate(v): if isinstance(v2, dict): v[ix] = from_dict(v2) - outdict[k] = v + outdict[k] = copy.deepcopy(v) return cls(**outdict) @@ -325,35 +410,6 @@ def from_json(injson): return from_dict(indict) -def from_schema(name, schema=None, schema_file=None, base_classes=None): - base_classes = base_classes or [] - base_classes.append(BaseModel) - schema_file = schema_file or '{}.json'.format(name) - class_name = '{}{}'.format(name[0].upper(), name[1:]) - if '/' not in 'schema_file': - schema_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'schemas', - schema_file) - - schema_path = 'file://' + schema_file - - with open(schema_file) as f: - schema = json.load(f) - - dct = {} - - resolver = jsonschema.RefResolver(schema_path, schema) - dct['@type'] = name - dct['_schema_file'] = schema_file - dct['schema'] = schema - dct['_validator'] = jsonschema.Draft4Validator(schema, resolver=resolver) - - newclass = type(class_name, tuple(base_classes), dct) - - register(newclass, name) - return newclass - - def _add_from_schema(*args, **kwargs): generatedClass = from_schema(*args, **kwargs) globals()[generatedClass.__name__] = generatedClass @@ -384,40 +440,14 @@ for i in [ _ErrorModel = from_schema('error') -class Error(SenpyMixin, Exception): +class Error(_ErrorModel, Exception): def __init__(self, message, *args, **kwargs): - super(Error, self).__init__(self, message, message) - self._error = _ErrorModel(message=message, *args, **kwargs) + Exception.__init__(self, message) + super(Error, self).__init__(*args, **kwargs) self.message = message - def validate(self, obj=None): - self._error.validate() - - def __getitem__(self, key): - return self._error[key] - - def __setitem__(self, key, value): - self._error[key] = value - - def __delitem__(self, key): - del self._error[key] - - def __getattr__(self, key): - if key != '_error' and hasattr(self._error, key): - return getattr(self._error, key) - raise AttributeError(key) - - def __setattr__(self, key, value): - if key != '_error': - return setattr(self._error, key, value) - else: - super(Error, self).__setattr__(key, value) - - def __delattr__(self, key): - delattr(self._error, key) - - def __str__(self): - return str(self.to_JSON(with_context=False)) + def __hash__(self): + return Exception.__hash__(self) register(Error, 'error') diff --git a/senpy/plugins/__init__.py b/senpy/plugins/__init__.py index a695d53..a14d7ae 100644 --- a/senpy/plugins/__init__.py +++ b/senpy/plugins/__init__.py @@ -1,5 +1,6 @@ from future import standard_library standard_library.install_aliases() +from future.utils import with_metaclass import os.path import os @@ -16,21 +17,33 @@ import yaml import threading from .. import models, utils -from ..api import API_PARAMS +from .. import api logger = logging.getLogger(__name__) -class Plugin(models.Plugin): - def __init__(self, info=None, data_folder=None): +class PluginMeta(models.BaseMeta): + + def __new__(mcs, name, bases, attrs, **kwargs): + plugin_type = [] + if hasattr(bases[0], 'plugin_type'): + plugin_type += bases[0].plugin_type + plugin_type.append(name) + attrs['plugin_type'] = plugin_type + return super(PluginMeta, mcs).__new__(mcs, name, bases, attrs) + + +class Plugin(with_metaclass(PluginMeta, models.Plugin)): + + def __init__(self, info=None, data_folder=None, **kwargs): """ Provides a canonical name for plugins and serves as base for other kinds of plugins. """ logger.debug("Initialising {}".format(info)) + super(Plugin, self).__init__(**kwargs) if info: self.update(info) - super(Plugin, self).__init__(**self) if not self.validate(): raise models.Error(message=("You need to provide configuration" "information for the plugin.")) @@ -57,7 +70,8 @@ class Plugin(models.Plugin): 'test cases').format(self.id, inspect.getfile(self.__class__))) for case in self.test_cases: entry = models.Entry(case['entry']) - params = case.get('params', {}) + given_parameters = case.get('params', {}) + params = api.parse_params(given_parameters, self.extra_params) fails = case.get('fails', False) try: res = list(self.analyse_entry(entry, params)) @@ -90,7 +104,7 @@ SenpyPlugin = Plugin class AnalysisPlugin(Plugin): def analyse(self, *args, **kwargs): - raise NotImplemented( + raise NotImplementedError( 'Your method should implement either analyse or analyse_entry') def analyse_entry(self, entry, parameters): @@ -118,17 +132,17 @@ class ConversionPlugin(Plugin): pass -class SentimentPlugin(models.SentimentPlugin, AnalysisPlugin): +class SentimentPlugin(AnalysisPlugin, models.SentimentPlugin): minPolarityValue = 0 maxPolarityValue = 1 -class EmotionPlugin(models.EmotionPlugin, AnalysisPlugin): +class EmotionPlugin(AnalysisPlugin, models.EmotionPlugin): minEmotionValue = 0 maxEmotionValue = 1 -class EmotionConversionPlugin(models.EmotionConversionPlugin, ConversionPlugin): +class EmotionConversionPlugin(ConversionPlugin): pass @@ -171,19 +185,18 @@ class ShelfMixin(object): pickle.dump(self._sh, f) -default_plugin_type = API_PARAMS['plugin_type']['default'] - - def pfilter(plugins, **kwargs): """ Filter plugins by different criteria """ if isinstance(plugins, models.Plugins): plugins = plugins.plugins elif isinstance(plugins, dict): plugins = plugins.values() - ptype = kwargs.pop('plugin_type', default_plugin_type) + ptype = kwargs.pop('plugin_type', AnalysisPlugin) logger.debug('#' * 100) logger.debug('ptype {}'.format(ptype)) if ptype: + if isinstance(ptype, PluginMeta): + ptype = ptype.__name__ try: ptype = ptype[0].upper() + ptype[1:] pclass = globals()[ptype] diff --git a/senpy/plugins/example/emoRand/emoRand.py b/senpy/plugins/example/emoRand/emoRand.py index 9212353..63c2e56 100644 --- a/senpy/plugins/example/emoRand/emoRand.py +++ b/senpy/plugins/example/emoRand/emoRand.py @@ -4,7 +4,7 @@ from senpy.plugins import EmotionPlugin from senpy.models import EmotionSet, Emotion, Entry -class RmoRandPlugin(EmotionPlugin): +class EmoRandPlugin(EmotionPlugin): def analyse_entry(self, entry, params): category = "emoml:big6happiness" number = max(-1, min(1, random.gauss(0, 0.5))) diff --git a/senpy/plugins/misc/split.py b/senpy/plugins/misc/split.py index b444f34..edf0e40 100644 --- a/senpy/plugins/misc/split.py +++ b/senpy/plugins/misc/split.py @@ -11,7 +11,7 @@ class SplitPlugin(AnalysisPlugin): nltk.download('punkt') def analyse_entry(self, entry, params): - chunker_type = params.get("delimiter", "sentence") + chunker_type = params["delimiter"] original_text = entry['nif:isString'] if chunker_type == "sentence": tokenizer = PunktSentenceTokenizer() diff --git a/senpy/plugins/sentiment/sentiment140/sentiment140.py b/senpy/plugins/sentiment/sentiment140/sentiment140.py index 31ff8a8..6a8426b 100644 --- a/senpy/plugins/sentiment/sentiment140/sentiment140.py +++ b/senpy/plugins/sentiment/sentiment140/sentiment140.py @@ -7,7 +7,7 @@ from senpy.models import Sentiment class Sentiment140Plugin(SentimentPlugin): def analyse_entry(self, entry, params): - lang = params.get("language", "auto") + lang = params["language"] res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson", json.dumps({ "language": lang, @@ -35,6 +35,18 @@ class Sentiment140Plugin(SentimentPlugin): entry.language = lang yield entry + def test(self, *args, **kwargs): + ''' + To avoid calling the sentiment140 API, we will mock the results + from requests. + ''' + from senpy.test import patch_requests + expected = {"data": [{"polarity": 10}]} + with patch_requests(expected) as (request, response): + super(Sentiment140Plugin, self).test(*args, **kwargs) + assert request.called + assert response.json.called + test_cases = [ { 'entry': { diff --git a/senpy/plugins/sentiment/sentiment140/sentiment140.senpy b/senpy/plugins/sentiment/sentiment140/sentiment140.senpy index f2c92b3..2b38283 100644 --- a/senpy/plugins/sentiment/sentiment140/sentiment140.senpy +++ b/senpy/plugins/sentiment/sentiment140/sentiment140.senpy @@ -16,6 +16,7 @@ extra_params: - es - en - auto + default: auto requirements: {} maxPolarityValue: 1 minPolarityValue: 0 \ No newline at end of file diff --git a/senpy/schemas/help.json b/senpy/schemas/help.json index 10348bf..55ee3c3 100644 --- a/senpy/schemas/help.json +++ b/senpy/schemas/help.json @@ -7,11 +7,11 @@ "description": "Help containing accepted parameters", "type": "object", "properties": { - "parameters": { + "valid_parameters": { "type": "object" } }, - "required": "parameters" + "required": "valid_parameters" } ] -} \ No newline at end of file +} diff --git a/senpy/schemas/plugin.json b/senpy/schemas/plugin.json index a2cb04d..f15f3f3 100644 --- a/senpy/schemas/plugin.json +++ b/senpy/schemas/plugin.json @@ -1,7 +1,7 @@ { "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", - "required": ["@id", "extra_params"], + "required": ["@id", "name", "description", "version", "plugin_type"], "properties": { "@id": { "type": "string", @@ -9,7 +9,19 @@ }, "name": { "type": "string", - "description": "The name of the plugin, which will be used in the algorithm detection phase" + "description": "The name of the plugin, which will be used in the algorithm detection phase." + }, + "description": { + "type": "string", + "description": "A summary of what the plugin does, and pointers to further information." + }, + "version": { + "type": "string", + "description": "The version of the plugin." + }, + "plugin_type": { + "type": "string", + "description": "Sub-type of plugin. e.g. sentimentPlugin" }, "extra_params": { "type": "object", diff --git a/senpy/static/css/main.css b/senpy/static/css/main.css index 112be75..7ea452d 100644 --- a/senpy/static/css/main.css +++ b/senpy/static/css/main.css @@ -152,3 +152,18 @@ textarea{ /* background: white; */ display: none; } + +.deco-none { + color: inherit; + text-decoration: inherit; +} + +.deco-none:link { + color: inherit; + text-decoration: inherit; +} + +.deco-none:hover { + color: inherit; + text-decoration: inherit; +} diff --git a/senpy/static/js/main.js b/senpy/static/js/main.js index 05d009d..933cb6a 100644 --- a/senpy/static/js/main.js +++ b/senpy/static/js/main.js @@ -1,7 +1,10 @@ var ONYX = "http://www.gsi.dit.upm.es/ontologies/onyx/ns#"; var RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"; -var plugins_params={}; -var default_params = JSON.parse($.ajax({type: "GET", url: "/api?help=true" , async: false}).responseText); +var plugins_params = default_params = {}; +var plugins = []; +var defaultPlugin = {}; +var gplugins = {}; + function replaceURLWithHTMLLinks(text) { console.log('Text: ' + text); var exp = /(\b(https?|ftp|file):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])/ig; @@ -25,21 +28,45 @@ function hashchanged(){ } } -$(document).ready(function() { - var response = JSON.parse($.ajax({type: "GET", url: "/api/plugins/" , async: false}).responseText); - var defaultPlugin= JSON.parse($.ajax({type: "GET", url: "/api/plugins/default" , async: false}).responseText); - html=""; - var availablePlugins = document.getElementById('availablePlugins'); + +function get_plugins(response){ plugins = response.plugins; - gplugins = {}; +} + +function group_plugins(){ for (r in plugins){ ptype = plugins[r]['@type']; if(gplugins[ptype] == undefined){ - gplugins[ptype] = [r] + gplugins[ptype] = [r]; }else{ - gplugins[ptype].push(r) + gplugins[ptype].push(r); } - } + } +} + +function get_parameters(){ + for (p in plugins){ + plugin = plugins[p]; + if (plugin["extra_params"]){ + plugins_params[plugin["name"]]={}; + for (param in plugin["extra_params"]){ + if (typeof plugin["extra_params"][param] !="string"){ + var params = new Array(); + var alias = plugin["extra_params"][param]["aliases"][0]; + params[alias]=new Array(); + for (option in plugin["extra_params"][param]["options"]){ + params[alias].push(plugin["extra_params"][param]["options"][option]) + } + plugins_params[plugin["name"]][alias] = (params[alias]) + } + } + } + } +} + +function draw_plugins_selection(){ + html=""; + group_plugins(); for (g in gplugins){ html += "" for (r in gplugins[g]){ @@ -49,7 +76,7 @@ $(document).ready(function() { continue; } - html+= "" + document.getElementById('plugins').innerHTML = html; +} + +function draw_plugins_list(){ + var availablePlugins = document.getElementById('availablePlugins'); + + for(p in plugins){ var pluginEntry = document.createElement('li'); - + plugin = plugins[p]; newHtml = "" if(plugin.url) { newHtml= "" + plugin.name + ""; @@ -85,110 +107,185 @@ $(document).ready(function() { pluginEntry.innerHTML = newHtml; availablePlugins.appendChild(pluginEntry) } - html += "" - document.getElementById('plugins').innerHTML = html; - change_params(); - +} + +$(document).ready(function() { + var response = JSON.parse($.ajax({type: "GET", url: "/api/plugins/" , async: false}).responseText); + defaultPlugin= JSON.parse($.ajax({type: "GET", url: "/api/plugins/default" , async: false}).responseText); + get_plugins(response); + get_default_parameters(); + + draw_plugins_list(); + draw_plugins_selection(); + draw_parameters(); + $(window).on('hashchange', hashchanged); hashchanged(); $('.tooltip-form').tooltip(); }); +function get_default_parameters(){ + default_params = JSON.parse($.ajax({type: "GET", url: "/api?help=true" , async: false}).responseText).valid_parameters; + // Remove the parameters that are always added + delete default_params["input"]; + delete default_params["algorithm"]; + delete default_params["help"]; -function change_params(){ - var plugin = document.getElementById("plugins").options[document.getElementById("plugins").selectedIndex].value; - html="" - for (param in default_params){ - if ((default_params[param]['options']) && (['help','conversion'].indexOf(param) < 0)){ - html+= "" - if (default_params[param]['options'].length < 1) { - html +=""; - } - else { - html+= "
" - } - } - for (param in plugins_params[plugin]){ - if (param || plugins_params[plugin][param].length > 1){ - html+= "" - param_opts = plugins_params[plugin][param] - if (param_opts.length > 0) { - html+= "" - } - else { - html +=""; - } - } - } - document.getElementById("params").innerHTML = html -}; +} -function load_JSON(){ - url = "/api"; - var container = document.getElementById('results'); - var rawcontainer = document.getElementById("jsonraw"); - rawcontainer.innerHTML = ''; - container.innerHTML = ''; - var plugin = document.getElementById("plugins").options[document.getElementById("plugins").selectedIndex].value; - var input = encodeURIComponent(document.getElementById("input").value); - url += "?algo="+plugin+"&i="+input - for (param in plugins_params[plugin]){ - if (param != null){ - field = document.getElementById(param); - if (plugins_params[plugin][param].length > 0){ - var param_value = encodeURIComponent(field.options[field.selectedIndex].text); - } else { - var param_value = encodeURIComponent(field.text); +function draw_default_parameters(){ + var basic_params = document.getElementById("basic_params"); + basic_params.innerHTML = params_div(default_params); +} + +function draw_extra_parameters(){ + var plugin = document.getElementById("plugins").options[document.getElementById("plugins").selectedIndex].value; + get_parameters(); + + var extra_params = document.getElementById("extra_params"); + extra_params.innerHTML = params_div(plugins_params[plugin]); +} + +function draw_parameters(){ + draw_default_parameters(); + draw_extra_parameters(); +} + + +function add_default_params(){ + var html = ""; + // html += 'Basic API parameters'; + html += ''; + html += '