balkian/senpy
1
0
Fork 0
mirror of https://github.com/gsi-upm/senpy synced 2024-11-22 00:02:28 +00:00
Code Issues Releases Wiki Activity

Several fixes

Browse Source 
* Refactored BaseModel for efficiency
* Added plugin metaclass to keep track of plugin types
* Moved plugins to examples dir (in a previous commit)
* Simplified validation in parse_params
* Added convenience methods to mock requests in tests
* Changed help schema to use `.valid_parameters` instead of `.parameters`,
which was used in results to show parameters provided by the user.
* Improved UI
    * Added basic parameters
    * Fixed bugs in parameter handling
    * Refactored and cleaned code
This commit is contained in:
J. Fernando Sánchez 2018-01-01 13:13:17 +01:00
parent f93eed2cf5
commit bfc588a915
35 changed files with 845 additions and 445 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
example-plugins/async_plugin/asyncplugin.py Normal file
View File

@ -0,0 +1,26 @@
from senpy.plugins import AnalysisPlugin
import multiprocessing
def _train(process_number):
return process_number
class AsyncPlugin(AnalysisPlugin):
def _do_async(self, num_processes):
pool = multiprocessing.Pool(processes=num_processes)
values = pool.map(_train, range(num_processes))
return values
def activate(self):
self.value = self._do_async(4)
def analyse_entry(self, entry, params):
values = self._do_async(2)
entry.async_values = values
yield entry
def test(self):
pass

8
example-plugins/async_plugin/asyncplugin.senpy Normal file
View File

@ -0,0 +1,8 @@
---
name: Async
module: asyncplugin
description: I am async
author: "@balkian"
version: '0.1'
async: true
extra_params: {}

11
example-plugins/dummy_plugin/dummy.py Normal file
View File

@ -0,0 +1,11 @@
from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
def analyse_entry(self, entry, params):
entry['nif:isString'] = entry['nif:isString'][::-1]
entry.reversed = entry.get('reversed', 0) + 1
yield entry
def test(self):
pass

15
example-plugins/dummy_plugin/dummy.senpy Normal file
View File

@ -0,0 +1,15 @@
{
"name": "Dummy",
"module": "dummy",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"extra_params": {
"example": {
"@id": "example_parameter",
"aliases": ["example", "ex"],
"required": false,
"default": 0
}
}
}

27
example-plugins/dummy_plugin/dummy_noinfo.py Normal file
View File

@ -0,0 +1,27 @@
from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
description = 'This is a dummy self-contained plugin'
author = '@balkian'
version = '0.1'
def analyse_entry(self, entry, params):
entry['nif:isString'] = entry['nif:isString'][::-1]
entry.reversed = entry.get('reversed', 0) + 1
yield entry
test_cases = [{
"entry": {
"nif:isString": "Hello world!"
},
"expected": [{
"nif:isString": "!dlrow olleH"
}]
}]
if __name__ == '__main__':
d = DummyPlugin()
d.test()

2
example-plugins/dummy_plugin/dummy_noinfo.senpy Normal file
View File

@ -0,0 +1,2 @@
name: DummyNoInfo
module: dummy_noinfo

14
example-plugins/dummy_plugin/dummy_required.senpy Normal file
View File

@ -0,0 +1,14 @@
{
"name": "DummyRequired",
"module": "dummy",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"extra_params": {
"example": {
"@id": "example_parameter",
"aliases": ["example", "ex"],
"required": true
}
}
}

5
example-plugins/noop/noop_plugin.py Normal file
View File

@ -0,0 +1,5 @@
from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
import noop

14
example-plugins/sleep_plugin/sleep.py Normal file
View File

@ -0,0 +1,14 @@
from senpy.plugins import AnalysisPlugin
from time import sleep
class SleepPlugin(AnalysisPlugin):
def activate(self, *args, **kwargs):
sleep(self.timeout)
def analyse_entry(self, entry, params):
sleep(float(params.get("timeout", self.timeout)))
yield entry
def test(self):
pass

16
example-plugins/sleep_plugin/sleep.senpy Normal file
View File

@ -0,0 +1,16 @@
{
"name": "Sleep",
"module": "sleep",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"timeout": 0.05,
"extra_params": {
"timeout": {
"@id": "timeout_sleep",
"aliases": ["timeout", "to"],
"required": false,
"default": 0
}
}
}

1
requirements.txt
View File

@ -3,7 +3,6 @@ requests>=2.4.1
tornado>=4.4.3
PyLD>=0.6.5
nltk
six
future
jsonschema
jsonref

3
senpy/__init__.py
View File

@ -19,6 +19,9 @@ Sentiment analysis server in Python
"""
from .version import __version__
from future.standard_library import install_aliases
install_aliases()
import logging
logger = logging.getLogger(__name__)

2
senpy/__main__.py
View File

@ -67,7 +67,7 @@ def main():
'--plugins-folder',
'-f',
type=str,
default='plugins',
default='.',
help='Where to look for plugins.')
parser.add_argument(
'--only-install',

62
senpy/api.py
View File

@ -13,8 +13,9 @@ API_PARAMS = {
"expanded-jsonld": {
"@id": "expanded-jsonld",
"aliases": ["expanded"],
"options": "boolean",
"required": True,
"default": 0
"default": False
},
"with_parameters": {
"aliases": ['withparameters',
@ -23,13 +24,6 @@ API_PARAMS = {
"default": False,
"required": True
},
"plugin_type": {
"@id": "pluginType",
"description": 'What kind of plugins to list',
"aliases": ["pluginType"],
"required": True,
"default": "analysisPlugin"
},
"outformat": {
"@id": "outformat",
"aliases": ["o"],
@ -59,6 +53,16 @@ API_PARAMS = {
}
}
PLUGINS_PARAMS = {
"plugin_type": {
"@id": "pluginType",
"description": 'What kind of plugins to list',
"aliases": ["pluginType"],
"required": True,
"default": 'analysisPlugin'
}
}
WEB_PARAMS = {
"inHeaders": {
"aliases": ["headers"],
@ -126,24 +130,26 @@ def parse_params(indict, *specs):
wrong_params = {}
for spec in specs:
for param, options in iteritems(spec):
if param[0] != "@": # Exclude json-ld properties
for alias in options.get("aliases", []):
# Replace each alias with the correct name of the parameter
if alias in indict and alias is not param:
outdict[param] = indict[alias]
del indict[alias]
continue
if param not in outdict:
if options.get("required", False) and "default" not in options:
wrong_params[param] = spec[param]
else:
if "default" in options:
outdict[param] = options["default"]
elif "options" in spec[param]:
if spec[param]["options"] == "boolean":
outdict[param] = outdict[param] in [None, True, 'true', '1']
elif outdict[param] not in spec[param]["options"]:
wrong_params[param] = spec[param]
if param[0] == "@": # Exclude json-ld properties
continue
for alias in options.get("aliases", []):
# Replace each alias with the correct name of the parameter
if alias in indict and alias is not param:
outdict[param] = indict[alias]
del indict[alias]
continue
if param not in outdict:
if "default" in options:
# We assume the default is correct
outdict[param] = options["default"]
elif options.get("required", False):
wrong_params[param] = spec[param]
continue
if "options" in options:
if options["options"] == "boolean":
outdict[param] = outdict[param] in [None, True, 'true', '1']
elif outdict[param] not in options["options"]:
wrong_params[param] = spec[param]
if wrong_params:
logger.debug("Error parsing: %s", wrong_params)
message = Error(
@ -158,7 +164,7 @@ def parse_params(indict, *specs):
return outdict
def get_extra_params(request, plugin=None):
def parse_extra_params(request, plugin=None):
params = request.parameters.copy()
if plugin:
extra_params = parse_params(params, plugin.get('extra_params', {}))
@ -177,6 +183,6 @@ def parse_call(params):
elif params['informat'] == 'json-ld':
results = from_string(params['input'], cls=Results)
else:
raise NotImplemented('Informat {} is not implemented'.format(params['informat']))
raise NotImplementedError('Informat {} is not implemented'.format(params['informat']))
results.parameters = params
return results

47
senpy/blueprints.py
View File

@ -25,6 +25,7 @@ from .version import __version__
from functools import wraps
import logging
import traceback
import json
logger = logging.getLogger(__name__)
@ -72,12 +73,19 @@ def schema(schema="definitions"):
def basic_api(f):
default_params = {
'inHeaders': False,
'expanded-jsonld': False,
'outformat': 'json-ld',
'with_parameters': True,
}
@wraps(f)
def decorated_function(*args, **kwargs):
raw_params = get_params(request)
headers = {'X-ORIGINAL-PARAMS': json.dumps(raw_params)}
params = default_params
outformat = 'json-ld'
try:
print('Getting request:')
print(request)
@ -87,26 +95,32 @@ def basic_api(f):
else:
request.parameters = params
response = f(*args, **kwargs)
except Error as ex:
response = ex
response.parameters = params
logger.error(ex)
except (Exception) as ex:
if current_app.debug:
raise
if not isinstance(ex, Error):
msg = "{}:\n\t{}".format(ex,
traceback.format_exc())
ex = Error(message=msg, status=500)
logger.exception('Error returning analysis result')
response = ex
response.parameters = raw_params
logger.error(ex)
in_headers = params['inHeaders']
expanded = params['expanded-jsonld']
outformat = params['outformat']
if 'parameters' in response and not params['with_parameters']:
print(response)
print(response.data)
del response.parameters
return response.flask(
in_headers=in_headers,
in_headers=params['inHeaders'],
headers=headers,
prefix=url_for('.api_root', _external=True),
context_uri=url_for('api.context',
entity=type(response).__name__,
_external=True),
outformat=outformat,
expanded=expanded)
outformat=params['outformat'],
expanded=params['expanded-jsonld'])
return decorated_function
@ -116,19 +130,18 @@ def basic_api(f):
def api_root():
if request.parameters['help']:
dic = dict(api.API_PARAMS, **api.NIF_PARAMS)
response = Help(parameters=dic)
return response
else:
req = api.parse_call(request.parameters)
response = current_app.senpy.analyse(req)
response = Help(valid_parameters=dic)
return response
req = api.parse_call(request.parameters)
return current_app.senpy.analyse(req)
@api_blueprint.route('/plugins/', methods=['POST', 'GET'])
@basic_api
def plugins():
sp = current_app.senpy
ptype = request.parameters.get('plugin_type')
params = api.parse_params(request.parameters, api.PLUGINS_PARAMS)
ptype = params.get('plugin_type')
plugins = sp.filter_plugins(plugin_type=ptype)
dic = Plugins(plugins=list(plugins.values()))
return dic

5
senpy/client.py
View File

@ -1,7 +1,6 @@
import requests
import logging
from . import models
from .plugins import default_plugin_type
logger = logging.getLogger(__name__)
@ -13,8 +12,8 @@ class Client(object):
def analyse(self, input, method='GET', **kwargs):
return self.request('/', method=method, input=input, **kwargs)
def plugins(self, ptype=default_plugin_type):
resp = self.request(path='/plugins', plugin_type=ptype).plugins
def plugins(self, *args, **kwargs):
resp = self.request(path='/plugins').plugins
return {p.name: p for p in resp}
def request(self, path=None, method='GET', **params):

27
senpy/extensions.py
View File

@ -123,7 +123,7 @@ class Senpy(object):
return
plugin = plugins[0]
self._activate(plugin) # Make sure the plugin is activated
specific_params = api.get_extra_params(req, plugin)
specific_params = api.parse_extra_params(req, plugin)
req.analysis.append({'plugin': plugin,
'parameters': specific_params})
results = plugin.analyse_entries(entries, specific_params)
@ -262,17 +262,11 @@ class Senpy(object):
with plugin._lock:
if plugin.is_activated:
return
try:
plugin.activate()
msg = "Plugin activated: {}".format(plugin.name)
logger.info(msg)
success = True
self._set_active(plugin, success)
except Exception as ex:
msg = "Error activating plugin {} - {} : \n\t{}".format(
plugin.name, ex, traceback.format_exc())
logger.error(msg)
raise Error(msg)
plugin.activate()
msg = "Plugin activated: {}".format(plugin.name)
logger.info(msg)
success = True
self._set_active(plugin, success)
def activate_plugin(self, plugin_name, sync=True):
try:
@ -294,13 +288,8 @@ class Senpy(object):
with plugin._lock:
if not plugin.is_activated:
return
try:
plugin.deactivate()
logger.info("Plugin deactivated: {}".format(plugin.name))
except Exception as ex:
logger.error(
"Error deactivating plugin {}: {}".format(plugin.name, ex))
logger.error("Trace: {}".format(traceback.format_exc()))
plugin.deactivate()
logger.info("Plugin deactivated: {}".format(plugin.name))
def deactivate_plugin(self, plugin_name, sync=True):
try:

364
senpy/models.py
View File

@ -6,7 +6,11 @@ For compatibility with Py3 and for easier debugging, this new version drops
introspection and adds all arguments to the models.
'''
from __future__ import print_function
from six import string_types
from future import standard_library
standard_library.install_aliases()
from future.utils import with_metaclass
from past.builtins import basestring
import time
import copy
@ -15,6 +19,8 @@ import os
import jsonref
import jsonschema
import inspect
from collections import UserDict
from abc import ABCMeta
from flask import Response as FlaskResponse
from pyld import jsonld
@ -62,7 +68,7 @@ class Context(dict):
return contexts
elif isinstance(context, dict):
return Context(context)
elif isinstance(context, string_types):
elif isinstance(context, basestring):
try:
with open(context) as f:
return Context(json.loads(f.read()))
@ -75,9 +81,154 @@ class Context(dict):
base_context = Context.load(CONTEXT_PATH)
class SenpyMixin(object):
class BaseMeta(ABCMeta):
'''
Metaclass for models. It extracts the default values for the fields in
the model.
For instance, instances of the following class wouldn't need to mark
their version or description on initialization:
.. code-block:: python
class MyPlugin(Plugin):
version=0.3
description='A dull plugin'
Note that these operations could be included in the __init__ of the
class, but it would be very inefficient.
'''
def __new__(mcs, name, bases, attrs, **kwargs):
defaults = {}
if 'schema' in attrs:
defaults = mcs.get_defaults(attrs['schema'])
for b in bases:
if hasattr(b, 'defaults'):
defaults.update(b.defaults)
info = mcs.attrs_to_dict(attrs)
defaults.update(info)
attrs['defaults'] = defaults
return super(BaseMeta, mcs).__new__(mcs, name, bases, attrs)
@staticmethod
def attrs_to_dict(attrs):
'''
Extract the attributes of the class.
This allows adding default values in the class definition.
e.g.:
'''
def is_attr(k, v):
return (not(inspect.isroutine(v) or
inspect.ismethod(v) or
inspect.ismodule(v) or
isinstance(v, property)) and
k[0] != '_' and
k != 'schema' and
k != 'data')
return {key: copy.deepcopy(value) for key, value in attrs.items() if is_attr(key, value)}
@staticmethod
def get_defaults(schema):
temp = {}
for obj in [
schema,
] + schema.get('allOf', []):
for k, v in obj.get('properties', {}).items():
if 'default' in v and k not in temp:
temp[k] = copy.deepcopy(v['default'])
return temp
class CustomDict(UserDict, object):
'''
A dictionary whose elements can also be accessed as attributes. Since some
characters are not valid in the dot-notation, the attribute names also
converted. e.g.:
> d = CustomDict()
> d.key = d['ns:name'] = 1
> d.key == d['key']
True
> d.ns__name == d['ns:name']
'''
defaults = []
def __init__(self, *args, **kwargs):
temp = copy.deepcopy(self.defaults)
for arg in args:
temp.update(copy.deepcopy(arg))
for k, v in kwargs.items():
temp[self._get_key(k)] = v
super(CustomDict, self).__init__(temp)
@staticmethod
def _get_key(key):
if key is 'id':
key = '@id'
key = key.replace("__", ":", 1)
return key
@staticmethod
def _internal_key(key):
return key[0] == '_' or key == 'data'
def __getattr__(self, key):
'''
__getattr__ only gets called when the attribute could not be found
in the __dict__. So we only need to look for the the element in the
dictionary, or raise an Exception.
'''
mkey = self._get_key(key)
if not self._internal_key(key) and mkey in self:
return self[mkey]
raise AttributeError(key)
def __setattr__(self, key, value):
# Work as usual for internal properties or already existing
# properties
if self._internal_key(key) or key in self.__dict__:
return super(CustomDict, self).__setattr__(key, value)
key = self._get_key(key)
return self.__setitem__(self._get_key(key), value)
def __delattr__(self, key):
if self._internal_key(key):
return object.__delattr__(self, key)
key = self._get_key(key)
self.__delitem__(self._get_key(key))
class BaseModel(with_metaclass(BaseMeta, CustomDict)):
'''
Entities of the base model are a special kind of dictionary that emulates
a JSON-LD object. The structure of the dictionary is checked via JSON-schema.
For convenience, the values can also be accessed as attributes
(a la Javascript). e.g.:
> myobject.key == myobject['key']
True
> myobject.ns__name == myobject['ns:name']
True
'''
schema = base_schema
_context = base_context["@context"]
def __init__(self, *args, **kwargs):
auto_id = kwargs.pop('_auto_id', True)
super(BaseModel, self).__init__(*args, **kwargs)
if '@id' not in self and auto_id:
self.id = ':{}_{}'.format(type(self).__name__, time.time())
if '@type' not in self:
logger.warn('Created an instance of an unknown model')
def flask(self,
in_headers=True,
headers=None,
@ -146,7 +297,7 @@ class SenpyMixin(object):
else:
return item
return ser_or_down(self._plain_dict())
return ser_or_down(self.data)
def jsonld(self,
with_context=True,
@ -188,150 +339,21 @@ class SenpyMixin(object):
return str(self.serialize())
class BaseModel(SenpyMixin, dict):
'''
Entities of the base model are a special kind of dictionary that emulates
a JSON-LD object. For convenience, the values can also be accessed as attributes
(a la Javascript). e.g.:
> myobject.key == myobject['key']
True
> myobject.ns__name == myobject['ns:name']
True
'''
schema = base_schema
def __init__(self, *args, **kwargs):
self.attrs_to_dict()
if 'id' in kwargs:
self.id = kwargs.pop('id')
elif kwargs.pop('_auto_id', True):
self.id = '_:{}_{}'.format(type(self).__name__, time.time())
temp = self.get_defaults()
temp.update(dict(*args))
for k, v in kwargs.items():
temp[self._get_key(k)] = v
super(BaseModel, self).__init__(temp)
if '@type' not in self:
logger.warn('Created an instance of an unknown model')
def get_defaults(self):
temp = {}
for obj in [
self.schema,
] + self.schema.get('allOf', []):
for k, v in obj.get('properties', {}).items():
if 'default' in v and k not in temp:
temp[k] = copy.deepcopy(v['default'])
return temp
def attrs_to_dict(self):
'''
Copy the attributes of the class to the instance.
This allows adding default values in the class definition.
e.g.:
class MyPlugin(Plugin):
version=0.3
description='A dull plugin'
'''
def is_attr(x):
return not(inspect.isroutine(x) or inspect.ismethod(x) or isinstance(x, property))
for key, value in inspect.getmembers(self.__class__, is_attr):
if key[0] != '_' and key != 'schema':
self[key] = value
def _get_key(self, key):
if key is 'id':
key = '@id'
key = key.replace("__", ":", 1)
return key
def __delitem__(self, key):
key = self._get_key(key)
dict.__delitem__(self, key)
def _internal_key(self, key):
return key[0] == '_' or key in self.__dict__
def _plain_dict(self):
d = {k: v for (k, v) in self.items() if k[0] != "_"}
return d
def __getattr__(self, key):
'''
__getattr__ only gets called when the attribute could not
be found in the __dict__. So we only need to look for the
the element in the dictionary, or raise an Exception.
'''
if self._internal_key(key):
raise AttributeError(key)
return self.__getitem__(self._get_key(key))
def __setattr__(self, key, value):
if self._internal_key(key):
return super(BaseModel, self).__setattr__(key, value)
key = self._get_key(key)
return self.__setitem__(self._get_key(key), value)
def __delattr__(self, key):
if self._internal_key(key):
return object.__delattr__(self, key)
key = self._get_key(key)
self.__delitem__(self._get_key(key))
_subtypes = {}
def register(rsubclass, rtype=None):
_subtypes[rtype or rsubclass.__name__] = rsubclass
_subtypes = {}
def from_dict(indict, cls=None):
if not cls:
target = indict.get('@type', None)
try:
if target and target in _subtypes:
cls = _subtypes[target]
else:
cls = BaseModel
except Exception:
cls = BaseModel
outdict = dict()
for k, v in indict.items():
if k == '@context':
pass
elif isinstance(v, dict):
v = from_dict(indict[k])
elif isinstance(v, list):
for ix, v2 in enumerate(v):
if isinstance(v2, dict):
v[ix] = from_dict(v2)
outdict[k] = v
return cls(**outdict)
def from_string(string, **kwargs):
return from_dict(json.loads(string), **kwargs)
def from_json(injson):
indict = json.loads(injson)
return from_dict(indict)
def from_schema(name, schema=None, schema_file=None, base_classes=None):
base_classes = base_classes or []
base_classes.append(BaseModel)
schema_file = schema_file or '{}.json'.format(name)
class_name = '{}{}'.format(name[0].upper(), name[1:])
if '/' not in 'schema_file':
schema_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
thisdir = os.path.dirname(os.path.realpath(__file__))
schema_file = os.path.join(thisdir,
'schemas',
schema_file)
@ -354,6 +376,40 @@ def from_schema(name, schema=None, schema_file=None, base_classes=None):
return newclass
def from_dict(indict, cls=None):
if not cls:
target = indict.get('@type', None)
try:
if target and target in _subtypes:
cls = _subtypes[target]
else:
cls = BaseModel
except Exception:
cls = BaseModel
outdict = dict()
for k, v in indict.items():
if k == '@context':
pass
elif isinstance(v, dict):
v = from_dict(indict[k])
elif isinstance(v, list):
v = v[:]
for ix, v2 in enumerate(v):
if isinstance(v2, dict):
v[ix] = from_dict(v2)
outdict[k] = copy.deepcopy(v)
return cls(**outdict)
def from_string(string, **kwargs):
return from_dict(json.loads(string), **kwargs)
def from_json(injson):
indict = json.loads(injson)
return from_dict(indict)
def _add_from_schema(*args, **kwargs):
generatedClass = from_schema(*args, **kwargs)
globals()[generatedClass.__name__] = generatedClass
@ -384,40 +440,14 @@ for i in [
_ErrorModel = from_schema('error')
class Error(SenpyMixin, Exception):
class Error(_ErrorModel, Exception):
def __init__(self, message, *args, **kwargs):
super(Error, self).__init__(self, message, message)
self._error = _ErrorModel(message=message, *args, **kwargs)
Exception.__init__(self, message)
super(Error, self).__init__(*args, **kwargs)
self.message = message
def validate(self, obj=None):
self._error.validate()
def __getitem__(self, key):
return self._error[key]
def __setitem__(self, key, value):
self._error[key] = value
def __delitem__(self, key):
del self._error[key]
def __getattr__(self, key):
if key != '_error' and hasattr(self._error, key):
return getattr(self._error, key)
raise AttributeError(key)
def __setattr__(self, key, value):
if key != '_error':
return setattr(self._error, key, value)
else:
super(Error, self).__setattr__(key, value)
def __delattr__(self, key):
delattr(self._error, key)
def __str__(self):
return str(self.to_JSON(with_context=False))
def __hash__(self):
return Exception.__hash__(self)
register(Error, 'error')

39
senpy/plugins/__init__.py
View File

@ -1,5 +1,6 @@
from future import standard_library
standard_library.install_aliases()
from future.utils import with_metaclass
import os.path
import os
@ -16,21 +17,33 @@ import yaml
import threading
from .. import models, utils
from ..api import API_PARAMS
from .. import api
logger = logging.getLogger(__name__)
class Plugin(models.Plugin):
def __init__(self, info=None, data_folder=None):
class PluginMeta(models.BaseMeta):
def __new__(mcs, name, bases, attrs, **kwargs):
plugin_type = []
if hasattr(bases[0], 'plugin_type'):
plugin_type += bases[0].plugin_type
plugin_type.append(name)
attrs['plugin_type'] = plugin_type
return super(PluginMeta, mcs).__new__(mcs, name, bases, attrs)
class Plugin(with_metaclass(PluginMeta, models.Plugin)):
def __init__(self, info=None, data_folder=None, **kwargs):
"""
Provides a canonical name for plugins and serves as base for other
kinds of plugins.
"""
logger.debug("Initialising {}".format(info))
super(Plugin, self).__init__(**kwargs)
if info:
self.update(info)
super(Plugin, self).__init__(**self)
if not self.validate():
raise models.Error(message=("You need to provide configuration"
"information for the plugin."))
@ -57,7 +70,8 @@ class Plugin(models.Plugin):
'test cases').format(self.id, inspect.getfile(self.__class__)))
for case in self.test_cases:
entry = models.Entry(case['entry'])
params = case.get('params', {})
given_parameters = case.get('params', {})
params = api.parse_params(given_parameters, self.extra_params)
fails = case.get('fails', False)
try:
res = list(self.analyse_entry(entry, params))
@ -90,7 +104,7 @@ SenpyPlugin = Plugin
class AnalysisPlugin(Plugin):
def analyse(self, *args, **kwargs):
raise NotImplemented(
raise NotImplementedError(
'Your method should implement either analyse or analyse_entry')
def analyse_entry(self, entry, parameters):
@ -118,17 +132,17 @@ class ConversionPlugin(Plugin):
pass
class SentimentPlugin(models.SentimentPlugin, AnalysisPlugin):
class SentimentPlugin(AnalysisPlugin, models.SentimentPlugin):
minPolarityValue = 0
maxPolarityValue = 1
class EmotionPlugin(models.EmotionPlugin, AnalysisPlugin):
class EmotionPlugin(AnalysisPlugin, models.EmotionPlugin):
minEmotionValue = 0
maxEmotionValue = 1
class EmotionConversionPlugin(models.EmotionConversionPlugin, ConversionPlugin):
class EmotionConversionPlugin(ConversionPlugin):
pass
@ -171,19 +185,18 @@ class ShelfMixin(object):
pickle.dump(self._sh, f)
default_plugin_type = API_PARAMS['plugin_type']['default']
def pfilter(plugins, **kwargs):
""" Filter plugins by different criteria """
if isinstance(plugins, models.Plugins):
plugins = plugins.plugins
elif isinstance(plugins, dict):
plugins = plugins.values()
ptype = kwargs.pop('plugin_type', default_plugin_type)
ptype = kwargs.pop('plugin_type', AnalysisPlugin)
logger.debug('#' * 100)
logger.debug('ptype {}'.format(ptype))
if ptype:
if isinstance(ptype, PluginMeta):
ptype = ptype.__name__
try:
ptype = ptype[0].upper() + ptype[1:]
pclass = globals()[ptype]

2
senpy/plugins/example/emoRand/emoRand.py
View File

@ -4,7 +4,7 @@ from senpy.plugins import EmotionPlugin
from senpy.models import EmotionSet, Emotion, Entry
class RmoRandPlugin(EmotionPlugin):
class EmoRandPlugin(EmotionPlugin):
def analyse_entry(self, entry, params):
category = "emoml:big6happiness"
number = max(-1, min(1, random.gauss(0, 0.5)))

2
senpy/plugins/misc/split.py
View File

@ -11,7 +11,7 @@ class SplitPlugin(AnalysisPlugin):
nltk.download('punkt')
def analyse_entry(self, entry, params):
chunker_type = params.get("delimiter", "sentence")
chunker_type = params["delimiter"]
original_text = entry['nif:isString']
if chunker_type == "sentence":
tokenizer = PunktSentenceTokenizer()

14
senpy/plugins/sentiment/sentiment140/sentiment140.py
View File

@ -7,7 +7,7 @@ from senpy.models import Sentiment
class Sentiment140Plugin(SentimentPlugin):
def analyse_entry(self, entry, params):
lang = params.get("language", "auto")
lang = params["language"]
res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
json.dumps({
"language": lang,
@ -35,6 +35,18 @@ class Sentiment140Plugin(SentimentPlugin):
entry.language = lang
yield entry
def test(self, *args, **kwargs):
'''
To avoid calling the sentiment140 API, we will mock the results
from requests.
'''
from senpy.test import patch_requests
expected = {"data": [{"polarity": 10}]}
with patch_requests(expected) as (request, response):
super(Sentiment140Plugin, self).test(*args, **kwargs)
assert request.called
assert response.json.called
test_cases = [
{
'entry': {

1
senpy/plugins/sentiment/sentiment140/sentiment140.senpy
View File

@ -16,6 +16,7 @@ extra_params:
- es
- en
- auto
default: auto
requirements: {}
maxPolarityValue: 1
minPolarityValue: 0

6
senpy/schemas/help.json
View File

@ -7,11 +7,11 @@
"description": "Help containing accepted parameters",
"type": "object",
"properties": {
"parameters": {
"valid_parameters": {
"type": "object"
}
},
"required": "parameters"
"required": "valid_parameters"
}
]
}
}

16
senpy/schemas/plugin.json
View File

@ -1,7 +1,7 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"required": ["@id", "extra_params"],
"required": ["@id", "name", "description", "version", "plugin_type"],
"properties": {
"@id": {
"type": "string",
@ -9,7 +9,19 @@
},
"name": {
"type": "string",
"description": "The name of the plugin, which will be used in the algorithm detection phase"
"description": "The name of the plugin, which will be used in the algorithm detection phase."
},
"description": {
"type": "string",
"description": "A summary of what the plugin does, and pointers to further information."
},
"version": {
"type": "string",
"description": "The version of the plugin."
},
"plugin_type": {
"type": "string",
"description": "Sub-type of plugin. e.g. sentimentPlugin"
},
"extra_params": {
"type": "object",

15
senpy/static/css/main.css
View File

@ -152,3 +152,18 @@ textarea{
/* background: white; */
display: none;
}
.deco-none {
color: inherit;
text-decoration: inherit;
}
.deco-none:link {
color: inherit;
text-decoration: inherit;
}
.deco-none:hover {
color: inherit;
text-decoration: inherit;
}

345
senpy/static/js/main.js
View File

@ -1,7 +1,10 @@
var ONYX = "http://www.gsi.dit.upm.es/ontologies/onyx/ns#";
var RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type";
var plugins_params={};
var default_params = JSON.parse($.ajax({type: "GET", url: "/api?help=true" , async: false}).responseText);
var plugins_params = default_params = {};
var plugins = [];
var defaultPlugin = {};
var gplugins = {};
function replaceURLWithHTMLLinks(text) {
console.log('Text: ' + text);
var exp = /(\b(https?|ftp|file):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])/ig;
@ -25,21 +28,45 @@ function hashchanged(){
}
}
$(document).ready(function() {
var response = JSON.parse($.ajax({type: "GET", url: "/api/plugins/" , async: false}).responseText);
var defaultPlugin= JSON.parse($.ajax({type: "GET", url: "/api/plugins/default" , async: false}).responseText);
html="";
var availablePlugins = document.getElementById('availablePlugins');
function get_plugins(response){
plugins = response.plugins;
gplugins = {};
}
function group_plugins(){
for (r in plugins){
ptype = plugins[r]['@type'];
if(gplugins[ptype] == undefined){
gplugins[ptype] = [r]
gplugins[ptype] = [r];
}else{
gplugins[ptype].push(r)
gplugins[ptype].push(r);
}
}
}
}
function get_parameters(){
for (p in plugins){
plugin = plugins[p];
if (plugin["extra_params"]){
plugins_params[plugin["name"]]={};
for (param in plugin["extra_params"]){
if (typeof plugin["extra_params"][param] !="string"){
var params = new Array();
var alias = plugin["extra_params"][param]["aliases"][0];
params[alias]=new Array();
for (option in plugin["extra_params"][param]["options"]){
params[alias].push(plugin["extra_params"][param]["options"][option])
}
plugins_params[plugin["name"]][alias] = (params[alias])
}
}
}
}
}
function draw_plugins_selection(){
html="";
group_plugins();
for (g in gplugins){
html += "<optgroup label=\""+g+"\">"
for (r in gplugins[g]){
@ -49,7 +76,7 @@ $(document).ready(function() {
continue;
}
html+= "<option value=\""+plugin["name"]+"\" "
html+= "<option value=\""+plugin.name+"\" "
if (plugin["name"] == defaultPlugin["name"]){
html+= " selected=\"selected\""
}
@ -58,23 +85,18 @@ $(document).ready(function() {
}
html+=">"+plugin["name"]+"</option>"
if (plugin["extra_params"]){
plugins_params[plugin["name"]]={};
for (param in plugin["extra_params"]){
if (typeof plugin["extra_params"][param] !="string"){
var params = new Array();
var alias = plugin["extra_params"][param]["aliases"][0];
params[alias]=new Array();
for (option in plugin["extra_params"][param]["options"]){
params[alias].push(plugin["extra_params"][param]["options"][option])
}
plugins_params[plugin["name"]][alias] = (params[alias])
}
}
}
}
}
html += "</optgroup>"
document.getElementById('plugins').innerHTML = html;
}
function draw_plugins_list(){
var availablePlugins = document.getElementById('availablePlugins');
for(p in plugins){
var pluginEntry = document.createElement('li');
plugin = plugins[p];
newHtml = ""
if(plugin.url) {
newHtml= "<a href="+plugin.url+">" + plugin.name + "</a>";
@ -85,110 +107,185 @@ $(document).ready(function() {
pluginEntry.innerHTML = newHtml;
availablePlugins.appendChild(pluginEntry)
}
html += "</optgroup>"
document.getElementById('plugins').innerHTML = html;
change_params();
}
$(document).ready(function() {
var response = JSON.parse($.ajax({type: "GET", url: "/api/plugins/" , async: false}).responseText);
defaultPlugin= JSON.parse($.ajax({type: "GET", url: "/api/plugins/default" , async: false}).responseText);
get_plugins(response);
get_default_parameters();
draw_plugins_list();
draw_plugins_selection();
draw_parameters();
$(window).on('hashchange', hashchanged);
hashchanged();
$('.tooltip-form').tooltip();
});
function change_params(){
var plugin = document.getElementById("plugins").options[document.getElementById("plugins").selectedIndex].value;
html=""
for (param in default_params){
if ((default_params[param]['options']) && (['help','conversion'].indexOf(param) < 0)){
html+= "<label> "+param+"</label>"
if (default_params[param]['options'].length < 1) {
html +="<input></input>";
}
else {
html+= "<select id=\""+param+"\" name=\""+param+"\">"
for (option in default_params[param]['options']){
if (default_params[param]['options'][option] == default_params[param]['default']){
html+="<option value \""+default_params[param]['options'][option]+"\" selected >"+default_params[param]['options'][option]+"</option>"
}
else{
html+="<option value \""+default_params[param]['options'][option]+"\">"+default_params[param]['options'][option]+"</option>"
}
}
}
html+="</select><br>"
}
}
for (param in plugins_params[plugin]){
if (param || plugins_params[plugin][param].length > 1){
html+= "<label> Parameter "+param+"</label>"
param_opts = plugins_params[plugin][param]
if (param_opts.length > 0) {
html+= "<select id=\""+param+"\" name=\""+param+"\">"
for (option in param_opts){
html+="<option value \""+param_opts[option]+"\">"+param_opts[option]+"</option>"
}
html+="</select>"
}
else {
html +="<input id=\""+param+"\" name=\""+param+"\"></input>";
}
}
}
document.getElementById("params").innerHTML = html
};
function load_JSON(){
url = "/api";
var container = document.getElementById('results');
var rawcontainer = document.getElementById("jsonraw");
rawcontainer.innerHTML = '';
container.innerHTML = '';
var plugin = document.getElementById("plugins").options[document.getElementById("plugins").selectedIndex].value;
var input = encodeURIComponent(document.getElementById("input").value);
url += "?algo="+plugin+"&i="+input
for (param in plugins_params[plugin]){
if (param != null){
field = document.getElementById(param);
if (plugins_params[plugin][param].length > 0){
var param_value = encodeURIComponent(field.options[field.selectedIndex].text);
} else {
var param_value = encodeURIComponent(field.text);
}
if (param_value !== "undefined" && param_value.length > 0){
url+="&"+param+"="+param_value
}
}
}
for (param in default_params){
if ((param != null) && (default_params[param]['options']) && (['help','conversion'].indexOf(param) < 0)){
var param_value = encodeURIComponent(document.getElementById(param).options[document.getElementById(param).selectedIndex].value);
if (param_value){
url+="&"+param+"="+param_value
}
}
}
var response = $.ajax({type: "GET", url: url , async: false}).responseText;
rawcontainer.innerHTML = replaceURLWithHTMLLinks(response)
document.getElementById("input_request").innerHTML = "<a href='"+url+"'>"+url+"</a>"
document.getElementById("results-div").style.display = 'block';
try {
response = JSON.parse(response);
var options = {
mode: 'view'
};
var editor = new JSONEditor(container, options, response);
editor.expandAll();
}
catch(err){
console.log("Error decoding JSON (got turtle?)");
}
function get_default_parameters(){
default_params = JSON.parse($.ajax({type: "GET", url: "/api?help=true" , async: false}).responseText).valid_parameters;
// Remove the parameters that are always added
delete default_params["input"];
delete default_params["algorithm"];
delete default_params["help"];
}
function draw_default_parameters(){
var basic_params = document.getElementById("basic_params");
basic_params.innerHTML = params_div(default_params);
}
function draw_extra_parameters(){
var plugin = document.getElementById("plugins").options[document.getElementById("plugins").selectedIndex].value;
get_parameters();
var extra_params = document.getElementById("extra_params");
extra_params.innerHTML = params_div(plugins_params[plugin]);
}
function draw_parameters(){
draw_default_parameters();
draw_extra_parameters();
}
function add_default_params(){
var html = "";
// html += '<a href="#basic_params" class="btn btn-info" data-toggle="collapse">Basic API parameters</a>';
html += '<span id="basic_params" class="panel-collapse collapse">';
html += '<ul class="list-group">'
html += params_div(default_params);
html += '</span>';
return html;
}
function params_div(params){
var html = '<div class="container-fluid">';
if (Object.keys(params).length === 0) {
html += '<p class="text text-muted text-center">This plugin does not take any extra parameters</p>';
}
// Iterate over the keys in order
pnames = Object.keys(params).sort()
for (ix in pnames){
pname = pnames[ix];
param = params[pname];
html+='<div class="form-group">';
html += '<div class="row">'
html+= '<label class="col-sm-2" for="'+pname+'">'+pname+'</label>'
if (param.options){
opts = param.options;
if(param.options.length == 1 && param.options[0] == 'boolean') {
opts = [true, false];
}
html+= '<select class="col-sm-10" id="'+pname+"\" name=\""+pname+"\">"
var defaultopt = param.default;
for (option in opts){
isselected = "";
if (defaultopt != undefined && opts[option] == defaultopt ){
isselected = ' selected="selected"'
}
html+="<option value=\""+opts[option]+'"' + isselected +
'>'+opts[option]+"</option>"
}
html+="</select>"
}
else {
default_value = "";
if(param.default != undefined){
default_value = param.default;
};
html +='<input class="col-sm-10" id="'+pname+'" name="'+pname+'" value="' + default_value + '"></input>';
}
html+='</div>';
html+='<div class="row">';
if ('description' in param){
html += '<p class="form-text sm-sm-12 text-muted text-center">' + param.description + '</p>';
}
html+='</div>';
html+='</div>';
}
html+='</div>';
return html;
}
function _get_form_parameters(id){
var element = document.getElementById(id);
params = {};
var selects = element.getElementsByTagName('select');
var inputs = element.getElementsByTagName('input');
Array.prototype.forEach.call(selects, function (sel) {
key = sel.name;
value = sel.options[sel.selectedIndex].value
params[key] = value;
});
Array.prototype.forEach.call(inputs, function (el) {
params[el.name] = el.value;
});
for (k in params){
value = params[k];
if (value == "" || value === "undefined"){
delete params[k];
}
}
return params;
}
function get_form_parameters(){
var p1 = _get_form_parameters("basic_params");
var p2 = _get_form_parameters("extra_params");
return Object.assign(p1, p2);
}
function add_param(key, value){
value = encodeURIComponent(value);
return "&"+key+"="+value;
}
function load_JSON(){
url = "/api";
var container = document.getElementById('results');
var rawcontainer = document.getElementById("jsonraw");
rawcontainer.innerHTML = '';
container.innerHTML = '';
var plugin = document.getElementById("plugins").options[document.getElementById("plugins").selectedIndex].value;
var input = encodeURIComponent(document.getElementById("input").value);
url += "?algo="+plugin+"&i="+input
params = get_form_parameters();
for (key in params){
url += add_param(key, params[key]);
}
var response = $.ajax({type: "GET", url: url , async: false}).responseText;
rawcontainer.innerHTML = replaceURLWithHTMLLinks(response);
document.getElementById("input_request").innerHTML = "<a href='"+url+"'>"+url+"</a>"
document.getElementById("results-div").style.display = 'block';
try {
response = JSON.parse(response);
var options = {
mode: 'view'
};
var editor = new JSONEditor(container, options, response);
editor.expandAll();
// $('#results-div a[href="#viewer"]').tab('show');
$('#results-div a[href="#viewer"]').click();
// location.hash = 'raw';
}
catch(err){
console.log("Error decoding JSON (got turtle?)");
$('#results-div a[href="#raw"]').click();
// location.hash = 'raw';
}
}

50
senpy/templates/index.html
View File

@ -88,28 +88,52 @@
<div class="tab-pane active" id="test">
<div class="well">
<form id="form" onsubmit="return getPlugins();" accept-charset="utf-8">
<div id="inputswrapper">
<form id="form" class="container" onsubmit="return getPlugins();" accept-charset="utf-8">
<div><textarea id="input" class="boxsizingBorder" rows="5" name="i">This text makes me sad.
whilst this text makes me happy and surprised at the same time.
I cannot believe it!</textarea></div>
<label>Select the plugin:</label>
<select id="plugins" name="plugins" onchange="change_params()">
</select>
</br>
<div id ="params">
I cannot believe it!</textarea>
</div>
<div>
<label>Select the plugin:</label>
<select id="plugins" name="plugins" onchange="draw_extra_parameters()">
</select>
</div>
<!-- PARAMETERS -->
<div class="panel-group" id="parameters">
<div class="panel panel-default">
<a data-toggle="collapse" class="deco-none" href="#basic_params">
<div class="panel-heading">
<h4 class="panel-title">
Basic API parameters
</h4>
</div>
</a>
<div id="basic_params" class="panel-collapse collapse panel-body">
</div>
</br>
</div>
<div class="panel panel-default">
<a data-toggle="collapse" class="deco-none" href="#extra_params">
<div class="panel-heading">
<h4 class="panel-title">
Plugin extra parameters
</h4>
</div>
</a>
<div id="extra_params" class="panel-collapse collapse in panel-body">
</div>
</div>
</div>
<!-- END PARAMETERS -->
<a id="preview" class="btn btn-lg btn-primary" onclick="load_JSON()">Analyse!</a>
<!--<button id="visualise" name="type" type="button">Visualise!</button>-->
</div>
</form>
</div>
<span id="input_request"></span>
<div id="results-div">
<ul class="nav nav-tabs" role="tablist">
<li role="presentation" class="active"><a class="active" href="#viewer">Viewer</a></li>
<li role="presentation"><a class="active" href="#raw">Raw</a></li>
<li role="presentation" class="active"><a data-toggle="tab" class="active" href="#viewer">Viewer</a></li>
<li role="presentation"><a data-toggle="tab" class="active" href="#raw">Raw</a></li>
</ul>
<div class="tab-content" id="results-container">
@ -119,7 +143,7 @@ I cannot believe it!</textarea></div>
</div>
</div>
<div class="tab-pane" id="raw">
<div class="tab-pane" id="raw">
<div id="content">
<pre id="jsonraw" class="results"></pre>
</div>

43
senpy/test.py Normal file
View File

@ -0,0 +1,43 @@
try:
from unittest.mock import patch, MagicMock
except ImportError:
from mock import patch, MagicMock
import json
from contextlib import contextmanager
from .models import BaseModel
class Call(dict):
def __init__(self, obj):
self.obj = obj.serialize()
self.status_code = 200
self.content = self.json()
def json(self):
return json.loads(self.obj)
@contextmanager
def patch_requests(value, code=200):
success = MagicMock()
if isinstance(value, BaseModel):
value = value.jsonld()
data = json.dumps(value)
success.json.return_value = value
success.data.return_value = data
success.status_code = code
if hasattr(value, 'jsonld'):
success.content = value.jsonld()
else:
success.content = json.dumps(value)
method_mocker = MagicMock()
method_mocker.return_value = success
with patch.multiple('requests', request=method_mocker,
get=method_mocker, post=method_mocker):
yield method_mocker, success
assert method_mocker.called

9
senpy/utils.py
View File

@ -1,15 +1,17 @@
from . import models
from collections import MutableMapping
# MutableMapping should be enough, but it causes problems with py2
DICTCLASSES = (MutableMapping, dict, models.BaseModel)
def check_template(indict, template):
if isinstance(template, dict) and isinstance(indict, dict):
if isinstance(template, DICTCLASSES) and isinstance(indict, DICTCLASSES):
for k, v in template.items():
if k not in indict:
return '{} not in {}'.format(k, indict)
check_template(indict[k], v)
elif isinstance(template, list) and isinstance(indict, list):
if len(indict) != len(template):
raise models.Error('Different size for {} and {}'.format(indict, template))
for e in template:
found = False
for i in indict:
@ -17,6 +19,7 @@ def check_template(indict, template):
check_template(i, e)
found = True
except models.Error as ex:
# raise
continue
if not found:
raise models.Error('{} not found in {}'.format(e, indict))

4
tests/test_blueprints.py
View File

@ -162,5 +162,5 @@ class BlueprintsTest(TestCase):
resp = self.client.get("/api/?help=true")
self.assertCode(resp, 200)
js = parse_resp(resp)
assert "parameters" in js
assert "help" in js["parameters"]
assert "valid_parameters" in js
assert "help" in js["valid_parameters"]

36
tests/test_client.py
View File

@ -1,24 +1,9 @@
from unittest import TestCase
try:
from unittest.mock import patch
except ImportError:
from mock import patch
import json
from senpy.test import patch_requests
from senpy.client import Client
from senpy.models import Results, Plugins, Error
from senpy.plugins import AnalysisPlugin, default_plugin_type
class Call(dict):
def __init__(self, obj):
self.obj = obj.serialize()
self.status_code = 200
self.content = self.json()
def json(self):
return json.loads(self.obj)
from senpy.plugins import AnalysisPlugin
class ModelsTest(TestCase):
@ -29,20 +14,18 @@ class ModelsTest(TestCase):
def test_client(self):
endpoint = 'http://dummy/'
client = Client(endpoint)
success = Call(Results())
with patch('requests.request', return_value=success) as patched:
with patch_requests(Results()) as (request, response):
resp = client.analyse('hello')
assert isinstance(resp, Results)
patched.assert_called_with(
request.assert_called_with(
url=endpoint + '/', method='GET', params={'input': 'hello'})
error = Call(Error('Nothing'))
with patch('requests.request', return_value=error) as patched:
with patch_requests(Error('Nothing')) as (request, response):
try:
client.analyse(input='hello', algorithm='NONEXISTENT')
raise Exception('Exceptions should be raised. This is not golang')
except Error:
pass
patched.assert_called_with(
request.assert_called_with(
url=endpoint + '/',
method='GET',
params={'input': 'hello',
@ -54,12 +37,11 @@ class ModelsTest(TestCase):
plugins = Plugins()
p1 = AnalysisPlugin({'name': 'AnalysisP1', 'version': 0, 'description': 'No'})
plugins.plugins = [p1, ]
success = Call(plugins)
with patch('requests.request', return_value=success) as patched:
with patch_requests(plugins) as (request, response):
response = client.plugins()
assert isinstance(response, dict)
assert len(response) == 1
assert "AnalysisP1" in response
patched.assert_called_with(
request.assert_called_with(
url=endpoint + '/plugins', method='GET',
params={'plugin_type': default_plugin_type})
params={})

6
tests/test_extensions.py
View File

@ -106,7 +106,7 @@ class ExtensionsTest(TestCase):
r2 = analyse(self.senpy, input="tupni", output="tuptuo")
assert r1.analysis[0] == "plugins/Dummy_0.1"
assert r2.analysis[0] == "plugins/Dummy_0.1"
assert r1.entries[0]['nif:iString'] == 'input'
assert r1.entries[0]['nif:isString'] == 'input'
def test_analyse_jsonld(self):
""" Using a plugin with JSON-LD input"""
@ -130,7 +130,7 @@ class ExtensionsTest(TestCase):
output="tuptuo")
assert r1.analysis[0] == "plugins/Dummy_0.1"
assert r2.analysis[0] == "plugins/Dummy_0.1"
assert r1.entries[0]['nif:iString'] == 'input'
assert r1.entries[0]['nif:isString'] == 'input'
def test_analyse_error(self):
mm = mock.MagicMock()
@ -185,7 +185,7 @@ class ExtensionsTest(TestCase):
response = Results({
'analysis': [{'plugin': plugin}],
'entries': [Entry({
'nif:iString': 'much ado about nothing',
'nif:isString': 'much ado about nothing',
'emotions': [eSet1]
})]
})

32
tests/test_models.py
View File

@ -12,8 +12,8 @@ from senpy.models import (Emotion,
Error,
Results,
Sentiment,
SentimentPlugin,
Plugins,
Plugin,
from_string,
from_dict)
from senpy import plugins
@ -99,19 +99,19 @@ class ModelsTest(TestCase):
def test_plugins(self):
self.assertRaises(Error, plugins.Plugin)
p = plugins.Plugin({"name": "dummy",
"description": "I do nothing",
"version": 0,
"extra_params": {
"none": {
"options": ["es", ],
"required": False,
"default": "0"
}
}})
p = plugins.SentimentPlugin({"name": "dummy",
"description": "I do nothing",
"version": 0,
"extra_params": {
"none": {
"options": ["es", ],
"required": False,
"default": "0"
}
}})
c = p.jsonld()
assert '@type' in c
assert c['@type'] == 'plugin'
assert c['@type'] == 'sentimentPlugin'
assert 'info' not in c
assert 'repo' not in c
assert 'extra_params' in c
@ -173,13 +173,14 @@ class ModelsTest(TestCase):
def test_single_plugin(self):
"""A response with a single plugin should still return a list"""
plugs = Plugins()
p = Plugin({'id': str(1),
'version': 0,
'description': 'dummy'})
p = SentimentPlugin({'id': str(1),
'version': 0,
'description': 'dummy'})
plugs.plugins.append(p)
assert isinstance(plugs.plugins, list)
js = plugs.jsonld()
assert isinstance(js['plugins'], list)
assert js['plugins'][0]['@type'] == 'sentimentPlugin'
def test_from_string(self):
results = {
@ -192,6 +193,7 @@ class ModelsTest(TestCase):
}]
}
recovered = from_dict(results)
assert recovered.id == results['@id']
assert isinstance(recovered, Results)
assert isinstance(recovered.entries[0], Entry)

21
tests/test_plugins.py
View File

@ -6,7 +6,7 @@ import shutil
import tempfile
from unittest import TestCase
from senpy.models import Results, Entry, EmotionSet, Emotion
from senpy.models import Results, Entry, EmotionSet, Emotion, Plugins
from senpy import plugins
from senpy.plugins.conversion.emotion.centroids import CentroidConversion
@ -49,6 +49,25 @@ class PluginsTest(TestCase):
assert os.path.isfile(a.shelf_file)
os.remove(a.shelf_file)
def test_plugin_filter(self):
ps = Plugins()
for i in (plugins.SentimentPlugin,
plugins.EmotionPlugin,
plugins.AnalysisPlugin):
p = i(name='Plugin_{}'.format(i.__name__),
description='TEST',
version=0,
author='NOBODY')
ps.plugins.append(p)
assert len(ps.plugins) == 3
cases = [('AnalysisPlugin', 3),
('SentimentPlugin', 1),
('EmotionPlugin', 1)]
for name, num in cases:
res = plugins.pfilter(ps.plugins, plugin_type=name)
assert len(res) == num
def test_shelf(self):
''' A shelf is created and the value is stored '''
newfile = self.shelf_file + "new"