From 0204e0b8e95811f7dee483cbd2424fbe9c52aef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Sat, 30 Dec 2017 18:59:58 +0100 Subject: [PATCH] Several changes * Simplified setattr * Added loading attributes in class * Added ability to specify failing test cases in plugins --- senpy/extensions.py | 2 +- senpy/models.py | 91 +++++++++++++------ senpy/plugins/__init__.py | 90 ++++++++++-------- senpy/plugins/misc/split.py | 2 +- senpy/utils.py | 20 ++++ setup.py | 5 + test-requirements.txt | 1 + tests/plugins/async_plugin/asyncplugin.py | 26 ------ tests/plugins/async_plugin/asyncplugin.senpy | 8 -- tests/plugins/dummy_plugin/dummy.py | 11 --- tests/plugins/dummy_plugin/dummy.senpy | 15 --- .../plugins/dummy_plugin/dummy_required.senpy | 14 --- tests/plugins/noop/noop_plugin.py | 5 - tests/plugins/sleep_plugin/sleep.py | 14 --- tests/plugins/sleep_plugin/sleep.senpy | 16 ---- tests/test_extensions.py | 10 +- tests/test_models.py | 5 +- tests/test_plugins.py | 9 +- 18 files changed, 164 insertions(+), 180 deletions(-) delete mode 100644 tests/plugins/async_plugin/asyncplugin.py delete mode 100644 tests/plugins/async_plugin/asyncplugin.senpy delete mode 100644 tests/plugins/dummy_plugin/dummy.py delete mode 100644 tests/plugins/dummy_plugin/dummy.senpy delete mode 100644 tests/plugins/dummy_plugin/dummy_required.senpy delete mode 100644 tests/plugins/noop/noop_plugin.py delete mode 100644 tests/plugins/sleep_plugin/sleep.py delete mode 100644 tests/plugins/sleep_plugin/sleep.senpy diff --git a/senpy/extensions.py b/senpy/extensions.py index a11dd5a..fc9068c 100644 --- a/senpy/extensions.py +++ b/senpy/extensions.py @@ -83,7 +83,7 @@ class Senpy(object): self._search_folders.add(folder) self._outdated = True else: - logger.debug("Not a folder: %s", folder) + raise AttributeError("Not a folder: %s", folder) def _get_plugins(self, request): if not self.analysis_plugins: diff --git a/senpy/models.py b/senpy/models.py index 0fae66e..884493d 100644 --- a/senpy/models.py +++ b/senpy/models.py @@ -14,6 +14,7 @@ import json import os import jsonref import jsonschema +import inspect from flask import Response as FlaskResponse from pyld import jsonld @@ -102,7 +103,7 @@ class SenpyMixin(object): }) return FlaskResponse( response=content, - status=getattr(self, "status", 200), + status=self.get('status', 200), headers=headers, mimetype=mimetype) @@ -188,34 +189,61 @@ class SenpyMixin(object): class BaseModel(SenpyMixin, dict): + ''' + Entities of the base model are a special kind of dictionary that emulates + a JSON-LD object. For convenience, the values can also be accessed as attributes + (a la Javascript). e.g.: + + > myobject.key == myobject['key'] + True + > myobject.ns__name == myobject['ns:name'] + True + ''' schema = base_schema def __init__(self, *args, **kwargs): + self.attrs_to_dict() if 'id' in kwargs: self.id = kwargs.pop('id') elif kwargs.pop('_auto_id', True): self.id = '_:{}_{}'.format(type(self).__name__, time.time()) - temp = dict(*args, **kwargs) + temp = self.get_defaults() + temp.update(dict(*args)) + for k, v in kwargs.items(): + temp[self._get_key(k)] = v + super(BaseModel, self).__init__(temp) + + if '@type' not in self: + logger.warn('Created an instance of an unknown model') + + def get_defaults(self): + temp = {} for obj in [ self.schema, ] + self.schema.get('allOf', []): for k, v in obj.get('properties', {}).items(): if 'default' in v and k not in temp: temp[k] = copy.deepcopy(v['default']) + return temp - for i in temp: - nk = self._get_key(i) - if nk != i: - temp[nk] = temp[i] - del temp[i] - try: - temp['@type'] = getattr(self, '@type') - except AttributeError: - logger.warn('Creating an instance of an unknown model') + def attrs_to_dict(self): + ''' + Copy the attributes of the class to the instance. - super(BaseModel, self).__init__(temp) + This allows adding default values in the class definition. + e.g.: + + class MyPlugin(Plugin): + version=0.3 + description='A dull plugin' + ''' + def is_attr(x): + return not(inspect.isroutine(x) or inspect.ismethod(x) or isinstance(x, property)) + for key, value in inspect.getmembers(self.__class__, is_attr): + if key[0] != '_' and key != 'schema': + self[key] = value def _get_key(self, key): if key is 'id': @@ -224,27 +252,38 @@ class BaseModel(SenpyMixin, dict): return key def __delitem__(self, key): + key = self._get_key(key) dict.__delitem__(self, key) - def __getattr__(self, key): - try: - return self.__getitem__(self._get_key(key)) - except KeyError: - raise AttributeError(key) - - def __setattr__(self, key, value): - self.__setitem__(self._get_key(key), value) - - def __delattr__(self, key): - try: - object.__delattr__(self, key) - except AttributeError: - self.__delitem__(self._get_key(key)) + def _internal_key(self, key): + return key[0] == '_' or key in self.__dict__ def _plain_dict(self): d = {k: v for (k, v) in self.items() if k[0] != "_"} return d + def __getattr__(self, key): + ''' + __getattr__ only gets called when the attribute could not + be found in the __dict__. So we only need to look for the + the element in the dictionary, or raise an Exception. + ''' + if self._internal_key(key): + raise AttributeError(key) + return self.__getitem__(self._get_key(key)) + + def __setattr__(self, key, value): + if self._internal_key(key): + return super(BaseModel, self).__setattr__(key, value) + key = self._get_key(key) + return self.__setitem__(self._get_key(key), value) + + def __delattr__(self, key): + if self._internal_key(key): + return object.__delattr__(self, key) + key = self._get_key(key) + self.__delitem__(self._get_key(key)) + def register(rsubclass, rtype=None): _subtypes[rtype or rsubclass.__name__] = rsubclass diff --git a/senpy/plugins/__init__.py b/senpy/plugins/__init__.py index c3abb0e..a695d53 100644 --- a/senpy/plugins/__init__.py +++ b/senpy/plugins/__init__.py @@ -15,8 +15,6 @@ import importlib import yaml import threading -from contextlib import contextmanager - from .. import models, utils from ..api import API_PARAMS @@ -29,16 +27,21 @@ class Plugin(models.Plugin): Provides a canonical name for plugins and serves as base for other kinds of plugins. """ - if not info: + logger.debug("Initialising {}".format(info)) + if info: + self.update(info) + super(Plugin, self).__init__(**self) + if not self.validate(): raise models.Error(message=("You need to provide configuration" "information for the plugin.")) - logger.debug("Initialising {}".format(info)) - id = 'plugins/{}_{}'.format(info['name'], info['version']) - super(Plugin, self).__init__(id=id, **info) + self.id = 'plugins/{}_{}'.format(self['name'], self['version']) self.is_activated = False self._lock = threading.Lock() self.data_folder = data_folder or os.getcwd() + def validate(self): + return all(x in self for x in ('name', 'description', 'version')) + def get_folder(self): return os.path.dirname(inspect.getfile(self.__class__)) @@ -50,12 +53,21 @@ class Plugin(models.Plugin): def test(self): if not hasattr(self, 'test_cases'): - import inspect raise AttributeError(('Plugin {} [{}] does not have any defined ' 'test cases').format(self.id, inspect.getfile(self.__class__))) for case in self.test_cases: - res = list(self.analyse_entry(models.Entry(case['entry']), - case['params'])) + entry = models.Entry(case['entry']) + params = case.get('params', {}) + fails = case.get('fails', False) + try: + res = list(self.analyse_entry(entry, params)) + except models.Error: + if fails: + continue + raise + if fails: + raise Exception('This test should have raised an exception.') + exp = case['expected'] if not isinstance(exp, list): exp = [exp] @@ -63,12 +75,13 @@ class Plugin(models.Plugin): for r in res: r.validate() - @contextmanager def open(self, fpath, *args, **kwargs): if not os.path.isabs(fpath): fpath = os.path.join(self.data_folder, fpath) - with open(fpath, *args, **kwargs) as f: - yield f + return open(fpath, *args, **kwargs) + + def serve(self, **kwargs): + utils.serve(plugin=self, **kwargs) SenpyPlugin = Plugin @@ -106,17 +119,13 @@ class ConversionPlugin(Plugin): class SentimentPlugin(models.SentimentPlugin, AnalysisPlugin): - def __init__(self, info, *args, **kwargs): - super(SentimentPlugin, self).__init__(info, *args, **kwargs) - self.minPolarityValue = float(info.get("minPolarityValue", 0)) - self.maxPolarityValue = float(info.get("maxPolarityValue", 1)) + minPolarityValue = 0 + maxPolarityValue = 1 class EmotionPlugin(models.EmotionPlugin, AnalysisPlugin): - def __init__(self, info, *args, **kwargs): - super(EmotionPlugin, self).__init__(info, *args, **kwargs) - self.minEmotionValue = float(info.get("minEmotionValue", -1)) - self.maxEmotionValue = float(info.get("maxEmotionValue", 1)) + minEmotionValue = 0 + maxEmotionValue = 1 class EmotionConversionPlugin(models.EmotionConversionPlugin, ConversionPlugin): @@ -127,11 +136,11 @@ class ShelfMixin(object): @property def sh(self): if not hasattr(self, '_sh') or self._sh is None: - self.__dict__['_sh'] = {} + self._sh = {} if os.path.isfile(self.shelf_file): try: with self.open(self.shelf_file, 'rb') as p: - self.__dict__['_sh'] = pickle.load(p) + self._sh = pickle.load(p) except (IndexError, EOFError, pickle.UnpicklingError): logger.warning('{} has a corrupted shelf file!'.format(self.id)) if not self.get('force_shelf', False): @@ -142,9 +151,13 @@ class ShelfMixin(object): def sh(self): if os.path.isfile(self.shelf_file): os.remove(self.shelf_file) - del self.__dict__['_sh'] + del self._sh self.save() + @sh.setter + def sh(self, value): + self._sh = value + @property def shelf_file(self): if 'shelf_file' not in self or not self['shelf_file']: @@ -196,7 +209,7 @@ def pfilter(plugins, **kwargs): def validate_info(info): - return all(x in info for x in ('name', 'module', 'description', 'version')) + return all(x in info for x in ('name',)) def load_module(name, root=None): @@ -235,6 +248,17 @@ def install_deps(*plugins): return installed +def get_plugin_class(module): + candidate = None + for _, obj in inspect.getmembers(module): + if inspect.isclass(obj) and inspect.getmodule(obj) == module: + logger.debug(("Found plugin class:" + " {}@{}").format(obj, inspect.getmodule(obj))) + candidate = obj + break + return candidate + + def load_plugin_from_info(info, root=None, validator=validate_info, install=True, *args, **kwargs): if not root and '_path' in info: root = os.path.dirname(info['_path']) @@ -249,18 +273,12 @@ def load_plugin_from_info(info, root=None, validator=validate_info, install=True raise install_deps(info) tmp = load_module(module, root) - candidate = None - for _, obj in inspect.getmembers(tmp): - if inspect.isclass(obj) and inspect.getmodule(obj) == tmp: - logger.debug(("Found plugin class:" - " {}@{}").format(obj, inspect.getmodule(obj))) - candidate = obj - break - if not candidate: - logger.debug("No valid plugin for: {}".format(module)) - return - module = candidate(info=info, *args, **kwargs) - return module + cls = None + if '@type' not in info: + cls = get_plugin_class(tmp) + if not cls: + raise Exception("No valid plugin for: {}".format(module)) + return cls(info=info, *args, **kwargs) def parse_plugin_info(fpath): diff --git a/senpy/plugins/misc/split.py b/senpy/plugins/misc/split.py index a809ffc..b444f34 100644 --- a/senpy/plugins/misc/split.py +++ b/senpy/plugins/misc/split.py @@ -45,7 +45,7 @@ class SplitPlugin(AnalysisPlugin): }, { 'entry': { - "id": ":test", + "@id": ":test", 'nif:isString': 'Hello\nWorld' }, 'params': { diff --git a/senpy/utils.py b/senpy/utils.py index 6dae8c4..cb04840 100644 --- a/senpy/utils.py +++ b/senpy/utils.py @@ -23,3 +23,23 @@ def check_template(indict, template): else: if indict != template: raise models.Error('{} and {} are different'.format(indict, template)) + + +def easy(app=None, plugin=None, host='0.0.0.0', port=5000, **kwargs): + ''' + Run a server with a specific plugin. + ''' + + from flask import Flask + from senpy.extensions import Senpy + + if not app: + app = Flask(__name__) + sp = Senpy(app) + if plugin: + sp.add_plugin(plugin) + sp.install_deps() + app.run(host, + port, + debug=app.debug, + **kwargs) diff --git a/setup.py b/setup.py index 7dee40a..61d17d5 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,11 @@ setup( install_requires=install_reqs, tests_require=test_reqs, setup_requires=['pytest-runner', ], + extras_require={ + 'evaluation': [ + 'gsitk' + ] + }, include_package_data=True, entry_points={ 'console_scripts': diff --git a/test-requirements.txt b/test-requirements.txt index 5ef1112..bd8891e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,4 @@ mock pytest-cov pytest +gsitk diff --git a/tests/plugins/async_plugin/asyncplugin.py b/tests/plugins/async_plugin/asyncplugin.py deleted file mode 100644 index a37f2cb..0000000 --- a/tests/plugins/async_plugin/asyncplugin.py +++ /dev/null @@ -1,26 +0,0 @@ -from senpy.plugins import AnalysisPlugin - -import multiprocessing - - -def _train(process_number): - return process_number - - -class AsyncPlugin(AnalysisPlugin): - def _do_async(self, num_processes): - pool = multiprocessing.Pool(processes=num_processes) - values = pool.map(_train, range(num_processes)) - - return values - - def activate(self): - self.value = self._do_async(4) - - def analyse_entry(self, entry, params): - values = self._do_async(2) - entry.async_values = values - yield entry - - def test(self): - pass diff --git a/tests/plugins/async_plugin/asyncplugin.senpy b/tests/plugins/async_plugin/asyncplugin.senpy deleted file mode 100644 index 8c71849..0000000 --- a/tests/plugins/async_plugin/asyncplugin.senpy +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: Async -module: asyncplugin -description: I am async -author: "@balkian" -version: '0.1' -async: true -extra_params: {} \ No newline at end of file diff --git a/tests/plugins/dummy_plugin/dummy.py b/tests/plugins/dummy_plugin/dummy.py deleted file mode 100644 index 25a8b73..0000000 --- a/tests/plugins/dummy_plugin/dummy.py +++ /dev/null @@ -1,11 +0,0 @@ -from senpy.plugins import SentimentPlugin - - -class DummyPlugin(SentimentPlugin): - def analyse_entry(self, entry, params): - entry['nif:iString'] = entry['nif:isString'][::-1] - entry.reversed = entry.get('reversed', 0) + 1 - yield entry - - def test(self): - pass diff --git a/tests/plugins/dummy_plugin/dummy.senpy b/tests/plugins/dummy_plugin/dummy.senpy deleted file mode 100644 index ea0c405..0000000 --- a/tests/plugins/dummy_plugin/dummy.senpy +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "Dummy", - "module": "dummy", - "description": "I am dummy", - "author": "@balkian", - "version": "0.1", - "extra_params": { - "example": { - "@id": "example_parameter", - "aliases": ["example", "ex"], - "required": false, - "default": 0 - } - } -} diff --git a/tests/plugins/dummy_plugin/dummy_required.senpy b/tests/plugins/dummy_plugin/dummy_required.senpy deleted file mode 100644 index 3e361f6..0000000 --- a/tests/plugins/dummy_plugin/dummy_required.senpy +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "DummyRequired", - "module": "dummy", - "description": "I am dummy", - "author": "@balkian", - "version": "0.1", - "extra_params": { - "example": { - "@id": "example_parameter", - "aliases": ["example", "ex"], - "required": true - } - } -} diff --git a/tests/plugins/noop/noop_plugin.py b/tests/plugins/noop/noop_plugin.py deleted file mode 100644 index ba851b5..0000000 --- a/tests/plugins/noop/noop_plugin.py +++ /dev/null @@ -1,5 +0,0 @@ -from senpy.plugins import SentimentPlugin - - -class DummyPlugin(SentimentPlugin): - import noop diff --git a/tests/plugins/sleep_plugin/sleep.py b/tests/plugins/sleep_plugin/sleep.py deleted file mode 100644 index 770dd3b..0000000 --- a/tests/plugins/sleep_plugin/sleep.py +++ /dev/null @@ -1,14 +0,0 @@ -from senpy.plugins import AnalysisPlugin -from time import sleep - - -class SleepPlugin(AnalysisPlugin): - def activate(self, *args, **kwargs): - sleep(self.timeout) - - def analyse_entry(self, entry, params): - sleep(float(params.get("timeout", self.timeout))) - yield entry - - def test(self): - pass diff --git a/tests/plugins/sleep_plugin/sleep.senpy b/tests/plugins/sleep_plugin/sleep.senpy deleted file mode 100644 index 166f234..0000000 --- a/tests/plugins/sleep_plugin/sleep.senpy +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "Sleep", - "module": "sleep", - "description": "I am dummy", - "author": "@balkian", - "version": "0.1", - "timeout": 0.05, - "extra_params": { - "timeout": { - "@id": "timeout_sleep", - "aliases": ["timeout", "to"], - "required": false, - "default": 0 - } - } -} diff --git a/tests/test_extensions.py b/tests/test_extensions.py index a9bf767..dd50122 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -25,8 +25,8 @@ def analyse(instance, **kwargs): class ExtensionsTest(TestCase): def setUp(self): self.app = Flask('test_extensions') - self.dir = os.path.dirname(__file__) - self.senpy = Senpy(plugin_folder=self.dir, + self.examples_dir = os.path.join(os.path.dirname(__file__), '..', 'example-plugins') + self.senpy = Senpy(plugin_folder=self.examples_dir, app=self.app, default_plugins=False) self.senpy.activate_plugin("Dummy", sync=True) @@ -41,7 +41,7 @@ class ExtensionsTest(TestCase): def test_discovery(self): """ Discovery of plugins in given folders. """ # noinspection PyProtectedMember - assert self.dir in self.senpy._search_folders + assert self.examples_dir in self.senpy._search_folders print(self.senpy.plugins) assert "Dummy" in self.senpy.plugins @@ -54,7 +54,7 @@ class ExtensionsTest(TestCase): 'requirements': ['noop'], 'version': 0 } - root = os.path.join(self.dir, 'plugins', 'noop') + root = os.path.join(self.examples_dir, 'noop') module = plugins.load_plugin_from_info(info, root=root, install=True) assert module.name == 'TestPip' assert module @@ -166,7 +166,7 @@ class ExtensionsTest(TestCase): self.senpy.filter_plugins(name="Dummy", is_activated=True)) def test_load_default_plugins(self): - senpy = Senpy(plugin_folder=self.dir, default_plugins=True) + senpy = Senpy(plugin_folder=self.examples_dir, default_plugins=True) assert len(senpy.plugins) > 1 def test_convert_emotions(self): diff --git a/tests/test_models.py b/tests/test_models.py index 64fa728..217511a 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -100,6 +100,7 @@ class ModelsTest(TestCase): def test_plugins(self): self.assertRaises(Error, plugins.Plugin) p = plugins.Plugin({"name": "dummy", + "description": "I do nothing", "version": 0, "extra_params": { "none": { @@ -123,7 +124,9 @@ class ModelsTest(TestCase): def test_str(self): """The string representation shouldn't include private variables""" r = Results() - p = plugins.Plugin({"name": "STR test", "version": 0}) + p = plugins.Plugin({"name": "STR test", + "description": "Test of private variables.", + "version": 0}) p._testing = 0 s = str(p) assert "_testing" not in s diff --git a/tests/test_plugins.py b/tests/test_plugins.py index e3cdb54..bf31223 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -43,6 +43,7 @@ class PluginsTest(TestCase): def test_shelf_file(self): a = ShelfDummyPlugin( info={'name': 'default_shelve_file', + 'description': 'Dummy plugin for tests', 'version': 'test'}) a.activate() assert os.path.isfile(a.shelf_file) @@ -53,6 +54,7 @@ class PluginsTest(TestCase): newfile = self.shelf_file + "new" a = ShelfDummyPlugin(info={ 'name': 'shelve', + 'description': 'Shelf plugin for tests', 'version': 'test', 'shelf_file': newfile }) @@ -75,6 +77,7 @@ class PluginsTest(TestCase): pickle.dump({'counter': 99}, f) a = ShelfDummyPlugin(info={ 'name': 'DummyShelf', + 'description': 'Dummy plugin for tests', 'shelf_file': self.shelf_file, 'version': 'test' }) @@ -105,7 +108,8 @@ class PluginsTest(TestCase): with open(fn, 'rb') as f: msg, error = files[fn] a = ShelfDummyPlugin(info={ - 'name': 'shelve', + 'name': 'test_corrupt_shelf_{}'.format(msg), + 'description': 'Dummy plugin for tests', 'version': 'test', 'shelf_file': f.name }) @@ -126,6 +130,7 @@ class PluginsTest(TestCase): ''' Reusing the values of a previous shelf ''' a = ShelfDummyPlugin(info={ 'name': 'shelve', + 'description': 'Dummy plugin for tests', 'version': 'test', 'shelf_file': self.shelf_file }) @@ -136,6 +141,7 @@ class PluginsTest(TestCase): b = ShelfDummyPlugin(info={ 'name': 'shelve', + 'description': 'Dummy plugin for tests', 'version': 'test', 'shelf_file': self.shelf_file }) @@ -148,6 +154,7 @@ class PluginsTest(TestCase): ''' Should be able to set extra parameters''' a = ShelfDummyPlugin(info={ 'name': 'shelve', + 'description': 'Dummy shelf plugin for tests', 'version': 'test', 'shelf_file': self.shelf_file, 'extra_params': {