From ff1492505633d769d2a0427eb1f3bbdb7318e234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Mon, 1 Dec 2014 18:27:20 +0100 Subject: [PATCH] Improved plugins, better tests, gevent Moved from Yapsy again (it is not flexible enough), now we use a custom solution. The activation and deactivation of plugins is asynchronous, so that plugins that take a long time don't interfere with the rest. --- app.py | 9 +- .../{__init__.py => sentiment140.py} | 15 --- plugins/sentiment140/sentiment140.senpy | 15 +++ requirements.txt | 4 +- senpy/blueprints.py | 19 ++- senpy/extensions.py | 120 ++++++++++++------ senpy/models.py | 11 +- senpy/plugins.py | 42 +++--- setup.py | 2 +- tests/__init__.py | 1 - tests/blueprints_test/__init__.py | 42 ++++++ tests/dummy_plugin/__init__.py | 3 - tests/dummy_plugin/dummy.py | 8 ++ tests/dummy_plugin/dummy.senpy | 7 + tests/extensions_test/__init__.py | 37 +++--- tests/sleep_plugin/sleep.py | 10 ++ tests/sleep_plugin/sleep.senpy | 8 ++ 17 files changed, 242 insertions(+), 111 deletions(-) rename plugins/sentiment140/{__init__.py => sentiment140.py} (70%) create mode 100644 plugins/sentiment140/sentiment140.senpy delete mode 100644 tests/dummy_plugin/__init__.py create mode 100644 tests/dummy_plugin/dummy.py create mode 100644 tests/dummy_plugin/dummy.senpy create mode 100644 tests/sleep_plugin/sleep.py create mode 100644 tests/sleep_plugin/sleep.senpy diff --git a/app.py b/app.py index 3aa175f..c11dd63 100644 --- a/app.py +++ b/app.py @@ -19,18 +19,25 @@ Simple Sentiment Analysis server for EUROSENTIMENT This class shows how to use the nif_server module to create custom services. """ +from gevent.monkey import patch_all; patch_all() +import gevent import config from flask import Flask from senpy.extensions import Senpy import logging import os +from gevent.wsgi import WSGIServer logging.basicConfig(level=logging.DEBUG) app = Flask(__name__) mypath = os.path.dirname(os.path.realpath(__file__)) sp = Senpy(app, os.path.join(mypath, "plugins")) +sp.activate_all() if __name__ == '__main__': + import logging + logging.basicConfig(level=config.DEBUG) app.debug = config.DEBUG - app.run(host="0.0.0.0", use_reloader=False) + http_server = WSGIServer(('', 5000), app) + http_server.serve_forever() \ No newline at end of file diff --git a/plugins/sentiment140/__init__.py b/plugins/sentiment140/sentiment140.py similarity index 70% rename from plugins/sentiment140/__init__.py rename to plugins/sentiment140/sentiment140.py index caf38f1..fa02c9d 100644 --- a/plugins/sentiment140/__init__.py +++ b/plugins/sentiment140/sentiment140.py @@ -6,19 +6,6 @@ from senpy.models import Response, Opinion, Entry class Sentiment140Plugin(SentimentPlugin): - EXTRA_PARAMS = { - "language": {"aliases": ["language", "l"], - "required": False, - "options": ["es", "en", "auto"], - } - } - - def __init__(self, **kwargs): - super(Sentiment140Plugin, self).__init__(name="sentiment140", - version="2.0", - extraparams=self.EXTRA_PARAMS, - **kwargs) - def analyse(self, **params): lang = params.get("language", "auto") res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson", @@ -45,5 +32,3 @@ class Sentiment140Plugin(SentimentPlugin): entry.language = lang response.entries.append(entry) return response - -plugin = Sentiment140Plugin() diff --git a/plugins/sentiment140/sentiment140.senpy b/plugins/sentiment140/sentiment140.senpy new file mode 100644 index 0000000..b0901fe --- /dev/null +++ b/plugins/sentiment140/sentiment140.senpy @@ -0,0 +1,15 @@ +{ + "name": "sentiment140", + "module": "sentiment140", + "description": "What my plugin broadly does", + "author": "@balkian", + "version": "0.1", + "extra_params": { + "language": { + "aliases": ["language", "l"], + "required": false, + "options": ["es", "en", "auto"] + } + }, + "requirements": {} +} diff --git a/requirements.txt b/requirements.txt index 1f91c73..533341b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ Flask==0.10.1 gunicorn==19.0.0 requests==2.4.1 -GitPython==0.3.2.RC1 \ No newline at end of file +GitPython==0.3.2.RC1 +Yapsy>=1.10.423 +gevent>=1.0.1 \ No newline at end of file diff --git a/senpy/blueprints.py b/senpy/blueprints.py index 0b781cb..4fd086b 100644 --- a/senpy/blueprints.py +++ b/senpy/blueprints.py @@ -110,23 +110,22 @@ def default(): @nif_blueprint.route('/plugins//', methods=['POST', 'GET']) def plugins(plugin=None, action="list"): filt = {} + sp = current_app.senpy if plugin: filt["name"] = plugin - plugs = current_app.senpy.filter_plugins(**filt) + plugs = sp.filter_plugins(**filt) if plugin and not plugs: return "Plugin not found", 400 if action == "list": with_params = request.args.get("params", "") == "1" - dic = {plug: plugs[plug].jsonable(with_params) for plug in plugs} + if plugin: + dic = plugs[plugin].jsonable(with_params) + else: + dic = {plug: plugs[plug].jsonable(with_params) for plug in plugs} return jsonify(dic) - if action == "disable": - current_app.senpy.disable_plugin(plugin) - return "Ok" - elif action == "enable": - current_app.senpy.enable_plugin(plugin) - return "Ok" - elif action == "reload": - current_app.senpy.reload_plugin(plugin) + method = "{}_plugin".format(action) + if(hasattr(sp, method)): + getattr(sp, method)(plugin) return "Ok" else: return "action '{}' not allowed".format(action), 400 diff --git a/senpy/extensions.py b/senpy/extensions.py index 14f3fb3..78a2dd4 100644 --- a/senpy/extensions.py +++ b/senpy/extensions.py @@ -1,16 +1,21 @@ """ """ import os +import fnmatch +import inspect import sys import imp import logging +import gevent +import json logger = logging.getLogger(__name__) -from .plugins import SentimentPlugin, EmotionPlugin +from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin from .blueprints import nif_blueprint from git import Repo, InvalidGitRepositoryError +from functools import partial class Senpy(object): @@ -45,11 +50,13 @@ class Senpy(object): app.register_blueprint(nif_blueprint) def add_folder(self, folder): + logger.debug("Adding folder: %s", folder) if os.path.isdir(folder): self._search_folders.add(folder) self._outdated = True return True else: + logger.debug("Not a folder: %s", folder) return False def analyse(self, **params): @@ -59,17 +66,20 @@ class Senpy(object): algo = params["algorithm"] elif self.plugins: algo = self.default_plugin - if algo in self.plugins and self.plugins[algo].enabled: - plug = self.plugins[algo] - resp = plug.analyse(**params) - resp.analysis.append(plug.jsonable()) - return resp + if algo in self.plugins: + if self.plugins[algo].is_activated: + plug = self.plugins[algo] + resp = plug.analyse(**params) + resp.analysis.append(plug.jsonable()) + return resp + logger.debug("Plugin not activated: {}".format(algo)) else: + logger.debug("The algorithm '{}' is not valid\nValid algorithms: {}".format(algo, self.plugins.keys())) return {"status": 400, "message": "The algorithm '{}' is not valid".format(algo)} @property def default_plugin(self): - candidates = self.filter_plugins(enabled=True) + candidates = self.filter_plugins(is_activated=True) if len(candidates) > 0: candidate = candidates.keys()[0] logger.debug("Default: {}".format(candidate)) @@ -80,11 +90,38 @@ class Senpy(object): def parameters(self, algo): return getattr(self.plugins.get(algo or self.default_plugin), "params", {}) - def enable_plugin(self, plugin): - self.plugins[plugin].enable() - - def disable_plugin(self, plugin): - self.plugins[plugin].disable() + def activate_all(self, sync=False): + ps = [] + for plug in self.plugins.keys(): + ps.append(self.activate_plugin(plug, sync=sync)) + return ps + + def deactivate_all(self, sync=False): + ps = [] + for plug in self.plugins.keys(): + ps.append(self.deactivate_plugin(plug, sync=sync)) + return ps + + def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs): + self.plugins[plugin_name].is_activated = active + + def activate_plugin(self, plugin_name, sync=False): + plugin = self.plugins[plugin_name] + th = gevent.spawn(plugin.activate) + th.link_value(partial(self._set_active_plugin, plugin_name, True)) + if sync: + th.join() + else: + return th + + def deactivate_plugin(self, plugin_name, sync=False): + plugin = self.plugins[plugin_name] + th = gevent.spawn(plugin.deactivate) + th.link_value(partial(self._set_active_plugin, plugin_name, False)) + if sync: + th.join() + else: + return th def reload_plugin(self, plugin): logger.debug("Reloading {}".format(plugin)) @@ -94,38 +131,47 @@ class Senpy(object): self.plugins[nplug.name] = nplug @staticmethod - def _load_plugin(plugin, search_folder, enabled=True): - logger.debug("Loading plugins") - sys.path.append(search_folder) - (fp, pathname, desc) = imp.find_module(plugin) + def _load_plugin(root, filename): + logger.debug("Loading plugin: {}".format(filename)) + fpath = os.path.join(root, filename) + with open(fpath,'r') as f: + info = json.load(f) + logger.debug("Info: {}".format(info)) + sys.path.append(root) + module = info["module"] + name = info["name"] + (fp, pathname, desc) = imp.find_module(module, [root,]) try: - tmp = imp.load_module(plugin, fp, pathname, desc).plugin - sys.path.remove(search_folder) - tmp.path = search_folder + tmp = imp.load_module(module, fp, pathname, desc) + sys.path.remove(root) + candidate = None + for _, obj in inspect.getmembers(tmp): + if inspect.isclass(obj) and inspect.getmodule(obj) == tmp: + logger.debug("Found plugin class: {}@{}".format(obj, inspect.getmodule(obj))) + candidate = obj + break + if not candidate: + logger.debug("No valid plugin for: {}".format(filename)) + return + module = candidate(info=info) try: - repo_path = os.path.join(search_folder, plugin) - tmp.repo = Repo(repo_path) + repo_path = root + module.repo = Repo(repo_path) except InvalidGitRepositoryError: - tmp.repo = None - if not hasattr(tmp, "enabled"): - tmp.enabled = enabled - tmp.module = plugin + module.repo = None except Exception as ex: - tmp = None - logger.debug("Exception importing {}: {}".format(plugin, ex)) - return tmp + logger.debug("Exception importing {}: {}".format(filename, ex)) + return None, None + return name, module def _load_plugins(self): plugins = {} for search_folder in self._search_folders: - for item in os.listdir(search_folder): - if os.path.isdir(os.path.join(search_folder, item)) \ - and os.path.exists(os.path.join(search_folder, - item, - "__init__.py")): - plugin = self._load_plugin(item, search_folder) + for root, dirnames, filenames in os.walk(search_folder): + for filename in fnmatch.filter(filenames, '*.senpy'): + name, plugin = self._load_plugin(root, filename) if plugin: - plugins[plugin.name] = plugin + plugins[name] = plugin self._outdated = False return plugins @@ -133,10 +179,6 @@ class Senpy(object): def teardown(self, exception): pass - def enable_all(self): - for plugin in self.plugins: - self.enable_plugin(plugin) - @property def plugins(self): """ Return the plugins registered for a given application. """ diff --git a/senpy/models.py b/senpy/models.py index dce3cbd..8b51cf9 100644 --- a/senpy/models.py +++ b/senpy/models.py @@ -77,7 +77,7 @@ class Entry(Leaf): class Opinion(Leaf): - opinionContext = {} + opinionContext = {"@vocab": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"} def __init__(self, polarityValue=None, hasPolarity=None, *args, **kwargs): super(Opinion, self).__init__(context=self.opinionContext, *args, @@ -93,7 +93,14 @@ class EmotionSet(Leaf): def __init__(self, emotions=None, *args, **kwargs): if not emotions: emotions = [] - super(EmotionSet, self).__init__(context=self.emotionContext, + super(EmotionSet, self).__init__(context=EmotionSet.emotionContext, *args, **kwargs) self.emotions = emotions or [] + +class Emotion(Leaf): + emotionContext = {} + def __init__(self, emotions=None, *args, **kwargs): + super(EmotionSet, self).__init__(context=Emotion.emotionContext, + *args, + **kwargs) diff --git a/senpy/plugins.py b/senpy/plugins.py index 6c3b73d..b9c3251 100644 --- a/senpy/plugins.py +++ b/senpy/plugins.py @@ -1,4 +1,5 @@ import logging +import ConfigParser logger = logging.getLogger(__name__) @@ -38,27 +39,28 @@ PARAMS = {"input": {"aliases": ["i", "input"], class SenpyPlugin(object): - def __init__(self, name=None, version=None, extraparams=None, params=None): - logger.debug("Initialising {}".format(name)) - self.name = name - self.version = version - if params: - self.params = params - else: - self.params = PARAMS.copy() - if extraparams: - self.params.update(extraparams) - self.extraparams = extraparams or {} - self.enabled = True + def __init__(self, info=None): + if not info: + raise ValueError("You need to provide configuration information for the plugin.") + logger.debug("Initialising {}".format(info)) + self.name = info["name"] + self.version = info["version"] + self.params = info.get("params", PARAMS.copy()) + self.extra_params = info.get("extra_params", {}) + self.params.update(self.extra_params) + self.is_activated = False + self.info = info def analyse(self, *args, **kwargs): + logger.debug("Analysing with: {} {}".format(self.name, self.version)) pass - def enable(self): - self.enabled = True + def activate(self): + pass + + def deactivate(self): + pass - def disable(self): - self.enabled = False @property def id(self): @@ -66,15 +68,15 @@ class SenpyPlugin(object): def jsonable(self, parameters=False): resp = { - "@id": self.id, - "enabled": self.enabled, + "@id": "{}_{}".format(self.name, self.version), + "is_activated": self.is_activated, } if hasattr(self, "repo") and self.repo: resp["repo"] = self.repo.remotes[0].url if parameters: resp["parameters"] = self.params - elif self.extraparams: - resp["extra_parameters"] = self.extraparams + elif self.extra_params: + resp["extra_parameters"] = self.extra_params return resp diff --git a/setup.py b/setup.py index e051456..b4352fa 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ install_reqs = parse_requirements("requirements.txt") # e.g. ['django==1.5.1', 'mezzanine==1.4.6'] reqs = [str(ir.req) for ir in install_reqs] -VERSION = "0.2.9" +VERSION = "0.3.0" print(reqs) diff --git a/tests/__init__.py b/tests/__init__.py index 8b13789..e69de29 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +0,0 @@ - diff --git a/tests/blueprints_test/__init__.py b/tests/blueprints_test/__init__.py index 3712682..49d9689 100644 --- a/tests/blueprints_test/__init__.py +++ b/tests/blueprints_test/__init__.py @@ -9,6 +9,7 @@ except ImportError: from senpy.extensions import Senpy from flask import Flask from flask.ext.testing import TestCase +from gevent import sleep def check_dict(indic, template): @@ -22,6 +23,7 @@ class BlueprintsTest(TestCase): self.senpy.init_app(self.app) self.dir = os.path.join(os.path.dirname(__file__), "..") self.senpy.add_folder(self.dir) + self.senpy.activate_plugin("Dummy", sync=True) return self.app def test_home(self): @@ -36,3 +38,43 @@ class BlueprintsTest(TestCase): } assert check_dict(resp.json, atleast) + def test_analysis(self): + """ The dummy plugin returns an empty response, it should contain the context """ + resp = self.client.get("/?i=My aloha mohame") + self.assert200(resp) + logging.debug(resp.json) + assert "@context" in resp.json + assert check_dict(resp.json["@context"], {"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"}) + assert "entries" in resp.json + + def test_list(self): + """ List the plugins """ + resp = self.client.get("/plugins/") + self.assert200(resp) + logging.debug(resp.json) + assert "Dummy" in resp.json + + def test_detail(self): + """ Show only one plugin""" + resp = self.client.get("/plugins/Dummy") + self.assert200(resp) + logging.debug(resp.json) + assert "@id" in resp.json + assert resp.json["@id"] == "Dummy_0.1" + + def test_activate(self): + """ Activate and deactivate one plugin """ + resp = self.client.get("/plugins/Dummy/deactivate") + self.assert200(resp) + sleep(0.5) + resp = self.client.get("/plugins/Dummy") + self.assert200(resp) + assert "is_activated" in resp.json + assert resp.json["is_activated"] == False + resp = self.client.get("/plugins/Dummy/activate") + self.assert200(resp) + sleep(0.5) + resp = self.client.get("/plugins/Dummy") + self.assert200(resp) + assert "is_activated" in resp.json + assert resp.json["is_activated"] == True diff --git a/tests/dummy_plugin/__init__.py b/tests/dummy_plugin/__init__.py deleted file mode 100644 index c9e674c..0000000 --- a/tests/dummy_plugin/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from senpy.plugins import SenpyPlugin - -plugin = SenpyPlugin("dummy") diff --git a/tests/dummy_plugin/dummy.py b/tests/dummy_plugin/dummy.py new file mode 100644 index 0000000..4a7419e --- /dev/null +++ b/tests/dummy_plugin/dummy.py @@ -0,0 +1,8 @@ +from senpy.plugins import SentimentPlugin +from senpy.models import Response + +class DummyPlugin(SentimentPlugin): + pass + + def analyse(self, *args, **kwargs): + return Response() \ No newline at end of file diff --git a/tests/dummy_plugin/dummy.senpy b/tests/dummy_plugin/dummy.senpy new file mode 100644 index 0000000..996e614 --- /dev/null +++ b/tests/dummy_plugin/dummy.senpy @@ -0,0 +1,7 @@ +{ + "name": "Dummy", + "module": "dummy", + "description": "I am dummy", + "author": "@balkian", + "version": "0.1" +} diff --git a/tests/extensions_test/__init__.py b/tests/extensions_test/__init__.py index 25ed292..305f61b 100644 --- a/tests/extensions_test/__init__.py +++ b/tests/extensions_test/__init__.py @@ -13,10 +13,10 @@ from flask.ext.testing import TestCase class ExtensionsTest(TestCase): def create_app(self): self.app = Flask("test_extensions") - self.senpy = Senpy() - self.senpy.init_app(self.app) self.dir = os.path.join(os.path.dirname(__file__), "..") - self.senpy.add_folder(self.dir) + self.senpy = Senpy(plugin_folder=self.dir) + self.senpy.init_app(self.app) + self.senpy.activate_plugin("Dummy", sync=True) return self.app def test_init(self): @@ -31,41 +31,42 @@ class ExtensionsTest(TestCase): # noinspection PyProtectedMember assert self.dir in self.senpy._search_folders print self.senpy.plugins - assert "dummy" in self.senpy.plugins + assert "Dummy" in self.senpy.plugins def test_enabling(self): """ Enabling a plugin """ - self.senpy.enable_plugin("dummy") - assert self.senpy.plugins["dummy"].enabled + self.senpy.activate_all(sync=True) + assert len(self.senpy.plugins) == 2 + assert self.senpy.plugins["Sleep"].is_activated def test_disabling(self): """ Disabling a plugin """ - self.senpy.enable_plugin("dummy") - self.senpy.disable_plugin("dummy") - assert not self.senpy.plugins["dummy"].enabled + self.senpy.deactivate_all(sync=True) + assert self.senpy.plugins["Dummy"].is_activated == False + assert self.senpy.plugins["Sleep"].is_activated == False def test_default(self): """ Default plugin should be set """ assert self.senpy.default_plugin - assert self.senpy.default_plugin == "dummy" + assert self.senpy.default_plugin == "Dummy" def test_analyse(self): """ Using a plugin """ - with mock.patch.object(self.senpy.plugins["dummy"], "analyse") as mocked: - self.senpy.analyse(algorithm="dummy", input="tupni", output="tuptuo") + with mock.patch.object(self.senpy.plugins["Dummy"], "analyse") as mocked: + self.senpy.analyse(algorithm="Dummy", input="tupni", output="tuptuo") self.senpy.analyse(input="tupni", output="tuptuo") - mocked.assert_any_call(input="tupni", output="tuptuo", algorithm="dummy") + mocked.assert_any_call(input="tupni", output="tuptuo", algorithm="Dummy") mocked.assert_any_call(input="tupni", output="tuptuo") for plug in self.senpy.plugins: - self.senpy.disable_plugin(plug) + self.senpy.deactivate_plugin(plug, sync=True) resp = self.senpy.analyse(input="tupni") logging.debug("Response: {}".format(resp)) assert resp["status"] == 400 def test_filtering(self): """ Filtering plugins """ - assert len(self.senpy.filter_plugins(name="dummy")) > 0 + assert len(self.senpy.filter_plugins(name="Dummy")) > 0 assert not len(self.senpy.filter_plugins(name="notdummy")) - assert self.senpy.filter_plugins(name="dummy", enabled=True) - self.senpy.disable_plugin("dummy") - assert not len(self.senpy.filter_plugins(name="dummy", enabled=True)) + assert self.senpy.filter_plugins(name="Dummy", is_activated=True) + self.senpy.deactivate_plugin("Dummy", sync=True) + assert not len(self.senpy.filter_plugins(name="Dummy", is_activated=True)) diff --git a/tests/sleep_plugin/sleep.py b/tests/sleep_plugin/sleep.py new file mode 100644 index 0000000..30a0157 --- /dev/null +++ b/tests/sleep_plugin/sleep.py @@ -0,0 +1,10 @@ +from senpy.plugins import SenpyPlugin +from time import sleep + +class SleepPlugin(SenpyPlugin): + def __init__(self, info, *args, **kwargs): + super(SleepPlugin, self).__init__(info, *args, **kwargs) + self.timeout = int(info["timeout"]) + + def activate(self, *args, **kwargs): + sleep(self.timeout) diff --git a/tests/sleep_plugin/sleep.senpy b/tests/sleep_plugin/sleep.senpy new file mode 100644 index 0000000..6920a77 --- /dev/null +++ b/tests/sleep_plugin/sleep.senpy @@ -0,0 +1,8 @@ +{ + "name": "Sleep", + "module": "sleep", + "description": "I am dummy", + "author": "@balkian", + "version": "0.1", + "timeout": "2" +}