From eaf65f0c6b5cd60919f4b0dc3660947835b28b8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Fri, 7 Nov 2014 19:12:21 +0100 Subject: [PATCH] First tests --- plugins-bak/prueba/__init__.py | 9 ++ .../sentiment140/__init__.py | 0 senpy/__init__.py | 2 +- senpy/blueprints.py | 11 +- senpy/extensions.py | 108 +++++++++++------- senpy/plugins.py | 6 +- senpy/plugins/sentiment140/sentiment140.py | 41 ------- tests/__init__.py | 0 tests/blueprints_test/__init__.py | 35 ++++++ tests/dummy_plugin/__init__.py | 3 + tests/extensions_test/__init__.py | 70 ++++++++++++ 11 files changed, 199 insertions(+), 86 deletions(-) create mode 100644 plugins-bak/prueba/__init__.py rename {senpy/plugins => plugins}/sentiment140/__init__.py (100%) delete mode 100644 senpy/plugins/sentiment140/sentiment140.py create mode 100644 tests/__init__.py create mode 100644 tests/blueprints_test/__init__.py create mode 100644 tests/dummy_plugin/__init__.py create mode 100644 tests/extensions_test/__init__.py diff --git a/plugins-bak/prueba/__init__.py b/plugins-bak/prueba/__init__.py new file mode 100644 index 0000000..af50e93 --- /dev/null +++ b/plugins-bak/prueba/__init__.py @@ -0,0 +1,9 @@ +from senpy.plugins import SenpyPlugin + +class Prueba(SenpyPlugin): + def __init__(self, **kwargs): + super(Prueba, self).__init__(name="prueba", + version="4.0", + **kwargs) + +plugin = Prueba() diff --git a/senpy/plugins/sentiment140/__init__.py b/plugins/sentiment140/__init__.py similarity index 100% rename from senpy/plugins/sentiment140/__init__.py rename to plugins/sentiment140/__init__.py diff --git a/senpy/__init__.py b/senpy/__init__.py index 6190b98..528ea3a 100644 --- a/senpy/__init__.py +++ b/senpy/__init__.py @@ -18,7 +18,7 @@ Sentiment analysis server in Python ''' -VERSION = "0.2.5" +VERSION = "0.2.6" import extensions import blueprints diff --git a/senpy/blueprints.py b/senpy/blueprints.py index 28b505b..e9340a8 100644 --- a/senpy/blueprints.py +++ b/senpy/blueprints.py @@ -17,8 +17,11 @@ ''' Simple Sentiment Analysis server ''' -from flask import Blueprint, render_template, request, jsonify, current_app import json +import logging +logger = logging.getLogger(__name__) + +from flask import Blueprint, render_template, request, jsonify, current_app nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__) @@ -86,10 +89,12 @@ def home(entries=None): algo = get_params(request).get("algorithm", None) specific_params = current_app.senpy.parameters(algo) params = get_params(request, specific_params) + response = current_app.senpy.analyse(**params) + return jsonify(response) except ValueError as ex: return ex.message - response = current_app.senpy.analyse(**params) - return jsonify(response) + except Exception as ex: + return jsonify(status="400", message=ex.message) @nif_blueprint.route("/default") def default(): diff --git a/senpy/extensions.py b/senpy/extensions.py index 3cc2cb8..b91823e 100644 --- a/senpy/extensions.py +++ b/senpy/extensions.py @@ -1,9 +1,10 @@ import os import sys import imp +import logging + +logger = logging.getLogger(__name__) -from flask import current_app -from collections import defaultdict from .plugins import SentimentPlugin, EmotionPlugin try: @@ -11,29 +12,33 @@ try: except ImportError: from flask import _request_ctx_stack as stack -from blueprints import nif_blueprint +from .blueprints import nif_blueprint from git import Repo, InvalidGitRepositoryError + class Senpy(object): + """ Default Senpy extension for Flask """ def __init__(self, app=None, plugin_folder="plugins"): self.app = app base_folder = os.path.join(os.path.dirname(__file__), "plugins") - self.search_folders = (folder for folder in (base_folder, plugin_folder, '/tmp/plugins') - if folder and os.path.isdir(folder)) + self._search_folders = set() + self._outdated = True + + for folder in (base_folder, plugin_folder): + self.add_folder(folder) if app is not None: self.init_app(app) - """ - + def init_app(self, app): + """ Initialise a flask app to add plugins to its context """ + """ Note: I'm not particularly fond of adding self.app and app.senpy, but I can't think of a better way to do it. - """ - def init_app(self, app, plugin_folder="plugins"): + """ app.senpy = self - #app.config.setdefault('SQLITE3_DATABASE', ':memory:') # Use the newstyle teardown_appcontext if it's available, # otherwise fall back to the request context if hasattr(app, 'teardown_appcontext'): @@ -42,9 +47,18 @@ class Senpy(object): app.teardown_request(self.teardown) app.register_blueprint(nif_blueprint) + def add_folder(self, folder): + if os.path.isdir(folder): + self._search_folders.add(folder) + self._outdated = True + return True + else: + return False + + def analyse(self, **params): algo = None - print("analysing with params: {}".format(params)) + logger.debug("analysing with params: {}".format(params)) if "algorithm" in params: algo = params["algorithm"] elif self.plugins: @@ -55,60 +69,68 @@ class Senpy(object): resp.analysis.append(plug.jsonable()) return resp else: - return {"status": 500, "message": "No valid algorithm"} + return {"status": 400, "message": "The algorithm '{}' is not valid".format(algo) } @property def default_plugin(self): - if self.plugins: - candidate = self.filter_plugins(enabled=True).keys()[0] - print("Default: {}".format(candidate)) + candidates = self.filter_plugins(enabled=True) + if len(candidates)>1: + candidate = candidates.keys()[0] + logger.debug("Default: {}".format(candidate)) return candidate else: - return Exception("No algorithm") + return None def parameters(self, algo): return getattr(self.plugins.get(algo or self.default_plugin), "params", {}) def enable_plugin(self, plugin): - self.plugins[plugin].disable() + self.plugins[plugin].enable() def disable_plugin(self, plugin): self.plugins[plugin].disable() def reload_plugin(self, plugin): - print("Reloading {}".format(plugin)) + logger.debug("Reloading {}".format(plugin)) plug = self.plugins[plugin] nplug = self._load_plugin(plug.module, plug.path) del self.plugins[plugin] self.plugins[nplug.name] = nplug def _load_plugin(self, plugin, search_folder, enabled=True): + logger.debug("Loading plugins") sys.path.append(search_folder) (fp, pathname, desc) = imp.find_module(plugin) - tmp = imp.load_module(plugin, fp, pathname, desc).plugin - sys.path.remove(search_folder) - tmp.path = search_folder try: - repo_path = os.path.join(search_folder, plugin) - tmp.repo = Repo(repo_path) - except InvalidGitRepositoryError: - tmp.repo = None - if not hasattr(tmp, "enabled"): - tmp.enabled = enabled - tmp.module = plugin + tmp = imp.load_module(plugin, fp, pathname, desc).plugin + sys.path.remove(search_folder) + tmp.path = search_folder + try: + repo_path = os.path.join(search_folder, plugin) + tmp.repo = Repo(repo_path) + except InvalidGitRepositoryError: + tmp.repo = None + if not hasattr(tmp, "enabled"): + tmp.enabled = enabled + tmp.module = plugin + except Exception as ex: + tmp = None + logger.debug("Exception importing {}: {}".format(plugin, ex)) return tmp def _load_plugins(self): plugins = {} - for search_folder in self.search_folders: + for search_folder in self._search_folders: for item in os.listdir(search_folder): if os.path.isdir(os.path.join(search_folder, item)) \ and os.path.exists( os.path.join(search_folder, item, "__init__.py")): plugin = self._load_plugin(item, search_folder) - plugins[plugin.name] = plugin + if plugin: + plugins[plugin.name] = plugin + self._outdated = False return plugins def teardown(self, exception): @@ -126,30 +148,36 @@ class Senpy(object): @property def plugins(self): + """ Return the plugins registered for a given application. """ ctx = stack.top if ctx is not None: - if not hasattr(self, '_plugins'): - self._plugins = self._load_plugins() - return self._plugins + if not hasattr(ctx, 'senpy_plugins') or self._outdated: + ctx.senpy_plugins = self._load_plugins() + return ctx.senpy_plugins def filter_plugins(self, **kwargs): + """ Filter plugins by different criteria """ def matches(plug): - res = all(getattr(plug, k, None)==v for (k,v) in kwargs.items()) - print("matching {} with {}: {}".format(plug.name, kwargs, res)) + res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items()) + logger.debug("matching {} with {}: {}".format(plug.name, + kwargs, + res)) return res if not kwargs: return self.plugins else: - return {n:p for n,p in self.plugins.items() if matches(p)} + return {n:p for n, p in self.plugins.items() if matches(p)} def sentiment_plugins(self): - return (plugin for plugin in self.plugins if - isinstance(plugin, SentimentPlugin)) + """ Return only the sentiment plugins """ + return {p:plugin for p, plugin in self.plugins.items() if + isinstance(plugin, SentimentPlugin)} + if __name__ == '__main__': from flask import Flask app = Flask(__name__) sp = Senpy() - sp.init_app(app) - with app.app_context(): + sp.init_app(APP) + with APP.app_context(): sp._load_plugins() diff --git a/senpy/plugins.py b/senpy/plugins.py index 8fe22ac..a3e2fd3 100644 --- a/senpy/plugins.py +++ b/senpy/plugins.py @@ -1,3 +1,7 @@ +import logging + +logger = logging.getLogger(__name__) + PARAMS = {"input": {"aliases": ["i", "input"], "required": True, "help": "Input text" @@ -30,7 +34,7 @@ PARAMS = {"input": {"aliases": ["i", "input"], class SenpyPlugin(object): def __init__(self, name=None, version=None, extraparams=None, params=None): - print("Initing {}".format(name)) + logger.debug("Initialising {}".format(name)) self.name = name self.version = version if params: diff --git a/senpy/plugins/sentiment140/sentiment140.py b/senpy/plugins/sentiment140/sentiment140.py deleted file mode 100644 index 2751727..0000000 --- a/senpy/plugins/sentiment140/sentiment140.py +++ /dev/null @@ -1,41 +0,0 @@ -''' -SENTIMENT140 -============= - -* http://www.sentiment140.com/api/bulkClassifyJson -* Method: POST -* Parameters: JSON Object (that is copied to the result) - * text - * query - * language - * topic - -* Example response: -```json -{"data": [{"text": "I love Titanic.", "id":1234, "polarity": 4}, - {"text": "I hate Titanic.", "id":4567, "polarity": 0}]} -``` -''' -import requests -import json - -ENDPOINT_URI = "http://www.sentiment140.com/api/bulkClassifyJson" - -def analyse(texts): - parameters = {"data": []} - if isinstance(texts, list): - for text in texts: - parameters["data"].append({"text": text}) - else: - parameters["data"].append({"text": texts}) - - res = requests.post(ENDPOINT_URI, json.dumps(parameters)) - res.json() - return res.json() - -def test(): - print analyse("I love Titanic") - print analyse(["I love Titanic", "I hate Titanic"]) - -if __name__ == "__main__": - test() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/blueprints_test/__init__.py b/tests/blueprints_test/__init__.py new file mode 100644 index 0000000..df0f1a4 --- /dev/null +++ b/tests/blueprints_test/__init__.py @@ -0,0 +1,35 @@ +import os +import logging +try: + import unittest.mock as mock +except ImportError: + import mock +from senpy.extensions import Senpy +from flask import Flask +from flask.ext.testing import TestCase + +def check_dict(indic, template): + return all(item in indic.items() for item in template.items()) + +class Blueprints_Test(TestCase): + def create_app(self): + self.app = Flask("test_extensions") + self.senpy = Senpy() + self.senpy.init_app(self.app) + self.dir = os.path.join(os.path.dirname(__file__), "..") + self.senpy.add_folder(self.dir) + return self.app + + + def test_home(self): + """ Calling with no arguments should ask the user for more arguments """ + resp = self.client.get("/") + self.assert200(resp) + logging.debug(resp.json) + assert resp.json["status"] == "failed" + atleast = { + "status": "failed", + "message": "Missing or invalid parameters", + } + assert check_dict(resp.json, atleast) + diff --git a/tests/dummy_plugin/__init__.py b/tests/dummy_plugin/__init__.py new file mode 100644 index 0000000..c9e674c --- /dev/null +++ b/tests/dummy_plugin/__init__.py @@ -0,0 +1,3 @@ +from senpy.plugins import SenpyPlugin + +plugin = SenpyPlugin("dummy") diff --git a/tests/extensions_test/__init__.py b/tests/extensions_test/__init__.py new file mode 100644 index 0000000..3a6f240 --- /dev/null +++ b/tests/extensions_test/__init__.py @@ -0,0 +1,70 @@ +import os +import logging +try: + import unittest.mock as mock +except ImportError: + import mock +from senpy.extensions import Senpy +from flask import Flask +from flask.ext.testing import TestCase + + +class Extensions_Test(TestCase): + def create_app(self): + self.app = Flask("test_extensions") + self.senpy = Senpy() + self.senpy.init_app(self.app) + self.dir = os.path.join(os.path.dirname(__file__), "..") + self.senpy.add_folder(self.dir) + return self.app + + def test_init(self): + """ Initialising the app with the extension. """ + assert hasattr(self.app, "senpy") + tapp = Flask("temp app") + tsen = Senpy(tapp) + assert hasattr(tapp, "senpy") + + def test_discovery(self): + """ Discovery of plugins in given folders. """ + assert self.dir in self.senpy._search_folders + print self.senpy.plugins + assert "dummy" in self.senpy.plugins + + + def test_enabling(self): + """ Enabling a plugin """ + self.senpy.enable_plugin("dummy") + assert self.senpy.plugins["dummy"].enabled + + def test_disabling(self): + """ Disabling a plugin """ + self.senpy.enable_plugin("dummy") + self.senpy.disable_plugin("dummy") + assert self.senpy.plugins["dummy"].enabled == False + + def test_default(self): + """ Default plugin should be set """ + assert self.senpy.default_plugin + assert self.senpy.default_plugin == "dummy" + + def test_analyse(self): + """ Using a plugin """ + with mock.patch.object(self.senpy.plugins["dummy"], "analyse") as mocked: + self.senpy.analyse(algorithm="dummy", input="tupni", output="tuptuo") + self.senpy.analyse(input="tupni", output="tuptuo") + mocked.assert_any_call(input="tupni", output="tuptuo", algorithm="dummy") + mocked.assert_any_call(input="tupni", output="tuptuo") + for plug in self.senpy.plugins: + self.senpy.disable_plugin(plug) + resp = self.senpy.analyse(input="tupni") + logging.debug("Response: {}".format(resp)) + assert resp["status"] == 400 + + def test_filtering(self): + """ Filtering plugins """ + assert len(self.senpy.filter_plugins(name="dummy"))>0 + assert not len(self.senpy.filter_plugins(name="notdummy")) + assert self.senpy.filter_plugins(name="dummy", enabled=True) + self.senpy.disable_plugin("dummy") + assert not len(self.senpy.filter_plugins(name="dummy", enabled=True))