First tests

plugins-bak/prueba/__init__.py

@@ -0,0 +1,9 @@
+from senpy.plugins import SenpyPlugin
+
+class Prueba(SenpyPlugin):
+    def __init__(self, **kwargs):
+        super(Prueba, self).__init__(name="prueba",
+                                                 version="4.0",
+                                                 **kwargs)
+
+plugin = Prueba()

senpy/__init__.py

@@ -18,7 +18,7 @@
 Sentiment analysis server in Python
 '''
 
-VERSION = "0.2.5"
+VERSION = "0.2.6"
 
 import extensions
 import blueprints

senpy/blueprints.py

@@ -17,8 +17,11 @@
 '''
 Simple Sentiment Analysis server
 '''
-from flask import Blueprint, render_template, request, jsonify, current_app
 import json
+import logging
+logger = logging.getLogger(__name__)
+
+from flask import Blueprint, render_template, request, jsonify, current_app
 
 nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__)
 
@@ -86,10 +89,12 @@ def home(entries=None):
         algo = get_params(request).get("algorithm", None)
         specific_params = current_app.senpy.parameters(algo)
         params = get_params(request, specific_params)
-    except ValueError as ex:
-        return ex.message
         response = current_app.senpy.analyse(**params)
         return jsonify(response)
+    except ValueError as ex:
+        return ex.message
+    except Exception as ex:
+        return jsonify(status="400", message=ex.message)
 
 @nif_blueprint.route("/default")
 def default():

senpy/extensions.py

@@ -1,9 +1,10 @@
 import os
 import sys
 import imp
+import logging
+
+logger = logging.getLogger(__name__)
 
-from flask import current_app
-from collections import defaultdict
 from .plugins import SentimentPlugin, EmotionPlugin
 
 try:
@@ -11,29 +12,33 @@ try:
 except ImportError:
     from flask import _request_ctx_stack as stack
 
-from blueprints import nif_blueprint
+from .blueprints import nif_blueprint
 from git import Repo, InvalidGitRepositoryError
 
+
 class Senpy(object):
+    """ Default Senpy extension for Flask """
 
     def __init__(self, app=None, plugin_folder="plugins"):
         self.app = app
         base_folder = os.path.join(os.path.dirname(__file__), "plugins")
 
-        self.search_folders = (folder for folder in (base_folder, plugin_folder, '/tmp/plugins')
-                                if folder and os.path.isdir(folder))
+        self._search_folders = set()
+        self._outdated = True
+
+        for folder in (base_folder, plugin_folder):
+            self.add_folder(folder)
 
         if app is not None:
             self.init_app(app)
 
+    def init_app(self, app):
+        """ Initialise a flask app to add plugins to its context """
         """
-
         Note: I'm not particularly fond of adding self.app and app.senpy, but
         I can't think of a better way to do it.
         """
-    def init_app(self, app, plugin_folder="plugins"):
         app.senpy = self
-        #app.config.setdefault('SQLITE3_DATABASE', ':memory:')
         # Use the newstyle teardown_appcontext if it's available,
         # otherwise fall back to the request context
         if hasattr(app, 'teardown_appcontext'):
@@ -42,9 +47,18 @@ class Senpy(object):
             app.teardown_request(self.teardown)
         app.register_blueprint(nif_blueprint)
 
+    def add_folder(self, folder):
+        if os.path.isdir(folder):
+            self._search_folders.add(folder)
+            self._outdated = True
+            return True
+        else:
+            return False
+
+
     def analyse(self, **params):
         algo = None
-        print("analysing with params: {}".format(params))
+        logger.debug("analysing with params: {}".format(params))
         if "algorithm" in params:
             algo = params["algorithm"]
         elif self.plugins:
@@ -55,36 +69,39 @@ class Senpy(object):
             resp.analysis.append(plug.jsonable())
             return resp
         else:
-            return {"status": 500, "message": "No valid algorithm"}
+            return {"status": 400, "message": "The algorithm '{}' is not valid".format(algo) }
 
     @property
     def default_plugin(self):
-        if self.plugins:
-            candidate = self.filter_plugins(enabled=True).keys()[0]
-            print("Default: {}".format(candidate))
+        candidates = self.filter_plugins(enabled=True)
+        if len(candidates)>1:
+            candidate = candidates.keys()[0]
+            logger.debug("Default: {}".format(candidate))
             return candidate
         else:
-            return Exception("No algorithm")
+            return None
 
     def parameters(self, algo):
         return getattr(self.plugins.get(algo or self.default_plugin), "params", {})
 
     def enable_plugin(self, plugin):
-        self.plugins[plugin].disable()
+        self.plugins[plugin].enable()
 
     def disable_plugin(self, plugin):
         self.plugins[plugin].disable()
 
     def reload_plugin(self, plugin):
-        print("Reloading {}".format(plugin))
+        logger.debug("Reloading {}".format(plugin))
         plug = self.plugins[plugin]
         nplug = self._load_plugin(plug.module, plug.path)
         del self.plugins[plugin]
         self.plugins[nplug.name] = nplug
 
     def _load_plugin(self, plugin, search_folder, enabled=True):
+        logger.debug("Loading plugins")
         sys.path.append(search_folder)
         (fp, pathname, desc) = imp.find_module(plugin)
+        try:
             tmp = imp.load_module(plugin, fp, pathname, desc).plugin
             sys.path.remove(search_folder)
             tmp.path = search_folder
@@ -96,19 +113,24 @@ class Senpy(object):
             if not hasattr(tmp, "enabled"):
                 tmp.enabled = enabled
             tmp.module = plugin
+        except Exception as ex:
+            tmp = None
+            logger.debug("Exception importing {}: {}".format(plugin, ex))
         return tmp
 
 
     def _load_plugins(self):
         plugins = {}
-        for search_folder in self.search_folders:
+        for search_folder in self._search_folders:
             for item in os.listdir(search_folder):
                 if os.path.isdir(os.path.join(search_folder, item)) \
                         and os.path.exists(
                             os.path.join(search_folder, item, "__init__.py")):
                     plugin = self._load_plugin(item, search_folder)
+                    if plugin:
                         plugins[plugin.name] = plugin
 
+        self._outdated = False
         return plugins
 
     def teardown(self, exception):
@@ -126,16 +148,20 @@ class Senpy(object):
 
     @property
     def plugins(self):
+        """ Return the plugins registered for a given application.  """
         ctx = stack.top
         if ctx is not None:
-            if not hasattr(self, '_plugins'):
-                self._plugins = self._load_plugins()
-            return self._plugins
+            if not hasattr(ctx, 'senpy_plugins') or self._outdated:
+                ctx.senpy_plugins = self._load_plugins()
+            return ctx.senpy_plugins
 
     def filter_plugins(self, **kwargs):
+        """ Filter plugins by different criteria """
         def matches(plug):
             res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
-            print("matching {} with {}: {}".format(plug.name, kwargs, res))
+            logger.debug("matching {} with {}: {}".format(plug.name,
+                                                          kwargs,
+                                                          res))
             return res
         if not kwargs:
             return self.plugins
@@ -143,13 +169,15 @@ class Senpy(object):
             return {n:p for n, p in self.plugins.items() if matches(p)}
 
     def sentiment_plugins(self):
-        return (plugin for plugin in self.plugins if
-                isinstance(plugin, SentimentPlugin))
+        """ Return only the sentiment plugins """
+        return {p:plugin for p, plugin in self.plugins.items() if
+                isinstance(plugin, SentimentPlugin)}
+
 
 if __name__ == '__main__':
     from flask import Flask
     app = Flask(__name__)
     sp = Senpy()
-    sp.init_app(app)
-    with app.app_context():
+    sp.init_app(APP)
+    with APP.app_context():
         sp._load_plugins()

senpy/plugins.py

@@ -1,3 +1,7 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
 PARAMS = {"input": {"aliases": ["i", "input"],
                     "required": True,
                     "help": "Input text"
@@ -30,7 +34,7 @@ PARAMS = {"input": {"aliases": ["i", "input"],
 
 class SenpyPlugin(object):
     def __init__(self, name=None, version=None, extraparams=None, params=None):
-        print("Initing {}".format(name))
+        logger.debug("Initialising {}".format(name))
         self.name = name
         self.version = version
         if params:

senpy/plugins/sentiment140/sentiment140.py

@@ -1,41 +0,0 @@
-'''
-SENTIMENT140
-=============
-
-* http://www.sentiment140.com/api/bulkClassifyJson
-* Method: POST
-* Parameters: JSON Object (that is copied to the result)
-    * text
-    * query
-    * language
-    * topic
-
-* Example response:
-```json
-{"data": [{"text": "I love Titanic.", "id":1234, "polarity": 4},
-          {"text": "I hate Titanic.", "id":4567, "polarity": 0}]}
-```
-'''
-import requests
-import json
-
-ENDPOINT_URI = "http://www.sentiment140.com/api/bulkClassifyJson"
-
-def analyse(texts):
-    parameters = {"data": []}
-    if isinstance(texts, list):
-        for text in texts:
-            parameters["data"].append({"text": text})
-    else:
-        parameters["data"].append({"text": texts})
-
-    res = requests.post(ENDPOINT_URI, json.dumps(parameters))
-    res.json()
-    return res.json()
-
-def test():
-    print analyse("I love Titanic")
-    print analyse(["I love Titanic", "I hate Titanic"])
-
-if __name__ == "__main__":
-    test()

tests/blueprints_test/__init__.py

@@ -0,0 +1,35 @@
+import os
+import logging
+try:
+    import unittest.mock as mock
+except ImportError:
+    import mock
+from senpy.extensions import Senpy
+from flask import Flask
+from flask.ext.testing import TestCase
+
+def check_dict(indic, template):
+        return all(item in indic.items() for item in template.items())
+
+class Blueprints_Test(TestCase):
+    def create_app(self):
+        self.app = Flask("test_extensions")
+        self.senpy = Senpy()
+        self.senpy.init_app(self.app)
+        self.dir = os.path.join(os.path.dirname(__file__), "..")
+        self.senpy.add_folder(self.dir)
+        return self.app
+
+
+    def test_home(self):
+        """ Calling with no arguments should ask the user for more arguments """
+        resp = self.client.get("/")
+        self.assert200(resp)
+        logging.debug(resp.json)
+        assert resp.json["status"] == "failed"
+        atleast = {
+            "status": "failed",
+            "message": "Missing or invalid parameters",
+        }
+        assert check_dict(resp.json, atleast)
+

tests/dummy_plugin/__init__.py

@@ -0,0 +1,3 @@
+from senpy.plugins import SenpyPlugin
+
+plugin = SenpyPlugin("dummy")

tests/extensions_test/__init__.py

@@ -0,0 +1,70 @@
+import os
+import logging
+try:
+    import unittest.mock as mock
+except ImportError:
+    import mock
+from senpy.extensions import Senpy
+from flask import Flask
+from flask.ext.testing import TestCase
+
+
+class Extensions_Test(TestCase):
+    def create_app(self):
+        self.app = Flask("test_extensions")
+        self.senpy = Senpy()
+        self.senpy.init_app(self.app)
+        self.dir = os.path.join(os.path.dirname(__file__), "..")
+        self.senpy.add_folder(self.dir)
+        return self.app
+
+    def test_init(self):
+        """ Initialising the app with the extension.  """
+        assert hasattr(self.app, "senpy")
+        tapp = Flask("temp app")
+        tsen = Senpy(tapp)
+        assert hasattr(tapp, "senpy")
+
+    def test_discovery(self):
+        """ Discovery of plugins in given folders.  """
+        assert self.dir in self.senpy._search_folders
+        print self.senpy.plugins
+        assert "dummy" in self.senpy.plugins
+
+
+    def test_enabling(self):
+        """ Enabling a plugin """
+        self.senpy.enable_plugin("dummy")
+        assert self.senpy.plugins["dummy"].enabled
+
+    def test_disabling(self):
+        """ Disabling a plugin """
+        self.senpy.enable_plugin("dummy")
+        self.senpy.disable_plugin("dummy")
+        assert self.senpy.plugins["dummy"].enabled == False
+
+    def test_default(self):
+        """ Default plugin should be set """
+        assert self.senpy.default_plugin
+        assert self.senpy.default_plugin == "dummy"
+
+    def test_analyse(self):
+        """ Using a plugin """
+        with mock.patch.object(self.senpy.plugins["dummy"], "analyse") as mocked:
+            self.senpy.analyse(algorithm="dummy", input="tupni", output="tuptuo")
+            self.senpy.analyse(input="tupni", output="tuptuo")
+        mocked.assert_any_call(input="tupni", output="tuptuo", algorithm="dummy")
+        mocked.assert_any_call(input="tupni", output="tuptuo")
+        for plug in self.senpy.plugins:
+            self.senpy.disable_plugin(plug)
+        resp = self.senpy.analyse(input="tupni")
+        logging.debug("Response: {}".format(resp))
+        assert resp["status"] == 400
+
+    def test_filtering(self):
+        """ Filtering plugins """
+        assert len(self.senpy.filter_plugins(name="dummy"))>0
+        assert not len(self.senpy.filter_plugins(name="notdummy"))
+        assert self.senpy.filter_plugins(name="dummy", enabled=True)
+        self.senpy.disable_plugin("dummy")
+        assert not len(self.senpy.filter_plugins(name="dummy", enabled=True))