First tests

senpy/__init__.py

@@ -18,7 +18,7 @@
 Sentiment analysis server in Python
 '''
 
-VERSION = "0.2.5"
+VERSION = "0.2.6"
 
 import extensions
 import blueprints

senpy/blueprints.py

@@ -17,8 +17,11 @@
 '''
 Simple Sentiment Analysis server
 '''
-from flask import Blueprint, render_template, request, jsonify, current_app
 import json
+import logging
+logger = logging.getLogger(__name__)
+
+from flask import Blueprint, render_template, request, jsonify, current_app
 
 nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__)
 
@@ -86,10 +89,12 @@ def home(entries=None):
         algo = get_params(request).get("algorithm", None)
         specific_params = current_app.senpy.parameters(algo)
         params = get_params(request, specific_params)
+        response = current_app.senpy.analyse(**params)
+        return jsonify(response)
     except ValueError as ex:
         return ex.message
-    response = current_app.senpy.analyse(**params)
-    return jsonify(response)
+    except Exception as ex:
+        return jsonify(status="400", message=ex.message)
 
 @nif_blueprint.route("/default")
 def default():

senpy/extensions.py

@@ -1,9 +1,10 @@
 import os
 import sys
 import imp
+import logging
+
+logger = logging.getLogger(__name__)
 
-from flask import current_app
-from collections import defaultdict
 from .plugins import SentimentPlugin, EmotionPlugin
 
 try:
@@ -11,29 +12,33 @@ try:
 except ImportError:
     from flask import _request_ctx_stack as stack
 
-from blueprints import nif_blueprint
+from .blueprints import nif_blueprint
 from git import Repo, InvalidGitRepositoryError
 
+
 class Senpy(object):
+    """ Default Senpy extension for Flask """
 
     def __init__(self, app=None, plugin_folder="plugins"):
         self.app = app
         base_folder = os.path.join(os.path.dirname(__file__), "plugins")
 
-        self.search_folders = (folder for folder in (base_folder, plugin_folder, '/tmp/plugins')
-                                if folder and os.path.isdir(folder))
+        self._search_folders = set()
+        self._outdated = True
+
+        for folder in (base_folder, plugin_folder):
+            self.add_folder(folder)
 
         if app is not None:
             self.init_app(app)
 
-    """
-
+    def init_app(self, app):
+        """ Initialise a flask app to add plugins to its context """
+        """
         Note: I'm not particularly fond of adding self.app and app.senpy, but
         I can't think of a better way to do it.
-    """
-    def init_app(self, app, plugin_folder="plugins"):
+        """
         app.senpy = self
-        #app.config.setdefault('SQLITE3_DATABASE', ':memory:')
         # Use the newstyle teardown_appcontext if it's available,
         # otherwise fall back to the request context
         if hasattr(app, 'teardown_appcontext'):
@@ -42,9 +47,18 @@ class Senpy(object):
             app.teardown_request(self.teardown)
         app.register_blueprint(nif_blueprint)
 
+    def add_folder(self, folder):
+        if os.path.isdir(folder):
+            self._search_folders.add(folder)
+            self._outdated = True
+            return True
+        else:
+            return False
+
+
     def analyse(self, **params):
         algo = None
-        print("analysing with params: {}".format(params))
+        logger.debug("analysing with params: {}".format(params))
         if "algorithm" in params:
             algo = params["algorithm"]
         elif self.plugins:
@@ -55,60 +69,68 @@ class Senpy(object):
             resp.analysis.append(plug.jsonable())
             return resp
         else:
-            return {"status": 500, "message": "No valid algorithm"}
+            return {"status": 400, "message": "The algorithm '{}' is not valid".format(algo) }
 
     @property
     def default_plugin(self):
-        if self.plugins:
-            candidate = self.filter_plugins(enabled=True).keys()[0]
-            print("Default: {}".format(candidate))
+        candidates = self.filter_plugins(enabled=True)
+        if len(candidates)>1:
+            candidate = candidates.keys()[0]
+            logger.debug("Default: {}".format(candidate))
             return candidate
         else:
-            return Exception("No algorithm")
+            return None
 
     def parameters(self, algo):
         return getattr(self.plugins.get(algo or self.default_plugin), "params", {})
 
     def enable_plugin(self, plugin):
-        self.plugins[plugin].disable()
+        self.plugins[plugin].enable()
 
     def disable_plugin(self, plugin):
         self.plugins[plugin].disable()
 
     def reload_plugin(self, plugin):
-        print("Reloading {}".format(plugin))
+        logger.debug("Reloading {}".format(plugin))
         plug = self.plugins[plugin]
         nplug = self._load_plugin(plug.module, plug.path)
         del self.plugins[plugin]
         self.plugins[nplug.name] = nplug
 
     def _load_plugin(self, plugin, search_folder, enabled=True):
+        logger.debug("Loading plugins")
         sys.path.append(search_folder)
         (fp, pathname, desc) = imp.find_module(plugin)
-        tmp = imp.load_module(plugin, fp, pathname, desc).plugin
-        sys.path.remove(search_folder)
-        tmp.path = search_folder
         try:
-            repo_path = os.path.join(search_folder, plugin)
-            tmp.repo = Repo(repo_path)
-        except InvalidGitRepositoryError:
-            tmp.repo = None
-        if not hasattr(tmp, "enabled"):
-            tmp.enabled = enabled
-        tmp.module = plugin
+            tmp = imp.load_module(plugin, fp, pathname, desc).plugin
+            sys.path.remove(search_folder)
+            tmp.path = search_folder
+            try:
+                repo_path = os.path.join(search_folder, plugin)
+                tmp.repo = Repo(repo_path)
+            except InvalidGitRepositoryError:
+                tmp.repo = None
+            if not hasattr(tmp, "enabled"):
+                tmp.enabled = enabled
+            tmp.module = plugin
+        except Exception as ex:
+            tmp = None
+            logger.debug("Exception importing {}: {}".format(plugin, ex))
         return tmp
 
 
     def _load_plugins(self):
         plugins = {}
-        for search_folder in self.search_folders:
+        for search_folder in self._search_folders:
             for item in os.listdir(search_folder):
                 if os.path.isdir(os.path.join(search_folder, item)) \
                         and os.path.exists(
                             os.path.join(search_folder, item, "__init__.py")):
                     plugin = self._load_plugin(item, search_folder)
-                    plugins[plugin.name] = plugin
+                    if plugin:
+                        plugins[plugin.name] = plugin
 
+        self._outdated = False
         return plugins
 
     def teardown(self, exception):
@@ -126,30 +148,36 @@ class Senpy(object):
 
     @property
     def plugins(self):
+        """ Return the plugins registered for a given application.  """
         ctx = stack.top
         if ctx is not None:
-            if not hasattr(self, '_plugins'):
-                self._plugins = self._load_plugins()
-            return self._plugins
+            if not hasattr(ctx, 'senpy_plugins') or self._outdated:
+                ctx.senpy_plugins = self._load_plugins()
+            return ctx.senpy_plugins
 
     def filter_plugins(self, **kwargs):
+        """ Filter plugins by different criteria """
         def matches(plug):
-            res = all(getattr(plug, k, None)==v for (k,v) in kwargs.items())
-            print("matching {} with {}: {}".format(plug.name, kwargs, res))
+            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
+            logger.debug("matching {} with {}: {}".format(plug.name,
+                                                          kwargs,
+                                                          res))
             return res
         if not kwargs:
             return self.plugins
         else:
-            return {n:p for n,p in self.plugins.items() if matches(p)}
+            return {n:p for n, p in self.plugins.items() if matches(p)}
 
     def sentiment_plugins(self):
-        return (plugin for plugin in self.plugins if
-                isinstance(plugin, SentimentPlugin))
+        """ Return only the sentiment plugins """
+        return {p:plugin for p, plugin in self.plugins.items() if
+                isinstance(plugin, SentimentPlugin)}
+
 
 if __name__ == '__main__':
     from flask import Flask
     app = Flask(__name__)
     sp = Senpy()
-    sp.init_app(app)
-    with app.app_context():
+    sp.init_app(APP)
+    with APP.app_context():
         sp._load_plugins()

senpy/plugins.py

@@ -1,3 +1,7 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
 PARAMS = {"input": {"aliases": ["i", "input"],
                     "required": True,
                     "help": "Input text"
@@ -30,7 +34,7 @@ PARAMS = {"input": {"aliases": ["i", "input"],
 
 class SenpyPlugin(object):
     def __init__(self, name=None, version=None, extraparams=None, params=None):
-        print("Initing {}".format(name))
+        logger.debug("Initialising {}".format(name))
         self.name = name
         self.version = version
         if params:

senpy/plugins/sentiment140/__init__.py

@@ -1,47 +0,0 @@
-import requests
-import json
-
-import sys
-
-from senpy.plugins import SentimentPlugin
-from senpy.models import Response, Opinion, Entry
-
-class Sentiment140Plugin(SentimentPlugin):
-    EXTRA_PARAMS = {
-        "language": {"aliases": ["language", "l"],
-                     "required": False,
-                     "options": ["es", "en", "auto"],
-                     }
-    }
-    def __init__(self, **kwargs):
-        super(Sentiment140Plugin, self).__init__(name="sentiment140",
-                                                 version="2.0",
-                                                 extraparams=self.EXTRA_PARAMS,
-                                                 **kwargs)
-
-    def analyse(self, **params):
-        lang = params.get("language", "auto")
-        res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
-                            json.dumps({
-                                "language": lang,
-                                "data": [{"text": params["input"]}]}
-                                    ))
-
-
-        response = Response()
-        polarityValue = int(res.json()["data"][0]["polarity"]) * 25
-        polarity = "marl:Neutral"
-        if polarityValue > 50:
-            polarity = "marl:Positive"
-        elif polarityValue < 50:
-            polarity = "marl:Negative"
-
-
-        entry = Entry(text=params["input"])
-        opinion = Opinion(polarity=polarity, polarityValue=polarityValue)
-        entry.opinions.append(opinion)
-        entry.language = lang
-        response.entries.append(entry)
-        return response
-
-plugin = Sentiment140Plugin()

senpy/plugins/sentiment140/sentiment140.py

@@ -1,41 +0,0 @@
-'''
-SENTIMENT140
-=============
-
-* http://www.sentiment140.com/api/bulkClassifyJson
-* Method: POST
-* Parameters: JSON Object (that is copied to the result)
-    * text
-    * query
-    * language
-    * topic
-
-* Example response:
-```json
-{"data": [{"text": "I love Titanic.", "id":1234, "polarity": 4},
-          {"text": "I hate Titanic.", "id":4567, "polarity": 0}]}
-```
-'''
-import requests
-import json
-
-ENDPOINT_URI = "http://www.sentiment140.com/api/bulkClassifyJson"
-
-def analyse(texts):
-    parameters = {"data": []}
-    if isinstance(texts, list):
-        for text in texts:
-            parameters["data"].append({"text": text})
-    else:
-        parameters["data"].append({"text": texts})
-
-    res = requests.post(ENDPOINT_URI, json.dumps(parameters))
-    res.json()
-    return res.json()
-
-def test():
-    print analyse("I love Titanic")
-    print analyse(["I love Titanic", "I hate Titanic"])
-
-if __name__ == "__main__":
-    test()