Improved plugins, better tests, gevent

Moved from Yapsy again (it is not flexible enough), now we use a
custom solution.
The activation and deactivation of plugins is asynchronous, so
that plugins that take a long time don't interfere with the rest.

senpy/blueprints.py

@@ -110,23 +110,22 @@ def default():
 @nif_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
 def plugins(plugin=None, action="list"):
     filt = {}
+    sp = current_app.senpy
     if plugin:
         filt["name"] = plugin
-    plugs = current_app.senpy.filter_plugins(**filt)
+    plugs = sp.filter_plugins(**filt)
     if plugin and not plugs:
         return "Plugin not found", 400
     if action == "list":
         with_params = request.args.get("params", "") == "1"
-        dic = {plug: plugs[plug].jsonable(with_params) for plug in plugs}
+        if plugin:
+            dic = plugs[plugin].jsonable(with_params)
+        else:
+            dic = {plug: plugs[plug].jsonable(with_params) for plug in plugs}
         return jsonify(dic)
-    if action == "disable":
-        current_app.senpy.disable_plugin(plugin)
-        return "Ok"
-    elif action == "enable":
-        current_app.senpy.enable_plugin(plugin)
-        return "Ok"
-    elif action == "reload":
-        current_app.senpy.reload_plugin(plugin)
+    method = "{}_plugin".format(action)
+    if(hasattr(sp, method)):
+        getattr(sp, method)(plugin)
         return "Ok"
     else:
         return "action '{}' not allowed".format(action), 400

senpy/extensions.py

@@ -1,16 +1,21 @@
 """
 """
 import os
+import fnmatch
+import inspect
 import sys
 import imp
 import logging
+import gevent
+import json
 
 logger = logging.getLogger(__name__)
 
-from .plugins import SentimentPlugin, EmotionPlugin
+from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
 
 from .blueprints import nif_blueprint
 from git import Repo, InvalidGitRepositoryError
+from functools import partial
 
 
 class Senpy(object):
@@ -45,11 +50,13 @@ class Senpy(object):
         app.register_blueprint(nif_blueprint)
 
     def add_folder(self, folder):
+        logger.debug("Adding folder: %s", folder)
         if os.path.isdir(folder):
             self._search_folders.add(folder)
             self._outdated = True
             return True
         else:
+            logger.debug("Not a folder: %s", folder)
             return False
 
     def analyse(self, **params):
@@ -59,17 +66,20 @@ class Senpy(object):
             algo = params["algorithm"]
         elif self.plugins:
             algo = self.default_plugin
-        if algo in self.plugins and self.plugins[algo].enabled:
-            plug = self.plugins[algo]
-            resp = plug.analyse(**params)
-            resp.analysis.append(plug.jsonable())
-            return resp
+        if algo in self.plugins:
+            if self.plugins[algo].is_activated:
+                plug = self.plugins[algo]
+                resp = plug.analyse(**params)
+                resp.analysis.append(plug.jsonable())
+                return resp
+            logger.debug("Plugin not activated: {}".format(algo))
         else:
+            logger.debug("The algorithm '{}' is not valid\nValid algorithms: {}".format(algo, self.plugins.keys()))
             return {"status": 400, "message": "The algorithm '{}' is not valid".format(algo)}
 
     @property
     def default_plugin(self):
-        candidates = self.filter_plugins(enabled=True)
+        candidates = self.filter_plugins(is_activated=True)
         if len(candidates) > 0:
             candidate = candidates.keys()[0]
             logger.debug("Default: {}".format(candidate))
@@ -80,11 +90,38 @@ class Senpy(object):
     def parameters(self, algo):
         return getattr(self.plugins.get(algo or self.default_plugin), "params", {})
 
-    def enable_plugin(self, plugin):
-        self.plugins[plugin].enable()
+    def activate_all(self, sync=False):
+        ps = []
+        for plug in self.plugins.keys():
+            ps.append(self.activate_plugin(plug, sync=sync))
+        return ps
 
-    def disable_plugin(self, plugin):
-        self.plugins[plugin].disable()
+    def deactivate_all(self, sync=False):
+        ps = []
+        for plug in self.plugins.keys():
+            ps.append(self.deactivate_plugin(plug, sync=sync))
+        return ps
+
+    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
+        self.plugins[plugin_name].is_activated = active
+
+    def activate_plugin(self, plugin_name, sync=False):
+        plugin = self.plugins[plugin_name]
+        th = gevent.spawn(plugin.activate)
+        th.link_value(partial(self._set_active_plugin, plugin_name, True))
+        if sync:
+            th.join()
+        else:
+            return th
+
+    def deactivate_plugin(self, plugin_name, sync=False):
+        plugin = self.plugins[plugin_name]
+        th = gevent.spawn(plugin.deactivate)
+        th.link_value(partial(self._set_active_plugin, plugin_name, False))
+        if sync:
+            th.join()
+        else:
+            return th
 
     def reload_plugin(self, plugin):
         logger.debug("Reloading {}".format(plugin))
@@ -94,38 +131,47 @@ class Senpy(object):
         self.plugins[nplug.name] = nplug
 
     @staticmethod
-    def _load_plugin(plugin, search_folder, enabled=True):
-        logger.debug("Loading plugins")
-        sys.path.append(search_folder)
-        (fp, pathname, desc) = imp.find_module(plugin)
+    def _load_plugin(root, filename):
+        logger.debug("Loading plugin: {}".format(filename))
+        fpath = os.path.join(root, filename)
+        with open(fpath,'r') as f:
+            info = json.load(f)
+        logger.debug("Info: {}".format(info))
+        sys.path.append(root)
+        module = info["module"]
+        name = info["name"]
+        (fp, pathname, desc) = imp.find_module(module, [root,])
         try:
-            tmp = imp.load_module(plugin, fp, pathname, desc).plugin
-            sys.path.remove(search_folder)
-            tmp.path = search_folder
+            tmp = imp.load_module(module, fp, pathname, desc)
+            sys.path.remove(root)
+            candidate = None
+            for _, obj in inspect.getmembers(tmp):
+                if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
+                    logger.debug("Found plugin class: {}@{}".format(obj, inspect.getmodule(obj)))
+                    candidate = obj
+                    break
+            if not candidate:
+                logger.debug("No valid plugin for: {}".format(filename))
+                return
+            module = candidate(info=info)
             try:
-                repo_path = os.path.join(search_folder, plugin)
-                tmp.repo = Repo(repo_path)
+                repo_path = root
+                module.repo = Repo(repo_path)
             except InvalidGitRepositoryError:
-                tmp.repo = None
-            if not hasattr(tmp, "enabled"):
-                tmp.enabled = enabled
-            tmp.module = plugin
+                module.repo = None
         except Exception as ex:
-            tmp = None
-            logger.debug("Exception importing {}: {}".format(plugin, ex))
-        return tmp
+            logger.debug("Exception importing {}: {}".format(filename, ex))
+            return None, None
+        return name, module
 
     def _load_plugins(self):
         plugins = {}
         for search_folder in self._search_folders:
-            for item in os.listdir(search_folder):
-                if os.path.isdir(os.path.join(search_folder, item)) \
-                        and os.path.exists(os.path.join(search_folder,
-                                                        item,
-                                                        "__init__.py")):
-                    plugin = self._load_plugin(item, search_folder)
+            for root, dirnames, filenames in os.walk(search_folder):
+                for filename in fnmatch.filter(filenames, '*.senpy'):
+                    name, plugin = self._load_plugin(root, filename)
                     if plugin:
-                        plugins[plugin.name] = plugin
+                        plugins[name] = plugin
 
         self._outdated = False
         return plugins
@@ -133,10 +179,6 @@ class Senpy(object):
     def teardown(self, exception):
         pass
 
-    def enable_all(self):
-        for plugin in self.plugins:
-            self.enable_plugin(plugin)
-
     @property
     def plugins(self):
         """ Return the plugins registered for a given application.  """

senpy/models.py

@@ -77,7 +77,7 @@ class Entry(Leaf):
 
 
 class Opinion(Leaf):
-    opinionContext = {}
+    opinionContext = {"@vocab": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"}
     def __init__(self, polarityValue=None, hasPolarity=None, *args, **kwargs):
         super(Opinion, self).__init__(context=self.opinionContext,
                                       *args,
@@ -93,7 +93,14 @@ class EmotionSet(Leaf):
     def __init__(self, emotions=None, *args, **kwargs):
         if not emotions:
             emotions = []
-        super(EmotionSet, self).__init__(context=self.emotionContext,
+        super(EmotionSet, self).__init__(context=EmotionSet.emotionContext,
                                          *args,
                                          **kwargs)
         self.emotions = emotions or []
+
+class Emotion(Leaf):
+    emotionContext = {}
+    def __init__(self, emotions=None, *args, **kwargs):
+        super(EmotionSet, self).__init__(context=Emotion.emotionContext,
+                                         *args,
+                                         **kwargs)

senpy/plugins.py

@@ -1,4 +1,5 @@
 import logging
+import ConfigParser
 
 logger = logging.getLogger(__name__)
 
@@ -38,27 +39,28 @@ PARAMS = {"input": {"aliases": ["i", "input"],
 
 
 class SenpyPlugin(object):
-    def __init__(self, name=None, version=None, extraparams=None, params=None):
-        logger.debug("Initialising {}".format(name))
-        self.name = name
-        self.version = version
-        if params:
-            self.params = params
-        else:
-            self.params = PARAMS.copy()
-            if extraparams:
-                self.params.update(extraparams)
-        self.extraparams = extraparams or {}
-        self.enabled = True
+    def __init__(self, info=None):
+        if not info:
+            raise ValueError("You need to provide configuration information for the plugin.")
+        logger.debug("Initialising {}".format(info))
+        self.name = info["name"]
+        self.version = info["version"]
+        self.params = info.get("params", PARAMS.copy())
+        self.extra_params = info.get("extra_params", {})
+        self.params.update(self.extra_params)
+        self.is_activated = False
+        self.info = info
 
     def analyse(self, *args, **kwargs):
+        logger.debug("Analysing with: {} {}".format(self.name, self.version))
         pass
 
-    def enable(self):
-        self.enabled = True
+    def activate(self):
+        pass
+
+    def deactivate(self):
+        pass
 
-    def disable(self):
-        self.enabled = False
 
     @property
     def id(self):
@@ -66,15 +68,15 @@ class SenpyPlugin(object):
 
     def jsonable(self, parameters=False):
         resp = {
-            "@id": self.id,
-            "enabled": self.enabled,
+            "@id": "{}_{}".format(self.name, self.version),
+            "is_activated": self.is_activated,
         }
         if hasattr(self, "repo") and self.repo:
             resp["repo"] = self.repo.remotes[0].url
         if parameters:
             resp["parameters"] = self.params
-        elif self.extraparams:
-            resp["extra_parameters"] = self.extraparams
+        elif self.extra_params:
+            resp["extra_parameters"] = self.extra_params
         return resp