Improved plugins API and loading

Also:

* added drone-ci integration: tests for py2.7 and py3

.drone.yml

@@ -0,0 +1,9 @@
+build:
+  image: python:$$PYTHON_VERSION
+  commands:
+    - python setup.py test
+
+matrix:
+  PYTHON_VERSION:
+    - 2.7
+    - 3.4

senpy/__main__.py

@@ -63,7 +63,9 @@ def main():
                         default="plugins",
                         help='Where to look for plugins.')
     args = parser.parse_args()
-    logging.basicConfig(level=getattr(logging, args.level))
+    logging.basicConfig()
+    rl = logging.getLogger()
+    rl.setLevel(getattr(logging, args.level))
     app = Flask(__name__)
     app.debug = args.debug
     sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins)

senpy/blueprints.py

@@ -18,7 +18,7 @@
 Blueprints for Senpy
 """
 from flask import Blueprint, request, current_app, render_template
-from .models import Error, Response
+from .models import Error, Response, Plugins
 from future.utils import iteritems
 
 import json
@@ -29,7 +29,7 @@ logger = logging.getLogger(__name__)
 nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__)
 demo_blueprint = Blueprint("Demo of the service. It includes an HTML+Javascript playground to test senpy", __name__)
 
-BASIC_PARAMS = {
+API_PARAMS = {
     "algorithm": {
         "aliases": ["algorithm", "a", "algo"],
         "required": False,
@@ -41,6 +41,63 @@ BASIC_PARAMS = {
     }
 }
 
+BASIC_PARAMS = {
+    "algorithm": {
+        "aliases": ["algorithm", "a", "algo"],
+        "required": False,
+    },
+    "inHeaders": {
+        "aliases": ["inHeaders", "headers"],
+        "required": True,
+        "default": "0"
+    },
+    "input": {
+        "@id": "input",
+        "aliases": ["i", "input"],
+        "required": True,
+        "help": "Input text"
+    },
+    "informat": {
+        "@id": "informat",
+        "aliases": ["f", "informat"],
+        "required": False,
+        "default": "text",
+        "options": ["turtle", "text"],
+    },
+    "intype": {
+        "@id": "intype",
+        "aliases": ["intype", "t"],
+        "required": False,
+        "default": "direct",
+        "options": ["direct", "url", "file"],
+    },
+    "outformat": {
+        "@id": "outformat",
+        "aliases": ["outformat", "o"],
+        "default": "json-ld",
+        "required": False,
+        "options": ["json-ld"],
+    },
+    "language": {
+        "@id": "language",
+        "aliases": ["language", "l"],
+        "required": False,
+    },
+    "prefix": {
+        "@id": "prefix",
+        "aliases": ["prefix", "p"],
+        "required": True,
+        "default": "",
+    },
+    "urischeme": {
+        "@id": "urischeme",
+        "aliases": ["urischeme", "u"],
+        "required": False,
+        "default": "RFC5147String",
+        "options": "RFC5147String"
+    },
+}
+
 def get_params(req, params=BASIC_PARAMS):
     if req.method == 'POST':
         indict = req.form
@@ -119,36 +176,29 @@ def api():
         return ex.message.flask()
 
 
-@nif_blueprint.route("/default")
-def default():
-    # return current_app.senpy.default_plugin
-    plug = current_app.senpy.default_plugin
-    if plug:
-        return plugins(action="list", plugin=plug.name)
-    else:
-        error = Error(status=404, message="No plugins found")
-        return error.flask()
-
-
 @nif_blueprint.route('/plugins/', methods=['POST', 'GET'])
+def plugins():
+    in_headers = get_params(request, API_PARAMS)["inHeaders"] != "0"
+    sp = current_app.senpy
+    dic = Plugins(plugins=list(sp.plugins.values()))
+    return dic.flask(in_headers=in_headers)
+    
 @nif_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
 @nif_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
-def plugins(plugin=None, action="list"):
+def plugin(plugin=None, action="list"):
     filt = {}
     sp = current_app.senpy
-    if plugin:
+    plugs = sp.filter_plugins(name=plugin)
-        filt["name"] = plugin
+    if plugin == 'default' and sp.default_plugin:
-    plugs = sp.filter_plugins(**filt)
+        response = sp.default_plugin
-    if plugin and not plugs:
+        plugin = response.name
-        return "Plugin not found", 400
+    elif plugin in sp.plugins:
+        response = sp.plugins[plugin]
+    else:
+        return Error(message="Plugin not found", status=404).flask()
     if action == "list":
-        in_headers = get_params(request, BASIC_PARAMS)["inHeaders"] != "0"
+        in_headers = get_params(request, API_PARAMS)["inHeaders"] != "0"
-        if plugin:
+        return response.flask(in_headers=in_headers)
-            dic = plugs[plugin]
-        else:
-            dic = Response(
-                    {plug: plugs[plug].serializable() for plug in plugs})
-        return dic.flask(in_headers=in_headers)
     method = "{}_plugin".format(action)
     if(hasattr(sp, method)):
         getattr(sp, method)(plugin)
@@ -156,7 +206,6 @@ def plugins(plugin=None, action="list"):
     else:
         return Error(message="action '{}' not allowed".format(action)).flask()
 
-
 if __name__ == '__main__':
     import config
 

senpy/extensions.py

@@ -34,6 +34,7 @@ class Senpy(object):
         self.app = app
 
         self._search_folders = set()
+        self._plugin_list = []
         self._outdated = True
 
         self.add_folder(plugin_folder)
@@ -65,10 +66,8 @@ class Senpy(object):
         if os.path.isdir(folder):
             self._search_folders.add(folder)
             self._outdated = True
-            return True
         else:
             logger.debug("Not a folder: %s", folder)
-            return False
 
     def analyse(self, **params):
         algo = None
@@ -113,7 +112,7 @@ class Senpy(object):
 
     def parameters(self, algo):
         return getattr(self.plugins.get(algo) or self.default_plugin,
-                       "params",
+                       "extra_params",
                        {})
 
     def activate_all(self, sync=False):
@@ -129,13 +128,18 @@ class Senpy(object):
         return ps
 
     def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
+        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
+        Note that plugins may activate themselves by setting this variable.
+        '''
         self.plugins[plugin_name].is_activated = active
 
     def activate_plugin(self, plugin_name, sync=False):
         plugin = self.plugins[plugin_name]
+        logger.info("Activating plugin: {}".format(plugin.name))
         def act():
             try:
                 plugin.activate()
+                logger.info("Plugin activated: {}".format(plugin.name))
             except Exception as ex:
                 logger.error("Error activating plugin {}: {}".format(plugin.name,
                                                                      ex))
@@ -149,19 +153,33 @@ class Senpy(object):
 
     def deactivate_plugin(self, plugin_name, sync=False):
         plugin = self.plugins[plugin_name]
-        th = gevent.spawn(plugin.deactivate)
+
+        def deact():
+            try:
+                plugin.deactivate()
+                logger.info("Plugin deactivated: {}".format(plugin.name))
+            except Exception as ex:
+                logger.error("Error deactivating plugin {}: {}".format(plugin.name,
+                                                                       ex))
+                logger.error("Trace: {}".format(traceback.format_exc()))
+
+        th = gevent.spawn(deact)
         th.link_value(partial(self._set_active_plugin, plugin_name, False))
         if sync:
             th.join()
         else:
             return th
 
-    def reload_plugin(self, plugin):
+    def reload_plugin(self, name):
-        logger.debug("Reloading {}".format(plugin))
+        logger.debug("Reloading {}".format(name))
-        plug = self.plugins[plugin]
+        plugin = self.plugins[name]
-        nplug = self._load_plugin(plug.module, plug.path)
+        try:
-        del self.plugins[plugin]
+            del self.plugins[name]
-        self.plugins[nplug.name] = nplug
+            nplug = self._load_plugin(plugin.module, plugin.path)
+            self.plugins[nplug.name] = nplug
+        except Exception as ex:
+            logger.error('Error reloading {}: {}'.format(name, ex))
+            self.plugins[name] = plugin
 
     @staticmethod
     def _load_plugin(root, filename):
@@ -206,7 +224,7 @@ class Senpy(object):
             for root, dirnames, filenames in os.walk(search_folder):
                 for filename in fnmatch.filter(filenames, '*.senpy'):
                     name, plugin = self._load_plugin(root, filename)
-                    if plugin:
+                    if plugin and name not in self._plugin_list:
                         plugins[name] = plugin
 
         self._outdated = False
@@ -218,9 +236,9 @@ class Senpy(object):
     @property
     def plugins(self):
         """ Return the plugins registered for a given application.  """
-        if not hasattr(self, 'senpy_plugins') or self._outdated:
+        if self._outdated:
-            self.senpy_plugins = self._load_plugins()
+            self._plugin_list = self._load_plugins()
-        return self.senpy_plugins
+        return self._plugin_list
 
     def filter_plugins(self, **kwargs):
         """ Filter plugins by different criteria """

senpy/models.py

@@ -117,11 +117,16 @@ class SenpyMixin(object):
                         sort_keys=True)
         return js
 
+    def validate(self, obj=None):
+        if not obj:
+            obj = self
+        if hasattr(obj, "jsonld"):
+            obj = obj.jsonld()
+        jsonschema.validate(obj, self.schema)
     
 class SenpyModel(SenpyMixin, dict):
 
     schema = base_schema
-    prefix = None
 
     def __init__(self, *args, **kwargs):
         temp = dict(*args, **kwargs)
@@ -163,14 +168,6 @@ class SenpyModel(SenpyMixin, dict):
         self.__delitem__(self._get_key(key))
         
     
-    def validate(self, obj=None):
-        if not obj:
-            obj = self
-        if hasattr(obj, "jsonld"):
-            obj = obj.jsonld()
-        jsonschema.validate(obj, self.schema)
-        
-    
     @classmethod
     def from_base(cls, name):
         subschema = base_schema[name]

senpy/plugins.py

@@ -9,55 +9,6 @@ from .models import Response, PluginModel, Error
 
 logger = logging.getLogger(__name__)
 
-PARAMS = {
-    "input": {
-        "@id": "input",
-        "aliases": ["i", "input"],
-        "required": True,
-        "help": "Input text"
-    },
-    "informat": {
-        "@id": "informat",
-        "aliases": ["f", "informat"],
-        "required": False,
-        "default": "text",
-        "options": ["turtle", "text"],
-    },
-    "intype": {
-        "@id": "intype",
-        "aliases": ["intype", "t"],
-        "required": False,
-        "default": "direct",
-        "options": ["direct", "url", "file"],
-    },
-    "outformat": {
-        "@id": "outformat",
-        "aliases": ["outformat", "o"],
-        "default": "json-ld",
-        "required": False,
-        "options": ["json-ld"],
-    },
-    "language": {
-        "@id": "language",
-        "aliases": ["language", "l"],
-        "required": False,
-    },
-    "prefix": {
-        "@id": "prefix",
-        "aliases": ["prefix", "p"],
-        "required": True,
-        "default": "",
-    },
-    "urischeme": {
-        "@id": "urischeme",
-        "aliases": ["urischeme", "u"],
-        "required": False,
-        "default": "RFC5147String",
-        "options": "RFC5147String"
-    },
-}
-
-
 class SenpyPlugin(PluginModel):
 
     def __init__(self, info=None):
@@ -65,14 +16,12 @@ class SenpyPlugin(PluginModel):
             raise Error(message=("You need to provide configuration"
                                  "information for the plugin."))
         logger.debug("Initialising {}".format(info))
-        self.name = info["name"]
+        super(SenpyPlugin, self).__init__(info)
-        self.version = info["version"]
+        self.params = info.get("extra_params", {})
-        self.params = info.get("params", PARAMS.copy())
+        self._info = info
         if "@id" not in self.params:
             self.params["@id"] = "params_%s" % self.id
         self.is_activated = False
-        self._info = info
-        super(SenpyPlugin, self).__init__()
 
     def get_folder(self):
         return os.path.dirname(inspect.getfile(self.__class__))

setup.py

@@ -15,7 +15,7 @@ except AttributeError:
 install_reqs = [str(ir.req) for ir in install_reqs]
 test_reqs = [str(ir.req) for ir in test_reqs]
 
-VERSION = "0.5"
+VERSION = "0.5.1"
 
 setup(
     name='senpy',

tests/test_blueprints.py

@@ -56,7 +56,10 @@ class BlueprintsTest(TestCase):
         resp = self.client.get("/api/plugins/")
         self.assert200(resp)
         logging.debug(resp.json)
-        assert "Dummy" in resp.json
+        assert 'plugins' in resp.json
+        plugins = resp.json['plugins']
+        assert len(plugins) > 1
+        assert list(p for p in plugins if p['name'] == "Dummy")
         assert "@context" in resp.json
 
     def test_headers(self):
@@ -98,7 +101,7 @@ class BlueprintsTest(TestCase):
 
     def test_default(self):
         """ Show only one plugin"""
-        resp = self.client.get("/api/default")
+        resp = self.client.get("/api/plugins/default/")
         self.assert200(resp)
         logging.debug(resp.json)
         assert "@id" in resp.json
@@ -106,5 +109,5 @@ class BlueprintsTest(TestCase):
         resp = self.client.get("/api/plugins/Dummy/deactivate")
         self.assert200(resp)
         sleep(0.5)
-        resp = self.client.get("/api/default")
+        resp = self.client.get("/api/plugins/default/")
         self.assert404(resp)