Improved plugins API and loading

Also: * added drone-ci integration: tests for py2.7 and py3
2024-11-21 07:42:28 +00:00 · 2016-02-20 18:15:04 +01:00 · 2016-02-20 18:15:04 +01:00 · 48d7d1d02e
commit 48d7d1d02e
parent 14c9f61864
8 changed files with 136 additions and 109 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -0,0 +1,9 @@
+build:
+  image: python:$$PYTHON_VERSION
+  commands:
+    - python setup.py test
+
+matrix:
+  PYTHON_VERSION:
+    - 2.7
+    - 3.4
--- a/senpy/main.py
+++ b/senpy/main.py
@ -63,7 +63,9 @@ def main():
                        default="plugins",
                        help='Where to look for plugins.')
    args = parser.parse_args()
-    logging.basicConfig(level=getattr(logging, args.level))
+    logging.basicConfig()
+    rl = logging.getLogger()
+    rl.setLevel(getattr(logging, args.level))
    app = Flask(__name__)
    app.debug = args.debug
    sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins)
--- a/senpy/blueprints.py
+++ b/senpy/blueprints.py
@ -18,7 +18,7 @@
 Blueprints for Senpy
 """
 from flask import Blueprint, request, current_app, render_template
-from .models import Error, Response
+from .models import Error, Response, Plugins
 from future.utils import iteritems

 import json
@ -29,7 +29,7 @@ logger = logging.getLogger(__name__)
 nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__)
 demo_blueprint = Blueprint("Demo of the service. It includes an HTML+Javascript playground to test senpy", __name__)

-BASIC_PARAMS = {
+API_PARAMS = {
    "algorithm": {
        "aliases": ["algorithm", "a", "algo"],
        "required": False,
@ -41,6 +41,63 @@ BASIC_PARAMS = {
    }
 }

+BASIC_PARAMS = {
+    "algorithm": {
+        "aliases": ["algorithm", "a", "algo"],
+        "required": False,
+    },
+    "inHeaders": {
+        "aliases": ["inHeaders", "headers"],
+        "required": True,
+        "default": "0"
+    },
+    "input": {
+        "@id": "input",
+        "aliases": ["i", "input"],
+        "required": True,
+        "help": "Input text"
+    },
+    "informat": {
+        "@id": "informat",
+        "aliases": ["f", "informat"],
+        "required": False,
+        "default": "text",
+        "options": ["turtle", "text"],
+    },
+    "intype": {
+        "@id": "intype",
+        "aliases": ["intype", "t"],
+        "required": False,
+        "default": "direct",
+        "options": ["direct", "url", "file"],
+    },
+    "outformat": {
+        "@id": "outformat",
+        "aliases": ["outformat", "o"],
+        "default": "json-ld",
+        "required": False,
+        "options": ["json-ld"],
+    },
+    "language": {
+        "@id": "language",
+        "aliases": ["language", "l"],
+        "required": False,
+    },
+    "prefix": {
+        "@id": "prefix",
+        "aliases": ["prefix", "p"],
+        "required": True,
+        "default": "",
+    },
+    "urischeme": {
+        "@id": "urischeme",
+        "aliases": ["urischeme", "u"],
+        "required": False,
+        "default": "RFC5147String",
+        "options": "RFC5147String"
+    },
+}
+
 def get_params(req, params=BASIC_PARAMS):
    if req.method == 'POST':
        indict = req.form
@ -119,36 +176,29 @@ def api():
        return ex.message.flask()


-@nif_blueprint.route("/default")
-def default():
-    # return current_app.senpy.default_plugin
-    plug = current_app.senpy.default_plugin
-    if plug:
-        return plugins(action="list", plugin=plug.name)
-    else:
-        error = Error(status=404, message="No plugins found")
-        return error.flask()
-
-
@nif_blueprint.route('/plugins/', methods=['POST', 'GET'])
+def plugins():
+    in_headers = get_params(request, API_PARAMS)["inHeaders"] != "0"
+    sp = current_app.senpy
+    dic = Plugins(plugins=list(sp.plugins.values()))
+    return dic.flask(in_headers=in_headers)
+    
@nif_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
@nif_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
-def plugins(plugin=None, action="list"):
+def plugin(plugin=None, action="list"):
    filt = {}
    sp = current_app.senpy
-    if plugin:
-        filt["name"] = plugin
-    plugs = sp.filter_plugins(**filt)
-    if plugin and not plugs:
-        return "Plugin not found", 400
+    plugs = sp.filter_plugins(name=plugin)
+    if plugin == 'default' and sp.default_plugin:
+        response = sp.default_plugin
+        plugin = response.name
+    elif plugin in sp.plugins:
+        response = sp.plugins[plugin]
+    else:
+        return Error(message="Plugin not found", status=404).flask()
    if action == "list":
-        in_headers = get_params(request, BASIC_PARAMS)["inHeaders"] != "0"
-        if plugin:
-            dic = plugs[plugin]
-        else:
-            dic = Response(
-                    {plug: plugs[plug].serializable() for plug in plugs})
-        return dic.flask(in_headers=in_headers)
+        in_headers = get_params(request, API_PARAMS)["inHeaders"] != "0"
+        return response.flask(in_headers=in_headers)
    method = "{}_plugin".format(action)
    if(hasattr(sp, method)):
        getattr(sp, method)(plugin)
@ -156,7 +206,6 @@ def plugins(plugin=None, action="list"):
    else:
        return Error(message="action '{}' not allowed".format(action)).flask()

-
 if __name__ == '__main__':
    import config

--- a/senpy/extensions.py
+++ b/senpy/extensions.py
@ -34,6 +34,7 @@ class Senpy(object):
        self.app = app

        self._search_folders = set()
+        self._plugin_list = []
        self._outdated = True

        self.add_folder(plugin_folder)
@ -65,10 +66,8 @@ class Senpy(object):
        if os.path.isdir(folder):
            self._search_folders.add(folder)
            self._outdated = True
-            return True
        else:
            logger.debug("Not a folder: %s", folder)
-            return False

    def analyse(self, **params):
        algo = None
@ -113,7 +112,7 @@ class Senpy(object):

    def parameters(self, algo):
        return getattr(self.plugins.get(algo) or self.default_plugin,
-                       "params",
+                       "extra_params",
                       {})

    def activate_all(self, sync=False):
@ -129,13 +128,18 @@ class Senpy(object):
        return ps

    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
+        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
+        Note that plugins may activate themselves by setting this variable.
+        '''
        self.plugins[plugin_name].is_activated = active

    def activate_plugin(self, plugin_name, sync=False):
        plugin = self.plugins[plugin_name]
+        logger.info("Activating plugin: {}".format(plugin.name))
        def act():
            try:
                plugin.activate()
+                logger.info("Plugin activated: {}".format(plugin.name))
            except Exception as ex:
                logger.error("Error activating plugin {}: {}".format(plugin.name,
                                                                     ex))
@ -149,19 +153,33 @@ class Senpy(object):

    def deactivate_plugin(self, plugin_name, sync=False):
        plugin = self.plugins[plugin_name]
-        th = gevent.spawn(plugin.deactivate)
+
+        def deact():
+            try:
+                plugin.deactivate()
+                logger.info("Plugin deactivated: {}".format(plugin.name))
+            except Exception as ex:
+                logger.error("Error deactivating plugin {}: {}".format(plugin.name,
+                                                                       ex))
+                logger.error("Trace: {}".format(traceback.format_exc()))
+
+        th = gevent.spawn(deact)
        th.link_value(partial(self._set_active_plugin, plugin_name, False))
        if sync:
            th.join()
        else:
            return th

-    def reload_plugin(self, plugin):
-        logger.debug("Reloading {}".format(plugin))
-        plug = self.plugins[plugin]
-        nplug = self._load_plugin(plug.module, plug.path)
-        del self.plugins[plugin]
-        self.plugins[nplug.name] = nplug
+    def reload_plugin(self, name):
+        logger.debug("Reloading {}".format(name))
+        plugin = self.plugins[name]
+        try:
+            del self.plugins[name]
+            nplug = self._load_plugin(plugin.module, plugin.path)
+            self.plugins[nplug.name] = nplug
+        except Exception as ex:
+            logger.error('Error reloading {}: {}'.format(name, ex))
+            self.plugins[name] = plugin

    @staticmethod
    def _load_plugin(root, filename):
@ -206,7 +224,7 @@ class Senpy(object):
            for root, dirnames, filenames in os.walk(search_folder):
                for filename in fnmatch.filter(filenames, '*.senpy'):
                    name, plugin = self._load_plugin(root, filename)
-                    if plugin:
+                    if plugin and name not in self._plugin_list:
                        plugins[name] = plugin

        self._outdated = False
@ -218,9 +236,9 @@ class Senpy(object):
    @property
    def plugins(self):
        """ Return the plugins registered for a given application.  """
-        if not hasattr(self, 'senpy_plugins') or self._outdated:
-            self.senpy_plugins = self._load_plugins()
-        return self.senpy_plugins
+        if self._outdated:
+            self._plugin_list = self._load_plugins()
+        return self._plugin_list

    def filter_plugins(self, **kwargs):
        """ Filter plugins by different criteria """
--- a/senpy/models.py
+++ b/senpy/models.py
@ -117,11 +117,16 @@ class SenpyMixin(object):
                        sort_keys=True)
        return js

+    def validate(self, obj=None):
+        if not obj:
+            obj = self
+        if hasattr(obj, "jsonld"):
+            obj = obj.jsonld()
+        jsonschema.validate(obj, self.schema)
    
 class SenpyModel(SenpyMixin, dict):

    schema = base_schema
-    prefix = None

    def __init__(self, *args, **kwargs):
        temp = dict(*args, **kwargs)
@ -161,14 +166,6 @@ class SenpyModel(SenpyMixin, dict):

    def __delattr__(self, key):
        self.__delitem__(self._get_key(key))
-
-
-    def validate(self, obj=None):
-        if not obj:
-            obj = self
-        if hasattr(obj, "jsonld"):
-            obj = obj.jsonld()
-        jsonschema.validate(obj, self.schema)
        
    
    @classmethod
--- a/senpy/plugins.py
+++ b/senpy/plugins.py
@ -9,55 +9,6 @@ from .models import Response, PluginModel, Error

 logger = logging.getLogger(__name__)

-PARAMS = {
-    "input": {
-        "@id": "input",
-        "aliases": ["i", "input"],
-        "required": True,
-        "help": "Input text"
-    },
-    "informat": {
-        "@id": "informat",
-        "aliases": ["f", "informat"],
-        "required": False,
-        "default": "text",
-        "options": ["turtle", "text"],
-    },
-    "intype": {
-        "@id": "intype",
-        "aliases": ["intype", "t"],
-        "required": False,
-        "default": "direct",
-        "options": ["direct", "url", "file"],
-    },
-    "outformat": {
-        "@id": "outformat",
-        "aliases": ["outformat", "o"],
-        "default": "json-ld",
-        "required": False,
-        "options": ["json-ld"],
-    },
-    "language": {
-        "@id": "language",
-        "aliases": ["language", "l"],
-        "required": False,
-    },
-    "prefix": {
-        "@id": "prefix",
-        "aliases": ["prefix", "p"],
-        "required": True,
-        "default": "",
-    },
-    "urischeme": {
-        "@id": "urischeme",
-        "aliases": ["urischeme", "u"],
-        "required": False,
-        "default": "RFC5147String",
-        "options": "RFC5147String"
-    },
-}
-
-
 class SenpyPlugin(PluginModel):

    def __init__(self, info=None):
@ -65,14 +16,12 @@ class SenpyPlugin(PluginModel):
            raise Error(message=("You need to provide configuration"
                                 "information for the plugin."))
        logger.debug("Initialising {}".format(info))
-        self.name = info["name"]
-        self.version = info["version"]
-        self.params = info.get("params", PARAMS.copy())
+        super(SenpyPlugin, self).__init__(info)
+        self.params = info.get("extra_params", {})
+        self._info = info
        if "@id" not in self.params:
            self.params["@id"] = "params_%s" % self.id
        self.is_activated = False
-        self._info = info
-        super(SenpyPlugin, self).__init__()

    def get_folder(self):
        return os.path.dirname(inspect.getfile(self.__class__))
--- a/setup.py
+++ b/setup.py
@ -15,7 +15,7 @@ except AttributeError:
 install_reqs = [str(ir.req) for ir in install_reqs]
 test_reqs = [str(ir.req) for ir in test_reqs]

-VERSION = "0.5"
+VERSION = "0.5.1"

 setup(
    name='senpy',
--- a/tests/test_blueprints.py
+++ b/tests/test_blueprints.py
@ -56,7 +56,10 @@ class BlueprintsTest(TestCase):
        resp = self.client.get("/api/plugins/")
        self.assert200(resp)
        logging.debug(resp.json)
-        assert "Dummy" in resp.json
+        assert 'plugins' in resp.json
+        plugins = resp.json['plugins']
+        assert len(plugins) > 1
+        assert list(p for p in plugins if p['name'] == "Dummy")
        assert "@context" in resp.json

    def test_headers(self):
@ -98,7 +101,7 @@ class BlueprintsTest(TestCase):

    def test_default(self):
        """ Show only one plugin"""
-        resp = self.client.get("/api/default")
+        resp = self.client.get("/api/plugins/default/")
        self.assert200(resp)
        logging.debug(resp.json)
        assert "@id" in resp.json
@ -106,5 +109,5 @@ class BlueprintsTest(TestCase):
        resp = self.client.get("/api/plugins/Dummy/deactivate")
        self.assert200(resp)
        sleep(0.5)
-        resp = self.client.get("/api/default")
+        resp = self.client.get("/api/plugins/default/")
        self.assert404(resp)