Several fixes and changes

* Added interactive debugging
* Better exception logging
* More tests for errors
* Added ONBUILD to dockerfile
  Now creating new images based on senpy's is as easy as:
  ```from senpy:<version>```. This will automatically mount the code to
  /senpy-plugins and install all dependencies
* Added /data as a VOLUME
* Added `--use-wheel` to pip install both on the image and in the
  auto-install function.
* Closes #9

Break compatibilitity:

* Removed ability to (de)activate plugins through the web

.gitlab-ci.yml

@@ -14,13 +14,20 @@ stages:
 
 .test: &test_definition
   variables:
-    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.eggs"
+    PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
   cache:
     paths:
       - .eggs/
+      - "$CI_PROJECT_DIR/pip-cache"
+      - .venv
     key: "$CI_PROJECT_NAME"
   stage: test
   script:
+    - pip install --use-wheel -U pip setuptools virtualenv
+    - virtualenv .venv/$PYTHON_VERSION
+    - source .venv/$PYTHON_VERSION/bin/activate
+    - pip install --use-wheel -r requirements.txt
+    - pip install --use-wheel -r test-requirements.txt
     - python setup.py test
  
 test-3.5:

Dockerfile-2.7

@@ -1,9 +1,21 @@
 from python:2.7
 
+RUN mkdir /cache/
+ENV PIP_CACHE_DIR=/cache/
+
 WORKDIR /usr/src/app
 ADD requirements.txt /usr/src/app/
-RUN pip install -r requirements.txt
+RUN pip install --use-wheel -r requirements.txt
 ADD . /usr/src/app/
 RUN pip install .
 
-ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]
+
+VOLUME /data/
+
+RUN mkdir /senpy-plugins/
+
+WORKDIR /senpy-plugins/
+ONBUILD ADD . /senpy-plugins/
+ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
+
+ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

Dockerfile-3.4

@@ -1,9 +1,18 @@
-from python:3.4
+from python:3.4-slim
 
 WORKDIR /usr/src/app
 ADD requirements.txt /usr/src/app/
-RUN pip install -r requirements.txt
+RUN pip install --use-wheel -r requirements.txt
 ADD . /usr/src/app/
-RUN pip install .
+RUN pip install --use-wheel .
 
-ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]
+
+VOLUME /data/
+
+RUN mkdir /senpy-plugins/
+
+WORKDIR /senpy-plugins/
+ONBUILD ADD . /senpy-plugins/
+ONBUILD RUN python -m senpy -f /senpy-plugins
+
+ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

Dockerfile-3.5

@@ -1,9 +1,21 @@
 from python:3.5
 
+RUN mkdir /cache/
+ENV PIP_CACHE_DIR=/cache/
+
 WORKDIR /usr/src/app
 ADD requirements.txt /usr/src/app/
-RUN pip install -r requirements.txt
+RUN pip install --use-wheel -r requirements.txt
 ADD . /usr/src/app/
 RUN pip install .
 
-ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]
+
+VOLUME /data/
+
+RUN mkdir /senpy-plugins/
+
+WORKDIR /senpy-plugins/
+ONBUILD ADD . /senpy-plugins/
+ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
+
+ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

Dockerfile.template

@@ -1,9 +1,21 @@
 from python:{{PYVERSION}}
 
+RUN mkdir /cache/
+ENV PIP_CACHE_DIR=/cache/
+
 WORKDIR /usr/src/app
 ADD requirements.txt /usr/src/app/
-RUN pip install -r requirements.txt
+RUN pip install --use-wheel -r requirements.txt
 ADD . /usr/src/app/
 RUN pip install .
 
-ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]
+
+VOLUME /data/
+
+RUN mkdir /senpy-plugins/
+
+WORKDIR /senpy-plugins/
+ONBUILD ADD . /senpy-plugins/
+ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
+
+ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

senpy/VERSION

@@ -1 +1 @@
-0.7.1
+0.7.2-dev1

senpy/__main__.py

@@ -26,6 +26,7 @@ from gevent.wsgi import WSGIServer
 from gevent.monkey import patch_all
 import logging
 import os
+import sys
 import argparse
 import senpy
 
@@ -34,6 +35,22 @@ patch_all(thread=False)
 SERVER_PORT = os.environ.get("PORT", 5000)
 
 
+def info(type, value, tb):
+    if hasattr(sys, 'ps1') or not sys.stderr.isatty():
+        #  we are in interactive mode or we don't have a tty-like
+        #  device, so we call the default hook
+        sys.__excepthook__(type, value, tb)
+    else:
+        import traceback
+        import pdb
+        # we are NOT in interactive mode, print the exception...
+        traceback.print_exception(type, value, tb)
+        print
+        # ...then start the debugger in post-mortem mode.
+        # pdb.pm() # deprecated
+        pdb.post_mortem(tb)  # more "modern"
+
+
 def main():
     parser = argparse.ArgumentParser(description='Run a Senpy server')
     parser.add_argument(
@@ -84,6 +101,8 @@ def main():
     rl.setLevel(getattr(logging, args.level))
     app = Flask(__name__)
     app.debug = args.debug
+    if args.debug:
+        sys.excepthook = info
     sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins)
     if args.only_install:
         sp.install_deps()

senpy/blueprints.py

@@ -102,9 +102,8 @@ def plugins():
 
 
 @api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
-@api_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
 @basic_api
-def plugin(plugin=None, action="list"):
+def plugin(plugin=None):
     sp = current_app.senpy
     if plugin == 'default' and sp.default_plugin:
         response = sp.default_plugin
@@ -113,11 +112,4 @@ def plugin(plugin=None, action="list"):
         response = sp.plugins[plugin]
     else:
         return Error(message="Plugin not found", status=404)
-    if action == "list":
-        return response
-    method = "{}_plugin".format(action)
-    if (hasattr(sp, method)):
-        getattr(sp, method)(plugin)
-        return Response(message="Ok")
-    else:
-        return Error(message="action '{}' not allowed".format(action))
+    return response

senpy/client.py

@@ -35,11 +35,3 @@ class Client(object):
                                        code=response.status_code,
                                        content=response.content))
             raise ex
-
-
-if __name__ == '__main__':
-    c = Client('http://senpy.cluster.gsi.dit.upm.es/api/')
-    resp = c.analyse('hello')
-    # print(resp)
-    print(resp.entries)
-    resp.validate()

senpy/extensions.py

@@ -106,8 +106,12 @@ class Senpy(object):
             resp = plug.analyse(**nif_params)
             resp.analysis.append(plug)
             logger.debug("Returning analysis result: {}".format(resp))
+        except Error as ex:
+            logger.exception('Error returning analysis result')
+            resp = ex
         except Exception as ex:
             resp = Error(message=str(ex), status=500)
+            logger.exception('Error returning analysis result')
         return resp
 
     @property
@@ -120,10 +124,6 @@ class Senpy(object):
         else:
             return None
 
-    def parameters(self, algo):
-        return getattr(
-            self.plugins.get(algo) or self.default_plugin, "extra_params", {})
-
     def activate_all(self, sync=False):
         ps = []
         for plug in self.plugins.keys():
@@ -194,17 +194,6 @@ class Senpy(object):
             th = Thread(target=deact)
             th.start()
 
-    def reload_plugin(self, name):
-        logger.debug("Reloading {}".format(name))
-        plugin = self.plugins[name]
-        try:
-            del self.plugins[name]
-            nplug = self._load_plugin(plugin.module, plugin.path)
-            self.plugins[nplug.name] = nplug
-        except Exception as ex:
-            logger.error('Error reloading {}: {}'.format(name, ex))
-            self.plugins[name] = plugin
-
     @classmethod
     def validate_info(cls, info):
         return all(x in info for x in ('name', 'module', 'version'))
@@ -219,6 +208,7 @@ class Senpy(object):
         if requirements:
             pip_args = []
             pip_args.append('install')
+            pip_args.append('--use-wheel')
             for req in requirements:
                 pip_args.append(req)
             logger.info('Installing requirements: ' + str(requirements))
@@ -233,32 +223,27 @@ class Senpy(object):
         name = info["name"]
         sys.path.append(root)
         (fp, pathname, desc) = imp.find_module(module, [root, ])
+        cls._install_deps(info)
+        tmp = imp.load_module(module, fp, pathname, desc)
+        sys.path.remove(root)
+        candidate = None
+        for _, obj in inspect.getmembers(tmp):
+            if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
+                logger.debug(("Found plugin class:"
+                              " {}@{}").format(obj, inspect.getmodule(obj)))
+                candidate = obj
+                break
+        if not candidate:
+            logger.debug("No valid plugin for: {}".format(module))
+            return
+        module = candidate(info=info)
+        repo_path = root
         try:
-            cls._install_deps(info)
-            tmp = imp.load_module(module, fp, pathname, desc)
-            sys.path.remove(root)
-            candidate = None
-            for _, obj in inspect.getmembers(tmp):
-                if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
-                    logger.debug(("Found plugin class:"
-                                  " {}@{}").format(obj, inspect.getmodule(
-                                      obj)))
-                    candidate = obj
-                    break
-            if not candidate:
-                logger.debug("No valid plugin for: {}".format(module))
-                return
-            module = candidate(info=info)
-            repo_path = root
             module._repo = Repo(repo_path)
         except InvalidGitRepositoryError:
             logger.debug("The plugin {} is not in a Git repository".format(
                 module))
             module._repo = None
-        except Exception as ex:
-            logger.error("Exception importing {}: {}".format(module, ex))
-            logger.error("Trace: {}".format(traceback.format_exc()))
-            return None, None
         return name, module
 
     @classmethod

senpy/models.py

@@ -115,7 +115,7 @@ class SenpyMixin(object):
 
         return ser_or_down(self._plain_dict())
 
-    def jsonld(self, with_context=True, context_uri=None):
+    def jsonld(self, with_context=False, context_uri=None):
         ser = self.serializable()
 
         if with_context:
@@ -230,6 +230,11 @@ def from_dict(indict):
     return cls(**indict)
 
 
+def from_json(injson):
+    indict = json.loads(injson)
+    return from_dict(indict)
+
+
 def from_schema(name, schema_file=None, base_classes=None):
     base_classes = base_classes or []
     base_classes.append(BaseModel)
@@ -275,6 +280,15 @@ class Error(SenpyMixin, BaseException):
         self._error = _ErrorModel(message=message, *args, **kwargs)
         self.message = message
 
+    def __getitem__(self, key):
+        return self._error[key]
+
+    def __setitem__(self, key, value):
+        self._error[key] = value
+
+    def __delitem__(self, key):
+        del self._error[key]
+
     def __getattr__(self, key):
         if key != '_error' and hasattr(self._error, key):
             return getattr(self._error, key)

senpy/plugins.py

@@ -50,6 +50,7 @@ class SentimentPlugin(SenpyPlugin, models.SentimentPlugin):
 
 class EmotionPlugin(SentimentPlugin, models.EmotionPlugin):
     def __init__(self, info, *args, **kwargs):
+        super(EmotionPlugin, self).__init__(info, *args, **kwargs)
         self.minEmotionValue = float(info.get("minEmotionValue", 0))
         self.maxEmotionValue = float(info.get("maxEmotionValue", 0))
         self["@type"] = "onyx:EmotionAnalysis"

senpy/schemas/error.json

@@ -13,7 +13,7 @@
           "type": "list",
           "items": {"type": "object"}
         },
-        "code": {
+        "status": {
           "type": "int"
         },
         "required": ["message"]

setup.py

@@ -12,8 +12,6 @@ except AttributeError:
     install_reqs = parse_requirements("requirements.txt")
     test_reqs = parse_requirements("test-requirements.txt")
 
-# reqs is a list of requirement
-# e.g. ['django==1.5.1', 'mezzanine==1.4.6']
 install_reqs = [str(ir.req) for ir in install_reqs]
 test_reqs = [str(ir.req) for ir in test_reqs]
 

tests/test_blueprints.py

@@ -1,11 +1,10 @@
 import os
 import logging
-import json
 
 from senpy.extensions import Senpy
+from senpy import models
 from flask import Flask
 from unittest import TestCase
-from gevent import sleep
 from itertools import product
 
 
@@ -14,7 +13,7 @@ def check_dict(indic, template):
 
 
 def parse_resp(resp):
-    return json.loads(resp.data.decode('utf-8'))
+    return models.from_json(resp.data.decode('utf-8'))
 
 
 class BlueprintsTest(TestCase):
@@ -57,6 +56,17 @@ class BlueprintsTest(TestCase):
         assert "@context" in js
         assert "entries" in js
 
+    def test_error(self):
+        """
+        The dummy plugin returns an empty response,\
+        it should contain the context
+        """
+        resp = self.client.get("/api/?i=My aloha mohame&algo=DOESNOTEXIST")
+        self.assertCode(resp, 404)
+        js = parse_resp(resp)
+        logging.debug("Got response: %s", js)
+        assert isinstance(js, models.Error)
+
     def test_list(self):
         """ List the plugins """
         resp = self.client.get("/api/plugins/")
@@ -94,25 +104,6 @@ class BlueprintsTest(TestCase):
         assert "@id" in js
         assert js["@id"] == "Dummy_0.1"
 
-    def test_activate(self):
-        """ Activate and deactivate one plugin """
-        resp = self.client.get("/api/plugins/Dummy/deactivate")
-        self.assertCode(resp, 200)
-        sleep(0.5)
-        resp = self.client.get("/api/plugins/Dummy/")
-        self.assertCode(resp, 200)
-        js = parse_resp(resp)
-        assert "is_activated" in js
-        assert not js["is_activated"]
-        resp = self.client.get("/api/plugins/Dummy/activate")
-        self.assertCode(resp, 200)
-        sleep(0.5)
-        resp = self.client.get("/api/plugins/Dummy/")
-        self.assertCode(resp, 200)
-        js = parse_resp(resp)
-        assert "is_activated" in js
-        assert js["is_activated"]
-
     def test_default(self):
         """ Show only one plugin"""
         resp = self.client.get("/api/plugins/default/")
@@ -121,11 +112,6 @@ class BlueprintsTest(TestCase):
         logging.debug(js)
         assert "@id" in js
         assert js["@id"] == "Dummy_0.1"
-        resp = self.client.get("/api/plugins/Dummy/deactivate")
-        self.assertCode(resp, 200)
-        sleep(0.5)
-        resp = self.client.get("/api/plugins/default/")
-        self.assertCode(resp, 404)
 
     def test_context(self):
         resp = self.client.get("/api/contexts/context.jsonld")

tests/test_extensions.py

@@ -2,6 +2,11 @@ from __future__ import print_function
 import os
 import logging
 
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
 from functools import partial
 from senpy.extensions import Senpy
 from senpy.models import Error
@@ -13,8 +18,9 @@ class ExtensionsTest(TestCase):
     def setUp(self):
         self.app = Flask("test_extensions")
         self.dir = os.path.join(os.path.dirname(__file__))
-        self.senpy = Senpy(plugin_folder=self.dir, default_plugins=False)
-        self.senpy.init_app(self.app)
+        self.senpy = Senpy(plugin_folder=self.dir,
+                           app=self.app,
+                           default_plugins=False)
         self.senpy.activate_plugin("Dummy", sync=True)
 
     def test_init(self):
@@ -69,7 +75,11 @@ class ExtensionsTest(TestCase):
     def test_noplugin(self):
         """ Don't analyse if there isn't any plugin installed """
         self.senpy.deactivate_all(sync=True)
-        self.assertRaises(Error, partial(self.senpy.analyse, input="tupni"))
+        self.assertRaises(Error, partial(self.senpy.analyse,
+                                         input="tupni"))
+        self.assertRaises(Error, partial(self.senpy.analyse,
+                                         input="tupni",
+                                         algorithm='Dummy'))
 
     def test_analyse(self):
         """ Using a plugin """
@@ -81,6 +91,18 @@ class ExtensionsTest(TestCase):
         assert r1.analysis[0].id[:5] == "Dummy"
         assert r2.analysis[0].id[:5] == "Dummy"
 
+    def test_analyse_error(self):
+        mm = mock.MagicMock()
+        mm.analyse.side_effect = Error('error on analysis', status=900)
+        self.senpy.plugins['MOCK'] = mm
+        resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
+        assert resp['message'] == 'error on analysis'
+        assert resp['status'] == 900
+        mm.analyse.side_effect = Exception('generic exception on analysis')
+        resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
+        assert resp['message'] == 'generic exception on analysis'
+        assert resp['status'] == 500
+
     def test_filtering(self):
         """ Filtering plugins """
         assert len(self.senpy.filter_plugins(name="Dummy")) > 0
@@ -90,3 +112,7 @@ class ExtensionsTest(TestCase):
         assert not len(
             self.senpy.filter_plugins(
                 name="Dummy", is_activated=True))
+
+    def test_load_default_plugins(self):
+        senpy = Senpy(plugin_folder=self.dir, default_plugins=True)
+        assert len(senpy.plugins) > 1