From b4ca5f4a7cbc49a3b7eec234bceebf3b2f3d2b72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Fri, 17 Feb 2017 01:53:02 +0100 Subject: [PATCH] Several fixes and changes * Added interactive debugging * Better exception logging * More tests for errors * Added ONBUILD to dockerfile Now creating new images based on senpy's is as easy as: ```from senpy:```. This will automatically mount the code to /senpy-plugins and install all dependencies * Added /data as a VOLUME * Added `--use-wheel` to pip install both on the image and in the auto-install function. * Closes #9 Break compatibilitity: * Removed ability to (de)activate plugins through the web --- .gitlab-ci.yml | 9 ++++++- Dockerfile-2.7 | 16 ++++++++++-- Dockerfile-3.4 | 17 ++++++++++--- Dockerfile-3.5 | 16 ++++++++++-- Dockerfile.template | 16 ++++++++++-- senpy/VERSION | 2 +- senpy/__main__.py | 19 ++++++++++++++ senpy/blueprints.py | 12 ++------- senpy/client.py | 8 ------ senpy/extensions.py | 55 +++++++++++++++------------------------- senpy/models.py | 16 +++++++++++- senpy/plugins.py | 1 + senpy/schemas/error.json | 2 +- setup.py | 2 -- tests/test_blueprints.py | 40 ++++++++++------------------- tests/test_extensions.py | 32 ++++++++++++++++++++--- 16 files changed, 164 insertions(+), 99 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b5070bc..8f2dd37 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,13 +14,20 @@ stages: .test: &test_definition variables: - PIP_CACHE_DIR: "$CI_PROJECT_DIR/.eggs" + PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache" cache: paths: - .eggs/ + - "$CI_PROJECT_DIR/pip-cache" + - .venv key: "$CI_PROJECT_NAME" stage: test script: + - pip install --use-wheel -U pip setuptools virtualenv + - virtualenv .venv/$PYTHON_VERSION + - source .venv/$PYTHON_VERSION/bin/activate + - pip install --use-wheel -r requirements.txt + - pip install --use-wheel -r test-requirements.txt - python setup.py test test-3.5: diff --git a/Dockerfile-2.7 b/Dockerfile-2.7 index 9d7cba9..5ec4a22 100644 --- a/Dockerfile-2.7 +++ b/Dockerfile-2.7 @@ -1,9 +1,21 @@ from python:2.7 +RUN mkdir /cache/ +ENV PIP_CACHE_DIR=/cache/ + WORKDIR /usr/src/app ADD requirements.txt /usr/src/app/ -RUN pip install -r requirements.txt +RUN pip install --use-wheel -r requirements.txt ADD . /usr/src/app/ RUN pip install . -ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"] + +VOLUME /data/ + +RUN mkdir /senpy-plugins/ + +WORKDIR /senpy-plugins/ +ONBUILD ADD . /senpy-plugins/ +ONBUILD RUN python -m senpy --only-install -f /senpy-plugins + +ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"] \ No newline at end of file diff --git a/Dockerfile-3.4 b/Dockerfile-3.4 index 92a6ef5..c631112 100644 --- a/Dockerfile-3.4 +++ b/Dockerfile-3.4 @@ -1,9 +1,18 @@ -from python:3.4 +from python:3.4-slim WORKDIR /usr/src/app ADD requirements.txt /usr/src/app/ -RUN pip install -r requirements.txt +RUN pip install --use-wheel -r requirements.txt ADD . /usr/src/app/ -RUN pip install . +RUN pip install --use-wheel . -ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"] + +VOLUME /data/ + +RUN mkdir /senpy-plugins/ + +WORKDIR /senpy-plugins/ +ONBUILD ADD . /senpy-plugins/ +ONBUILD RUN python -m senpy -f /senpy-plugins + +ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"] \ No newline at end of file diff --git a/Dockerfile-3.5 b/Dockerfile-3.5 index aca0a22..1899c5b 100644 --- a/Dockerfile-3.5 +++ b/Dockerfile-3.5 @@ -1,9 +1,21 @@ from python:3.5 +RUN mkdir /cache/ +ENV PIP_CACHE_DIR=/cache/ + WORKDIR /usr/src/app ADD requirements.txt /usr/src/app/ -RUN pip install -r requirements.txt +RUN pip install --use-wheel -r requirements.txt ADD . /usr/src/app/ RUN pip install . -ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"] + +VOLUME /data/ + +RUN mkdir /senpy-plugins/ + +WORKDIR /senpy-plugins/ +ONBUILD ADD . /senpy-plugins/ +ONBUILD RUN python -m senpy --only-install -f /senpy-plugins + +ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"] \ No newline at end of file diff --git a/Dockerfile.template b/Dockerfile.template index 06c2f45..79d1fc1 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -1,9 +1,21 @@ from python:{{PYVERSION}} +RUN mkdir /cache/ +ENV PIP_CACHE_DIR=/cache/ + WORKDIR /usr/src/app ADD requirements.txt /usr/src/app/ -RUN pip install -r requirements.txt +RUN pip install --use-wheel -r requirements.txt ADD . /usr/src/app/ RUN pip install . -ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"] + +VOLUME /data/ + +RUN mkdir /senpy-plugins/ + +WORKDIR /senpy-plugins/ +ONBUILD ADD . /senpy-plugins/ +ONBUILD RUN python -m senpy --only-install -f /senpy-plugins + +ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"] \ No newline at end of file diff --git a/senpy/VERSION b/senpy/VERSION index 39e898a..a48e2db 100644 --- a/senpy/VERSION +++ b/senpy/VERSION @@ -1 +1 @@ -0.7.1 +0.7.2-dev1 diff --git a/senpy/__main__.py b/senpy/__main__.py index 3d95f6d..72cec87 100644 --- a/senpy/__main__.py +++ b/senpy/__main__.py @@ -26,6 +26,7 @@ from gevent.wsgi import WSGIServer from gevent.monkey import patch_all import logging import os +import sys import argparse import senpy @@ -34,6 +35,22 @@ patch_all(thread=False) SERVER_PORT = os.environ.get("PORT", 5000) +def info(type, value, tb): + if hasattr(sys, 'ps1') or not sys.stderr.isatty(): + # we are in interactive mode or we don't have a tty-like + # device, so we call the default hook + sys.__excepthook__(type, value, tb) + else: + import traceback + import pdb + # we are NOT in interactive mode, print the exception... + traceback.print_exception(type, value, tb) + print + # ...then start the debugger in post-mortem mode. + # pdb.pm() # deprecated + pdb.post_mortem(tb) # more "modern" + + def main(): parser = argparse.ArgumentParser(description='Run a Senpy server') parser.add_argument( @@ -84,6 +101,8 @@ def main(): rl.setLevel(getattr(logging, args.level)) app = Flask(__name__) app.debug = args.debug + if args.debug: + sys.excepthook = info sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins) if args.only_install: sp.install_deps() diff --git a/senpy/blueprints.py b/senpy/blueprints.py index c1a2f30..bcc8b63 100644 --- a/senpy/blueprints.py +++ b/senpy/blueprints.py @@ -102,9 +102,8 @@ def plugins(): @api_blueprint.route('/plugins//', methods=['POST', 'GET']) -@api_blueprint.route('/plugins//', methods=['POST', 'GET']) @basic_api -def plugin(plugin=None, action="list"): +def plugin(plugin=None): sp = current_app.senpy if plugin == 'default' and sp.default_plugin: response = sp.default_plugin @@ -113,11 +112,4 @@ def plugin(plugin=None, action="list"): response = sp.plugins[plugin] else: return Error(message="Plugin not found", status=404) - if action == "list": - return response - method = "{}_plugin".format(action) - if (hasattr(sp, method)): - getattr(sp, method)(plugin) - return Response(message="Ok") - else: - return Error(message="action '{}' not allowed".format(action)) + return response diff --git a/senpy/client.py b/senpy/client.py index b9dcb3e..53668ff 100644 --- a/senpy/client.py +++ b/senpy/client.py @@ -35,11 +35,3 @@ class Client(object): code=response.status_code, content=response.content)) raise ex - - -if __name__ == '__main__': - c = Client('http://senpy.cluster.gsi.dit.upm.es/api/') - resp = c.analyse('hello') - # print(resp) - print(resp.entries) - resp.validate() diff --git a/senpy/extensions.py b/senpy/extensions.py index e32082b..2bc4e74 100644 --- a/senpy/extensions.py +++ b/senpy/extensions.py @@ -106,8 +106,12 @@ class Senpy(object): resp = plug.analyse(**nif_params) resp.analysis.append(plug) logger.debug("Returning analysis result: {}".format(resp)) + except Error as ex: + logger.exception('Error returning analysis result') + resp = ex except Exception as ex: resp = Error(message=str(ex), status=500) + logger.exception('Error returning analysis result') return resp @property @@ -120,10 +124,6 @@ class Senpy(object): else: return None - def parameters(self, algo): - return getattr( - self.plugins.get(algo) or self.default_plugin, "extra_params", {}) - def activate_all(self, sync=False): ps = [] for plug in self.plugins.keys(): @@ -194,17 +194,6 @@ class Senpy(object): th = Thread(target=deact) th.start() - def reload_plugin(self, name): - logger.debug("Reloading {}".format(name)) - plugin = self.plugins[name] - try: - del self.plugins[name] - nplug = self._load_plugin(plugin.module, plugin.path) - self.plugins[nplug.name] = nplug - except Exception as ex: - logger.error('Error reloading {}: {}'.format(name, ex)) - self.plugins[name] = plugin - @classmethod def validate_info(cls, info): return all(x in info for x in ('name', 'module', 'version')) @@ -219,6 +208,7 @@ class Senpy(object): if requirements: pip_args = [] pip_args.append('install') + pip_args.append('--use-wheel') for req in requirements: pip_args.append(req) logger.info('Installing requirements: ' + str(requirements)) @@ -233,32 +223,27 @@ class Senpy(object): name = info["name"] sys.path.append(root) (fp, pathname, desc) = imp.find_module(module, [root, ]) + cls._install_deps(info) + tmp = imp.load_module(module, fp, pathname, desc) + sys.path.remove(root) + candidate = None + for _, obj in inspect.getmembers(tmp): + if inspect.isclass(obj) and inspect.getmodule(obj) == tmp: + logger.debug(("Found plugin class:" + " {}@{}").format(obj, inspect.getmodule(obj))) + candidate = obj + break + if not candidate: + logger.debug("No valid plugin for: {}".format(module)) + return + module = candidate(info=info) + repo_path = root try: - cls._install_deps(info) - tmp = imp.load_module(module, fp, pathname, desc) - sys.path.remove(root) - candidate = None - for _, obj in inspect.getmembers(tmp): - if inspect.isclass(obj) and inspect.getmodule(obj) == tmp: - logger.debug(("Found plugin class:" - " {}@{}").format(obj, inspect.getmodule( - obj))) - candidate = obj - break - if not candidate: - logger.debug("No valid plugin for: {}".format(module)) - return - module = candidate(info=info) - repo_path = root module._repo = Repo(repo_path) except InvalidGitRepositoryError: logger.debug("The plugin {} is not in a Git repository".format( module)) module._repo = None - except Exception as ex: - logger.error("Exception importing {}: {}".format(module, ex)) - logger.error("Trace: {}".format(traceback.format_exc())) - return None, None return name, module @classmethod diff --git a/senpy/models.py b/senpy/models.py index 776492d..ec6ef80 100644 --- a/senpy/models.py +++ b/senpy/models.py @@ -115,7 +115,7 @@ class SenpyMixin(object): return ser_or_down(self._plain_dict()) - def jsonld(self, with_context=True, context_uri=None): + def jsonld(self, with_context=False, context_uri=None): ser = self.serializable() if with_context: @@ -230,6 +230,11 @@ def from_dict(indict): return cls(**indict) +def from_json(injson): + indict = json.loads(injson) + return from_dict(indict) + + def from_schema(name, schema_file=None, base_classes=None): base_classes = base_classes or [] base_classes.append(BaseModel) @@ -275,6 +280,15 @@ class Error(SenpyMixin, BaseException): self._error = _ErrorModel(message=message, *args, **kwargs) self.message = message + def __getitem__(self, key): + return self._error[key] + + def __setitem__(self, key, value): + self._error[key] = value + + def __delitem__(self, key): + del self._error[key] + def __getattr__(self, key): if key != '_error' and hasattr(self._error, key): return getattr(self._error, key) diff --git a/senpy/plugins.py b/senpy/plugins.py index af82700..512b125 100644 --- a/senpy/plugins.py +++ b/senpy/plugins.py @@ -50,6 +50,7 @@ class SentimentPlugin(SenpyPlugin, models.SentimentPlugin): class EmotionPlugin(SentimentPlugin, models.EmotionPlugin): def __init__(self, info, *args, **kwargs): + super(EmotionPlugin, self).__init__(info, *args, **kwargs) self.minEmotionValue = float(info.get("minEmotionValue", 0)) self.maxEmotionValue = float(info.get("maxEmotionValue", 0)) self["@type"] = "onyx:EmotionAnalysis" diff --git a/senpy/schemas/error.json b/senpy/schemas/error.json index 966303f..7cdf582 100644 --- a/senpy/schemas/error.json +++ b/senpy/schemas/error.json @@ -13,7 +13,7 @@ "type": "list", "items": {"type": "object"} }, - "code": { + "status": { "type": "int" }, "required": ["message"] diff --git a/setup.py b/setup.py index f33fe4f..9044fe3 100644 --- a/setup.py +++ b/setup.py @@ -12,8 +12,6 @@ except AttributeError: install_reqs = parse_requirements("requirements.txt") test_reqs = parse_requirements("test-requirements.txt") -# reqs is a list of requirement -# e.g. ['django==1.5.1', 'mezzanine==1.4.6'] install_reqs = [str(ir.req) for ir in install_reqs] test_reqs = [str(ir.req) for ir in test_reqs] diff --git a/tests/test_blueprints.py b/tests/test_blueprints.py index 0527af1..6674992 100644 --- a/tests/test_blueprints.py +++ b/tests/test_blueprints.py @@ -1,11 +1,10 @@ import os import logging -import json from senpy.extensions import Senpy +from senpy import models from flask import Flask from unittest import TestCase -from gevent import sleep from itertools import product @@ -14,7 +13,7 @@ def check_dict(indic, template): def parse_resp(resp): - return json.loads(resp.data.decode('utf-8')) + return models.from_json(resp.data.decode('utf-8')) class BlueprintsTest(TestCase): @@ -57,6 +56,17 @@ class BlueprintsTest(TestCase): assert "@context" in js assert "entries" in js + def test_error(self): + """ + The dummy plugin returns an empty response,\ + it should contain the context + """ + resp = self.client.get("/api/?i=My aloha mohame&algo=DOESNOTEXIST") + self.assertCode(resp, 404) + js = parse_resp(resp) + logging.debug("Got response: %s", js) + assert isinstance(js, models.Error) + def test_list(self): """ List the plugins """ resp = self.client.get("/api/plugins/") @@ -94,25 +104,6 @@ class BlueprintsTest(TestCase): assert "@id" in js assert js["@id"] == "Dummy_0.1" - def test_activate(self): - """ Activate and deactivate one plugin """ - resp = self.client.get("/api/plugins/Dummy/deactivate") - self.assertCode(resp, 200) - sleep(0.5) - resp = self.client.get("/api/plugins/Dummy/") - self.assertCode(resp, 200) - js = parse_resp(resp) - assert "is_activated" in js - assert not js["is_activated"] - resp = self.client.get("/api/plugins/Dummy/activate") - self.assertCode(resp, 200) - sleep(0.5) - resp = self.client.get("/api/plugins/Dummy/") - self.assertCode(resp, 200) - js = parse_resp(resp) - assert "is_activated" in js - assert js["is_activated"] - def test_default(self): """ Show only one plugin""" resp = self.client.get("/api/plugins/default/") @@ -121,11 +112,6 @@ class BlueprintsTest(TestCase): logging.debug(js) assert "@id" in js assert js["@id"] == "Dummy_0.1" - resp = self.client.get("/api/plugins/Dummy/deactivate") - self.assertCode(resp, 200) - sleep(0.5) - resp = self.client.get("/api/plugins/default/") - self.assertCode(resp, 404) def test_context(self): resp = self.client.get("/api/contexts/context.jsonld") diff --git a/tests/test_extensions.py b/tests/test_extensions.py index de95e3e..5e1faa0 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -2,6 +2,11 @@ from __future__ import print_function import os import logging +try: + from unittest import mock +except ImportError: + import mock + from functools import partial from senpy.extensions import Senpy from senpy.models import Error @@ -13,8 +18,9 @@ class ExtensionsTest(TestCase): def setUp(self): self.app = Flask("test_extensions") self.dir = os.path.join(os.path.dirname(__file__)) - self.senpy = Senpy(plugin_folder=self.dir, default_plugins=False) - self.senpy.init_app(self.app) + self.senpy = Senpy(plugin_folder=self.dir, + app=self.app, + default_plugins=False) self.senpy.activate_plugin("Dummy", sync=True) def test_init(self): @@ -69,7 +75,11 @@ class ExtensionsTest(TestCase): def test_noplugin(self): """ Don't analyse if there isn't any plugin installed """ self.senpy.deactivate_all(sync=True) - self.assertRaises(Error, partial(self.senpy.analyse, input="tupni")) + self.assertRaises(Error, partial(self.senpy.analyse, + input="tupni")) + self.assertRaises(Error, partial(self.senpy.analyse, + input="tupni", + algorithm='Dummy')) def test_analyse(self): """ Using a plugin """ @@ -81,6 +91,18 @@ class ExtensionsTest(TestCase): assert r1.analysis[0].id[:5] == "Dummy" assert r2.analysis[0].id[:5] == "Dummy" + def test_analyse_error(self): + mm = mock.MagicMock() + mm.analyse.side_effect = Error('error on analysis', status=900) + self.senpy.plugins['MOCK'] = mm + resp = self.senpy.analyse(input='nothing', algorithm='MOCK') + assert resp['message'] == 'error on analysis' + assert resp['status'] == 900 + mm.analyse.side_effect = Exception('generic exception on analysis') + resp = self.senpy.analyse(input='nothing', algorithm='MOCK') + assert resp['message'] == 'generic exception on analysis' + assert resp['status'] == 500 + def test_filtering(self): """ Filtering plugins """ assert len(self.senpy.filter_plugins(name="Dummy")) > 0 @@ -90,3 +112,7 @@ class ExtensionsTest(TestCase): assert not len( self.senpy.filter_plugins( name="Dummy", is_activated=True)) + + def test_load_default_plugins(self): + senpy = Senpy(plugin_folder=self.dir, default_plugins=True) + assert len(senpy.plugins) > 1