From 908090f63425ea15d13edd5d407eb4272d3ca74f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Wed, 8 Feb 2017 21:55:17 +0100 Subject: [PATCH] Released v0.7 Bug-fixes and improvements: * Closes #5 * Closes #1 * Adds Client (beta) * Added several schemas * Lighter string representation -> should avoid delays in the analysis with plugins that have 'heavy' attributes Backwards-incompatible changes: * Context in headers by default * All schemas include a "@type" argument that is used for autodetection in the client ... And possibly many more, this is still <1.0 --- Dockerfile-2.7 | 2 +- Dockerfile-3.4 | 2 +- Dockerfile-3.5 | 2 +- Dockerfile-debug-3.4 | 5 -- Dockerfile.template | 2 +- Makefile | 31 ++++--- senpy/VERSION | 2 +- senpy/client.py | 45 ++++++++++ senpy/extensions.py | 2 +- senpy/models.py | 129 ++++++++++++++++------------- senpy/plugins.py | 19 ++--- "senpy/schemas/\\" | 7 ++ senpy/schemas/atom.json | 15 ++++ senpy/schemas/emotionPlugin.json | 19 +++++ senpy/schemas/entry.json | 3 - senpy/schemas/error.json | 23 +++++ senpy/schemas/plugin.json | 7 +- senpy/schemas/sentimentPlugin.json | 19 +++++ tests/test_cli.py | 20 +++-- tests/test_client.py | 42 ++++++++++ tests/test_models.py | 14 ++-- tests/test_plugins.py | 1 - 22 files changed, 298 insertions(+), 113 deletions(-) delete mode 100644 Dockerfile-debug-3.4 create mode 100644 senpy/client.py create mode 100644 "senpy/schemas/\\" create mode 100644 senpy/schemas/atom.json create mode 100644 senpy/schemas/emotionPlugin.json create mode 100644 senpy/schemas/error.json create mode 100644 senpy/schemas/sentimentPlugin.json create mode 100644 tests/test_client.py diff --git a/Dockerfile-2.7 b/Dockerfile-2.7 index 8589bcf..9d7cba9 100644 --- a/Dockerfile-2.7 +++ b/Dockerfile-2.7 @@ -1,4 +1,4 @@ -from python:2.7-slim +from python:2.7 WORKDIR /usr/src/app ADD requirements.txt /usr/src/app/ diff --git a/Dockerfile-3.4 b/Dockerfile-3.4 index 15867bc..92a6ef5 100644 --- a/Dockerfile-3.4 +++ b/Dockerfile-3.4 @@ -1,4 +1,4 @@ -from python:3.4-slim +from python:3.4 WORKDIR /usr/src/app ADD requirements.txt /usr/src/app/ diff --git a/Dockerfile-3.5 b/Dockerfile-3.5 index 52dd691..aca0a22 100644 --- a/Dockerfile-3.5 +++ b/Dockerfile-3.5 @@ -1,4 +1,4 @@ -from python:3.5-slim +from python:3.5 WORKDIR /usr/src/app ADD requirements.txt /usr/src/app/ diff --git a/Dockerfile-debug-3.4 b/Dockerfile-debug-3.4 deleted file mode 100644 index 86ab349..0000000 --- a/Dockerfile-debug-3.4 +++ /dev/null @@ -1,5 +0,0 @@ -from python:3.4 - -RUN pip install pytest -ADD requirements.txt /usr/src/app/ -RUN pip install -r /usr/src/app/requirements.txt diff --git a/Dockerfile.template b/Dockerfile.template index cc04038..06c2f45 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -1,4 +1,4 @@ -from python:{{PYVERSION}}-slim +from python:{{PYVERSION}} WORKDIR /usr/src/app ADD requirements.txt /usr/src/app/ diff --git a/Makefile b/Makefile index 45a06e5..b86fcd9 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ NAME=senpy REPO=gsiupm VERSION=$(shell cat $(NAME)/VERSION) TARNAME=$(NAME)-$(subst -,.,$(VERSION)).tar.gz +IMAGENAME=$(REPO)/$(NAME):$(VERSION) all: build run @@ -22,27 +23,24 @@ dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS)) Dockerfile-%: Dockerfile.template sed "s/{{PYVERSION}}/$*/" Dockerfile.template > Dockerfile-$* -build: $(addprefix build-, $(PYMAIN)) +quick_build: $(addprefix build-, $(PYMAIN)) -buildall: $(addprefix build-, $(PYVERSIONS)) +build: $(addprefix build-, $(PYVERSIONS)) build-%: Dockerfile-% - docker build -t '$(REPO)/$(NAME):$(VERSION)-python$*' -f Dockerfile-$* .; + docker build -t '$(IMAGENAME)-python$*' -f Dockerfile-$* .; -build-debug-%: - docker build -t '$(NAME)-debug' -f Dockerfile-debug-$* .; +quick_test: $(addprefix test-,$(PYMAIN)) -test: $(addprefix test-,$(PYMAIN)) - -testall: $(addprefix test-,$(PYVERSIONS)) +test: $(addprefix test-,$(PYVERSIONS)) debug-%: - docker run --rm -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti $(NAME)-debug ; + (docker start $(NAME)-debug && docker attach $(NAME)-debug) || docker run -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti --name $(NAME)-debug '$(IMAGENAME)-python$*' debug: debug-$(PYMAIN) test-%: build-% - docker run --rm -w /usr/src/app/ --entrypoint=/usr/local/bin/python -ti '$(REPO)/$(NAME):$(VERSION)-python$*' setup.py test --addopts "-vvv -s" ; + docker run --rm -w /usr/src/app/ --entrypoint=/usr/local/bin/python -ti '$(IMAGENAME)-python$*' setup.py test --addopts "-vvv -s" ; dist/$(TARNAME): docker run --rm -ti -v $$PWD:/usr/src/app/ -w /usr/src/app/ python:$(PYMAIN) python setup.py sdist; @@ -55,12 +53,13 @@ pip_test-%: sdist pip_test: $(addprefix pip_test-,$(PYVERSIONS)) upload-%: test-% - docker push '$(REPO)/$(NAME):$(VERSION)-python$*' + docker push '$(IMAGENAME)-python$*' -upload: testall $(addprefix upload-,$(PYVERSIONS)) - docker tag '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)' '$(REPO)/$(NAME):$(VERSION)' - docker tag '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)' '$(REPO)/$(NAME)' - docker push '$(REPO)/$(NAME):$(VERSION)' +upload: test $(addprefix upload-,$(PYVERSIONS)) + docker tag '$(IMAGENAME)-python$(PYMAIN)' '$(IMAGENAME)' + docker tag '$(IMAGENAME)-python$(PYMAIN)' '$(REPO)/$(NAME)' + docker push '$(IMAGENAME)' + docker push '$(REPO)/$(NAME)' clean: @docker ps -a | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[1] != "${VERSION}"){ print $$1;}}' | xargs docker rm 2>/dev/null|| true @@ -78,6 +77,6 @@ pip_upload: pip_test: $(addprefix pip_test-,$(PYVERSIONS)) run: build - docker run --rm -p 5000:5000 -ti '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)' + docker run --rm -p 5000:5000 -ti '$(IMAGENAME)-python$(PYMAIN)' .PHONY: test test-% build-% build test pip_test run yapf dev diff --git a/senpy/VERSION b/senpy/VERSION index c4fdcc6..faef31a 100644 --- a/senpy/VERSION +++ b/senpy/VERSION @@ -1 +1 @@ -0.7.0-dev3 +0.7.0 diff --git a/senpy/client.py b/senpy/client.py new file mode 100644 index 0000000..b9dcb3e --- /dev/null +++ b/senpy/client.py @@ -0,0 +1,45 @@ +import requests +import logging +from . import models + +logger = logging.getLogger(__name__) + + +class Client(object): + + def __init__(self, endpoint): + self.endpoint = endpoint + + def analyse(self, input, method='GET', **kwargs): + return self.request('/', method=method, input=input, **kwargs) + + def request(self, path=None, method='GET', **params): + url = '{}{}'.format(self.endpoint, path) + response = requests.request(method=method, + url=url, + params=params) + try: + resp = models.from_dict(response.json()) + resp.validate(resp) + return resp + except Exception as ex: + logger.error(('There seems to be a problem with the response:\n' + '\tURL: {url}\n' + '\tError: {error}\n' + '\t\n' + '#### Response:\n' + '\tCode: {code}' + '\tContent: {content}' + '\n').format(error=ex, + url=url, + code=response.status_code, + content=response.content)) + raise ex + + +if __name__ == '__main__': + c = Client('http://senpy.cluster.gsi.dit.upm.es/api/') + resp = c.analyse('hello') + # print(resp) + print(resp.entries) + resp.validate() diff --git a/senpy/extensions.py b/senpy/extensions.py index 1583d96..e32082b 100644 --- a/senpy/extensions.py +++ b/senpy/extensions.py @@ -161,7 +161,7 @@ class Senpy(object): self._set_active_plugin(plugin_name, success) except Exception as ex: msg = "Error activating plugin {} - {} : \n\t{}".format( - plugin.name, ex, ex.format_exc()) + plugin.name, ex, traceback.format_exc()) logger.error(msg) raise Error(msg) if sync: diff --git a/senpy/models.py b/senpy/models.py index e6ca1f5..776492d 100644 --- a/senpy/models.py +++ b/senpy/models.py @@ -12,12 +12,15 @@ import time import copy import json import os -import logging import jsonref import jsonschema from flask import Response as FlaskResponse +import logging + +logger = logging.getLogger(__name__) + DEFINITIONS_FILE = 'definitions.json' CONTEXT_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld') @@ -40,7 +43,6 @@ def read_schema(schema_file, absolute=False): base_schema = read_schema(DEFINITIONS_FILE) -logging.debug(base_schema) class Context(dict): @@ -72,7 +74,7 @@ base_context = Context.load(CONTEXT_PATH) class SenpyMixin(object): context = base_context["@context"] - def flask(self, in_headers=False, headers=None, **kwargs): + def flask(self, in_headers=True, headers=None, **kwargs): """ Return the values and error to be used in flask. So far, it returns a fixed context. We should store/generate different @@ -151,14 +153,16 @@ class SenpyMixin(object): return str(self.to_JSON()) -class SenpyModel(SenpyMixin, dict): +class BaseModel(SenpyMixin, dict): schema = base_schema def __init__(self, *args, **kwargs): - self.id = kwargs.pop('id', '{}_{}'.format( - type(self).__name__, time.time())) - + if 'id' in kwargs: + self.id = kwargs.pop('id') + elif kwargs.pop('_auto_id', True): + self.id = '_:{}_{}'.format( + type(self).__name__, time.time()) temp = dict(*args, **kwargs) for obj in [self.schema, ] + self.schema.get('allOf', []): @@ -175,7 +179,11 @@ class SenpyModel(SenpyMixin, dict): context = temp['context'] del temp['context'] self.__dict__['context'] = Context.load(context) - super(SenpyModel, self).__init__(temp) + try: + temp['@type'] = getattr(self, '@type') + except AttributeError: + logger.warn('Creating an instance of an unknown model') + super(BaseModel, self).__init__(temp) def _get_key(self, key): key = key.replace("__", ":", 1) @@ -206,73 +214,80 @@ class SenpyModel(SenpyMixin, dict): return d -class Response(SenpyModel): - schema = read_schema('response.json') +_subtypes = {} -class Results(SenpyModel): - schema = read_schema('results.json') +def register(rsubclass, rtype=None): + _subtypes[rtype or rsubclass.__name__] = rsubclass -class Entry(SenpyModel): - schema = read_schema('entry.json') +def from_dict(indict): + target = indict.get('@type', None) + if target and target in _subtypes: + cls = _subtypes[target] + else: + cls = BaseModel + return cls(**indict) -class Sentiment(SenpyModel): - schema = read_schema('sentiment.json') +def from_schema(name, schema_file=None, base_classes=None): + base_classes = base_classes or [] + base_classes.append(BaseModel) + schema_file = schema_file or '{}.json'.format(name) + class_name = '{}{}'.format(i[0].upper(), i[1:]) + newclass = type(class_name, tuple(base_classes), {}) + setattr(newclass, '@type', name) + setattr(newclass, 'schema', read_schema(schema_file)) + register(newclass, name) + return newclass -class Analysis(SenpyModel): - schema = read_schema('analysis.json') +def _add_from_schema(*args, **kwargs): + generatedClass = from_schema(*args, **kwargs) + globals()[generatedClass.__name__] = generatedClass + del generatedClass -class EmotionSet(SenpyModel): - schema = read_schema('emotionSet.json') +for i in ['response', + 'results', + 'entry', + 'sentiment', + 'analysis', + 'emotionSet', + 'emotion', + 'emotionModel', + 'suggestion', + 'plugin', + 'emotionPlugin', + 'sentimentPlugin', + 'plugins']: + _add_from_schema(i) - -class Emotion(SenpyModel): - schema = read_schema('emotion.json') - - -class EmotionModel(SenpyModel): - schema = read_schema('emotionModel.json') - - -class Suggestion(SenpyModel): - schema = read_schema('suggestion.json') - - -class PluginModel(SenpyModel): - schema = read_schema('plugin.json') - - -class EmotionPluginModel(SenpyModel): - schema = read_schema('plugin.json') - - -class SentimentPluginModel(SenpyModel): - schema = read_schema('plugin.json') - - -class Plugins(SenpyModel): - schema = read_schema('plugins.json') +_ErrorModel = from_schema('error') class Error(SenpyMixin, BaseException): def __init__(self, message, - status=500, - params=None, - errors=None, *args, **kwargs): + super(Error, self).__init__(self, message, message) + self._error = _ErrorModel(message=message, *args, **kwargs) self.message = message - self.status = status - self.params = params or {} - self.errors = errors or "" - def _plain_dict(self): - return self.__dict__ + def __getattr__(self, key): + if key != '_error' and hasattr(self._error, key): + return getattr(self._error, key) + raise AttributeError(key) - def __str__(self): - return str(self.jsonld()) + def __setattr__(self, key, value): + if key != '_error': + return setattr(self._error, key, value) + else: + super(Error, self).__setattr__(key, value) + + def __delattr__(self, key): + delattr(self._error, key) + + +register(Error, 'error') diff --git a/senpy/plugins.py b/senpy/plugins.py index a35fae9..af82700 100644 --- a/senpy/plugins.py +++ b/senpy/plugins.py @@ -6,16 +6,16 @@ import os.path import pickle import logging import tempfile -from .models import PluginModel, Error +from . import models logger = logging.getLogger(__name__) -class SenpyPlugin(PluginModel): +class SenpyPlugin(models.Plugin): def __init__(self, info=None): if not info: - raise Error(message=("You need to provide configuration" - "information for the plugin.")) + raise models.Error(message=("You need to provide configuration" + "information for the plugin.")) logger.debug("Initialising {}".format(info)) super(SenpyPlugin, self).__init__(info) self.id = '{}_{}'.format(self.name, self.version) @@ -40,7 +40,7 @@ class SenpyPlugin(PluginModel): self.deactivate() -class SentimentPlugin(SenpyPlugin): +class SentimentPlugin(SenpyPlugin, models.SentimentPlugin): def __init__(self, info, *args, **kwargs): super(SentimentPlugin, self).__init__(info, *args, **kwargs) self.minPolarityValue = float(info.get("minPolarityValue", 0)) @@ -48,7 +48,7 @@ class SentimentPlugin(SenpyPlugin): self["@type"] = "marl:SentimentAnalysis" -class EmotionPlugin(SenpyPlugin): +class EmotionPlugin(SentimentPlugin, models.EmotionPlugin): def __init__(self, info, *args, **kwargs): self.minEmotionValue = float(info.get("minEmotionValue", 0)) self.maxEmotionValue = float(info.get("maxEmotionValue", 0)) @@ -71,10 +71,6 @@ class ShelfMixin(object): del self.__dict__['_sh'] self.save() - def __del__(self): - self.save() - super(ShelfMixin, self).__del__() - @property def shelf_file(self): if not hasattr(self, '_shelf_file') or not self._shelf_file: @@ -86,8 +82,7 @@ class ShelfMixin(object): return self._shelf_file def save(self): - logger.debug('closing pickle') + logger.debug('saving pickle') if hasattr(self, '_sh') and self._sh is not None: with open(self.shelf_file, 'wb') as f: pickle.dump(self._sh, f) - del (self.__dict__['_sh']) diff --git "a/senpy/schemas/\\" "b/senpy/schemas/\\" new file mode 100644 index 0000000..dd76fa5 --- /dev/null +++ "b/senpy/schemas/\\" @@ -0,0 +1,7 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "Senpy analysis", + "allOf": [{ + "$ref": "atom.json" + }] +} diff --git a/senpy/schemas/atom.json b/senpy/schemas/atom.json new file mode 100644 index 0000000..baa43fd --- /dev/null +++ b/senpy/schemas/atom.json @@ -0,0 +1,15 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "Base schema for all Senpy objects", + "type": "object", + "properties": { + "@id": { + "type": "string" + }, + "@type": { + "type": "string", + "description": "Type of the atom. e.g., 'onyx:EmotionAnalysis', 'nif:Entry'" + } + }, + "required": ["@id", "@type"] +} diff --git a/senpy/schemas/emotionPlugin.json b/senpy/schemas/emotionPlugin.json new file mode 100644 index 0000000..c36a859 --- /dev/null +++ b/senpy/schemas/emotionPlugin.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "$allOf": [ + { + "$ref": "plugin.json" + }, + { + "properties": { + "onyx:usesEmotionModel": { + "type": "array", + "items": { + "$ref": "emotionModel.json" + } + } + } + } + ] +} diff --git a/senpy/schemas/entry.json b/senpy/schemas/entry.json index d90160b..9eef1e2 100644 --- a/senpy/schemas/entry.json +++ b/senpy/schemas/entry.json @@ -5,9 +5,6 @@ "@id": { "type": "string" }, - "@type": { - "enum": [["nif:RFC5147String", "nif:Context"]] - }, "nif:isString": { "description": "String contained in this Context", "type": "string" diff --git a/senpy/schemas/error.json b/senpy/schemas/error.json new file mode 100644 index 0000000..966303f --- /dev/null +++ b/senpy/schemas/error.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "Base schema for all Senpy objects", + "type": "object", + "$allOf": [ + {"$ref": "atom.json"}, + { + "properties": { + "message": { + "type": "string" + }, + "errors": { + "type": "list", + "items": {"type": "object"} + }, + "code": { + "type": "int" + }, + "required": ["message"] + } + } + ] +} diff --git a/senpy/schemas/plugin.json b/senpy/schemas/plugin.json index aa24ae0..a2cb04d 100644 --- a/senpy/schemas/plugin.json +++ b/senpy/schemas/plugin.json @@ -4,7 +4,12 @@ "required": ["@id", "extra_params"], "properties": { "@id": { - "type": "string" + "type": "string", + "description": "Unique identifier for the plugin, usually comprised of the name of the plugin and the version." + }, + "name": { + "type": "string", + "description": "The name of the plugin, which will be used in the algorithm detection phase" }, "extra_params": { "type": "object", diff --git a/senpy/schemas/sentimentPlugin.json b/senpy/schemas/sentimentPlugin.json new file mode 100644 index 0000000..26830cd --- /dev/null +++ b/senpy/schemas/sentimentPlugin.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "$allOf": [ + { + "$ref": "plugin.json" + }, + { + "properties": { + "marl:minPolarityValue": { + "type": "number" + }, + "marl:maxPolarityValue": { + "type": "number" + } + } + } + ] +} diff --git a/tests/test_cli.py b/tests/test_cli.py index 27c9ae6..cd433a1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,11 @@ import logging from functools import partial +try: + from unittest.mock import patch +except ImportError: + from mock import patch + logger = logging.getLogger(__name__) from unittest import TestCase @@ -11,9 +16,12 @@ from senpy.models import Error class CLITest(TestCase): def test_basic(self): self.assertRaises(Error, partial(main_function, [])) - res = main_function(['--input', 'test']) - assert 'entries' in res - res = main_function(['--input', 'test', '--algo', 'rand']) - assert 'entries' in res - assert 'analysis' in res - assert res['analysis'][0]['name'] == 'rand' + + with patch('senpy.extensions.Senpy.analyse') as patched: + main_function(['--input', 'test']) + + patched.assert_called_with(input='test') + with patch('senpy.extensions.Senpy.analyse') as patched: + main_function(['--input', 'test', '--algo', 'rand']) + + patched.assert_called_with(input='test', algo='rand') diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 0000000..2690d88 --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,42 @@ +from unittest import TestCase +try: + from unittest.mock import patch +except ImportError: + from mock import patch + +from senpy.client import Client +from senpy.models import Results, Error + + +class Call(dict): + + def __init__(self, obj): + self.obj = obj.jsonld() + + def json(self): + return self.obj + + +class ModelsTest(TestCase): + def setUp(self): + self.host = '0.0.0.0' + self.port = 5000 + + def test_client(self): + endpoint = 'http://dummy/' + client = Client(endpoint) + success = Call(Results()) + with patch('requests.request', return_value=success) as patched: + resp = client.analyse('hello') + assert isinstance(resp, Results) + patched.assert_called_with(url=endpoint + '/', + method='GET', + params={'input': 'hello'}) + error = Call(Error('Nothing')) + with patch('requests.request', return_value=error) as patched: + resp = client.analyse(input='hello', algorithm='NONEXISTENT') + assert isinstance(resp, Error) + patched.assert_called_with(url=endpoint + '/', + method='GET', + params={'input': 'hello', + 'algorithm': 'NONEXISTENT'}) diff --git a/tests/test_models.py b/tests/test_models.py index a21b240..5ced7f3 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -11,7 +11,9 @@ from pprint import pprint class ModelsTest(TestCase): def test_jsonld(self): - prueba = {"id": "test", "analysis": [], "entries": []} + prueba = {"id": "test", + "analysis": [], + "entries": []} r = Results(**prueba) print("Response's context: ") pprint(r.context) @@ -28,11 +30,11 @@ class ModelsTest(TestCase): assert "id" not in j r6 = Results(**prueba) - r6.entries.append( - Entry({ - "@id": "ohno", - "nif:isString": "Just testing" - })) + e = Entry({ + "@id": "ohno", + "nif:isString": "Just testing" + }) + r6.entries.append(e) logging.debug("Reponse 6: %s", r6) assert ("marl" in r6.context) assert ("entries" in r6.context) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 52f49d4..734e643 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -44,7 +44,6 @@ class PluginsTest(TestCase): a = ShelfDummyPlugin( info={'name': 'default_shelve_file', 'version': 'test'}) - assert os.path.dirname(a.shelf_file) == tempfile.gettempdir() a.activate() assert os.path.isfile(a.shelf_file) os.remove(a.shelf_file)