diff --git a/Makefile b/Makefile index 96633ce..91fcd5c 100644 --- a/Makefile +++ b/Makefile @@ -73,8 +73,8 @@ pip_test: $(addprefix pip_test-,$(PYVERSIONS)) clean: @docker ps -a | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true @docker images | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true - @docker rmi $(NAME)-dev 2>/dev/null || true - + @docker stop $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true + @docker rm $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true git_commit: git commit -a diff --git a/docs/bad-examples/results/example-analysis-as-id-FAIL.json b/docs/bad-examples/results/example-analysis-as-id-FAIL.json new file mode 100644 index 0000000..a24e565 --- /dev/null +++ b/docs/bad-examples/results/example-analysis-as-id-FAIL.json @@ -0,0 +1,78 @@ +{ + "@context": "http://mixedemotions-project.eu/ns/context.jsonld", + "@id": "me:Result1", + "@type": "results", + "analysis": [ + "me:SAnalysis1", + "me:SgAnalysis1", + "me:EmotionAnalysis1", + "me:NER1", + { + "@type": "analysis", + "@id": "wrong" + } + ], + "entries": [ + { + "@id": "http://micro.blog/status1", + "@type": [ + "nif:RFC5147String", + "nif:Context" + ], + "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource", + "entities": [ + { + "@id": "http://micro.blog/status1#char=5,13", + "nif:beginIndex": 5, + "nif:endIndex": 13, + "nif:anchorOf": "Microsoft", + "me:references": "http://dbpedia.org/page/Microsoft", + "prov:wasGeneratedBy": "me:NER1" + }, + { + "@id": "http://micro.blog/status1#char=25,37", + "nif:beginIndex": 25, + "nif:endIndex": 37, + "nif:anchorOf": "Windows Phone", + "me:references": "http://dbpedia.org/page/Windows_Phone", + "prov:wasGeneratedBy": "me:NER1" + } + ], + "suggestions": [ + { + "@id": "http://micro.blog/status1#char=16,77", + "nif:beginIndex": 16, + "nif:endIndex": 77, + "nif:anchorOf": "put your Windows Phone on your newest #open technology program", + "prov:wasGeneratedBy": "me:SgAnalysis1" + } + ], + "sentiments": [ + { + "@id": "http://micro.blog/status1#char=80,97", + "nif:beginIndex": 80, + "nif:endIndex": 97, + "nif:anchorOf": "You'll be awesome.", + "marl:hasPolarity": "marl:Positive", + "marl:polarityValue": 0.9, + "prov:wasGeneratedBy": "me:SAnalysis1" + } + ], + "emotions": [ + { + "@id": "http://micro.blog/status1#char=0,109", + "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource", + "prov:wasGeneratedBy": "me:EAnalysis1", + "onyx:hasEmotion": [ + { + "onyx:hasEmotionCategory": "wna:liking" + }, + { + "onyx:hasEmotionCategory": "wna:excitement" + } + ] + } + ] + } + ] +} diff --git a/docs/examples/results/example-analysis-as-id.json b/docs/examples/results/example-analysis-as-id.json new file mode 100644 index 0000000..95ee7de --- /dev/null +++ b/docs/examples/results/example-analysis-as-id.json @@ -0,0 +1,74 @@ +{ + "@context": "http://mixedemotions-project.eu/ns/context.jsonld", + "@id": "me:Result1", + "@type": "results", + "analysis": [ + "me:SAnalysis1", + "me:SgAnalysis1", + "me:EmotionAnalysis1", + "me:NER1" + ], + "entries": [ + { + "@id": "http://micro.blog/status1", + "@type": [ + "nif:RFC5147String", + "nif:Context" + ], + "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource", + "entities": [ + { + "@id": "http://micro.blog/status1#char=5,13", + "nif:beginIndex": 5, + "nif:endIndex": 13, + "nif:anchorOf": "Microsoft", + "me:references": "http://dbpedia.org/page/Microsoft", + "prov:wasGeneratedBy": "me:NER1" + }, + { + "@id": "http://micro.blog/status1#char=25,37", + "nif:beginIndex": 25, + "nif:endIndex": 37, + "nif:anchorOf": "Windows Phone", + "me:references": "http://dbpedia.org/page/Windows_Phone", + "prov:wasGeneratedBy": "me:NER1" + } + ], + "suggestions": [ + { + "@id": "http://micro.blog/status1#char=16,77", + "nif:beginIndex": 16, + "nif:endIndex": 77, + "nif:anchorOf": "put your Windows Phone on your newest #open technology program", + "prov:wasGeneratedBy": "me:SgAnalysis1" + } + ], + "sentiments": [ + { + "@id": "http://micro.blog/status1#char=80,97", + "nif:beginIndex": 80, + "nif:endIndex": 97, + "nif:anchorOf": "You'll be awesome.", + "marl:hasPolarity": "marl:Positive", + "marl:polarityValue": 0.9, + "prov:wasGeneratedBy": "me:SAnalysis1" + } + ], + "emotions": [ + { + "@id": "http://micro.blog/status1#char=0,109", + "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource", + "prov:wasGeneratedBy": "me:EAnalysis1", + "onyx:hasEmotion": [ + { + "onyx:hasEmotionCategory": "wna:liking" + }, + { + "onyx:hasEmotionCategory": "wna:excitement" + } + ] + } + ] + } + ] +} diff --git a/requirements.txt b/requirements.txt index 0dcc894..db3c57b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ Flask>=0.10.1 requests>=2.4.1 -gevent>=1.1rc4 +tornado>=4.4.3 PyLD>=0.6.5 six future diff --git a/senpy/__main__.py b/senpy/__main__.py index dd711e4..4358174 100644 --- a/senpy/__main__.py +++ b/senpy/__main__.py @@ -22,15 +22,16 @@ the server. from flask import Flask from senpy.extensions import Senpy -from gevent.wsgi import WSGIServer -from gevent.monkey import patch_all +from tornado.wsgi import WSGIContainer +from tornado.httpserver import HTTPServer +from tornado.ioloop import IOLoop + + import logging import os import argparse import senpy -patch_all(thread=False) - SERVER_PORT = os.environ.get("PORT", 5000) @@ -92,9 +93,10 @@ def main(): print('Server running on port %s:%d. Ctrl+C to quit' % (args.host, args.port)) if not app.debug: - http_server = WSGIServer((args.host, args.port), app) + http_server = HTTPServer(WSGIContainer(app)) + http_server.listen(args.port, address=args.host) try: - http_server.serve_forever() + IOLoop.instance().start() except KeyboardInterrupt: print('Bye!') http_server.stop() diff --git a/senpy/client.py b/senpy/client.py index e2810f2..5867715 100644 --- a/senpy/client.py +++ b/senpy/client.py @@ -1,6 +1,7 @@ import requests import logging from . import models +from .plugins import default_plugin_type logger = logging.getLogger(__name__) @@ -12,6 +13,10 @@ class Client(object): def analyse(self, input, method='GET', **kwargs): return self.request('/', method=method, input=input, **kwargs) + def plugins(self, ptype=default_plugin_type): + resp = self.request(path='/plugins', plugin_type=ptype).plugins + return {p.name: p for p in resp} + def request(self, path=None, method='GET', **params): url = '{}{}'.format(self.endpoint, path) response = requests.request(method=method, url=url, params=params) diff --git a/senpy/extensions.py b/senpy/extensions.py index 36e2654..e30fa14 100644 --- a/senpy/extensions.py +++ b/senpy/extensions.py @@ -183,7 +183,7 @@ class Senpy(object): return resp def _conversion_candidates(self, fromModel, toModel): - candidates = self.filter_plugins(**{'@type': 'emotionConversionPlugin'}) + candidates = self.filter_plugins(plugin_type='emotionConversionPlugin') for name, candidate in candidates.items(): for pair in candidate.onyx__doesConversion: logging.debug(pair) @@ -303,6 +303,7 @@ class Senpy(object): else: th = Thread(target=act) th.start() + return th def deactivate_plugin(self, plugin_name, sync=False): try: @@ -327,6 +328,7 @@ class Senpy(object): else: th = Thread(target=deact) th.start() + return th @classmethod def validate_info(cls, info): @@ -417,33 +419,7 @@ class Senpy(object): return self._plugin_list def filter_plugins(self, **kwargs): - """ Filter plugins by different criteria """ - ptype = kwargs.pop('plugin_type', None) - logger.debug('#' * 100) - logger.debug('ptype {}'.format(ptype)) - if ptype: - try: - ptype = ptype[0].upper() + ptype[1:] - pclass = getattr(plugins, ptype) - logger.debug('Class: {}'.format(pclass)) - candidates = filter(lambda x: isinstance(x, pclass), - self.plugins.values()) - except AttributeError: - raise Error('{} is not a valid type'.format(ptype)) - else: - candidates = self.plugins.values() - - logger.debug(candidates) - - def matches(plug): - res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items()) - logger.debug( - "matching {} with {}: {}".format(plug.name, kwargs, res)) - return res - - if kwargs: - candidates = filter(matches, candidates) - return {p.name: p for p in candidates} + return plugins.pfilter(self.plugins, **kwargs) @property def analysis_plugins(self): diff --git a/senpy/plugins/__init__.py b/senpy/plugins/__init__.py index bad49ee..b45a7f4 100644 --- a/senpy/plugins/__init__.py +++ b/senpy/plugins/__init__.py @@ -9,6 +9,7 @@ import logging import tempfile import copy from .. import models +from ..api import API_PARAMS logger = logging.getLogger(__name__) @@ -117,3 +118,40 @@ class ShelfMixin(object): if hasattr(self, '_sh') and self._sh is not None: with open(self.shelf_file, 'wb') as f: pickle.dump(self._sh, f) + + +default_plugin_type = API_PARAMS['plugin_type']['default'] + + +def pfilter(plugins, **kwargs): + """ Filter plugins by different criteria """ + if isinstance(plugins, models.Plugins): + plugins = plugins.plugins + elif isinstance(plugins, dict): + plugins = plugins.values() + ptype = kwargs.pop('plugin_type', default_plugin_type) + logger.debug('#' * 100) + logger.debug('ptype {}'.format(ptype)) + if ptype: + try: + ptype = ptype[0].upper() + ptype[1:] + pclass = globals()[ptype] + logger.debug('Class: {}'.format(pclass)) + candidates = filter(lambda x: isinstance(x, pclass), + plugins) + except KeyError: + raise models.Error('{} is not a valid type'.format(ptype)) + else: + candidates = plugins + + logger.debug(candidates) + + def matches(plug): + res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items()) + logger.debug( + "matching {} with {}: {}".format(plug.name, kwargs, res)) + return res + + if kwargs: + candidates = filter(matches, candidates) + return {p.name: p for p in candidates} diff --git a/senpy/schemas/context.jsonld b/senpy/schemas/context.jsonld index 4b77adb..86d6c92 100644 --- a/senpy/schemas/context.jsonld +++ b/senpy/schemas/context.jsonld @@ -37,12 +37,12 @@ "@type": "@id", "@container": "@set" }, - "plugins": { - "@container": "@list" - }, "options": { "@container": "@set" }, + "plugins": { + "@container": "@set" + }, "prov:wasGeneratedBy": { "@type": "@id" }, diff --git a/senpy/schemas/plugins.json b/senpy/schemas/plugins.json index cd7c937..7cafd9b 100644 --- a/senpy/schemas/plugins.json +++ b/senpy/schemas/plugins.json @@ -10,8 +10,6 @@ "items": { "$ref": "plugin.json" } - }, - "@type": { } } } diff --git a/senpy/schemas/results.json b/senpy/schemas/results.json index d699b56..06ad88e 100644 --- a/senpy/schemas/results.json +++ b/senpy/schemas/results.json @@ -18,10 +18,16 @@ "type": "string" }, "analysis": { - "type": "array", "default": [], + "type": "array", "items": { - "$ref": "analysis.json" + "anyOf": [ + { + "$ref": "analysis.json" + },{ + "type": "string" + } + ] } }, "entries": { diff --git a/tests/plugins/async_plugin/asyncplugin.py b/tests/plugins/async_plugin/asyncplugin.py new file mode 100644 index 0000000..976e6c8 --- /dev/null +++ b/tests/plugins/async_plugin/asyncplugin.py @@ -0,0 +1,23 @@ +from senpy.plugins import AnalysisPlugin + +import multiprocessing + + +def _train(process_number): + return process_number + + +class AsyncPlugin(AnalysisPlugin): + def _do_async(self, num_processes): + pool = multiprocessing.Pool(processes=num_processes) + values = pool.map(_train, range(num_processes)) + + return values + + def activate(self): + self.value = self._do_async(4) + + def analyse_entry(self, entry, params): + values = self._do_async(2) + entry.async_values = values + yield entry diff --git a/tests/plugins/async_plugin/asyncplugin.senpy b/tests/plugins/async_plugin/asyncplugin.senpy new file mode 100644 index 0000000..8c71849 --- /dev/null +++ b/tests/plugins/async_plugin/asyncplugin.senpy @@ -0,0 +1,8 @@ +--- +name: Async +module: asyncplugin +description: I am async +author: "@balkian" +version: '0.1' +async: true +extra_params: {} \ No newline at end of file diff --git a/tests/test_client.py b/tests/test_client.py index c94af96..90ad7fa 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -4,18 +4,21 @@ try: except ImportError: from mock import patch +import json + from senpy.client import Client -from senpy.models import Results, Error +from senpy.models import Results, Plugins, Error +from senpy.plugins import AnalysisPlugin, default_plugin_type class Call(dict): def __init__(self, obj): - self.obj = obj.jsonld() + self.obj = obj.serialize() self.status_code = 200 self.content = self.json() def json(self): - return self.obj + return json.loads(self.obj) class ModelsTest(TestCase): @@ -44,3 +47,19 @@ class ModelsTest(TestCase): method='GET', params={'input': 'hello', 'algorithm': 'NONEXISTENT'}) + + def test_plugins(self): + endpoint = 'http://dummy/' + client = Client(endpoint) + plugins = Plugins() + p1 = AnalysisPlugin({'name': 'AnalysisP1', 'version': 0, 'description': 'No'}) + plugins.plugins = [p1, ] + success = Call(plugins) + with patch('requests.request', return_value=success) as patched: + response = client.plugins() + assert isinstance(response, dict) + assert len(response) == 1 + assert "AnalysisP1" in response + patched.assert_called_with( + url=endpoint + '/plugins', method='GET', + params={'plugin_type': default_plugin_type}) diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 0523789..067f697 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -167,7 +167,7 @@ class ExtensionsTest(TestCase): assert len(senpy.plugins) > 1 def test_convert_emotions(self): - self.senpy.activate_all() + self.senpy.activate_all(sync=True) plugin = Plugin({ 'id': 'imaginary', 'onyx:usesEmotionModel': 'emoml:fsre-dimensions' @@ -205,3 +205,14 @@ class ExtensionsTest(TestCase): [plugin, ], params) assert len(r3.entries[0].emotions) == 1 + + # def test_async_plugin(self): + # """ We should accept multiprocessing plugins with async=False""" + # thread1 = self.senpy.activate_plugin("Async", sync=False) + # thread1.join(timeout=1) + # assert len(self.senpy.plugins['Async'].value) == 4 + + # resp = self.senpy.analyse(input='nothing', algorithm='Async') + + # assert len(resp.entries[0].async_values) == 2 + # self.senpy.activate_plugin("Async", sync=True) diff --git a/tests/test_models.py b/tests/test_models.py index 7e3f020..c0c7034 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -109,13 +109,15 @@ class ModelsTest(TestCase): } }}) c = p.jsonld() - assert "info" not in c - assert "repo" not in c - assert "extra_params" in c - logging.debug("Framed:") + assert '@type' in c + assert c['@type'] == 'plugin' + assert 'info' not in c + assert 'repo' not in c + assert 'extra_params' in c + logging.debug('Framed:') logging.debug(c) p.validate() - assert "es" in c['extra_params']['none']['options'] + assert 'es' in c['extra_params']['none']['options'] assert isinstance(c['extra_params']['none']['options'], list) def test_str(self): @@ -158,14 +160,20 @@ class ModelsTest(TestCase): g = rdflib.Graph().parse(data=t, format='turtle') assert len(g) == len(triples) + def test_plugin_list(self): + """The plugin list should be of type \"plugins\"""" + plugs = Plugins() + c = plugs.jsonld() + assert '@type' in c + assert c['@type'] == 'plugins' + def test_single_plugin(self): """A response with a single plugin should still return a list""" plugs = Plugins() - for i in range(10): - p = Plugin({'id': str(i), - 'version': 0, - 'description': 'dummy'}) - plugs.plugins.append(p) + p = Plugin({'id': str(1), + 'version': 0, + 'description': 'dummy'}) + plugs.plugins.append(p) assert isinstance(plugs.plugins, list) js = plugs.jsonld() assert isinstance(js['plugins'], list)