balkian/senpy
1
0
Fork 0
mirror of https://github.com/gsi-upm/senpy synced 2024-12-22 04:58:12 +00:00
Code Issues Releases Wiki Activity

Merge branches into 0.8.x

Browse Source 
'25-validation-errors'
'27-add-method-to-get-list-of-plugins'
'28-fix-multiprocessing-issues'
This commit is contained in:
J. Fernando Sánchez 2017-04-10 20:31:26 +02:00
commit 4ba9535d56
12 changed files with 142 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
requirements.txt
View File

@ -1,6 +1,6 @@
Flask>=0.10.1
requests>=2.4.1
gevent>=1.1rc4
tornado>=4.4.3
PyLD>=0.6.5
six
future

14
senpy/__main__.py
View File

@ -22,15 +22,16 @@ the server.
from flask import Flask
from senpy.extensions import Senpy
from gevent.wsgi import WSGIServer
from gevent.monkey import patch_all
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
import logging
import os
import argparse
import senpy
patch_all(thread=False)
SERVER_PORT = os.environ.get("PORT", 5000)
@ -92,9 +93,10 @@ def main():
print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,
args.port))
if not app.debug:
http_server = WSGIServer((args.host, args.port), app)
http_server = HTTPServer(WSGIContainer(app))
http_server.listen(args.port, address=args.host)
try:
http_server.serve_forever()
IOLoop.instance().start()
except KeyboardInterrupt:
print('Bye!')
http_server.stop()

5
senpy/client.py
View File

@ -1,6 +1,7 @@
import requests
import logging
from . import models
from .plugins import default_plugin_type
logger = logging.getLogger(__name__)
@ -12,6 +13,10 @@ class Client(object):
def analyse(self, input, method='GET', **kwargs):
return self.request('/', method=method, input=input, **kwargs)
def plugins(self, ptype=default_plugin_type):
resp = self.request(path='/plugins', plugin_type=ptype).plugins
return {p.name: p for p in resp}
def request(self, path=None, method='GET', **params):
url = '{}{}'.format(self.endpoint, path)
response = requests.request(method=method, url=url, params=params)

32
senpy/extensions.py
View File

@ -183,7 +183,7 @@ class Senpy(object):
return resp
def _conversion_candidates(self, fromModel, toModel):
candidates = self.filter_plugins(**{'@type': 'emotionConversionPlugin'})
candidates = self.filter_plugins(plugin_type='emotionConversionPlugin')
for name, candidate in candidates.items():
for pair in candidate.onyx__doesConversion:
logging.debug(pair)
@ -303,6 +303,7 @@ class Senpy(object):
else:
th = Thread(target=act)
th.start()
return th
def deactivate_plugin(self, plugin_name, sync=False):
try:
@ -327,6 +328,7 @@ class Senpy(object):
else:
th = Thread(target=deact)
th.start()
return th
@classmethod
def validate_info(cls, info):
@ -417,33 +419,7 @@ class Senpy(object):
return self._plugin_list
def filter_plugins(self, **kwargs):
""" Filter plugins by different criteria """
ptype = kwargs.pop('plugin_type', None)
logger.debug('#' * 100)
logger.debug('ptype {}'.format(ptype))
if ptype:
try:
ptype = ptype[0].upper() + ptype[1:]
pclass = getattr(plugins, ptype)
logger.debug('Class: {}'.format(pclass))
candidates = filter(lambda x: isinstance(x, pclass),
self.plugins.values())
except AttributeError:
raise Error('{} is not a valid type'.format(ptype))
else:
candidates = self.plugins.values()
logger.debug(candidates)
def matches(plug):
res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
logger.debug(
"matching {} with {}: {}".format(plug.name, kwargs, res))
return res
if kwargs:
candidates = filter(matches, candidates)
return {p.name: p for p in candidates}
return plugins.pfilter(self.plugins, **kwargs)
@property
def analysis_plugins(self):

38
senpy/plugins/__init__.py
View File

@ -9,6 +9,7 @@ import logging
import tempfile
import copy
from .. import models
from ..api import API_PARAMS
logger = logging.getLogger(__name__)
@ -117,3 +118,40 @@ class ShelfMixin(object):
if hasattr(self, '_sh') and self._sh is not None:
with open(self.shelf_file, 'wb') as f:
pickle.dump(self._sh, f)
default_plugin_type = API_PARAMS['plugin_type']['default']
def pfilter(plugins, **kwargs):
""" Filter plugins by different criteria """
if isinstance(plugins, models.Plugins):
plugins = plugins.plugins
elif isinstance(plugins, dict):
plugins = plugins.values()
ptype = kwargs.pop('plugin_type', default_plugin_type)
logger.debug('#' * 100)
logger.debug('ptype {}'.format(ptype))
if ptype:
try:
ptype = ptype[0].upper() + ptype[1:]
pclass = globals()[ptype]
logger.debug('Class: {}'.format(pclass))
candidates = filter(lambda x: isinstance(x, pclass),
plugins)
except KeyError:
raise models.Error('{} is not a valid type'.format(ptype))
else:
candidates = plugins
logger.debug(candidates)
def matches(plug):
res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
logger.debug(
"matching {} with {}: {}".format(plug.name, kwargs, res))
return res
if kwargs:
candidates = filter(matches, candidates)
return {p.name: p for p in candidates}

6
senpy/schemas/context.jsonld
View File

@ -37,12 +37,12 @@
"@type": "@id",
"@container": "@set"
},
"plugins": {
"@container": "@list"
},
"options": {
"@container": "@set"
},
"plugins": {
"@container": "@set"
},
"prov:wasGeneratedBy": {
"@type": "@id"
},

2
senpy/schemas/plugins.json
View File

@ -10,8 +10,6 @@
"items": {
"$ref": "plugin.json"
}
},
"@type": {
}
}
}

23
tests/plugins/async_plugin/asyncplugin.py Normal file
View File

@ -0,0 +1,23 @@
from senpy.plugins import AnalysisPlugin
import multiprocessing
def _train(process_number):
return process_number
class AsyncPlugin(AnalysisPlugin):
def _do_async(self, num_processes):
pool = multiprocessing.Pool(processes=num_processes)
values = pool.map(_train, range(num_processes))
return values
def activate(self):
self.value = self._do_async(4)
def analyse_entry(self, entry, params):
values = self._do_async(2)
entry.async_values = values
yield entry

8
tests/plugins/async_plugin/asyncplugin.senpy Normal file
View File

@ -0,0 +1,8 @@
---
name: Async
module: asyncplugin
description: I am async
author: "@balkian"
version: '0.1'
async: true
extra_params: {}

25
tests/test_client.py
View File

@ -4,18 +4,21 @@ try:
except ImportError:
from mock import patch
import json
from senpy.client import Client
from senpy.models import Results, Error
from senpy.models import Results, Plugins, Error
from senpy.plugins import AnalysisPlugin, default_plugin_type
class Call(dict):
def __init__(self, obj):
self.obj = obj.jsonld()
self.obj = obj.serialize()
self.status_code = 200
self.content = self.json()
def json(self):
return self.obj
return json.loads(self.obj)
class ModelsTest(TestCase):
@ -44,3 +47,19 @@ class ModelsTest(TestCase):
method='GET',
params={'input': 'hello',
'algorithm': 'NONEXISTENT'})
def test_plugins(self):
endpoint = 'http://dummy/'
client = Client(endpoint)
plugins = Plugins()
p1 = AnalysisPlugin({'name': 'AnalysisP1', 'version': 0, 'description': 'No'})
plugins.plugins = [p1, ]
success = Call(plugins)
with patch('requests.request', return_value=success) as patched:
response = client.plugins()
assert isinstance(response, dict)
assert len(response) == 1
assert "AnalysisP1" in response
patched.assert_called_with(
url=endpoint + '/plugins', method='GET',
params={'plugin_type': default_plugin_type})

13
tests/test_extensions.py
View File

@ -167,7 +167,7 @@ class ExtensionsTest(TestCase):
assert len(senpy.plugins) > 1
def test_convert_emotions(self):
self.senpy.activate_all()
self.senpy.activate_all(sync=True)
plugin = Plugin({
'id': 'imaginary',
'onyx:usesEmotionModel': 'emoml:fsre-dimensions'
@ -205,3 +205,14 @@ class ExtensionsTest(TestCase):
[plugin, ],
params)
assert len(r3.entries[0].emotions) == 1
# def test_async_plugin(self):
# """ We should accept multiprocessing plugins with async=False"""
# thread1 = self.senpy.activate_plugin("Async", sync=False)
# thread1.join(timeout=1)
# assert len(self.senpy.plugins['Async'].value) == 4
# resp = self.senpy.analyse(input='nothing', algorithm='Async')
# assert len(resp.entries[0].async_values) == 2
# self.senpy.activate_plugin("Async", sync=True)

28
tests/test_models.py
View File

@ -109,13 +109,15 @@ class ModelsTest(TestCase):
}
}})
c = p.jsonld()
assert "info" not in c
assert "repo" not in c
assert "extra_params" in c
logging.debug("Framed:")
assert '@type' in c
assert c['@type'] == 'plugin'
assert 'info' not in c
assert 'repo' not in c
assert 'extra_params' in c
logging.debug('Framed:')
logging.debug(c)
p.validate()
assert "es" in c['extra_params']['none']['options']
assert 'es' in c['extra_params']['none']['options']
assert isinstance(c['extra_params']['none']['options'], list)
def test_str(self):
@ -158,14 +160,20 @@ class ModelsTest(TestCase):
g = rdflib.Graph().parse(data=t, format='turtle')
assert len(g) == len(triples)
def test_plugin_list(self):
"""The plugin list should be of type \"plugins\""""
plugs = Plugins()
c = plugs.jsonld()
assert '@type' in c
assert c['@type'] == 'plugins'
def test_single_plugin(self):
"""A response with a single plugin should still return a list"""
plugs = Plugins()
for i in range(10):
p = Plugin({'id': str(i),
'version': 0,
'description': 'dummy'})
plugs.plugins.append(p)
p = Plugin({'id': str(1),
'version': 0,
'description': 'dummy'})
plugs.plugins.append(p)
assert isinstance(plugs.plugins, list)
js = plugs.jsonld()
assert isinstance(js['plugins'], list)