balkian/senpy
1
0
Fork 0
mirror of https://github.com/gsi-upm/senpy synced 2024-11-22 00:02:28 +00:00
Code Issues Releases Wiki Activity

Merge branch 'master' 0.8.7 into patch-6

Browse Source
This commit is contained in:
Ian Wood 2017-05-05 15:02:15 +01:00
commit 1a9dd07f7e
16 changed files with 304 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -73,8 +73,8 @@ pip_test: $(addprefix pip_test-,$(PYVERSIONS))
clean:
@docker ps -a | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true
@docker images | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true
@docker rmi $(NAME)-dev 2>/dev/null || true
@docker stop $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true
@docker rm $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true
git_commit:
git commit -a

78
docs/bad-examples/results/example-analysis-as-id-FAIL.json Normal file
View File

@ -0,0 +1,78 @@
{
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1",
"@type": "results",
"analysis": [
"me:SAnalysis1",
"me:SgAnalysis1",
"me:EmotionAnalysis1",
"me:NER1",
{
"@type": "analysis",
"@id": "wrong"
}
],
"entries": [
{
"@id": "http://micro.blog/status1",
"@type": [
"nif:RFC5147String",
"nif:Context"
],
"nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"entities": [
{
"@id": "http://micro.blog/status1#char=5,13",
"nif:beginIndex": 5,
"nif:endIndex": 13,
"nif:anchorOf": "Microsoft",
"me:references": "http://dbpedia.org/page/Microsoft",
"prov:wasGeneratedBy": "me:NER1"
},
{
"@id": "http://micro.blog/status1#char=25,37",
"nif:beginIndex": 25,
"nif:endIndex": 37,
"nif:anchorOf": "Windows Phone",
"me:references": "http://dbpedia.org/page/Windows_Phone",
"prov:wasGeneratedBy": "me:NER1"
}
],
"suggestions": [
{
"@id": "http://micro.blog/status1#char=16,77",
"nif:beginIndex": 16,
"nif:endIndex": 77,
"nif:anchorOf": "put your Windows Phone on your newest #open technology program",
"prov:wasGeneratedBy": "me:SgAnalysis1"
}
],
"sentiments": [
{
"@id": "http://micro.blog/status1#char=80,97",
"nif:beginIndex": 80,
"nif:endIndex": 97,
"nif:anchorOf": "You'll be awesome.",
"marl:hasPolarity": "marl:Positive",
"marl:polarityValue": 0.9,
"prov:wasGeneratedBy": "me:SAnalysis1"
}
],
"emotions": [
{
"@id": "http://micro.blog/status1#char=0,109",
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"prov:wasGeneratedBy": "me:EAnalysis1",
"onyx:hasEmotion": [
{
"onyx:hasEmotionCategory": "wna:liking"
},
{
"onyx:hasEmotionCategory": "wna:excitement"
}
]
}
]
}
]
}

74
docs/examples/results/example-analysis-as-id.json Normal file
View File

@ -0,0 +1,74 @@
{
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1",
"@type": "results",
"analysis": [
"me:SAnalysis1",
"me:SgAnalysis1",
"me:EmotionAnalysis1",
"me:NER1"
],
"entries": [
{
"@id": "http://micro.blog/status1",
"@type": [
"nif:RFC5147String",
"nif:Context"
],
"nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"entities": [
{
"@id": "http://micro.blog/status1#char=5,13",
"nif:beginIndex": 5,
"nif:endIndex": 13,
"nif:anchorOf": "Microsoft",
"me:references": "http://dbpedia.org/page/Microsoft",
"prov:wasGeneratedBy": "me:NER1"
},
{
"@id": "http://micro.blog/status1#char=25,37",
"nif:beginIndex": 25,
"nif:endIndex": 37,
"nif:anchorOf": "Windows Phone",
"me:references": "http://dbpedia.org/page/Windows_Phone",
"prov:wasGeneratedBy": "me:NER1"
}
],
"suggestions": [
{
"@id": "http://micro.blog/status1#char=16,77",
"nif:beginIndex": 16,
"nif:endIndex": 77,
"nif:anchorOf": "put your Windows Phone on your newest #open technology program",
"prov:wasGeneratedBy": "me:SgAnalysis1"
}
],
"sentiments": [
{
"@id": "http://micro.blog/status1#char=80,97",
"nif:beginIndex": 80,
"nif:endIndex": 97,
"nif:anchorOf": "You'll be awesome.",
"marl:hasPolarity": "marl:Positive",
"marl:polarityValue": 0.9,
"prov:wasGeneratedBy": "me:SAnalysis1"
}
],
"emotions": [
{
"@id": "http://micro.blog/status1#char=0,109",
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"prov:wasGeneratedBy": "me:EAnalysis1",
"onyx:hasEmotion": [
{
"onyx:hasEmotionCategory": "wna:liking"
},
{
"onyx:hasEmotionCategory": "wna:excitement"
}
]
}
]
}
]
}

2
requirements.txt
View File

@ -1,6 +1,6 @@
Flask>=0.10.1
requests>=2.4.1
gevent>=1.1rc4
tornado>=4.4.3
PyLD>=0.6.5
six
future

14
senpy/__main__.py
View File

@ -22,15 +22,16 @@ the server.
from flask import Flask
from senpy.extensions import Senpy
from gevent.wsgi import WSGIServer
from gevent.monkey import patch_all
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
import logging
import os
import argparse
import senpy
patch_all(thread=False)
SERVER_PORT = os.environ.get("PORT", 5000)
@ -92,9 +93,10 @@ def main():
print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,
args.port))
if not app.debug:
http_server = WSGIServer((args.host, args.port), app)
http_server = HTTPServer(WSGIContainer(app))
http_server.listen(args.port, address=args.host)
try:
http_server.serve_forever()
IOLoop.instance().start()
except KeyboardInterrupt:
print('Bye!')
http_server.stop()

5
senpy/client.py
View File

@ -1,6 +1,7 @@
import requests
import logging
from . import models
from .plugins import default_plugin_type
logger = logging.getLogger(__name__)
@ -12,6 +13,10 @@ class Client(object):
def analyse(self, input, method='GET', **kwargs):
return self.request('/', method=method, input=input, **kwargs)
def plugins(self, ptype=default_plugin_type):
resp = self.request(path='/plugins', plugin_type=ptype).plugins
return {p.name: p for p in resp}
def request(self, path=None, method='GET', **params):
url = '{}{}'.format(self.endpoint, path)
response = requests.request(method=method, url=url, params=params)

32
senpy/extensions.py
View File

@ -183,7 +183,7 @@ class Senpy(object):
return resp
def _conversion_candidates(self, fromModel, toModel):
candidates = self.filter_plugins(**{'@type': 'emotionConversionPlugin'})
candidates = self.filter_plugins(plugin_type='emotionConversionPlugin')
for name, candidate in candidates.items():
for pair in candidate.onyx__doesConversion:
logging.debug(pair)
@ -303,6 +303,7 @@ class Senpy(object):
else:
th = Thread(target=act)
th.start()
return th
def deactivate_plugin(self, plugin_name, sync=False):
try:
@ -327,6 +328,7 @@ class Senpy(object):
else:
th = Thread(target=deact)
th.start()
return th
@classmethod
def validate_info(cls, info):
@ -417,33 +419,7 @@ class Senpy(object):
return self._plugin_list
def filter_plugins(self, **kwargs):
""" Filter plugins by different criteria """
ptype = kwargs.pop('plugin_type', None)
logger.debug('#' * 100)
logger.debug('ptype {}'.format(ptype))
if ptype:
try:
ptype = ptype[0].upper() + ptype[1:]
pclass = getattr(plugins, ptype)
logger.debug('Class: {}'.format(pclass))
candidates = filter(lambda x: isinstance(x, pclass),
self.plugins.values())
except AttributeError:
raise Error('{} is not a valid type'.format(ptype))
else:
candidates = self.plugins.values()
logger.debug(candidates)
def matches(plug):
res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
logger.debug(
"matching {} with {}: {}".format(plug.name, kwargs, res))
return res
if kwargs:
candidates = filter(matches, candidates)
return {p.name: p for p in candidates}
return plugins.pfilter(self.plugins, **kwargs)
@property
def analysis_plugins(self):

38
senpy/plugins/__init__.py
View File

@ -9,6 +9,7 @@ import logging
import tempfile
import copy
from .. import models
from ..api import API_PARAMS
logger = logging.getLogger(__name__)
@ -117,3 +118,40 @@ class ShelfMixin(object):
if hasattr(self, '_sh') and self._sh is not None:
with open(self.shelf_file, 'wb') as f:
pickle.dump(self._sh, f)
default_plugin_type = API_PARAMS['plugin_type']['default']
def pfilter(plugins, **kwargs):
""" Filter plugins by different criteria """
if isinstance(plugins, models.Plugins):
plugins = plugins.plugins
elif isinstance(plugins, dict):
plugins = plugins.values()
ptype = kwargs.pop('plugin_type', default_plugin_type)
logger.debug('#' * 100)
logger.debug('ptype {}'.format(ptype))
if ptype:
try:
ptype = ptype[0].upper() + ptype[1:]
pclass = globals()[ptype]
logger.debug('Class: {}'.format(pclass))
candidates = filter(lambda x: isinstance(x, pclass),
plugins)
except KeyError:
raise models.Error('{} is not a valid type'.format(ptype))
else:
candidates = plugins
logger.debug(candidates)
def matches(plug):
res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
logger.debug(
"matching {} with {}: {}".format(plug.name, kwargs, res))
return res
if kwargs:
candidates = filter(matches, candidates)
return {p.name: p for p in candidates}

6
senpy/schemas/context.jsonld
View File

@ -37,12 +37,12 @@
"@type": "@id",
"@container": "@set"
},
"plugins": {
"@container": "@list"
},
"options": {
"@container": "@set"
},
"plugins": {
"@container": "@set"
},
"prov:wasGeneratedBy": {
"@type": "@id"
},

2
senpy/schemas/plugins.json
View File

@ -10,8 +10,6 @@
"items": {
"$ref": "plugin.json"
}
},
"@type": {
}
}
}

8
senpy/schemas/results.json
View File

@ -18,10 +18,16 @@
"type": "string"
},
"analysis": {
"type": "array",
"default": [],
"type": "array",
"items": {
"anyOf": [
{
"$ref": "analysis.json"
},{
"type": "string"
}
]
}
},
"entries": {

23
tests/plugins/async_plugin/asyncplugin.py Normal file
View File

@ -0,0 +1,23 @@
from senpy.plugins import AnalysisPlugin
import multiprocessing
def _train(process_number):
return process_number
class AsyncPlugin(AnalysisPlugin):
def _do_async(self, num_processes):
pool = multiprocessing.Pool(processes=num_processes)
values = pool.map(_train, range(num_processes))
return values
def activate(self):
self.value = self._do_async(4)
def analyse_entry(self, entry, params):
values = self._do_async(2)
entry.async_values = values
yield entry

8
tests/plugins/async_plugin/asyncplugin.senpy Normal file
View File

@ -0,0 +1,8 @@
---
name: Async
module: asyncplugin
description: I am async
author: "@balkian"
version: '0.1'
async: true
extra_params: {}

25
tests/test_client.py
View File

@ -4,18 +4,21 @@ try:
except ImportError:
from mock import patch
import json
from senpy.client import Client
from senpy.models import Results, Error
from senpy.models import Results, Plugins, Error
from senpy.plugins import AnalysisPlugin, default_plugin_type
class Call(dict):
def __init__(self, obj):
self.obj = obj.jsonld()
self.obj = obj.serialize()
self.status_code = 200
self.content = self.json()
def json(self):
return self.obj
return json.loads(self.obj)
class ModelsTest(TestCase):
@ -44,3 +47,19 @@ class ModelsTest(TestCase):
method='GET',
params={'input': 'hello',
'algorithm': 'NONEXISTENT'})
def test_plugins(self):
endpoint = 'http://dummy/'
client = Client(endpoint)
plugins = Plugins()
p1 = AnalysisPlugin({'name': 'AnalysisP1', 'version': 0, 'description': 'No'})
plugins.plugins = [p1, ]
success = Call(plugins)
with patch('requests.request', return_value=success) as patched:
response = client.plugins()
assert isinstance(response, dict)
assert len(response) == 1
assert "AnalysisP1" in response
patched.assert_called_with(
url=endpoint + '/plugins', method='GET',
params={'plugin_type': default_plugin_type})

13
tests/test_extensions.py
View File

@ -167,7 +167,7 @@ class ExtensionsTest(TestCase):
assert len(senpy.plugins) > 1
def test_convert_emotions(self):
self.senpy.activate_all()
self.senpy.activate_all(sync=True)
plugin = Plugin({
'id': 'imaginary',
'onyx:usesEmotionModel': 'emoml:fsre-dimensions'
@ -205,3 +205,14 @@ class ExtensionsTest(TestCase):
[plugin, ],
params)
assert len(r3.entries[0].emotions) == 1
# def test_async_plugin(self):
# """ We should accept multiprocessing plugins with async=False"""
# thread1 = self.senpy.activate_plugin("Async", sync=False)
# thread1.join(timeout=1)
# assert len(self.senpy.plugins['Async'].value) == 4
# resp = self.senpy.analyse(input='nothing', algorithm='Async')
# assert len(resp.entries[0].async_values) == 2
# self.senpy.activate_plugin("Async", sync=True)

22
tests/test_models.py
View File

@ -109,13 +109,15 @@ class ModelsTest(TestCase):
}
}})
c = p.jsonld()
assert "info" not in c
assert "repo" not in c
assert "extra_params" in c
logging.debug("Framed:")
assert '@type' in c
assert c['@type'] == 'plugin'
assert 'info' not in c
assert 'repo' not in c
assert 'extra_params' in c
logging.debug('Framed:')
logging.debug(c)
p.validate()
assert "es" in c['extra_params']['none']['options']
assert 'es' in c['extra_params']['none']['options']
assert isinstance(c['extra_params']['none']['options'], list)
def test_str(self):
@ -158,11 +160,17 @@ class ModelsTest(TestCase):
g = rdflib.Graph().parse(data=t, format='turtle')
assert len(g) == len(triples)
def test_plugin_list(self):
"""The plugin list should be of type \"plugins\""""
plugs = Plugins()
c = plugs.jsonld()
assert '@type' in c
assert c['@type'] == 'plugins'
def test_single_plugin(self):
"""A response with a single plugin should still return a list"""
plugs = Plugins()
for i in range(10):
p = Plugin({'id': str(i),
p = Plugin({'id': str(1),
'version': 0,
'description': 'dummy'})
plugs.plugins.append(p)