balkian/senpy
1
0
Fork 0
mirror of https://github.com/gsi-upm/senpy synced 2024-11-21 07:42:28 +00:00
Code Issues Releases Wiki Activity

Improve extra requirement handling

Browse Source 
This commit adds a new mechanism to handle parameters beforehand in chained
calls, and the ability to get help on available parameters in chained
calls (through `?help`).
It also includes tests for this feature.

Closes #51
This commit is contained in:
J. Fernando Sánchez 2018-08-20 15:44:54 +02:00
parent aa35e62a27
commit ca69bddc17
5 changed files with 326 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
senpy/api.py
View File

@ -175,23 +175,117 @@ def parse_params(indict, *specs):
parameters=outdict,
errors=wrong_params)
raise message
if 'algorithm' in outdict and not isinstance(outdict['algorithm'], list):
outdict['algorithm'] = list(outdict['algorithm'].split(','))
return outdict
def parse_extra_params(request, plugin=None):
params = request.parameters.copy()
if plugin:
extra_params = parse_params(params, plugin.get('extra_params', {}))
params.update(extra_params)
def get_all_params(plugins, *specs):
'''Return a list of parameters for a given set of specifications and plugins.'''
dic = {}
for s in specs:
dic.update(s)
dic.update(get_extra_params(plugins))
return dic
def get_extra_params(plugins):
'''Get a list of possible parameters given a list of plugins'''
params = {}
extra_params = {}
for i, plugin in enumerate(plugins):
this_params = plugin.get('extra_params', {})
for k, v in this_params.items():
if k not in extra_params:
extra_params[k] = []
extra_params[k].append(v)
params['{}.{}'.format(plugin.name, k)] = v
params['{}.{}'.format(i, k)] = v
for k, v in extra_params.items(): # Resolve conflicts
if len(v) == 1: # Add the extra options that do not collide
params[k] = v[0]
else:
required = False
aliases = None
options = None
default = None
nodefault = False # Set when defaults are not compatible
for opt in v:
required = required or opt.get('required', False)
newaliases = set(opt.get('aliases', []))
if aliases is None:
aliases = newaliases
else:
aliases = aliases & newaliases
if 'options' in opt:
newoptions = set(opt['options'])
options = newoptions if options is None else options & newoptions
if 'default' in opt:
newdefault = opt['default']
if newdefault:
if default is None and not nodefault:
default = newdefault
elif newdefault != default:
nodefault = True
default = None
# Check for incompatibilities
if options != set():
params[k] = {
'default': default,
'aliases': list(aliases),
'required': required,
'options': list(options)
}
return params
def parse_extra_params(params, plugins):
'''
Parse the given parameters individually for each plugin, and get a list of the parameters that
belong to each of the plugins. Each item can then be used in the plugin.analyse_entries method.
'''
extra_params = []
for i, plugin in enumerate(plugins):
this_params = filter_params(params, plugin, i)
parsed = parse_params(this_params, plugin.get('extra_params', {}))
extra_params.append(parsed)
return extra_params
def filter_params(params, plugin, ith=-1):
'''
Get the values within params that apply to a plugin.
More specific names override more general names, in this order:
<index_order>.parameter > <plugin.name>.parameter > parameter
Example:
>>> filter_params({'0.hello': True, 'hello': False}, Plugin(), 0)
{ '0.hello': True, 'hello': True}
'''
thisparams = {}
if ith >= 0:
ith = '{}.'.format(ith)
else:
ith = ""
for k, v in params.items():
if ith and k.startswith(str(ith)):
thisparams[k[len(ith):]] = v
elif k.startswith(plugin.name):
thisparams[k[len(plugin.name) + 1:]] = v
elif k not in thisparams:
thisparams[k] = v
return thisparams
def parse_call(params):
'''Return a results object based on the parameters used in a call/request.
'''
params = parse_params(params, NIF_PARAMS)
if 'algorithm' in params and not isinstance(params['algorithm'], list):
params['algorithm'] = list(params['algorithm'].split(','))
if params['informat'] == 'text':
results = Results()
entry = Entry(nif__isString=params['input'],

17
senpy/blueprints.py
View File

@ -188,15 +188,20 @@ def basic_api(f):
@api_blueprint.route('/<path:plugin>', methods=['POST', 'GET'])
@basic_api
def api_root(plugin):
if plugin:
if 'algorithm' in request.parameters:
raise Error('You cannot specify the algorithm with a parameter and a URL variable.'
' Please, remove one of them')
plugin = plugin.replace('+', '/')
request.parameters['algorithm'] = plugin.split('/')
if request.parameters['help']:
dic = dict(api.API_PARAMS, **api.NIF_PARAMS)
response = Help(valid_parameters=dic)
sp = current_app.senpy
plugins = sp._get_plugins(request)
allparameters = api.get_all_params(plugins, api.WEB_PARAMS, api.API_PARAMS, api.NIF_PARAMS)
response = Help(valid_parameters=allparameters)
return response
req = api.parse_call(request.parameters)
if plugin:
plugin = plugin.replace('+', '/')
plugin = plugin.split('/')
req.parameters['algorithm'] = plugin
return current_app.senpy.analyse(req)

16
senpy/extensions.py
View File

@ -143,7 +143,7 @@ class Senpy(object):
plugins.append(self._plugins[algo])
return plugins
def _process_entries(self, entries, req, plugins):
def _process_entries(self, entries, req, plugins, extra, ith=0):
"""
Recursively process the entries with the first plugin in the list, and pass the results
to the rest of the plugins.
@ -153,11 +153,11 @@ class Senpy(object):
yield i
return
plugin = plugins[0]
specific_params = api.parse_extra_params(req, plugin)
specific_params = extra[ith]
req.analysis.append({'plugin': plugin,
'parameters': specific_params})
results = plugin.analyse_entries(entries, specific_params)
for i in self._process_entries(results, req, plugins[1:]):
for i in self._process_entries(results, req, plugins[1:], extra, ith=ith + 1):
yield i
def install_deps(self):
@ -169,12 +169,16 @@ class Senpy(object):
It takes a processed request, provided by the user, as returned
by api.parse_call().
"""
logger.debug("analysing request: {}".format(request))
entries = request.entries
request.entries = []
plugins = self._get_plugins(request)
extra = api.parse_extra_params(request.parameters, plugins)
entries = request.entries
results = request
for i in self._process_entries(entries, results, plugins):
results.entries = []
for i in self._process_entries(entries, results, plugins, extra):
results.entries.append(i)
self.convert_emotions(results)
logger.debug("Returning analysis result: {}".format(results))

159
tests/test_api.py
View File

@ -3,8 +3,9 @@ import logging
logger = logging.getLogger(__name__)
from unittest import TestCase
from senpy.api import parse_params, API_PARAMS, NIF_PARAMS, WEB_PARAMS
from senpy.models import Error
from senpy.api import (boolean, parse_params, get_extra_params, parse_extra_params,
API_PARAMS, NIF_PARAMS, WEB_PARAMS)
from senpy.models import Error, Plugin
class APITest(TestCase):
@ -89,3 +90,157 @@ class APITest(TestCase):
assert "Dummy" in p['algorithm']
assert 'input' in p
assert p['input'] == 'Aloha my friend'
def test_parse_extra_params(self):
'''The API should parse user parameters and return them in a format that plugins can use'''
plugins = [
Plugin({
'name': 'plugin1',
'extra_params': {
# Incompatible parameter
'param0': {
'aliases': ['p1', 'parameter1'],
'options': ['option1', 'option2'],
'default': 'option1',
'required': True
},
'param1': {
'aliases': ['p1', 'parameter1'],
'options': ['en', 'es'],
'default': 'en',
'required': False
},
'param2': {
'aliases': ['p2', 'parameter2'],
'required': False,
'options': ['value2_1', 'value2_2', 'value3_3']
}
}
}), Plugin({
'name': 'plugin2',
'extra_params': {
'param0': {
'aliases': ['parameter1'],
'options': ['new option', 'new option2'],
'default': 'new option',
'required': False
},
'param1': {
'aliases': ['myparam1', 'p1'],
'options': ['en', 'de', 'auto'],
'default': 'de',
'required': True
},
'param3': {
'aliases': ['p3', 'parameter3'],
'options': boolean,
'default': True
}
}
})
]
call = {
'param1': 'en',
'0.param0': 'option1',
'0.param1': 'en',
'param2': 'value2_1',
'param0': 'new option',
'1.param1': 'de',
'param3': False,
}
expected = [
{
'param0': 'option1',
'param1': 'en',
'param2': 'value2_1',
}, {
'param0': 'new option',
'param1': 'de',
'param3': False,
}
]
p = parse_extra_params(call, plugins)
for i, arg in enumerate(expected):
for k, v in arg.items():
assert p[i][k] == v
def test_get_extra_params(self):
'''The API should return the list of valid parameters for a set of plugins'''
plugins = [
Plugin({
'name': 'plugin1',
'extra_params': {
# Incompatible parameter
'param0': {
'aliases': ['p1', 'parameter1'],
'options': ['option1', 'option2'],
'default': 'option1',
'required': True
},
'param1': {
'aliases': ['p1', 'parameter1'],
'options': ['en', 'es'],
'default': 'en',
'required': False
},
'param2': {
'aliases': ['p2', 'parameter2'],
'required': False,
'options': ['value2_1', 'value2_2', 'value3_3']
}
}
}), Plugin({
'name': 'plugin2',
'extra_params': {
'param0': {
'aliases': ['parameter1'],
'options': ['new option', 'new option2'],
'default': 'new option',
'required': False
},
'param1': {
'aliases': ['myparam1', 'p1'],
'options': ['en', 'de', 'auto'],
'default': 'de',
'required': True
},
'param3': {
'aliases': ['p3', 'parameter3'],
'options': boolean,
'default': True
}
}
})
]
expected = {
# Each plugin's parameters
'0.param0': plugins[0]['extra_params']['param0'],
'0.param1': plugins[0]['extra_params']['param1'],
'0.param2': plugins[0]['extra_params']['param2'],
'1.param0': plugins[1]['extra_params']['param0'],
'1.param1': plugins[1]['extra_params']['param1'],
'1.param3': plugins[1]['extra_params']['param3'],
# Non-overlapping parameters
'param2': plugins[0]['extra_params']['param2'],
'param3': plugins[1]['extra_params']['param3'],
# Intersection of overlapping parameters
'param1': {
'aliases': ['p1'],
'options': ['en'],
'default': None,
'required': True
}
}
result = get_extra_params(plugins)
for ik, iv in expected.items():
assert ik in result
for jk, jv in iv.items():
assert jk in result[ik]
assert expected[ik][jk] == result[ik][jk]

47
tests/test_blueprints.py
View File

@ -133,6 +133,53 @@ class BlueprintsTest(TestCase):
assert len(js['analysis']) == 2
assert js['entries'][0]['nif:isString'] == 'My aloha mohame'
def test_requirements_chain_help(self):
'''The extra parameters of each plugin should be merged if they are in a chain '''
resp = self.client.get("/api/split/DummyRequired?help=true")
self.assertCode(resp, 200)
js = parse_resp(resp)
assert 'valid_parameters' in js
vp = js['valid_parameters']
assert 'example' in vp
def test_requirements_chain_repeat_help(self):
'''
If a plugin appears several times in a chain, there should be a way to set different
parameters for each.
'''
resp = self.client.get("/api/split/split?help=true")
self.assertCode(resp, 200)
js = parse_resp(resp)
assert 'valid_parameters' in js
vp = js['valid_parameters']
assert '0.delimiter' in vp
assert '1.delimiter' in vp
assert 'delimiter' in vp
def test_requirements_chain(self):
"""
It should be possible to specify different parameters for each step in the chain.
"""
# First, we split by sentence twice. Each call should generate 3 additional entries
# (one per sentence in the original).
resp = self.client.get('/api/split/split?i=The first sentence. The second sentence.'
'\nA new paragraph&delimiter=sentence')
js = parse_resp(resp)
assert len(js['analysis']) == 2
assert len(js['entries']) == 7
# Now, we split by sentence. This produces 3 additional entries.
# Then, we split by paragraph. This should create 2 additional entries (One per paragraph
# in the original text)
resp = self.client.get('/api/split/split?i=The first sentence. The second sentence.'
'\nA new paragraph&0.delimiter=sentence&1.delimiter=paragraph')
# Calling dummy twice, should return the same string
self.assertCode(resp, 200)
js = parse_resp(resp)
assert len(js['analysis']) == 2
assert len(js['entries']) == 6
def test_error(self):
"""
The dummy plugin returns an empty response,\