New schema for parameters

* Improve extra requirement handling
* New mechanism to handle parameters beforehand in chained
  calls, and the ability to get help on available parameters in chained
  calls (through `?help`).
* Redefined Analysis, to reflect the new ontology
* Add parameters as an entity in the schema
* Update examples to include analyses and parameters
* Add processing plugins, with an interface similar to analysis plugins
* Update tests
* Avoid duplication in split plugin

Closes #51

Squashed commit of the following:

commit d145a852e7
commit 6a1069780b
commit ca69bddc17
commit aa35e62a27
master
J. Fernando Sánchez 5 years ago
parent 41aa142ce0
commit 4ba30304a4

@ -6,13 +6,9 @@
], ],
"entries": [ "entries": [
{ {
"@type": [
"nif:RFC5147String",
"nif:Context"
],
"nif:beginIndex": 0, "nif:beginIndex": 0,
"nif:endIndex": 40, "nif:endIndex": 40,
"nif:isString": "My favourite actress is Natalie Portman" "text": "An entry should have a nif:isString key"
} }
] ]
} }

@ -3,10 +3,21 @@
"@id": "me:Result1", "@id": "me:Result1",
"@type": "results", "@type": "results",
"analysis": [ "analysis": [
"me:SAnalysis1", {
"me:SgAnalysis1", "@id": "_:SAnalysis1_Activity",
"me:EmotionAnalysis1", "@type": "marl:SentimentAnalysis",
"me:NER1" "prov:wasAssociatedWith": "me:SAnalysis1"
},
{
"@id": "_:EmotionAnalysis1_Activity",
"@type": "onyx:EmotionAnalysis",
"prov:wasAssociatedWith": "me:EmotionAnalysis1"
},
{
"@id": "_:NER1_Activity",
"@type": "me:NER",
"prov:wasAssociatedWith": "me:NER1"
}
], ],
"entries": [ "entries": [
{ {
@ -23,7 +34,7 @@
"nif:endIndex": 13, "nif:endIndex": 13,
"nif:anchorOf": "Microsoft", "nif:anchorOf": "Microsoft",
"me:references": "http://dbpedia.org/page/Microsoft", "me:references": "http://dbpedia.org/page/Microsoft",
"prov:wasGeneratedBy": "me:NER1" "prov:wasGeneratedBy": "_:NER1_Activity"
}, },
{ {
"@id": "http://micro.blog/status1#char=25,37", "@id": "http://micro.blog/status1#char=25,37",
@ -31,7 +42,7 @@
"nif:endIndex": 37, "nif:endIndex": 37,
"nif:anchorOf": "Windows Phone", "nif:anchorOf": "Windows Phone",
"me:references": "http://dbpedia.org/page/Windows_Phone", "me:references": "http://dbpedia.org/page/Windows_Phone",
"prov:wasGeneratedBy": "me:NER1" "prov:wasGeneratedBy": "_:NER1_Activity"
} }
], ],
"suggestions": [ "suggestions": [
@ -40,7 +51,7 @@
"nif:beginIndex": 16, "nif:beginIndex": 16,
"nif:endIndex": 77, "nif:endIndex": 77,
"nif:anchorOf": "put your Windows Phone on your newest #open technology program", "nif:anchorOf": "put your Windows Phone on your newest #open technology program",
"prov:wasGeneratedBy": "me:SgAnalysis1" "prov:wasGeneratedBy": "_:SgAnalysis1_Activity"
} }
], ],
"sentiments": [ "sentiments": [
@ -51,14 +62,14 @@
"nif:anchorOf": "You'll be awesome.", "nif:anchorOf": "You'll be awesome.",
"marl:hasPolarity": "marl:Positive", "marl:hasPolarity": "marl:Positive",
"marl:polarityValue": 0.9, "marl:polarityValue": 0.9,
"prov:wasGeneratedBy": "me:SAnalysis1" "prov:wasGeneratedBy": "_:SgAnalysis1_Activity"
} }
], ],
"emotions": [ "emotions": [
{ {
"@id": "http://micro.blog/status1#char=0,109", "@id": "http://micro.blog/status1#char=0,109",
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource", "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"prov:wasGeneratedBy": "me:EAnalysis1", "prov:wasGeneratedBy": "_:EmotionAnalysis1_Activity",
"onyx:hasEmotion": [ "onyx:hasEmotion": [
{ {
"onyx:hasEmotionCategory": "wna:liking" "onyx:hasEmotionCategory": "wna:liking"

@ -1,78 +0,0 @@
{
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1",
"@type": "results",
"analysis": [
"me:SAnalysis1",
"me:SgAnalysis1",
"me:EmotionAnalysis1",
"me:NER1",
{
"@type": "analysis",
"@id": "anonymous"
}
],
"entries": [
{
"@id": "http://micro.blog/status1",
"@type": [
"nif:RFC5147String",
"nif:Context"
],
"nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"entities": [
{
"@id": "http://micro.blog/status1#char=5,13",
"nif:beginIndex": 5,
"nif:endIndex": 13,
"nif:anchorOf": "Microsoft",
"me:references": "http://dbpedia.org/page/Microsoft",
"prov:wasGeneratedBy": "me:NER1"
},
{
"@id": "http://micro.blog/status1#char=25,37",
"nif:beginIndex": 25,
"nif:endIndex": 37,
"nif:anchorOf": "Windows Phone",
"me:references": "http://dbpedia.org/page/Windows_Phone",
"prov:wasGeneratedBy": "me:NER1"
}
],
"suggestions": [
{
"@id": "http://micro.blog/status1#char=16,77",
"nif:beginIndex": 16,
"nif:endIndex": 77,
"nif:anchorOf": "put your Windows Phone on your newest #open technology program",
"prov:wasGeneratedBy": "me:SgAnalysis1"
}
],
"sentiments": [
{
"@id": "http://micro.blog/status1#char=80,97",
"nif:beginIndex": 80,
"nif:endIndex": 97,
"nif:anchorOf": "You'll be awesome.",
"marl:hasPolarity": "marl:Positive",
"marl:polarityValue": 0.9,
"prov:wasGeneratedBy": "me:SAnalysis1"
}
],
"emotions": [
{
"@id": "http://micro.blog/status1#char=0,109",
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"prov:wasGeneratedBy": "me:EAnalysis1",
"onyx:hasEmotion": [
{
"onyx:hasEmotionCategory": "wna:liking"
},
{
"onyx:hasEmotionCategory": "wna:excitement"
}
]
}
]
}
]
}

@ -1,19 +1,18 @@
{ {
"@context": "http://mixedemotions-project.eu/ns/context.jsonld", "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "http://example.com#NIFExample", "@id": "me:Result1",
"@type": "results", "@type": "results",
"analysis": [ "analysis": [ ],
], "entries": [
"entries": [ {
{ "@id": "http://example.org#char=0,40",
"@id": "http://example.org#char=0,40", "@type": [
"@type": [ "nif:RFC5147String",
"nif:RFC5147String", "nif:Context"
"nif:Context" ],
], "nif:beginIndex": 0,
"nif:beginIndex": 0, "nif:endIndex": 40,
"nif:endIndex": 40, "nif:isString": "My favourite actress is Natalie Portman"
"nif:isString": "My favourite actress is Natalie Portman" }
} ]
]
} }

@ -1,88 +1,100 @@
{ {
"@context": "http://mixedemotions-project.eu/ns/context.jsonld", "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1", "@id": "me:Result1",
"@type": "results", "@type": "results",
"analysis": [ "analysis": [
{
"@id": "me:SAnalysis1",
"@type": "marl:SentimentAnalysis",
"marl:maxPolarityValue": 1,
"marl:minPolarityValue": 0
},
{
"@id": "me:SgAnalysis1",
"@type": "me:SuggestionAnalysis"
},
{
"@id": "me:EmotionAnalysis1",
"@type": "me:EmotionAnalysis"
},
{
"@id": "me:NER1",
"@type": "me:NER"
}
],
"entries": [
{
"@id": "http://micro.blog/status1",
"@type": [
"nif:RFC5147String",
"nif:Context"
],
"nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"entities": [
{ {
"@id": "http://micro.blog/status1#char=5,13", "@id": "_:SAnalysis1_Activity",
"nif:beginIndex": 5, "@type": "marl:SentimentAnalysis",
"nif:endIndex": 13, "prov:wasAssociatedWith": "me:SentimentAnalysis",
"nif:anchorOf": "Microsoft", "prov:used": [
"me:references": "http://dbpedia.org/page/Microsoft", {
"prov:wasGeneratedBy": "me:NER1" "name": "marl:maxPolarityValue",
"prov:value": "1"
},
{
"name": "marl:minPolarityValue",
"prov:value": "0"
}
]
}, },
{ {
"@id": "http://micro.blog/status1#char=25,37", "@id": "_:SgAnalysis1_Activity",
"nif:beginIndex": 25, "prov:wasAssociatedWith": "me:SgAnalysis1",
"nif:endIndex": 37, "@type": "me:SuggestionAnalysis"
"nif:anchorOf": "Windows Phone", },
"me:references": "http://dbpedia.org/page/Windows_Phone",
"prov:wasGeneratedBy": "me:NER1"
}
],
"suggestions": [
{ {
"@id": "http://micro.blog/status1#char=16,77", "@id": "_:EmotionAnalysis1_Activity",
"nif:beginIndex": 16, "@type": "me:EmotionAnalysis",
"nif:endIndex": 77, "prov:wasAssociatedWith": "me:EmotionAnalysis1"
"nif:anchorOf": "put your Windows Phone on your newest #open technology program", },
"prov:wasGeneratedBy": "me:SgAnalysis1"
}
],
"sentiments": [
{ {
"@id": "http://micro.blog/status1#char=80,97", "@id": "_:NER1_Activity",
"nif:beginIndex": 80, "@type": "me:NER",
"nif:endIndex": 97, "prov:wasAssociatedWith": "me:EmotionNER1"
"nif:anchorOf": "You'll be awesome.",
"marl:hasPolarity": "marl:Positive",
"marl:polarityValue": 0.9,
"prov:wasGeneratedBy": "me:SAnalysis1"
} }
], ],
"emotions": [ "entries": [
{ {
"@id": "http://micro.blog/status1#char=0,109", "@id": "http://micro.blog/status1",
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource", "@type": [
"prov:wasGeneratedBy": "me:EAnalysis1", "nif:RFC5147String",
"onyx:hasEmotion": [ "nif:Context"
{ ],
"onyx:hasEmotionCategory": "wna:liking" "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
}, "entities": [
{ {
"onyx:hasEmotionCategory": "wna:excitement" "@id": "http://micro.blog/status1#char=5,13",
} "nif:beginIndex": 5,
] "nif:endIndex": 13,
"nif:anchorOf": "Microsoft",
"me:references": "http://dbpedia.org/page/Microsoft",
"prov:wasGeneratedBy": "me:NER1"
},
{
"@id": "http://micro.blog/status1#char=25,37",
"nif:beginIndex": 25,
"nif:endIndex": 37,
"nif:anchorOf": "Windows Phone",
"me:references": "http://dbpedia.org/page/Windows_Phone",
"prov:wasGeneratedBy": "me:NER1"
}
],
"suggestions": [
{
"@id": "http://micro.blog/status1#char=16,77",
"nif:beginIndex": 16,
"nif:endIndex": 77,
"nif:anchorOf": "put your Windows Phone on your newest #open technology program",
"prov:wasGeneratedBy": "me:SgAnalysis1"
}
],
"sentiments": [
{
"@id": "http://micro.blog/status1#char=80,97",
"nif:beginIndex": 80,
"nif:endIndex": 97,
"nif:anchorOf": "You'll be awesome.",
"marl:hasPolarity": "marl:Positive",
"marl:polarityValue": 0.9,
"prov:wasGeneratedBy": "me:SAnalysis1"
}
],
"emotions": [
{
"@id": "http://micro.blog/status1#char=0,109",
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"prov:wasGeneratedBy": "me:EAnalysis1",
"onyx:hasEmotion": [
{
"onyx:hasEmotionCategory": "wna:liking"
},
{
"onyx:hasEmotionCategory": "wna:excitement"
}
]
}
]
} }
] ]
}
]
} }

@ -4,8 +4,9 @@
"@type": "results", "@type": "results",
"analysis": [ "analysis": [
{ {
"@id": "me:EmotionAnalysis1", "@id": "me:EmotionAnalysis1_Activity",
"@type": "onyx:EmotionAnalysis" "@type": "me:EmotionAnalysis1",
"prov:wasAssociatedWith": "me:EmotionAnalysis1"
} }
], ],
"entries": [ "entries": [
@ -26,7 +27,7 @@
{ {
"@id": "http://micro.blog/status1#char=0,109", "@id": "http://micro.blog/status1#char=0,109",
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource", "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"prov:wasGeneratedBy": "me:EmotionAnalysis1", "prov:wasGeneratedBy": "_:EmotionAnalysis1_Activity",
"onyx:hasEmotion": [ "onyx:hasEmotion": [
{ {
"onyx:hasEmotionCategory": "wna:liking" "onyx:hasEmotionCategory": "wna:liking"

@ -4,8 +4,9 @@
"@type": "results", "@type": "results",
"analysis": [ "analysis": [
{ {
"@id": "me:NER1", "@id": "_:NER1_Activity",
"@type": "me:NERAnalysis" "@type": "me:NERAnalysis",
"prov:wasAssociatedWith": "me:NER1"
} }
], ],
"entries": [ "entries": [

@ -9,9 +9,15 @@
"@type": "results", "@type": "results",
"analysis": [ "analysis": [
{ {
"@id": "me:HesamsAnalysis", "@id": "me:HesamsAnalysis_Activity",
"@type": "onyx:EmotionAnalysis", "@type": "onyx:EmotionAnalysis",
"onyx:usesEmotionModel": "emovoc:pad-dimensions" "prov:wasAssociatedWith": "me:HesamsAnalysis",
"prov:used": [
{
"name": "emotion-model",
"prov:value": "emovoc:pad-dimensions"
}
]
} }
], ],
"entries": [ "entries": [
@ -32,7 +38,7 @@
{ {
"@id": "Entry1#char=0,21", "@id": "Entry1#char=0,21",
"nif:anchorOf": "This is a test string", "nif:anchorOf": "This is a test string",
"prov:wasGeneratedBy": "me:HesamAnalysis", "prov:wasGeneratedBy": "_:HesamAnalysis_Activity",
"onyx:hasEmotion": [ "onyx:hasEmotion": [
{ {
"emovoc:pleasure": 0.5, "emovoc:pleasure": 0.5,

@ -4,10 +4,9 @@
"@type": "results", "@type": "results",
"analysis": [ "analysis": [
{ {
"@id": "me:SAnalysis1", "@id": "_:SAnalysis1_Activity",
"@type": "marl:SentimentAnalysis", "@type": "marl:SentimentAnalysis",
"marl:maxPolarityValue": 1, "prov:wasAssociatedWith": "me:SAnalysis1"
"marl:minPolarityValue": 0
} }
], ],
"entries": [ "entries": [
@ -30,7 +29,7 @@
"nif:anchorOf": "You'll be awesome.", "nif:anchorOf": "You'll be awesome.",
"marl:hasPolarity": "marl:Positive", "marl:hasPolarity": "marl:Positive",
"marl:polarityValue": 0.9, "marl:polarityValue": 0.9,
"prov:wasGeneratedBy": "me:SAnalysis1" "prov:wasGeneratedBy": "_:SAnalysis1_Activity"
} }
], ],
"emotionSets": [ "emotionSets": [

@ -3,7 +3,11 @@
"@id": "me:Result1", "@id": "me:Result1",
"@type": "results", "@type": "results",
"analysis": [ "analysis": [
"me:SgAnalysis1" {
"@id": "_:SgAnalysis1_Activity",
"@type": "me:SuggestionAnalysis",
"prov:wasAssociatedWith": "me:SgAnalysis1"
}
], ],
"entries": [ "entries": [
{ {
@ -12,7 +16,6 @@
"nif:RFC5147String", "nif:RFC5147String",
"nif:Context" "nif:Context"
], ],
"prov:wasGeneratedBy": "me:SAnalysis1",
"nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource", "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"entities": [ "entities": [
], ],
@ -22,7 +25,7 @@
"nif:beginIndex": 16, "nif:beginIndex": 16,
"nif:endIndex": 77, "nif:endIndex": 77,
"nif:anchorOf": "put your Windows Phone on your newest #open technology program", "nif:anchorOf": "put your Windows Phone on your newest #open technology program",
"prov:wasGeneratedBy": "me:SgAnalysis1" "prov:wasGeneratedBy": "_:SgAnalysis1_Activity"
} }
], ],
"sentiments": [ "sentiments": [

@ -8,7 +8,8 @@ boolean = [True, False]
API_PARAMS = { API_PARAMS = {
"algorithm": { "algorithm": {
"aliases": ["algorithms", "a", "algo"], "aliases": ["algorithms", "a", "algo"],
"required": False, "required": True,
"default": 'default',
"description": ("Algorithms that will be used to process the request." "description": ("Algorithms that will be used to process the request."
"It may be a list of comma-separated names."), "It may be a list of comma-separated names."),
}, },
@ -41,6 +42,15 @@ API_PARAMS = {
"options": boolean, "options": boolean,
"default": False "default": False
}, },
"verbose": {
"@id": "verbose",
"description": ("Show all help, including the common API parameters, or "
"only plugin-related info"),
"aliases": ["v"],
"required": True,
"options": boolean,
"default": True
},
"emotionModel": { "emotionModel": {
"@id": "emotionModel", "@id": "emotionModel",
"aliases": ["emoModel"], "aliases": ["emoModel"],
@ -168,8 +178,7 @@ def parse_params(indict, *specs):
outdict[param] = options["default"] outdict[param] = options["default"]
elif options.get("required", False): elif options.get("required", False):
wrong_params[param] = spec[param] wrong_params[param] = spec[param]
continue elif "options" in options:
if "options" in options:
if options["options"] == boolean: if options["options"] == boolean:
outdict[param] = str(outdict[param]).lower() in ['true', '1'] outdict[param] = str(outdict[param]).lower() in ['true', '1']
elif outdict[param] not in options["options"]: elif outdict[param] not in options["options"]:
@ -182,29 +191,116 @@ def parse_params(indict, *specs):
parameters=outdict, parameters=outdict,
errors=wrong_params) errors=wrong_params)
raise message raise message
if 'algorithm' in outdict and not isinstance(outdict['algorithm'], tuple):
outdict['algorithm'] = tuple(outdict['algorithm'].split(','))
return outdict return outdict
def parse_extra_params(request, plugins=None): def get_all_params(plugins, *specs):
plugins = plugins or [] '''Return a list of parameters for a given set of specifications and plugins.'''
params = request.parameters.copy() dic = {}
for s in specs:
dic.update(s)
dic.update(get_extra_params(plugins))
return dic
def get_extra_params(plugins):
'''Get a list of possible parameters given a list of plugins'''
params = {}
extra_params = {}
for plugin in plugins: for plugin in plugins:
if plugin: this_params = plugin.get('extra_params', {})
extra_params = parse_params(params, plugin.get('extra_params', {})) for k, v in this_params.items():
for k, v in extra_params.items(): if k not in extra_params:
if k not in BUILTIN_PARAMS: extra_params[k] = {}
if k in params: # Set by another plugin extra_params[k][plugin.name] = v
del params[k] for k, v in extra_params.items(): # Resolve conflicts
else: if len(v) == 1: # Add the extra options that do not collide
params[k] = v params[k] = list(v.values())[0]
params['{}.{}'.format(plugin.name, k)] = v else:
required = False
aliases = None
options = None
default = None
nodefault = False # Set when defaults are not compatible
for plugin, opt in v.items():
params['{}.{}'.format(plugin, k)] = opt
required = required or opt.get('required', False)
newaliases = set(opt.get('aliases', []))
if aliases is None:
aliases = newaliases
else:
aliases = aliases & newaliases
if 'options' in opt:
newoptions = set(opt['options'])
options = newoptions if options is None else options & newoptions
if 'default' in opt:
newdefault = opt['default']
if newdefault:
if default is None and not nodefault:
default = newdefault
elif newdefault != default:
nodefault = True
default = None
# Check for incompatibilities
if options != set():
params[k] = {
'default': default,
'aliases': list(aliases),
'required': required,
'options': list(options)
}
return params return params
def parse_analysis(params, plugins):
'''
Parse the given parameters individually for each plugin, and get a list of the parameters that
belong to each of the plugins. Each item can then be used in the plugin.analyse_entries method.
'''
analysis_list = []
for i, plugin in enumerate(plugins):
if not plugin:
continue
this_params = filter_params(params, plugin, i)
parsed = parse_params(this_params, plugin.get('extra_params', {}))
analysis = plugin.activity(parsed)
analysis_list.append(analysis)
return analysis_list
def filter_params(params, plugin, ith=-1):
'''
Get the values within params that apply to a plugin.
More specific names override more general names, in this order:
<index_order>.parameter > <plugin.name>.parameter > parameter
Example:
>>> filter_params({'0.hello': True, 'hello': False}, Plugin(), 0)
{ '0.hello': True, 'hello': True}
'''
thisparams = {}
if ith >= 0:
ith = '{}.'.format(ith)
else:
ith = ""
for k, v in params.items():
if ith and k.startswith(str(ith)):
thisparams[k[len(ith):]] = v
elif k.startswith(plugin.name):
thisparams[k[len(plugin.name) + 1:]] = v
elif k not in thisparams:
thisparams[k] = v
return thisparams
def parse_call(params): def parse_call(params):
'''Return a results object based on the parameters used in a call/request. '''
Return a results object based on the parameters used in a call/request.
''' '''
params = parse_params(params, NIF_PARAMS) params = parse_params(params, NIF_PARAMS)
if params['informat'] == 'text': if params['informat'] == 'text':

@ -188,17 +188,27 @@ def basic_api(f):
@api_blueprint.route('/<path:plugin>', methods=['POST', 'GET']) @api_blueprint.route('/<path:plugin>', methods=['POST', 'GET'])
@basic_api @basic_api
def api_root(plugin): def api_root(plugin):
if plugin:
if request.parameters['algorithm'] != api.API_PARAMS['algorithm']['default']:
raise Error('You cannot specify the algorithm with a parameter and a URL variable.'
' Please, remove one of them')
request.parameters['algorithm'] = tuple(plugin.replace('+', '/').split('/'))
plugin = request.parameters['algorithm']
sp = current_app.senpy
plugins = sp.get_plugins(plugin)
if request.parameters['help']: if request.parameters['help']:
dic = dict(api.API_PARAMS, **api.NIF_PARAMS) apis = []
response = Help(valid_parameters=dic) if request.parameters['verbose']:
apis.append(api.BUILTIN_PARAMS)
allparameters = api.get_all_params(plugins, *apis)
response = Help(valid_parameters=allparameters)
return response return response
req = api.parse_call(request.parameters) req = api.parse_call(request.parameters)
if plugin: analysis = api.parse_analysis(req.parameters, plugins)
plugin = plugin.replace('+', '/') results = current_app.senpy.analyse(req, analysis)
plugin = plugin.split('/')
req.parameters['algorithm'] = tuple(plugin)
results = current_app.senpy.analyse(req)
results.analysis = set(i.id for i in results.analysis)
return results return results

@ -31,10 +31,10 @@ def main_function(argv):
default_plugins = params.get('default-plugins', False) default_plugins = params.get('default-plugins', False)
sp = Senpy(default_plugins=default_plugins, plugin_folder=plugin_folder) sp = Senpy(default_plugins=default_plugins, plugin_folder=plugin_folder)
request = api.parse_call(params) request = api.parse_call(params)
algos = request.parameters.get('algorithm', None) algos = sp.get_plugins(request.parameters.get('algorithm', None))
if algos: if algos:
for algo in algos: for algo in algos:
sp.activate_plugin(algo) sp.activate_plugin(algo.name)
else: else:
sp.activate_all() sp.activate_all()
res = sp.analyse(request) res = sp.analyse(request)

@ -78,27 +78,47 @@ class Senpy(object):
def delete_plugin(self, plugin): def delete_plugin(self, plugin):
del self._plugins[plugin.name.lower()] del self._plugins[plugin.name.lower()]
def plugins(self, **kwargs): def plugins(self, plugin_type=None, is_activated=True, **kwargs):
""" Return the plugins registered for a given application. Filtered by criteria """ """ Return the plugins registered for a given application. Filtered by criteria """
return list(plugins.pfilter(self._plugins, **kwargs)) return list(plugins.pfilter(self._plugins, plugin_type=plugin_type,
is_activated=is_activated, **kwargs))
def get_plugin(self, name, default=None): def get_plugin(self, name, default=None):
if name == 'default': if name == 'default':
return self.default_plugin return self.default_plugin
plugin = name.lower() elif name == 'conversion':
if plugin in self._plugins: return None
return self._plugins[plugin]
results = self.plugins(id='endpoint:plugins/{}'.format(name)) if name.lower() in self._plugins:
return self._plugins[name.lower()]
if not results: results = self.plugins(id='endpoint:plugins/{}'.format(name.lower()),
return Error(message="Plugin not found", status=404) plugin_type=None)
return results[0] if results:
return results[0]
results = self.plugins(id=name,
plugin_type=None)
if results:
return results[0]
msg = ("Plugin not found: '{}'\n"
"Make sure it is ACTIVATED\n"
"Valid algorithms: {}").format(name,
self._plugins.keys())
raise Error(message=msg, status=404)
def get_plugins(self, name):
try:
name = name.split(',')
except AttributeError:
pass # Assume it is a tuple or a list
return tuple(self.get_plugin(n) for n in name)
@property @property
def analysis_plugins(self): def analysis_plugins(self):
""" Return only the analysis plugins """ """ Return only the analysis plugins that are active"""
return self.plugins(plugin_type='analysisPlugin') return self.plugins(plugin_type='analysisPlugin', is_activated=True)
def add_folder(self, folder, from_root=False): def add_folder(self, folder, from_root=False):
""" Find plugins in this folder and add them to this instance """ """ Find plugins in this folder and add them to this instance """
@ -113,37 +133,6 @@ class Senpy(object):
else: else:
raise AttributeError("Not a folder or does not exist: %s", folder) raise AttributeError("Not a folder or does not exist: %s", folder)
def _get_plugins(self, request):
'''Get a list of plugins that should be run for a specific request'''
if not self.analysis_plugins:
raise Error(
status=404,
message=("No plugins found."
" Please install one."))
algos = request.parameters.get('algorithm', None)
if not algos:
if self.default_plugin:
algos = [self.default_plugin.name, ]
else:
raise Error(
status=404,
message="No default plugin found, and None provided")
plugins = list()
for algo in algos:
algo = algo.lower()
if algo == 'conversion':
continue # Allow 'conversion' as a virtual plugin, which does nothing
if algo not in self._plugins:
msg = ("The algorithm '{}' is not valid\n"
"Valid algorithms: {}").format(algo,
self._plugins.keys())
logger.debug(msg)
raise Error(status=404, message=msg)
plugins.append(self._plugins[algo])
return plugins
def _process(self, req, pending, done=None): def _process(self, req, pending, done=None):
""" """
Recursively process the entries with the first plugin in the list, and pass the results Recursively process the entries with the first plugin in the list, and pass the results
@ -153,25 +142,31 @@ class Senpy(object):
if not pending: if not pending:
return req return req
plugin = pending[0] analysis = pending[0]
results = plugin.process(req, conversions_applied=done) results = analysis.run(req)
if plugin not in results.analysis: results.analysis.append(analysis)
results.analysis.append(plugin) done += analysis
return self._process(results, pending[1:], done) return self._process(results, pending[1:], done)
def install_deps(self): def install_deps(self):
plugins.install_deps(*self.plugins()) plugins.install_deps(*self.plugins())
def analyse(self, request): def analyse(self, request, analysis=None):
""" """
Main method that analyses a request, either from CLI or HTTP. Main method that analyses a request, either from CLI or HTTP.
It takes a processed request, provided by the user, as returned It takes a processed request, provided by the user, as returned
by api.parse_call(). by api.parse_call().
""" """
if not self.plugins():
raise Error(
status=404,
message=("No plugins found."
" Please install one."))
if analysis is None:
plugins = self.get_plugins(request.parameters['algorithm'])
analysis = api.parse_analysis(request.parameters, plugins)
logger.debug("analysing request: {}".format(request)) logger.debug("analysing request: {}".format(request))
plugins = self._get_plugins(request) results = self._process(request, analysis)
request.parameters = api.parse_extra_params(request, plugins)
results = self._process(request, plugins)
logger.debug("Got analysis result: {}".format(results)) logger.debug("Got analysis result: {}".format(results))
results = self.postprocess(results) results = self.postprocess(results)
logger.debug("Returning post-processed result: {}".format(results)) logger.debug("Returning post-processed result: {}".format(results))
@ -187,7 +182,10 @@ class Senpy(object):
""" """
plugins = resp.analysis plugins = resp.analysis
params = resp.parameters if 'parameters' not in resp:
return resp
params = resp['parameters']
toModel = params.get('emotionModel', None) toModel = params.get('emotionModel', None)
if not toModel: if not toModel:
return resp return resp
@ -288,7 +286,10 @@ class Senpy(object):
results = AggregatedEvaluation() results = AggregatedEvaluation()
results.parameters = params results.parameters = params
datasets = self._get_datasets(results) datasets = self._get_datasets(results)
plugins = self._get_plugins(results) plugins = []
for plugname in params.algorithm:
plugins = self.get_plugin(plugname)
for eval in plugins.evaluate(plugins, datasets): for eval in plugins.evaluate(plugins, datasets):
results.evaluations.append(eval) results.evaluations.append(eval)
if 'with_parameters' not in results.parameters: if 'with_parameters' not in results.parameters:

@ -85,7 +85,8 @@ class BaseMeta(ABCMeta):
schema = json.load(f) schema = json.load(f)
resolver = jsonschema.RefResolver(schema_path, schema) resolver = jsonschema.RefResolver(schema_path, schema)
attrs['@type'] = "".join((name[0].lower(), name[1:])) if '@type' not in attrs:
attrs['@type'] = "".join((name[0].lower(), name[1:]))
attrs['_schema_file'] = schema_file attrs['_schema_file'] = schema_file
attrs['schema'] = schema attrs['schema'] = schema
attrs['_validator'] = jsonschema.Draft4Validator(schema, resolver=resolver) attrs['_validator'] = jsonschema.Draft4Validator(schema, resolver=resolver)
@ -244,10 +245,10 @@ class CustomDict(MutableMapping, object):
return key[0] == '_' return key[0] == '_'
def __str__(self): def __str__(self):
return str(self.serializable()) return json.dumps(self.serializable(), sort_keys=True, indent=4)
def __repr__(self): def __repr__(self):
return str(self.serializable()) return json.dumps(self.serializable(), sort_keys=True, indent=4)
_Alias = namedtuple('Alias', 'indict') _Alias = namedtuple('Alias', 'indict')

@ -121,11 +121,11 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)):
''' '''
schema_file = DEFINITIONS_FILE # schema_file = DEFINITIONS_FILE
_context = base_context["@context"] _context = base_context["@context"]
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
auto_id = kwargs.pop('_auto_id', True) auto_id = kwargs.pop('_auto_id', False)
super(BaseModel, self).__init__(*args, **kwargs) super(BaseModel, self).__init__(*args, **kwargs)
@ -133,7 +133,7 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)):
self.id self.id
if '@type' not in self: if '@type' not in self:
logger.warn('Created an instance of an unknown model') logger.warning('Created an instance of an unknown model')
@property @property
def id(self): def id(self):
@ -325,7 +325,6 @@ def _add_class_from_schema(*args, **kwargs):
for i in [ for i in [
'aggregatedEvaluation', 'aggregatedEvaluation',
'analysis',
'dataset', 'dataset',
'datasets', 'datasets',
'emotion', 'emotion',
@ -339,7 +338,7 @@ for i in [
'entity', 'entity',
'help', 'help',
'metric', 'metric',
'plugin', 'parameter',
'plugins', 'plugins',
'response', 'response',
'results', 'results',
@ -349,3 +348,54 @@ for i in [
]: ]:
_add_class_from_schema(i) _add_class_from_schema(i)
class Analysis(BaseModel):
schema = 'analysis'
parameters = alias('prov:used')
@property
def params(self):
outdict = {}
outdict['algorithm'] = self.algorithm
for param in self.parameters:
outdict[param['name']] = param['value']
return outdict
@params.setter
def params(self, value):
for k, v in value.items():
for param in self.parameters:
if param.name == k:
param.value = v
break
else:
self.parameters.append(Parameter(name=k, value=v)) # noqa: F821
@property
def algorithm(self):
return self['prov:wasAssociatedWith']
@property
def plugin(self):
return self._plugin
@plugin.setter
def plugin(self, value):
self._plugin = value
self['prov:wasAssociatedWith'] = value.id
def run(self, request):
return self.plugin.process(request, self.params)
class Plugin(BaseModel):
schema = 'plugin'
def activity(self, parameters):
'''Generate a prov:Activity from this plugin and the '''
a = Analysis()
a.plugin = self
a.params = parameters
return a

@ -17,7 +17,7 @@ import subprocess
import importlib import importlib
import yaml import yaml
import threading import threading
import nltk from nltk import download
from .. import models, utils from .. import models, utils
from .. import api from .. import api
@ -132,12 +132,12 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)):
def deactivate(self): def deactivate(self):
pass pass
def process(self, request, **kwargs): def process(self, request, parameters, **kwargs):
""" """
An implemented plugin should override this method. An implemented plugin should override this method.
Here, we assume that a process_entries method exists.""" Here, we assume that a process_entries method exists."""
newentries = list( newentries = list(
self.process_entries(request.entries, request.parameters)) self.process_entries(request.entries, parameters))
request.entries = newentries request.entries = newentries
return request return request
@ -194,13 +194,13 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)):
try: try:
request = models.Response() request = models.Response()
request.parameters = api.parse_params(given_parameters, parameters = api.parse_params(given_parameters,
self.extra_params) self.extra_params)
request.entries = [ request.entries = [
entry, entry,
] ]
method = partial(self.process, request) method = partial(self.process, request, parameters)
if mock: if mock:
res = method() res = method()
@ -249,14 +249,14 @@ class Analysis(Plugin):
''' '''
def analyse(self, request, parameters): def analyse(self, request, parameters):
return super(Analysis, self).process(request) return super(Analysis, self).process(request, parameters)
def analyse_entries(self, entries, parameters): def analyse_entries(self, entries, parameters):
for i in super(Analysis, self).process_entries(entries, parameters): for i in super(Analysis, self).process_entries(entries, parameters):
yield i yield i
def process(self, request, **kwargs): def process(self, request, parameters, **kwargs):
return self.analyse(request, request.parameters) return self.analyse(request, parameters)
def process_entries(self, entries, parameters): def process_entries(self, entries, parameters):
for i in self.analyse_entries(entries, parameters): for i in self.analyse_entries(entries, parameters):
@ -279,12 +279,12 @@ class Conversion(Plugin):
e.g. a conversion of emotion models, or normalization of sentiment values. e.g. a conversion of emotion models, or normalization of sentiment values.
''' '''
def process(self, response, plugins=None, **kwargs): def process(self, response, parameters, plugins=None, **kwargs):
plugins = plugins or [] plugins = plugins or []
newentries = [] newentries = []
for entry in response.entries: for entry in response.entries:
newentries.append( newentries.append(
self.convert_entry(entry, response.parameters, plugins)) self.convert_entry(entry, parameters, plugins))
response.entries = newentries response.entries = newentries
return response return response
@ -574,7 +574,7 @@ def install_deps(*plugins):
"Dependencies not properly installed: {}".format(pip_args)) "Dependencies not properly installed: {}".format(pip_args))
nltk_resources |= set(info.get('nltk_resources', [])) nltk_resources |= set(info.get('nltk_resources', []))
installed |= nltk.download(list(nltk_resources)) installed |= download(list(nltk_resources))
return installed return installed

@ -9,7 +9,7 @@ class Split(AnalysisPlugin):
'''description: A sample plugin that chunks input text''' '''description: A sample plugin that chunks input text'''
author = ["@militarpancho", '@balkian'] author = ["@militarpancho", '@balkian']
version = '0.2' version = '0.3'
url = "https://github.com/gsi-upm/senpy" url = "https://github.com/gsi-upm/senpy"
extra_params = { extra_params = {
@ -33,12 +33,15 @@ class Split(AnalysisPlugin):
if chunker_type == "paragraph": if chunker_type == "paragraph":
tokenizer = LineTokenizer() tokenizer = LineTokenizer()
chars = list(tokenizer.span_tokenize(original_text)) chars = list(tokenizer.span_tokenize(original_text))
for i, chunk in enumerate(tokenizer.tokenize(original_text)): if len(chars) == 1:
print(chunk) # This sentence was already split
return
for i, chunk in enumerate(chars):
start, end = chunk
e = Entry() e = Entry()
e['nif:isString'] = chunk e['nif:isString'] = original_text[start:end]
if entry.id: if entry.id:
e.id = entry.id + "#char={},{}".format(chars[i][0], chars[i][1]) e.id = entry.id + "#char={},{}".format(start, end)
yield e yield e
test_cases = [ test_cases = [

@ -9,7 +9,20 @@
"@type": { "@type": {
"type": "string", "type": "string",
"description": "Type of the analysis. e.g. marl:SentimentAnalysis" "description": "Type of the analysis. e.g. marl:SentimentAnalysis"
},
"prov:wasAssociatedWith": {
"@type": "string",
"description": "Algorithm/plugin that was used"
},
"prov:used": {
"description": "Parameters of the algorithm",
"@type": "array",
"default": [],
"type": "array",
"items": {
"$ref": "parameter.json"
}
} }
}, },
"required": ["@id", "@type"] "required": ["@type", "prov:wasAssociatedWith"]
} }

@ -41,7 +41,7 @@
"@container": "@set" "@container": "@set"
}, },
"analysis": { "analysis": {
"@id": "AnalysisInvolved", "@id": "prov:wasInformedBy",
"@type": "@id", "@type": "@id",
"@container": "@set" "@container": "@set"
}, },

@ -20,5 +20,5 @@
"description": "The ID of the analysis that generated this Emotion. The full object should be included in the \"analysis\" property of the root object" "description": "The ID of the analysis that generated this Emotion. The full object should be included in the \"analysis\" property of the root object"
} }
}, },
"required": ["@id", "prov:wasGeneratedBy", "onyx:hasEmotion"] "required": ["prov:wasGeneratedBy", "onyx:hasEmotion"]
} }

@ -35,5 +35,5 @@
"default": [] "default": []
} }
}, },
"required": ["@id", "nif:isString"] "required": ["nif:isString"]
} }

@ -0,0 +1,16 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Parameters for a senpy analysis",
"type": "object",
"properties": {
"name": {
"type": "string",
"description": "Name of the parameter"
},
"prov:value": {
"@type": "any",
"description": "Value of the parameter"
}
},
"required": ["name", "prov:value"]
}

@ -21,13 +21,7 @@
"default": [], "default": [],
"type": "array", "type": "array",
"items": { "items": {
"anyOf": [ "$ref": "analysis.json"
{
"$ref": "analysis.json"
},{
"type": "string"
}
]
} }
}, },
"entries": { "entries": {

@ -19,5 +19,5 @@
"description": "The ID of the analysis that generated this Sentiment. The full object should be included in the \"analysis\" property of the root object" "description": "The ID of the analysis that generated this Sentiment. The full object should be included in the \"analysis\" property of the root object"
} }
}, },
"required": ["@id", "prov:wasGeneratedBy"] "required": ["prov:wasGeneratedBy"]
} }

@ -12,6 +12,7 @@ max-line-length = 100
universal=1 universal=1
[tool:pytest] [tool:pytest]
addopts = --cov=senpy --cov-report term-missing addopts = --cov=senpy --cov-report term-missing
filterwarnings =
ignore:the matrix subclass:PendingDeprecationWarning
[coverage:report] [coverage:report]
omit = senpy/__main__.py omit = senpy/__main__.py

@ -3,8 +3,9 @@ import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
from unittest import TestCase from unittest import TestCase
from senpy.api import parse_params, API_PARAMS, NIF_PARAMS, WEB_PARAMS from senpy.api import (boolean, parse_params, get_extra_params, parse_analysis,
from senpy.models import Error API_PARAMS, NIF_PARAMS, WEB_PARAMS)
from senpy.models import Error, Plugin
class APITest(TestCase): class APITest(TestCase):
@ -89,3 +90,156 @@ class APITest(TestCase):
assert "Dummy" in p['algorithm'] assert "Dummy" in p['algorithm']
assert 'input' in p assert 'input' in p
assert p['input'] == 'Aloha my friend' assert p['input'] == 'Aloha my friend'
def test_parse_analysis(self):
'''The API should parse user parameters and return them in a format that plugins can use'''
plugins = [
Plugin({
'name': 'plugin1',
'extra_params': {
# Incompatible parameter
'param0': {
'aliases': ['p1', 'parameter1'],
'options': ['option1', 'option2'],
'default': 'option1',
'required': True
},
'param1': {
'aliases': ['p1', 'parameter1'],
'options': ['en', 'es'],
'default': 'en',
'required': False
},
'param2': {
'aliases': ['p2', 'parameter2'],
'required': False,
'options': ['value2_1', 'value2_2', 'value3_3']
}
}
}), Plugin({
'name': 'plugin2',
'extra_params': {
'param0': {
'aliases': ['parameter1'],
'options': ['new option', 'new option2'],
'default': 'new option',
'required': False
},
'param1': {
'aliases': ['myparam1', 'p1'],
'options': ['en', 'de', 'auto'],
'default': 'de',
'required': True
},
'param3': {
'aliases': ['p3', 'parameter3'],
'options': boolean,
'default': True
}
}
})
]
call = {
'param1': 'en',
'0.param0': 'option1',
'0.param1': 'en',
'param2': 'value2_1',
'param0': 'new option',
'1.param1': 'de',
'param3': False,
}
expected = [
{
'param0': 'option1',
'param1': 'en',
'param2': 'value2_1',
}, {
'param0': 'new option',
'param1': 'de',
'param3': False,
}
]
p = parse_analysis(call, plugins)
for i, arg in enumerate(expected):
params = p[i].params
for k, v in arg.items():
assert params[k] == v
def test_get_extra_params(self):
'''The API should return the list of valid parameters for a set of plugins'''
plugins = [
Plugin({
'name': 'plugin1',
'extra_params': {
# Incompatible parameter
'param0': {
'aliases': ['p1', 'parameter1'],
'options': ['option1', 'option2'],
'default': 'option1',
'required': True
},
'param1': {
'aliases': ['p1', 'parameter1'],
'options': ['en', 'es'],
'default': 'en',
'required': False
},
'param2': {
'aliases': ['p2', 'parameter2'],
'required': False,
'options': ['value2_1', 'value2_2', 'value3_3']
}
}
}), Plugin({
'name': 'plugin2',
'extra_params': {
'param0': {
'aliases': ['parameter1'],
'options': ['new option', 'new option2'],
'default': 'new option',
'required': False
},
'param1': {
'aliases': ['myparam1', 'p1'],
'options': ['en', 'de', 'auto'],
'default': 'de',
'required': True
},
'param3': {
'aliases': ['p3', 'parameter3'],
'options': boolean,
'default': True
}
}
})
]
expected = {
# Overlapping parameters
'plugin1.param0': plugins[0]['extra_params']['param0'],
'plugin1.param1': plugins[0]['extra_params']['param1'],
'plugin2.param0': plugins[1]['extra_params']['param0'],
'plugin2.param1': plugins[1]['extra_params']['param1'],
# Non-overlapping parameters
'param2': plugins[0]['extra_params']['param2'],
'param3': plugins[1]['extra_params']['param3'],
# Intersection of overlapping parameters
'param1': {
'aliases': ['p1'],
'options': ['en'],
'default': None,
'required': True
}
}
result = get_extra_params(plugins)
for ik, iv in expected.items():
assert ik in result
for jk, jv in iv.items():
assert jk in result[ik]
assert expected[ik][jk] == result[ik][jk]

@ -26,8 +26,7 @@ class BlueprintsTest(TestCase):
cls.senpy.init_app(cls.app) cls.senpy.init_app(cls.app)
cls.dir = os.path.join(os.path.dirname(__file__), "..") cls.dir = os.path.join(os.path.dirname(__file__), "..")
cls.senpy.add_folder(cls.dir) cls.senpy.add_folder(cls.dir)
cls.senpy.activate_plugin("Dummy", sync=True) cls.senpy.activate_all()
cls.senpy.activate_plugin("DummyRequired", sync=True)
cls.senpy.default_plugin = 'Dummy' cls.senpy.default_plugin = 'Dummy'
def setUp(self): def setUp(self):
@ -107,6 +106,7 @@ class BlueprintsTest(TestCase):
assert isinstance(js, models.Error) assert isinstance(js, models.Error)
resp = self.client.get("/api/?i=My aloha mohame&algo=DummyRequired&example=notvalid") resp = self.client.get("/api/?i=My aloha mohame&algo=DummyRequired&example=notvalid")
self.assertCode(resp, 400) self.assertCode(resp, 400)
self.app.config['TESTING'] = True
resp = self.client.get("/api/?i=My aloha mohame&algo=DummyRequired&example=a") resp = self.client.get("/api/?i=My aloha mohame&algo=DummyRequired&example=a")
self.assertCode(resp, 200) self.assertCode(resp, 200)
@ -138,15 +138,76 @@ class BlueprintsTest(TestCase):
# Calling dummy twice, should return the same string # Calling dummy twice, should return the same string
self.assertCode(resp, 200) self.assertCode(resp, 200)
js = parse_resp(resp) js = parse_resp(resp)
assert len(js['analysis']) == 1 assert len(js['analysis']) == 2
assert js['entries'][0]['nif:isString'] == 'My aloha mohame' assert js['entries'][0]['nif:isString'] == 'My aloha mohame'
resp = self.client.get("/api/Dummy+Dummy?i=My aloha mohame") resp = self.client.get("/api/Dummy+Dummy?i=My aloha mohame")
# Same with pluses instead of slashes # Same with pluses instead of slashes
self.assertCode(resp, 200) self.assertCode(resp, 200)
js = parse_resp(resp) js = parse_resp(resp)
assert len(js['analysis']) == 1 assert len(js['analysis']) == 2
assert js['entries'][0]['nif:isString'] == 'My aloha mohame'
def test_analysis_chain_required(self):
"""
If a parameter is required and duplicated (because two plugins require it), specifying
it once should suffice
"""
resp = self.client.get("/api/DummyRequired/DummyRequired?i=My aloha mohame&example=a")
js = parse_resp(resp)
assert len(js['analysis']) == 2
assert js['entries'][0]['nif:isString'] == 'My aloha mohame' assert js['entries'][0]['nif:isString'] == 'My aloha mohame'
assert js['entries'][0]['reversed'] == 2
def test_requirements_chain_help(self):
'''The extra parameters of each plugin should be merged if they are in a chain '''
resp = self.client.get("/api/split/DummyRequired?help=true")
self.assertCode(resp, 200)
js = parse_resp(resp)
assert 'valid_parameters' in js
vp = js['valid_parameters']
assert 'example' in vp
assert 'delimiter' in vp
def test_requirements_chain_repeat_help(self):
'''
If a plugin appears several times in a chain, there should be a way to set different
parameters for each.
'''
resp = self.client.get("/api/split/split?help=true")
self.assertCode(resp, 200)
js = parse_resp(resp)
assert 'valid_parameters' in js
vp = js['valid_parameters']
assert 'delimiter' in vp
resp = self.client.get("/api/split/split?help=true&verbose=false")
js = parse_resp(resp)
vp = js['valid_parameters']
assert len(vp.keys()) == 1
def test_requirements_chain(self):
"""
It should be possible to specify different parameters for each step in the chain.
"""
# First, we split by sentence twice. Each call should generate 3 additional entries
# (one per sentence in the original).
resp = self.client.get('/api/split/split?i=The first sentence. The second sentence.'
'\nA new paragraph&delimiter=sentence')
js = parse_resp(resp)
assert len(js['analysis']) == 2
assert len(js['entries']) == 7
# Now, we split by sentence. This produces 3 additional entries.
# Then, we split by paragraph. This should create 2 additional entries (One per paragraph
# in the original text)
resp = self.client.get('/api/split/split?i=The first sentence. The second sentence.'
'\nA new paragraph&0.delimiter=sentence&1.delimiter=paragraph')
# Calling dummy twice, should return the same string
self.assertCode(resp, 200)
js = parse_resp(resp)
assert len(js['analysis']) == 2
assert len(js['entries']) == 6
def test_error(self): def test_error(self):
""" """

@ -3,10 +3,6 @@ import os
from copy import deepcopy from copy import deepcopy
import logging import logging
try:
from unittest import mock
except ImportError:
import mock
from functools import partial from functools import partial
from senpy.extensions import Senpy from senpy.extensions import Senpy
@ -18,7 +14,8 @@ from unittest import TestCase
def analyse(instance, **kwargs): def analyse(instance, **kwargs):
request = api.parse_call(kwargs) basic = api.parse_params(kwargs, api.API_PARAMS)
request = api.parse_call(basic)
return instance.analyse(request) return instance.analyse(request)
@ -49,9 +46,9 @@ class ExtensionsTest(TestCase):
'''Should be able to add and delete new plugins. ''' '''Should be able to add and delete new plugins. '''
new = plugins.Analysis(name='new', description='new', version=0) new = plugins.Analysis(name='new', description='new', version=0)
self.senpy.add_plugin(new) self.senpy.add_plugin(new)
assert new in self.senpy.plugins() assert new in self.senpy.plugins(is_activated=False)
self.senpy.delete_plugin(new) self.senpy.delete_plugin(new)
assert new not in self.senpy.plugins() assert new not in self.senpy.plugins(is_activated=False)
def test_adding_folder(self): def test_adding_folder(self):
""" It should be possible for senpy to look for plugins in more folders. """ """ It should be possible for senpy to look for plugins in more folders. """
@ -60,7 +57,7 @@ class ExtensionsTest(TestCase):
default_plugins=False) default_plugins=False)
assert not senpy.analysis_plugins assert not senpy.analysis_plugins
senpy.add_folder(self.examples_dir) senpy.add_folder(self.examples_dir)
assert senpy.analysis_plugins assert senpy.plugins(plugin_type=plugins.AnalysisPlugin, is_activated=False)
self.assertRaises(AttributeError, senpy.add_folder, 'DOES NOT EXIST') self.assertRaises(AttributeError, senpy.add_folder, 'DOES NOT EXIST')
def test_installing(self): def test_installing(self):
@ -121,8 +118,8 @@ class ExtensionsTest(TestCase):
# Leaf (defaultdict with __setattr__ and __getattr__. # Leaf (defaultdict with __setattr__ and __getattr__.
r1 = analyse(self.senpy, algorithm="Dummy", input="tupni", output="tuptuo") r1 = analyse(self.senpy, algorithm="Dummy", input="tupni", output="tuptuo")
r2 = analyse(self.senpy, input="tupni", output="tuptuo") r2 = analyse(self.senpy, input="tupni", output="tuptuo")
assert r1.analysis[0].id == "endpoint:plugins/Dummy_0.1" assert r1.analysis[0].algorithm == "endpoint:plugins/Dummy_0.1"
assert r2.analysis[0].id == "endpoint:plugins/Dummy_0.1" assert r2.analysis[0].algorithm == "endpoint:plugins/Dummy_0.1"
assert r1.entries[0]['nif:isString'] == 'input' assert r1.entries[0]['nif:isString'] == 'input'
def test_analyse_empty(self): def test_analyse_empty(self):
@ -130,7 +127,7 @@ class ExtensionsTest(TestCase):
senpy = Senpy(plugin_folder=None, senpy = Senpy(plugin_folder=None,
app=self.app, app=self.app,
default_plugins=False) default_plugins=False)
self.assertRaises(Error, senpy.analyse, Results()) self.assertRaises(Error, senpy.analyse, Results(), [])
def test_analyse_wrong(self): def test_analyse_wrong(self):
""" Trying to analyse with a non-existent plugin should raise an error.""" """ Trying to analyse with a non-existent plugin should raise an error."""
@ -156,29 +153,32 @@ class ExtensionsTest(TestCase):
r2 = analyse(self.senpy, r2 = analyse(self.senpy,
input="tupni", input="tupni",
output="tuptuo") output="tuptuo")
assert r1.analysis[0].id == "endpoint:plugins/Dummy_0.1" assert r1.analysis[0].algorithm == "endpoint:plugins/Dummy_0.1"
assert r2.analysis[0].id == "endpoint:plugins/Dummy_0.1" assert r2.analysis[0].algorithm == "endpoint:plugins/Dummy_0.1"
assert r1.entries[0]['nif:isString'] == 'input' assert r1.entries[0]['nif:isString'] == 'input'
def test_analyse_error(self): def test_analyse_error(self):
mm = mock.MagicMock() class ErrorPlugin(plugins.Analysis):
mm.id = 'magic_mock' author = 'nobody'
mm.name = 'mock' version = 0
mm.is_activated = True ex = Error()
mm.process.side_effect = Error('error in analysis', status=500)
self.senpy.add_plugin(mm) def process(self, *args, **kwargs):
raise self.ex
m = ErrorPlugin(ex=Error('error in analysis', status=500))
self.senpy.add_plugin(m)
try: try:
analyse(self.senpy, input='nothing', algorithm='MOCK') analyse(self.senpy, input='nothing', algorithm='ErrorPlugin')
assert False assert False
except Error as ex: except Error as ex:
assert 'error in analysis' in ex['message'] assert 'error in analysis' in ex['message']
assert ex['status'] == 500 assert ex['status'] == 500
ex = Exception('generic exception on analysis') m.ex = Exception('generic exception on analysis')
mm.process.side_effect = ex
try: try:
analyse(self.senpy, input='nothing', algorithm='MOCK') analyse(self.senpy, input='nothing', algorithm='ErrorPlugin')
assert False assert False
except Exception as ex: except Exception as ex:
assert 'generic exception on analysis' in str(ex) assert 'generic exception on analysis' in str(ex)
@ -194,7 +194,7 @@ class ExtensionsTest(TestCase):
def test_load_default_plugins(self): def test_load_default_plugins(self):
senpy = Senpy(plugin_folder=self.examples_dir, default_plugins=True) senpy = Senpy(plugin_folder=self.examples_dir, default_plugins=True)
assert len(senpy.plugins()) > 1 assert len(senpy.plugins(is_activated=False)) > 1
def test_convert_emotions(self): def test_convert_emotions(self):
self.senpy.activate_all(sync=True) self.senpy.activate_all(sync=True)

@ -5,7 +5,8 @@ import jsonschema
import json import json
import rdflib import rdflib
from unittest import TestCase from unittest import TestCase
from senpy.models import (Emotion, from senpy.models import (Analysis,
Emotion,
EmotionAnalysis, EmotionAnalysis,
EmotionSet, EmotionSet,
Entry, Entry,
@ -61,7 +62,7 @@ class ModelsTest(TestCase):
def test_id(self): def test_id(self):
""" Adding the id after creation should overwrite the automatic ID """ Adding the id after creation should overwrite the automatic ID
""" """
r = Entry() r = Entry(_auto_id=True)
j = r.jsonld() j = r.jsonld()
assert '@id' in j assert '@id' in j
r.id = "test" r.id = "test"
@ -189,6 +190,19 @@ class ModelsTest(TestCase):
assert isinstance(js['plugins'], list) assert isinstance(js['plugins'], list)
assert js['plugins'][0]['@type'] == 'sentimentPlugin' assert js['plugins'][0]['@type'] == 'sentimentPlugin'
def test_parameters(self):
'''An Analysis should contain the algorithm and the list of parameters to be used'''
a = Analysis()
a.params = {'param1': 1, 'param2': 2}
assert len(a.parameters) == 2
for param in a.parameters:
if param.name == 'param1':
assert param.value == 1
elif param.name == 'param2':
assert param.value == 2
else:
raise Exception('Unknown value %s' % param)
def test_from_string(self): def test_from_string(self):
results = { results = {
'@type': 'results', '@type': 'results',

Loading…
Cancel
Save