balkian/senpy
1
0
Fork 0
mirror of https://github.com/gsi-upm/senpy synced 2024-11-21 15:52:28 +00:00
Code Issues Releases Wiki Activity

New schema for parameters

Browse Source 
* Add parameters as an entity in the schema
* Update examples to include parameters
* Change the API for processing plugins, params is a parameter again, instead of
only adding the request.
* Update tests
This commit is contained in:
J. Fernando Sánchez 2018-11-23 20:29:08 +01:00
parent 6a1069780b
commit c090501534
29 changed files with 446 additions and 355 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/bad-examples/results/example-basic-FAIL.json
View File

@ -6,13 +6,9 @@
],
"entries": [
{
"@type": [
"nif:RFC5147String",
"nif:Context"
],
"nif:beginIndex": 0,
"nif:endIndex": 40,
"nif:isString": "My favourite actress is Natalie Portman"
"text": "An entry should have a nif:isString key"
}
]
}

29
docs/examples/results/example-analysis-as-id.json
View File

@ -3,10 +3,21 @@
"@id": "me:Result1",
"@type": "results",
"analysis": [
"me:SAnalysis1",
"me:SgAnalysis1",
"me:EmotionAnalysis1",
"me:NER1"
{
"@id": "_:SAnalysis1_Activity",
"@type": "marl:SentimentAnalysis",
"prov:wasAssociatedWith": "me:SAnalysis1"
},
{
"@id": "_:EmotionAnalysis1_Activity",
"@type": "onyx:EmotionAnalysis",
"prov:wasAssociatedWith": "me:EmotionAnalysis1"
},
{
"@id": "_:NER1_Activity",
"@type": "me:NER",
"prov:wasAssociatedWith": "me:NER1"
}
],
"entries": [
{
@ -23,7 +34,7 @@
"nif:endIndex": 13,
"nif:anchorOf": "Microsoft",
"me:references": "http://dbpedia.org/page/Microsoft",
"prov:wasGeneratedBy": "me:NER1"
"prov:wasGeneratedBy": "_:NER1_Activity"
},
{
"@id": "http://micro.blog/status1#char=25,37",
@ -31,7 +42,7 @@
"nif:endIndex": 37,
"nif:anchorOf": "Windows Phone",
"me:references": "http://dbpedia.org/page/Windows_Phone",
"prov:wasGeneratedBy": "me:NER1"
"prov:wasGeneratedBy": "_:NER1_Activity"
}
],
"suggestions": [
@ -40,7 +51,7 @@
"nif:beginIndex": 16,
"nif:endIndex": 77,
"nif:anchorOf": "put your Windows Phone on your newest #open technology program",
"prov:wasGeneratedBy": "me:SgAnalysis1"
"prov:wasGeneratedBy": "_:SgAnalysis1_Activity"
}
],
"sentiments": [
@ -51,14 +62,14 @@
"nif:anchorOf": "You'll be awesome.",
"marl:hasPolarity": "marl:Positive",
"marl:polarityValue": 0.9,
"prov:wasGeneratedBy": "me:SAnalysis1"
"prov:wasGeneratedBy": "_:SgAnalysis1_Activity"
}
],
"emotions": [
{
"@id": "http://micro.blog/status1#char=0,109",
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"prov:wasGeneratedBy": "me:EAnalysis1",
"prov:wasGeneratedBy": "_:EmotionAnalysis1_Activity",
"onyx:hasEmotion": [
{
"onyx:hasEmotionCategory": "wna:liking"

78
docs/examples/results/example-analysis-mixed-ids.json
View File

@ -1,78 +0,0 @@
{
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1",
"@type": "results",
"analysis": [
"me:SAnalysis1",
"me:SgAnalysis1",
"me:EmotionAnalysis1",
"me:NER1",
{
"@type": "analysis",
"@id": "anonymous"
}
],
"entries": [
{
"@id": "http://micro.blog/status1",
"@type": [
"nif:RFC5147String",
"nif:Context"
],
"nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"entities": [
{
"@id": "http://micro.blog/status1#char=5,13",
"nif:beginIndex": 5,
"nif:endIndex": 13,
"nif:anchorOf": "Microsoft",
"me:references": "http://dbpedia.org/page/Microsoft",
"prov:wasGeneratedBy": "me:NER1"
},
{
"@id": "http://micro.blog/status1#char=25,37",
"nif:beginIndex": 25,
"nif:endIndex": 37,
"nif:anchorOf": "Windows Phone",
"me:references": "http://dbpedia.org/page/Windows_Phone",
"prov:wasGeneratedBy": "me:NER1"
}
],
"suggestions": [
{
"@id": "http://micro.blog/status1#char=16,77",
"nif:beginIndex": 16,
"nif:endIndex": 77,
"nif:anchorOf": "put your Windows Phone on your newest #open technology program",
"prov:wasGeneratedBy": "me:SgAnalysis1"
}
],
"sentiments": [
{
"@id": "http://micro.blog/status1#char=80,97",
"nif:beginIndex": 80,
"nif:endIndex": 97,
"nif:anchorOf": "You'll be awesome.",
"marl:hasPolarity": "marl:Positive",
"marl:polarityValue": 0.9,
"prov:wasGeneratedBy": "me:SAnalysis1"
}
],
"emotions": [
{
"@id": "http://micro.blog/status1#char=0,109",
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"prov:wasGeneratedBy": "me:EAnalysis1",
"onyx:hasEmotion": [
{
"onyx:hasEmotionCategory": "wna:liking"
},
{
"onyx:hasEmotionCategory": "wna:excitement"
}
]
}
]
}
]
}

33
docs/examples/results/example-basic.json
View File

@ -1,19 +1,18 @@
{
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "http://example.com#NIFExample",
"@type": "results",
"analysis": [
],
"entries": [
{
"@id": "http://example.org#char=0,40",
"@type": [
"nif:RFC5147String",
"nif:Context"
],
"nif:beginIndex": 0,
"nif:endIndex": 40,
"nif:isString": "My favourite actress is Natalie Portman"
}
]
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1",
"@type": "results",
"analysis": [ ],
"entries": [
{
"@id": "http://example.org#char=0,40",
"@type": [
"nif:RFC5147String",
"nif:Context"
],
"nif:beginIndex": 0,
"nif:endIndex": 40,
"nif:isString": "My favourite actress is Natalie Portman"
}
]
}

168
docs/examples/results/example-complete.json
View File

@ -1,88 +1,100 @@
{
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1",
"@type": "results",
"analysis": [
{
"@id": "me:SAnalysis1",
"@type": "marl:SentimentAnalysis",
"marl:maxPolarityValue": 1,
"marl:minPolarityValue": 0
},
{
"@id": "me:SgAnalysis1",
"@type": "me:SuggestionAnalysis"
},
{
"@id": "me:EmotionAnalysis1",
"@type": "me:EmotionAnalysis"
},
{
"@id": "me:NER1",
"@type": "me:NER"
}
],
"entries": [
{
"@id": "http://micro.blog/status1",
"@type": [
"nif:RFC5147String",
"nif:Context"
],
"nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"entities": [
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1",
"@type": "results",
"analysis": [
{
"@id": "http://micro.blog/status1#char=5,13",
"nif:beginIndex": 5,
"nif:endIndex": 13,
"nif:anchorOf": "Microsoft",
"me:references": "http://dbpedia.org/page/Microsoft",
"prov:wasGeneratedBy": "me:NER1"
"@id": "_:SAnalysis1_Activity",
"@type": "marl:SentimentAnalysis",
"prov:wasAssociatedWith": "me:SentimentAnalysis",
"prov:used": [
{
"name": "marl:maxPolarityValue",
"prov:value": "1"
},
{
"name": "marl:minPolarityValue",
"prov:value": "0"
}
]
},
{
"@id": "http://micro.blog/status1#char=25,37",
"nif:beginIndex": 25,
"nif:endIndex": 37,
"nif:anchorOf": "Windows Phone",
"me:references": "http://dbpedia.org/page/Windows_Phone",
"prov:wasGeneratedBy": "me:NER1"
}
],
"suggestions": [
"@id": "_:SgAnalysis1_Activity",
"prov:wasAssociatedWith": "me:SgAnalysis1",
"@type": "me:SuggestionAnalysis"
},
{
"@id": "http://micro.blog/status1#char=16,77",
"nif:beginIndex": 16,
"nif:endIndex": 77,
"nif:anchorOf": "put your Windows Phone on your newest #open technology program",
"prov:wasGeneratedBy": "me:SgAnalysis1"
}
],
"sentiments": [
"@id": "_:EmotionAnalysis1_Activity",
"@type": "me:EmotionAnalysis",
"prov:wasAssociatedWith": "me:EmotionAnalysis1"
},
{
"@id": "http://micro.blog/status1#char=80,97",
"nif:beginIndex": 80,
"nif:endIndex": 97,
"nif:anchorOf": "You'll be awesome.",
"marl:hasPolarity": "marl:Positive",
"marl:polarityValue": 0.9,
"prov:wasGeneratedBy": "me:SAnalysis1"
"@id": "_:NER1_Activity",
"@type": "me:NER",
"prov:wasAssociatedWith": "me:EmotionNER1"
}
],
"emotions": [
],
"entries": [
{
"@id": "http://micro.blog/status1#char=0,109",
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"prov:wasGeneratedBy": "me:EAnalysis1",
"onyx:hasEmotion": [
{
"onyx:hasEmotionCategory": "wna:liking"
},
{
"onyx:hasEmotionCategory": "wna:excitement"
}
]
"@id": "http://micro.blog/status1",
"@type": [
"nif:RFC5147String",
"nif:Context"
],
"nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"entities": [
{
"@id": "http://micro.blog/status1#char=5,13",
"nif:beginIndex": 5,
"nif:endIndex": 13,
"nif:anchorOf": "Microsoft",
"me:references": "http://dbpedia.org/page/Microsoft",
"prov:wasGeneratedBy": "me:NER1"
},
{
"@id": "http://micro.blog/status1#char=25,37",
"nif:beginIndex": 25,
"nif:endIndex": 37,
"nif:anchorOf": "Windows Phone",
"me:references": "http://dbpedia.org/page/Windows_Phone",
"prov:wasGeneratedBy": "me:NER1"
}
],
"suggestions": [
{
"@id": "http://micro.blog/status1#char=16,77",
"nif:beginIndex": 16,
"nif:endIndex": 77,
"nif:anchorOf": "put your Windows Phone on your newest #open technology program",
"prov:wasGeneratedBy": "me:SgAnalysis1"
}
],
"sentiments": [
{
"@id": "http://micro.blog/status1#char=80,97",
"nif:beginIndex": 80,
"nif:endIndex": 97,
"nif:anchorOf": "You'll be awesome.",
"marl:hasPolarity": "marl:Positive",
"marl:polarityValue": 0.9,
"prov:wasGeneratedBy": "me:SAnalysis1"
}
],
"emotions": [
{
"@id": "http://micro.blog/status1#char=0,109",
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"prov:wasGeneratedBy": "me:EAnalysis1",
"onyx:hasEmotion": [
{
"onyx:hasEmotionCategory": "wna:liking"
},
{
"onyx:hasEmotionCategory": "wna:excitement"
}
]
}
]
}
]
}
]
]
}

7
docs/examples/results/example-emotion.json
View File

@ -4,8 +4,9 @@
"@type": "results",
"analysis": [
{
"@id": "me:EmotionAnalysis1",
"@type": "onyx:EmotionAnalysis"
"@id": "me:EmotionAnalysis1_Activity",
"@type": "me:EmotionAnalysis1",
"prov:wasAssociatedWith": "me:EmotionAnalysis1"
}
],
"entries": [
@ -26,7 +27,7 @@
{
"@id": "http://micro.blog/status1#char=0,109",
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"prov:wasGeneratedBy": "me:EmotionAnalysis1",
"prov:wasGeneratedBy": "_:EmotionAnalysis1_Activity",
"onyx:hasEmotion": [
{
"onyx:hasEmotionCategory": "wna:liking"

5
docs/examples/results/example-ner.json
View File

@ -4,8 +4,9 @@
"@type": "results",
"analysis": [
{
"@id": "me:NER1",
"@type": "me:NERAnalysis"
"@id": "_:NER1_Activity",
"@type": "me:NERAnalysis",
"prov:wasAssociatedWith": "me:NER1"
}
],
"entries": [

12
docs/examples/results/example-pad.json
View File

@ -9,9 +9,15 @@
"@type": "results",
"analysis": [
{
"@id": "me:HesamsAnalysis",
"@id": "me:HesamsAnalysis_Activity",
"@type": "onyx:EmotionAnalysis",
"onyx:usesEmotionModel": "emovoc:pad-dimensions"
"prov:wasAssociatedWith": "me:HesamsAnalysis",
"prov:used": [
{
"name": "emotion-model",
"prov:value": "emovoc:pad-dimensions"
}
]
}
],
"entries": [
@ -32,7 +38,7 @@
{
"@id": "Entry1#char=0,21",
"nif:anchorOf": "This is a test string",
"prov:wasGeneratedBy": "me:HesamAnalysis",
"prov:wasGeneratedBy": "_:HesamAnalysis_Activity",
"onyx:hasEmotion": [
{
"emovoc:pleasure": 0.5,

7
docs/examples/results/example-sentiment.json
View File

@ -4,10 +4,9 @@
"@type": "results",
"analysis": [
{
"@id": "me:SAnalysis1",
"@id": "_:SAnalysis1_Activity",
"@type": "marl:SentimentAnalysis",
"marl:maxPolarityValue": 1,
"marl:minPolarityValue": 0
"prov:wasAssociatedWith": "me:SAnalysis1"
}
],
"entries": [
@ -30,7 +29,7 @@
"nif:anchorOf": "You'll be awesome.",
"marl:hasPolarity": "marl:Positive",
"marl:polarityValue": 0.9,
"prov:wasGeneratedBy": "me:SAnalysis1"
"prov:wasGeneratedBy": "_:SAnalysis1_Activity"
}
],
"emotionSets": [

9
docs/examples/results/example-suggestion.json
View File

@ -3,7 +3,11 @@
"@id": "me:Result1",
"@type": "results",
"analysis": [
"me:SgAnalysis1"
{
"@id": "_:SgAnalysis1_Activity",
"@type": "me:SuggestionAnalysis",
"prov:wasAssociatedWith": "me:SgAnalysis1"
}
],
"entries": [
{
@ -12,7 +16,6 @@
"nif:RFC5147String",
"nif:Context"
],
"prov:wasGeneratedBy": "me:SAnalysis1",
"nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
"entities": [
],
@ -22,7 +25,7 @@
"nif:beginIndex": 16,
"nif:endIndex": 77,
"nif:anchorOf": "put your Windows Phone on your newest #open technology program",
"prov:wasGeneratedBy": "me:SgAnalysis1"
"prov:wasGeneratedBy": "_:SgAnalysis1_Activity"
}
],
"sentiments": [

45
senpy/api.py
View File

@ -1,5 +1,5 @@
from future.utils import iteritems
from .models import Error, Results, Entry, from_string
from .models import Analysis, Error, Results, Entry, from_string
import logging
logger = logging.getLogger(__name__)
@ -8,7 +8,8 @@ boolean = [True, False]
API_PARAMS = {
"algorithm": {
"aliases": ["algorithms", "a", "algo"],
"required": False,
"required": True,
"default": 'default',
"description": ("Algorithms that will be used to process the request."
"It may be a list of comma-separated names."),
},
@ -41,6 +42,14 @@ API_PARAMS = {
"options": boolean,
"default": False
},
"verbose": {
"@id": "verbose",
"description": "Show all help, including the common API parameters, or only plugin-related info",
"aliases": ["v"],
"required": True,
"options": boolean,
"default": True
},
"emotionModel": {
"@id": "emotionModel",
"aliases": ["emoModel"],
@ -168,8 +177,7 @@ def parse_params(indict, *specs):
outdict[param] = options["default"]
elif options.get("required", False):
wrong_params[param] = spec[param]
continue
if "options" in options:
elif "options" in options:
if options["options"] == boolean:
outdict[param] = str(outdict[param]).lower() in ['true', '1']
elif outdict[param] not in options["options"]:
@ -182,8 +190,6 @@ def parse_params(indict, *specs):
parameters=outdict,
errors=wrong_params)
raise message
if 'algorithm' in outdict and not isinstance(outdict['algorithm'], tuple):
outdict['algorithm'] = tuple(outdict['algorithm'].split(','))
return outdict
@ -200,17 +206,15 @@ def get_extra_params(plugins):
'''Get a list of possible parameters given a list of plugins'''
params = {}
extra_params = {}
for i, plugin in enumerate(plugins):
for plugin in plugins:
this_params = plugin.get('extra_params', {})
for k, v in this_params.items():
if k not in extra_params:
extra_params[k] = []
extra_params[k].append(v)
params['{}.{}'.format(plugin.name, k)] = v
params['{}.{}'.format(i, k)] = v
extra_params[k] = {}
extra_params[k][plugin.name] = v
for k, v in extra_params.items(): # Resolve conflicts
if len(v) == 1: # Add the extra options that do not collide
params[k] = v[0]
params[k] = list(v.values())[0]
else:
required = False
aliases = None
@ -218,7 +222,8 @@ def get_extra_params(plugins):
default = None
nodefault = False # Set when defaults are not compatible
for opt in v:
for plugin, opt in v.items():
params['{}.{}'.format(plugin, k)] = opt
required = required or opt.get('required', False)
newaliases = set(opt.get('aliases', []))
if aliases is None:
@ -247,17 +252,20 @@ def get_extra_params(plugins):
return params
def parse_extra_params(params, plugins):
def parse_analysis(params, plugins):
'''
Parse the given parameters individually for each plugin, and get a list of the parameters that
belong to each of the plugins. Each item can then be used in the plugin.analyse_entries method.
'''
extra_params = []
analysis_list = []
for i, plugin in enumerate(plugins):
if not plugin:
continue
this_params = filter_params(params, plugin, i)
parsed = parse_params(this_params, plugin.get('extra_params', {}))
extra_params.append(parsed)
return extra_params
analysis = plugin.activity(parsed)
analysis_list.append(analysis)
return analysis_list
def filter_params(params, plugin, ith=-1):
@ -290,7 +298,8 @@ def filter_params(params, plugin, ith=-1):
def parse_call(params):
'''Return a results object based on the parameters used in a call/request.
'''
Return a results object based on the parameters used in a call/request.
'''
params = parse_params(params, NIF_PARAMS)
if params['informat'] == 'text':

22
senpy/blueprints.py
View File

@ -189,21 +189,27 @@ def basic_api(f):
@basic_api
def api_root(plugin):
if plugin:
if 'algorithm' in request.parameters:
if request.parameters['algorithm'] != api.API_PARAMS['algorithm']['default']:
raise Error('You cannot specify the algorithm with a parameter and a URL variable.'
' Please, remove one of them')
plugin = plugin.replace('+', '/')
request.parameters['algorithm'] = tuple(plugin.split('/'))
request.parameters['algorithm'] = tuple(plugin.replace('+', '/').split('/'))
params = request.parameters
plugin = request.parameters['algorithm']
sp = current_app.senpy
plugins = sp.get_plugins(plugin)
if request.parameters['help']:
sp = current_app.senpy
plugins = sp._get_plugins(request)
allparameters = api.get_all_params(plugins, api.WEB_PARAMS, api.API_PARAMS, api.NIF_PARAMS)
apis = []
if request.parameters['verbose']:
apis.append(api.BUILTIN_PARAMS)
allparameters = api.get_all_params(plugins, *apis)
response = Help(valid_parameters=allparameters)
return response
req = api.parse_call(request.parameters)
results = current_app.senpy.analyse(req)
results.analysis = set(i.id for i in results.analysis)
analysis = api.parse_analysis(req.parameters, plugins)
results = current_app.senpy.analyse(req, analysis)
return results

4
senpy/cli.py
View File

@ -31,10 +31,10 @@ def main_function(argv):
default_plugins = params.get('default-plugins', False)
sp = Senpy(default_plugins=default_plugins, plugin_folder=plugin_folder)
request = api.parse_call(params)
algos = request.parameters.get('algorithm', None)
algos = sp.get_plugins(request.parameters.get('algorithm', None))
if algos:
for algo in algos:
sp.activate_plugin(algo)
sp.activate_plugin(algo.name)
else:
sp.activate_all()
res = sp.analyse(request)

125
senpy/extensions.py
View File

@ -78,27 +78,47 @@ class Senpy(object):
def delete_plugin(self, plugin):
del self._plugins[plugin.name.lower()]
def plugins(self, **kwargs):
def plugins(self, plugin_type=None, is_activated=True, **kwargs):
""" Return the plugins registered for a given application. Filtered by criteria """
return list(plugins.pfilter(self._plugins, **kwargs))
return list(plugins.pfilter(self._plugins, plugin_type=plugin_type,
is_activated=is_activated, **kwargs))
def get_plugin(self, name, default=None):
if name == 'default':
return self.default_plugin
plugin = name.lower()
if plugin in self._plugins:
return self._plugins[plugin]
elif name == 'conversion':
return None
results = self.plugins(id='endpoint:plugins/{}'.format(name))
if name.lower() in self._plugins:
return self._plugins[name.lower()]
if not results:
return Error(message="Plugin not found", status=404)
return results[0]
results = self.plugins(id='endpoint:plugins/{}'.format(name.lower()),
plugin_type=None)
if results:
return results[0]
results = self.plugins(id=name,
plugin_type=None)
if results:
return results[0]
msg = ("Plugin not found: '{}'\n"
"Make sure it is ACTIVATED\n"
"Valid algorithms: {}").format(name,
self._plugins.keys())
raise Error(message=msg, status=404)
def get_plugins(self, name):
try:
name = name.split(',')
except AttributeError:
pass # Assume it is a tuple or a list
return tuple(self.get_plugin(n) for n in name)
@property
def analysis_plugins(self):
""" Return only the analysis plugins """
return self.plugins(plugin_type='analysisPlugin')
""" Return only the analysis plugins that are active"""
return self.plugins(plugin_type='analysisPlugin', is_activated=True)
def add_folder(self, folder, from_root=False):
""" Find plugins in this folder and add them to this instance """
@ -113,38 +133,24 @@ class Senpy(object):
else:
raise AttributeError("Not a folder or does not exist: %s", folder)
def _get_plugins(self, request):
'''Get a list of plugins that should be run for a specific request'''
if not self.analysis_plugins:
raise Error(
status=404,
message=("No plugins found."
" Please install one."))
algos = request.parameters.get('algorithm', None)
if not algos:
if self.default_plugin:
algos = [self.default_plugin.name, ]
else:
raise Error(
status=404,
message="No default plugin found, and None provided")
# def check_analysis_request(self, analysis):
# '''Check if the analysis request can be fulfilled'''
# if not self.plugins():
# raise Error(
# status=404,
# message=("No plugins found."
# " Please install one."))
# for a in analysis:
# algo = a.algorithm
# if algo == 'default' and not self.default_plugin:
# raise Error(
# status=404,
# message="No default plugin found, and None provided")
# else:
# self.get_plugin(algo)
plugins = list()
for algo in algos:
algo = algo.lower()
if algo == 'conversion':
continue # Allow 'conversion' as a virtual plugin, which does nothing
if algo not in self._plugins:
msg = ("The algorithm '{}' is not valid\n"
"Valid algorithms: {}").format(algo,
self._plugins.keys())
logger.debug(msg)
raise Error(status=404, message=msg)
plugins.append(self._plugins[algo])
return plugins
def _process(self, req, parameters, pending, done=None):
def _process(self, req, pending, done=None):
"""
Recursively process the entries with the first plugin in the list, and pass the results
to the rest of the plugins.
@ -153,27 +159,32 @@ class Senpy(object):
if not pending:
return req
plugin = pending[0]
req.parameters = parameters[0]
results = plugin.process(req, conversions_applied=done)
if plugin not in results.analysis:
results.analysis.append(plugin)
return self._process(results, parameters[1:], pending[1:], done)
analysis = pending[0]
results = analysis.run(req)
results.analysis.append(analysis)
done += analysis
return self._process(results, pending[1:], done)
def install_deps(self):
plugins.install_deps(*self.plugins())
def analyse(self, request):
def analyse(self, request, analysis=None):
"""
Main method that analyses a request, either from CLI or HTTP.
It takes a processed request, provided by the user, as returned
by api.parse_call().
"""
if not self.plugins():
raise Error(
status=404,
message=("No plugins found."
" Please install one."))
if analysis is None:
params = str(request)
plugins = self.get_plugins(request.parameters['algorithm'])
analysis = api.parse_analysis(request.parameters, plugins)
logger.debug("analysing request: {}".format(request))
plugins = self._get_plugins(request)
parameters = api.parse_extra_params(request.parameters, plugins)
results = self._process(request, parameters, plugins)
results = self._process(request, analysis)
logger.debug("Got analysis result: {}".format(results))
results = self.postprocess(results)
logger.debug("Returning post-processed result: {}".format(results))
@ -189,7 +200,10 @@ class Senpy(object):
"""
plugins = resp.analysis
params = resp.parameters
if 'parameters' not in resp:
return resp
params = resp['parameters']
toModel = params.get('emotionModel', None)
if not toModel:
return resp
@ -290,7 +304,10 @@ class Senpy(object):
results = AggregatedEvaluation()
results.parameters = params
datasets = self._get_datasets(results)
plugins = self._get_plugins(results)
plugins = []
for plugname in params.algorithm:
plugins = self.get_plugin(plugname)
for eval in plugins.evaluate(plugins, datasets):
results.evaluations.append(eval)
if 'with_parameters' not in results.parameters:

7
senpy/meta.py
View File

@ -85,7 +85,8 @@ class BaseMeta(ABCMeta):
schema = json.load(f)
resolver = jsonschema.RefResolver(schema_path, schema)
attrs['@type'] = "".join((name[0].lower(), name[1:]))
if '@type' not in attrs:
attrs['@type'] = "".join((name[0].lower(), name[1:]))
attrs['_schema_file'] = schema_file
attrs['schema'] = schema
attrs['_validator'] = jsonschema.Draft4Validator(schema, resolver=resolver)
@ -244,10 +245,10 @@ class CustomDict(MutableMapping, object):
return key[0] == '_'
def __str__(self):
return str(self.serializable())
return json.dumps(self.serializable(), sort_keys=True, indent=4)
def __repr__(self):
return str(self.serializable())
return json.dumps(self.serializable(), sort_keys=True, indent=4)
_Alias = namedtuple('Alias', 'indict')

61
senpy/models.py
View File

@ -121,11 +121,11 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)):
'''
schema_file = DEFINITIONS_FILE
# schema_file = DEFINITIONS_FILE
_context = base_context["@context"]
def __init__(self, *args, **kwargs):
auto_id = kwargs.pop('_auto_id', True)
auto_id = kwargs.pop('_auto_id', False)
super(BaseModel, self).__init__(*args, **kwargs)
@ -133,7 +133,7 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)):
self.id
if '@type' not in self:
logger.warn('Created an instance of an unknown model')
logger.warning('Created an instance of an unknown model')
@property
def id(self):
@ -325,7 +325,6 @@ def _add_class_from_schema(*args, **kwargs):
for i in [
'aggregatedEvaluation',
'analysis',
'dataset',
'datasets',
'emotion',
@ -339,7 +338,7 @@ for i in [
'entity',
'help',
'metric',
'plugin',
'parameter',
'plugins',
'response',
'results',
@ -349,3 +348,55 @@ for i in [
]:
_add_class_from_schema(i)
class Analysis(BaseModel):
schema = 'analysis'
parameters = alias('prov:used')
@property
def params(self):
outdict = {}
outdict['algorithm'] = self.algorithm
for param in self.parameters:
outdict[param['name']] = param['value']
return outdict
@params.setter
def params(self, value):
for k, v in value.items():
for param in self.parameters:
if param.name == k:
param.value = v
break
else:
self.parameters.append(Parameter(name=k, value=v))
@property
def algorithm(self):
return self['prov:wasAssociatedWith']
@property
def plugin(self):
return self._plugin
@plugin.setter
def plugin(self, value):
self._plugin = value
self['prov:wasAssociatedWith'] = value.id
def run(self, request):
return self.plugin.process(request, self.params)
class Plugin(BaseModel):
schema = 'plugin'
def activity(self, parameters):
'''Generate a prov:Activity from this plugin and the '''
a = Analysis()
a.plugin = self
a.params = parameters
return a

18
senpy/plugins/__init__.py
View File

@ -132,12 +132,12 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)):
def deactivate(self):
pass
def process(self, request, **kwargs):
def process(self, request, parameters, **kwargs):
"""
An implemented plugin should override this method.
Here, we assume that a process_entries method exists."""
newentries = list(
self.process_entries(request.entries, request.parameters))
self.process_entries(request.entries, parameters))
request.entries = newentries
return request
@ -194,13 +194,13 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)):
try:
request = models.Response()
request.parameters = api.parse_params(given_parameters,
parameters = api.parse_params(given_parameters,
self.extra_params)
request.entries = [
entry,
]
method = partial(self.process, request)
method = partial(self.process, request, parameters)
if mock:
res = method()
@ -249,14 +249,14 @@ class Analysis(Plugin):
'''
def analyse(self, request, parameters):
return super(Analysis, self).process(request)
return super(Analysis, self).process(request, parameters)
def analyse_entries(self, entries, parameters):
for i in super(Analysis, self).process_entries(entries, parameters):
yield i
def process(self, request, **kwargs):
return self.analyse(request, request.parameters)
def process(self, request, parameters, **kwargs):
return self.analyse(request, parameters)
def process_entries(self, entries, parameters):
for i in self.analyse_entries(entries, parameters):
@ -279,12 +279,12 @@ class Conversion(Plugin):
e.g. a conversion of emotion models, or normalization of sentiment values.
'''
def process(self, response, plugins=None, **kwargs):
def process(self, response, parameters, plugins=None, **kwargs):
plugins = plugins or []
newentries = []
for entry in response.entries:
newentries.append(
self.convert_entry(entry, response.parameters, plugins))
self.convert_entry(entry, parameters, plugins))
response.entries = newentries
return response

15
senpy/schemas/analysis.json
View File

@ -9,7 +9,20 @@
"@type": {
"type": "string",
"description": "Type of the analysis. e.g. marl:SentimentAnalysis"
},
"prov:wasAssociatedWith": {
"@type": "string",
"description": "Algorithm/plugin that was used"
},
"prov:used": {
"description": "Parameters of the algorithm",
"@type": "array",
"default": [],
"type": "array",
"items": {
"$ref": "parameter.json"
}
}
},
"required": ["@id", "@type"]
"required": ["@type", "prov:wasAssociatedWith"]
}

2
senpy/schemas/context.jsonld
View File

@ -41,7 +41,7 @@
"@container": "@set"
},
"analysis": {
"@id": "AnalysisInvolved",
"@id": "prov:wasInformedBy",
"@type": "@id",
"@container": "@set"
},

2
senpy/schemas/emotionSet.json
View File

@ -20,5 +20,5 @@
"description": "The ID of the analysis that generated this Emotion. The full object should be included in the \"analysis\" property of the root object"
}
},
"required": ["@id", "prov:wasGeneratedBy", "onyx:hasEmotion"]
"required": ["prov:wasGeneratedBy", "onyx:hasEmotion"]
}

2
senpy/schemas/entry.json
View File

@ -35,5 +35,5 @@
"default": []
}
},
"required": ["@id", "nif:isString"]
"required": ["nif:isString"]
}

16
senpy/schemas/parameter.json Normal file
View File

@ -0,0 +1,16 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Parameters for a senpy analysis",
"type": "object",
"properties": {
"name": {
"type": "string",
"description": "Name of the parameter"
},
"prov:value": {
"@type": "any",
"description": "Value of the parameter"
}
},
"required": ["name", "prov:value"]
}

8
senpy/schemas/results.json
View File

@ -21,13 +21,7 @@
"default": [],
"type": "array",
"items": {
"anyOf": [
{
"$ref": "analysis.json"
},{
"type": "string"
}
]
"$ref": "analysis.json"
}
},
"entries": {

2
senpy/schemas/sentiment.json
View File

@ -19,5 +19,5 @@
"description": "The ID of the analysis that generated this Sentiment. The full object should be included in the \"analysis\" property of the root object"
}
},
"required": ["@id", "prov:wasGeneratedBy"]
"required": ["prov:wasGeneratedBy"]
}

6
setup.cfg
View File

@ -12,6 +12,8 @@ max-line-length = 100
universal=1
[tool:pytest]
addopts = --cov=senpy --cov-report term-missing
filterwarnings =
error
ignore:the matrix subclass:PendingDeprecationWarning
[coverage:report]
omit = senpy/__main__.py
omit = senpy/__main__.py

21
tests/test_api.py
View File

@ -3,7 +3,7 @@ import logging
logger = logging.getLogger(__name__)
from unittest import TestCase
from senpy.api import (boolean, parse_params, get_extra_params, parse_extra_params,
from senpy.api import (boolean, parse_params, get_extra_params, parse_analysis,
API_PARAMS, NIF_PARAMS, WEB_PARAMS)
from senpy.models import Error, Plugin
@ -91,7 +91,7 @@ class APITest(TestCase):
assert 'input' in p
assert p['input'] == 'Aloha my friend'
def test_parse_extra_params(self):
def test_parse_analysis(self):
'''The API should parse user parameters and return them in a format that plugins can use'''
plugins = [
Plugin({
@ -161,10 +161,11 @@ class APITest(TestCase):
}
]
p = parse_extra_params(call, plugins)
p = parse_analysis(call, plugins)
for i, arg in enumerate(expected):
params = p[i].params
for k, v in arg.items():
assert p[i][k] == v
assert params[k] == v
def test_get_extra_params(self):
'''The API should return the list of valid parameters for a set of plugins'''
@ -216,13 +217,11 @@ class APITest(TestCase):
]
expected = {
# Each plugin's parameters
'0.param0': plugins[0]['extra_params']['param0'],
'0.param1': plugins[0]['extra_params']['param1'],
'0.param2': plugins[0]['extra_params']['param2'],
'1.param0': plugins[1]['extra_params']['param0'],
'1.param1': plugins[1]['extra_params']['param1'],
'1.param3': plugins[1]['extra_params']['param3'],
# Overlapping parameters
'plugin1.param0': plugins[0]['extra_params']['param0'],
'plugin1.param1': plugins[0]['extra_params']['param1'],
'plugin2.param0': plugins[1]['extra_params']['param0'],
'plugin2.param1': plugins[1]['extra_params']['param1'],
# Non-overlapping parameters
'param2': plugins[0]['extra_params']['param2'],

27
tests/test_blueprints.py
View File

@ -26,8 +26,7 @@ class BlueprintsTest(TestCase):
cls.senpy.init_app(cls.app)
cls.dir = os.path.join(os.path.dirname(__file__), "..")
cls.senpy.add_folder(cls.dir)
cls.senpy.activate_plugin("Dummy", sync=True)
cls.senpy.activate_plugin("DummyRequired", sync=True)
cls.senpy.activate_all()
cls.senpy.default_plugin = 'Dummy'
def setUp(self):
@ -139,16 +138,27 @@ class BlueprintsTest(TestCase):
# Calling dummy twice, should return the same string
self.assertCode(resp, 200)
js = parse_resp(resp)
assert len(js['analysis']) == 1
assert len(js['analysis']) == 2
assert js['entries'][0]['nif:isString'] == 'My aloha mohame'
resp = self.client.get("/api/Dummy+Dummy?i=My aloha mohame")
# Same with pluses instead of slashes
self.assertCode(resp, 200)
js = parse_resp(resp)
assert len(js['analysis']) == 1
assert len(js['analysis']) == 2
assert js['entries'][0]['nif:isString'] == 'My aloha mohame'
def test_analysis_chain_required(self):
"""
If a parameter is required and duplicated (because two plugins require it), specifying
it once should suffice
"""
resp = self.client.get("/api/DummyRequired/DummyRequired?i=My aloha mohame&example=a")
js = parse_resp(resp)
assert len(js['analysis']) == 2
assert js['entries'][0]['nif:isString'] == 'My aloha mohame'
assert js['entries'][0]['reversed'] == 2
def test_requirements_chain_help(self):
'''The extra parameters of each plugin should be merged if they are in a chain '''
resp = self.client.get("/api/split/DummyRequired?help=true")
@ -157,6 +167,7 @@ class BlueprintsTest(TestCase):
assert 'valid_parameters' in js
vp = js['valid_parameters']
assert 'example' in vp
assert 'delimiter' in vp
def test_requirements_chain_repeat_help(self):
'''
@ -168,10 +179,14 @@ class BlueprintsTest(TestCase):
js = parse_resp(resp)
assert 'valid_parameters' in js
vp = js['valid_parameters']
assert '0.delimiter' in vp
assert '1.delimiter' in vp
assert 'delimiter' in vp
resp = self.client.get("/api/split/split?help=true&verbose=false")
js = parse_resp(resp)
vp = js['valid_parameters']
assert len(vp.keys()) == 1
def test_requirements_chain(self):
"""
It should be possible to specify different parameters for each step in the chain.

46
tests/test_extensions.py
View File

@ -11,14 +11,15 @@ except ImportError:
from functools import partial
from senpy.extensions import Senpy
from senpy import plugins
from senpy.models import Error, Results, Entry, EmotionSet, Emotion, Plugin
from senpy.models import Analysis, Error, Results, Entry, EmotionSet, Emotion, Plugin
from senpy import api
from flask import Flask
from unittest import TestCase
def analyse(instance, **kwargs):
request = api.parse_call(kwargs)
basic = api.parse_params(kwargs, api.API_PARAMS)
request = api.parse_call(basic)
return instance.analyse(request)
@ -49,9 +50,9 @@ class ExtensionsTest(TestCase):
'''Should be able to add and delete new plugins. '''
new = plugins.Analysis(name='new', description='new', version=0)
self.senpy.add_plugin(new)
assert new in self.senpy.plugins()
assert new in self.senpy.plugins(is_activated=False)
self.senpy.delete_plugin(new)
assert new not in self.senpy.plugins()
assert new not in self.senpy.plugins(is_activated=False)
def test_adding_folder(self):
""" It should be possible for senpy to look for plugins in more folders. """
@ -60,7 +61,7 @@ class ExtensionsTest(TestCase):
default_plugins=False)
assert not senpy.analysis_plugins
senpy.add_folder(self.examples_dir)
assert senpy.analysis_plugins
assert senpy.plugins(plugin_type=plugins.AnalysisPlugin, is_activated=False)
self.assertRaises(AttributeError, senpy.add_folder, 'DOES NOT EXIST')
def test_installing(self):
@ -121,8 +122,8 @@ class ExtensionsTest(TestCase):
# Leaf (defaultdict with __setattr__ and __getattr__.
r1 = analyse(self.senpy, algorithm="Dummy", input="tupni", output="tuptuo")
r2 = analyse(self.senpy, input="tupni", output="tuptuo")
assert r1.analysis[0].id == "endpoint:plugins/Dummy_0.1"
assert r2.analysis[0].id == "endpoint:plugins/Dummy_0.1"
assert r1.analysis[0].algorithm == "endpoint:plugins/Dummy_0.1"
assert r2.analysis[0].algorithm == "endpoint:plugins/Dummy_0.1"
assert r1.entries[0]['nif:isString'] == 'input'
def test_analyse_empty(self):
@ -130,7 +131,7 @@ class ExtensionsTest(TestCase):
senpy = Senpy(plugin_folder=None,
app=self.app,
default_plugins=False)
self.assertRaises(Error, senpy.analyse, Results())
self.assertRaises(Error, senpy.analyse, Results(), [])
def test_analyse_wrong(self):
""" Trying to analyse with a non-existent plugin should raise an error."""
@ -156,29 +157,32 @@ class ExtensionsTest(TestCase):
r2 = analyse(self.senpy,
input="tupni",
output="tuptuo")
assert r1.analysis[0].id == "endpoint:plugins/Dummy_0.1"
assert r2.analysis[0].id == "endpoint:plugins/Dummy_0.1"
assert r1.analysis[0].algorithm == "endpoint:plugins/Dummy_0.1"
assert r2.analysis[0].algorithm == "endpoint:plugins/Dummy_0.1"
assert r1.entries[0]['nif:isString'] == 'input'
def test_analyse_error(self):
mm = mock.MagicMock()
mm.id = 'magic_mock'
mm.name = 'mock'
mm.is_activated = True
mm.process.side_effect = Error('error in analysis', status=500)
self.senpy.add_plugin(mm)
class ErrorPlugin(plugins.Analysis):
author = 'nobody'
version = 0
ex = Error()
def process(self, *args, **kwargs):
raise self.ex
m = ErrorPlugin(ex=Error('error in analysis', status=500))
self.senpy.add_plugin(m)
try:
analyse(self.senpy, input='nothing', algorithm='MOCK')
analyse(self.senpy, input='nothing', algorithm='ErrorPlugin')
assert False
except Error as ex:
assert 'error in analysis' in ex['message']
assert ex['status'] == 500
ex = Exception('generic exception on analysis')
mm.process.side_effect = ex
m.ex = Exception('generic exception on analysis')
try:
analyse(self.senpy, input='nothing', algorithm='MOCK')
analyse(self.senpy, input='nothing', algorithm='ErrorPlugin')
assert False
except Exception as ex:
assert 'generic exception on analysis' in str(ex)
@ -194,7 +198,7 @@ class ExtensionsTest(TestCase):
def test_load_default_plugins(self):
senpy = Senpy(plugin_folder=self.examples_dir, default_plugins=True)
assert len(senpy.plugins()) > 1
assert len(senpy.plugins(is_activated=False)) > 1
def test_convert_emotions(self):
self.senpy.activate_all(sync=True)

18
tests/test_models.py
View File

@ -5,7 +5,8 @@ import jsonschema
import json
import rdflib
from unittest import TestCase
from senpy.models import (Emotion,
from senpy.models import (Analysis,
Emotion,
EmotionAnalysis,
EmotionSet,
Entry,
@ -61,7 +62,7 @@ class ModelsTest(TestCase):
def test_id(self):
""" Adding the id after creation should overwrite the automatic ID
"""
r = Entry()
r = Entry(_auto_id=True)
j = r.jsonld()
assert '@id' in j
r.id = "test"
@ -189,6 +190,19 @@ class ModelsTest(TestCase):
assert isinstance(js['plugins'], list)
assert js['plugins'][0]['@type'] == 'sentimentPlugin'
def test_parameters(self):
'''An Analysis should contain the algorithm and the list of parameters to be used'''
a = Analysis()
a.params = {'param1': 1, 'param2': 2}
assert len(a.parameters) == 2
for param in a.parameters:
if param.name == 'param1':
assert param.value == 1
elif param.name == 'param2':
assert param.value == 2
else:
raise Exception('Unknown value %s' % param)
def test_from_string(self):
results = {
'@type': 'results',