diff --git a/docs/bad-examples/results/example-basic-FAIL.json b/docs/bad-examples/results/example-basic-FAIL.json index 288843c..606a0df 100644 --- a/docs/bad-examples/results/example-basic-FAIL.json +++ b/docs/bad-examples/results/example-basic-FAIL.json @@ -6,13 +6,9 @@ ], "entries": [ { - "@type": [ - "nif:RFC5147String", - "nif:Context" - ], "nif:beginIndex": 0, "nif:endIndex": 40, - "nif:isString": "My favourite actress is Natalie Portman" + "text": "An entry should have a nif:isString key" } ] } diff --git a/docs/examples/results/example-analysis-as-id.json b/docs/examples/results/example-analysis-as-id.json index 95ee7de..4b1dfb0 100644 --- a/docs/examples/results/example-analysis-as-id.json +++ b/docs/examples/results/example-analysis-as-id.json @@ -3,10 +3,21 @@ "@id": "me:Result1", "@type": "results", "analysis": [ - "me:SAnalysis1", - "me:SgAnalysis1", - "me:EmotionAnalysis1", - "me:NER1" + { + "@id": "_:SAnalysis1_Activity", + "@type": "marl:SentimentAnalysis", + "prov:wasAssociatedWith": "me:SAnalysis1" + }, + { + "@id": "_:EmotionAnalysis1_Activity", + "@type": "onyx:EmotionAnalysis", + "prov:wasAssociatedWith": "me:EmotionAnalysis1" + }, + { + "@id": "_:NER1_Activity", + "@type": "me:NER", + "prov:wasAssociatedWith": "me:NER1" + } ], "entries": [ { @@ -23,7 +34,7 @@ "nif:endIndex": 13, "nif:anchorOf": "Microsoft", "me:references": "http://dbpedia.org/page/Microsoft", - "prov:wasGeneratedBy": "me:NER1" + "prov:wasGeneratedBy": "_:NER1_Activity" }, { "@id": "http://micro.blog/status1#char=25,37", @@ -31,7 +42,7 @@ "nif:endIndex": 37, "nif:anchorOf": "Windows Phone", "me:references": "http://dbpedia.org/page/Windows_Phone", - "prov:wasGeneratedBy": "me:NER1" + "prov:wasGeneratedBy": "_:NER1_Activity" } ], "suggestions": [ @@ -40,7 +51,7 @@ "nif:beginIndex": 16, "nif:endIndex": 77, "nif:anchorOf": "put your Windows Phone on your newest #open technology program", - "prov:wasGeneratedBy": "me:SgAnalysis1" + "prov:wasGeneratedBy": "_:SgAnalysis1_Activity" } ], "sentiments": [ @@ -51,14 +62,14 @@ "nif:anchorOf": "You'll be awesome.", "marl:hasPolarity": "marl:Positive", "marl:polarityValue": 0.9, - "prov:wasGeneratedBy": "me:SAnalysis1" + "prov:wasGeneratedBy": "_:SgAnalysis1_Activity" } ], "emotions": [ { "@id": "http://micro.blog/status1#char=0,109", "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource", - "prov:wasGeneratedBy": "me:EAnalysis1", + "prov:wasGeneratedBy": "_:EmotionAnalysis1_Activity", "onyx:hasEmotion": [ { "onyx:hasEmotionCategory": "wna:liking" diff --git a/docs/examples/results/example-analysis-mixed-ids.json b/docs/examples/results/example-analysis-mixed-ids.json deleted file mode 100644 index 46c1465..0000000 --- a/docs/examples/results/example-analysis-mixed-ids.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "@context": "http://mixedemotions-project.eu/ns/context.jsonld", - "@id": "me:Result1", - "@type": "results", - "analysis": [ - "me:SAnalysis1", - "me:SgAnalysis1", - "me:EmotionAnalysis1", - "me:NER1", - { - "@type": "analysis", - "@id": "anonymous" - } - ], - "entries": [ - { - "@id": "http://micro.blog/status1", - "@type": [ - "nif:RFC5147String", - "nif:Context" - ], - "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource", - "entities": [ - { - "@id": "http://micro.blog/status1#char=5,13", - "nif:beginIndex": 5, - "nif:endIndex": 13, - "nif:anchorOf": "Microsoft", - "me:references": "http://dbpedia.org/page/Microsoft", - "prov:wasGeneratedBy": "me:NER1" - }, - { - "@id": "http://micro.blog/status1#char=25,37", - "nif:beginIndex": 25, - "nif:endIndex": 37, - "nif:anchorOf": "Windows Phone", - "me:references": "http://dbpedia.org/page/Windows_Phone", - "prov:wasGeneratedBy": "me:NER1" - } - ], - "suggestions": [ - { - "@id": "http://micro.blog/status1#char=16,77", - "nif:beginIndex": 16, - "nif:endIndex": 77, - "nif:anchorOf": "put your Windows Phone on your newest #open technology program", - "prov:wasGeneratedBy": "me:SgAnalysis1" - } - ], - "sentiments": [ - { - "@id": "http://micro.blog/status1#char=80,97", - "nif:beginIndex": 80, - "nif:endIndex": 97, - "nif:anchorOf": "You'll be awesome.", - "marl:hasPolarity": "marl:Positive", - "marl:polarityValue": 0.9, - "prov:wasGeneratedBy": "me:SAnalysis1" - } - ], - "emotions": [ - { - "@id": "http://micro.blog/status1#char=0,109", - "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource", - "prov:wasGeneratedBy": "me:EAnalysis1", - "onyx:hasEmotion": [ - { - "onyx:hasEmotionCategory": "wna:liking" - }, - { - "onyx:hasEmotionCategory": "wna:excitement" - } - ] - } - ] - } - ] -} diff --git a/docs/examples/results/example-basic.json b/docs/examples/results/example-basic.json index c308f0f..e78c2e5 100644 --- a/docs/examples/results/example-basic.json +++ b/docs/examples/results/example-basic.json @@ -1,19 +1,18 @@ { - "@context": "http://mixedemotions-project.eu/ns/context.jsonld", - "@id": "http://example.com#NIFExample", - "@type": "results", - "analysis": [ - ], - "entries": [ - { - "@id": "http://example.org#char=0,40", - "@type": [ - "nif:RFC5147String", - "nif:Context" - ], - "nif:beginIndex": 0, - "nif:endIndex": 40, - "nif:isString": "My favourite actress is Natalie Portman" - } - ] + "@context": "http://mixedemotions-project.eu/ns/context.jsonld", + "@id": "me:Result1", + "@type": "results", + "analysis": [ ], + "entries": [ + { + "@id": "http://example.org#char=0,40", + "@type": [ + "nif:RFC5147String", + "nif:Context" + ], + "nif:beginIndex": 0, + "nif:endIndex": 40, + "nif:isString": "My favourite actress is Natalie Portman" + } + ] } diff --git a/docs/examples/results/example-complete.json b/docs/examples/results/example-complete.json index d735e6e..dcdebd0 100644 --- a/docs/examples/results/example-complete.json +++ b/docs/examples/results/example-complete.json @@ -1,88 +1,100 @@ { - "@context": "http://mixedemotions-project.eu/ns/context.jsonld", - "@id": "me:Result1", - "@type": "results", - "analysis": [ - { - "@id": "me:SAnalysis1", - "@type": "marl:SentimentAnalysis", - "marl:maxPolarityValue": 1, - "marl:minPolarityValue": 0 - }, - { - "@id": "me:SgAnalysis1", - "@type": "me:SuggestionAnalysis" - }, - { - "@id": "me:EmotionAnalysis1", - "@type": "me:EmotionAnalysis" - }, - { - "@id": "me:NER1", - "@type": "me:NER" - } - ], - "entries": [ - { - "@id": "http://micro.blog/status1", - "@type": [ - "nif:RFC5147String", - "nif:Context" - ], - "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource", - "entities": [ + "@context": "http://mixedemotions-project.eu/ns/context.jsonld", + "@id": "me:Result1", + "@type": "results", + "analysis": [ { - "@id": "http://micro.blog/status1#char=5,13", - "nif:beginIndex": 5, - "nif:endIndex": 13, - "nif:anchorOf": "Microsoft", - "me:references": "http://dbpedia.org/page/Microsoft", - "prov:wasGeneratedBy": "me:NER1" + "@id": "_:SAnalysis1_Activity", + "@type": "marl:SentimentAnalysis", + "prov:wasAssociatedWith": "me:SentimentAnalysis", + "prov:used": [ + { + "name": "marl:maxPolarityValue", + "prov:value": "1" + }, + { + "name": "marl:minPolarityValue", + "prov:value": "0" + } + ] }, { - "@id": "http://micro.blog/status1#char=25,37", - "nif:beginIndex": 25, - "nif:endIndex": 37, - "nif:anchorOf": "Windows Phone", - "me:references": "http://dbpedia.org/page/Windows_Phone", - "prov:wasGeneratedBy": "me:NER1" - } - ], - "suggestions": [ + "@id": "_:SgAnalysis1_Activity", + "prov:wasAssociatedWith": "me:SgAnalysis1", + "@type": "me:SuggestionAnalysis" + }, { - "@id": "http://micro.blog/status1#char=16,77", - "nif:beginIndex": 16, - "nif:endIndex": 77, - "nif:anchorOf": "put your Windows Phone on your newest #open technology program", - "prov:wasGeneratedBy": "me:SgAnalysis1" - } - ], - "sentiments": [ + "@id": "_:EmotionAnalysis1_Activity", + "@type": "me:EmotionAnalysis", + "prov:wasAssociatedWith": "me:EmotionAnalysis1" + }, { - "@id": "http://micro.blog/status1#char=80,97", - "nif:beginIndex": 80, - "nif:endIndex": 97, - "nif:anchorOf": "You'll be awesome.", - "marl:hasPolarity": "marl:Positive", - "marl:polarityValue": 0.9, - "prov:wasGeneratedBy": "me:SAnalysis1" + "@id": "_:NER1_Activity", + "@type": "me:NER", + "prov:wasAssociatedWith": "me:EmotionNER1" } - ], - "emotions": [ + ], + "entries": [ { - "@id": "http://micro.blog/status1#char=0,109", - "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource", - "prov:wasGeneratedBy": "me:EAnalysis1", - "onyx:hasEmotion": [ - { - "onyx:hasEmotionCategory": "wna:liking" - }, - { - "onyx:hasEmotionCategory": "wna:excitement" - } - ] + "@id": "http://micro.blog/status1", + "@type": [ + "nif:RFC5147String", + "nif:Context" + ], + "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource", + "entities": [ + { + "@id": "http://micro.blog/status1#char=5,13", + "nif:beginIndex": 5, + "nif:endIndex": 13, + "nif:anchorOf": "Microsoft", + "me:references": "http://dbpedia.org/page/Microsoft", + "prov:wasGeneratedBy": "me:NER1" + }, + { + "@id": "http://micro.blog/status1#char=25,37", + "nif:beginIndex": 25, + "nif:endIndex": 37, + "nif:anchorOf": "Windows Phone", + "me:references": "http://dbpedia.org/page/Windows_Phone", + "prov:wasGeneratedBy": "me:NER1" + } + ], + "suggestions": [ + { + "@id": "http://micro.blog/status1#char=16,77", + "nif:beginIndex": 16, + "nif:endIndex": 77, + "nif:anchorOf": "put your Windows Phone on your newest #open technology program", + "prov:wasGeneratedBy": "me:SgAnalysis1" + } + ], + "sentiments": [ + { + "@id": "http://micro.blog/status1#char=80,97", + "nif:beginIndex": 80, + "nif:endIndex": 97, + "nif:anchorOf": "You'll be awesome.", + "marl:hasPolarity": "marl:Positive", + "marl:polarityValue": 0.9, + "prov:wasGeneratedBy": "me:SAnalysis1" + } + ], + "emotions": [ + { + "@id": "http://micro.blog/status1#char=0,109", + "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource", + "prov:wasGeneratedBy": "me:EAnalysis1", + "onyx:hasEmotion": [ + { + "onyx:hasEmotionCategory": "wna:liking" + }, + { + "onyx:hasEmotionCategory": "wna:excitement" + } + ] + } + ] } - ] - } - ] + ] } diff --git a/docs/examples/results/example-emotion.json b/docs/examples/results/example-emotion.json index 452606a..5eaa8d3 100644 --- a/docs/examples/results/example-emotion.json +++ b/docs/examples/results/example-emotion.json @@ -4,8 +4,9 @@ "@type": "results", "analysis": [ { - "@id": "me:EmotionAnalysis1", - "@type": "onyx:EmotionAnalysis" + "@id": "me:EmotionAnalysis1_Activity", + "@type": "me:EmotionAnalysis1", + "prov:wasAssociatedWith": "me:EmotionAnalysis1" } ], "entries": [ @@ -26,7 +27,7 @@ { "@id": "http://micro.blog/status1#char=0,109", "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource", - "prov:wasGeneratedBy": "me:EmotionAnalysis1", + "prov:wasGeneratedBy": "_:EmotionAnalysis1_Activity", "onyx:hasEmotion": [ { "onyx:hasEmotionCategory": "wna:liking" diff --git a/docs/examples/results/example-ner.json b/docs/examples/results/example-ner.json index d177bf8..39215ce 100644 --- a/docs/examples/results/example-ner.json +++ b/docs/examples/results/example-ner.json @@ -4,8 +4,9 @@ "@type": "results", "analysis": [ { - "@id": "me:NER1", - "@type": "me:NERAnalysis" + "@id": "_:NER1_Activity", + "@type": "me:NERAnalysis", + "prov:wasAssociatedWith": "me:NER1" } ], "entries": [ diff --git a/docs/examples/results/example-pad.json b/docs/examples/results/example-pad.json index bf90bbc..d092f55 100644 --- a/docs/examples/results/example-pad.json +++ b/docs/examples/results/example-pad.json @@ -9,9 +9,15 @@ "@type": "results", "analysis": [ { - "@id": "me:HesamsAnalysis", + "@id": "me:HesamsAnalysis_Activity", "@type": "onyx:EmotionAnalysis", - "onyx:usesEmotionModel": "emovoc:pad-dimensions" + "prov:wasAssociatedWith": "me:HesamsAnalysis", + "prov:used": [ + { + "name": "emotion-model", + "prov:value": "emovoc:pad-dimensions" + } + ] } ], "entries": [ @@ -32,7 +38,7 @@ { "@id": "Entry1#char=0,21", "nif:anchorOf": "This is a test string", - "prov:wasGeneratedBy": "me:HesamAnalysis", + "prov:wasGeneratedBy": "_:HesamAnalysis_Activity", "onyx:hasEmotion": [ { "emovoc:pleasure": 0.5, diff --git a/docs/examples/results/example-sentiment.json b/docs/examples/results/example-sentiment.json index 175cb56..318c55c 100644 --- a/docs/examples/results/example-sentiment.json +++ b/docs/examples/results/example-sentiment.json @@ -4,10 +4,9 @@ "@type": "results", "analysis": [ { - "@id": "me:SAnalysis1", + "@id": "_:SAnalysis1_Activity", "@type": "marl:SentimentAnalysis", - "marl:maxPolarityValue": 1, - "marl:minPolarityValue": 0 + "prov:wasAssociatedWith": "me:SAnalysis1" } ], "entries": [ @@ -30,7 +29,7 @@ "nif:anchorOf": "You'll be awesome.", "marl:hasPolarity": "marl:Positive", "marl:polarityValue": 0.9, - "prov:wasGeneratedBy": "me:SAnalysis1" + "prov:wasGeneratedBy": "_:SAnalysis1_Activity" } ], "emotionSets": [ diff --git a/docs/examples/results/example-suggestion.json b/docs/examples/results/example-suggestion.json index e209f03..2d6903e 100644 --- a/docs/examples/results/example-suggestion.json +++ b/docs/examples/results/example-suggestion.json @@ -3,7 +3,11 @@ "@id": "me:Result1", "@type": "results", "analysis": [ - "me:SgAnalysis1" + { + "@id": "_:SgAnalysis1_Activity", + "@type": "me:SuggestionAnalysis", + "prov:wasAssociatedWith": "me:SgAnalysis1" + } ], "entries": [ { @@ -12,7 +16,6 @@ "nif:RFC5147String", "nif:Context" ], - "prov:wasGeneratedBy": "me:SAnalysis1", "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource", "entities": [ ], @@ -22,7 +25,7 @@ "nif:beginIndex": 16, "nif:endIndex": 77, "nif:anchorOf": "put your Windows Phone on your newest #open technology program", - "prov:wasGeneratedBy": "me:SgAnalysis1" + "prov:wasGeneratedBy": "_:SgAnalysis1_Activity" } ], "sentiments": [ diff --git a/senpy/api.py b/senpy/api.py index ee35dad..68d51bc 100644 --- a/senpy/api.py +++ b/senpy/api.py @@ -1,5 +1,5 @@ from future.utils import iteritems -from .models import Error, Results, Entry, from_string +from .models import Analysis, Error, Results, Entry, from_string import logging logger = logging.getLogger(__name__) @@ -8,7 +8,8 @@ boolean = [True, False] API_PARAMS = { "algorithm": { "aliases": ["algorithms", "a", "algo"], - "required": False, + "required": True, + "default": 'default', "description": ("Algorithms that will be used to process the request." "It may be a list of comma-separated names."), }, @@ -41,6 +42,14 @@ API_PARAMS = { "options": boolean, "default": False }, + "verbose": { + "@id": "verbose", + "description": "Show all help, including the common API parameters, or only plugin-related info", + "aliases": ["v"], + "required": True, + "options": boolean, + "default": True + }, "emotionModel": { "@id": "emotionModel", "aliases": ["emoModel"], @@ -168,8 +177,7 @@ def parse_params(indict, *specs): outdict[param] = options["default"] elif options.get("required", False): wrong_params[param] = spec[param] - continue - if "options" in options: + elif "options" in options: if options["options"] == boolean: outdict[param] = str(outdict[param]).lower() in ['true', '1'] elif outdict[param] not in options["options"]: @@ -182,8 +190,6 @@ def parse_params(indict, *specs): parameters=outdict, errors=wrong_params) raise message - if 'algorithm' in outdict and not isinstance(outdict['algorithm'], tuple): - outdict['algorithm'] = tuple(outdict['algorithm'].split(',')) return outdict @@ -200,17 +206,15 @@ def get_extra_params(plugins): '''Get a list of possible parameters given a list of plugins''' params = {} extra_params = {} - for i, plugin in enumerate(plugins): + for plugin in plugins: this_params = plugin.get('extra_params', {}) for k, v in this_params.items(): if k not in extra_params: - extra_params[k] = [] - extra_params[k].append(v) - params['{}.{}'.format(plugin.name, k)] = v - params['{}.{}'.format(i, k)] = v + extra_params[k] = {} + extra_params[k][plugin.name] = v for k, v in extra_params.items(): # Resolve conflicts if len(v) == 1: # Add the extra options that do not collide - params[k] = v[0] + params[k] = list(v.values())[0] else: required = False aliases = None @@ -218,7 +222,8 @@ def get_extra_params(plugins): default = None nodefault = False # Set when defaults are not compatible - for opt in v: + for plugin, opt in v.items(): + params['{}.{}'.format(plugin, k)] = opt required = required or opt.get('required', False) newaliases = set(opt.get('aliases', [])) if aliases is None: @@ -247,17 +252,20 @@ def get_extra_params(plugins): return params -def parse_extra_params(params, plugins): +def parse_analysis(params, plugins): ''' Parse the given parameters individually for each plugin, and get a list of the parameters that belong to each of the plugins. Each item can then be used in the plugin.analyse_entries method. ''' - extra_params = [] + analysis_list = [] for i, plugin in enumerate(plugins): + if not plugin: + continue this_params = filter_params(params, plugin, i) parsed = parse_params(this_params, plugin.get('extra_params', {})) - extra_params.append(parsed) - return extra_params + analysis = plugin.activity(parsed) + analysis_list.append(analysis) + return analysis_list def filter_params(params, plugin, ith=-1): @@ -290,7 +298,8 @@ def filter_params(params, plugin, ith=-1): def parse_call(params): - '''Return a results object based on the parameters used in a call/request. + ''' + Return a results object based on the parameters used in a call/request. ''' params = parse_params(params, NIF_PARAMS) if params['informat'] == 'text': diff --git a/senpy/blueprints.py b/senpy/blueprints.py index 0bc6e40..d4ba8aa 100644 --- a/senpy/blueprints.py +++ b/senpy/blueprints.py @@ -189,21 +189,27 @@ def basic_api(f): @basic_api def api_root(plugin): if plugin: - if 'algorithm' in request.parameters: + if request.parameters['algorithm'] != api.API_PARAMS['algorithm']['default']: raise Error('You cannot specify the algorithm with a parameter and a URL variable.' ' Please, remove one of them') - plugin = plugin.replace('+', '/') - request.parameters['algorithm'] = tuple(plugin.split('/')) + request.parameters['algorithm'] = tuple(plugin.replace('+', '/').split('/')) + + params = request.parameters + plugin = request.parameters['algorithm'] + + sp = current_app.senpy + plugins = sp.get_plugins(plugin) if request.parameters['help']: - sp = current_app.senpy - plugins = sp._get_plugins(request) - allparameters = api.get_all_params(plugins, api.WEB_PARAMS, api.API_PARAMS, api.NIF_PARAMS) + apis = [] + if request.parameters['verbose']: + apis.append(api.BUILTIN_PARAMS) + allparameters = api.get_all_params(plugins, *apis) response = Help(valid_parameters=allparameters) return response req = api.parse_call(request.parameters) - results = current_app.senpy.analyse(req) - results.analysis = set(i.id for i in results.analysis) + analysis = api.parse_analysis(req.parameters, plugins) + results = current_app.senpy.analyse(req, analysis) return results diff --git a/senpy/cli.py b/senpy/cli.py index 9be9325..af1eed9 100644 --- a/senpy/cli.py +++ b/senpy/cli.py @@ -31,10 +31,10 @@ def main_function(argv): default_plugins = params.get('default-plugins', False) sp = Senpy(default_plugins=default_plugins, plugin_folder=plugin_folder) request = api.parse_call(params) - algos = request.parameters.get('algorithm', None) + algos = sp.get_plugins(request.parameters.get('algorithm', None)) if algos: for algo in algos: - sp.activate_plugin(algo) + sp.activate_plugin(algo.name) else: sp.activate_all() res = sp.analyse(request) diff --git a/senpy/extensions.py b/senpy/extensions.py index 054bded..3fd2298 100644 --- a/senpy/extensions.py +++ b/senpy/extensions.py @@ -78,27 +78,47 @@ class Senpy(object): def delete_plugin(self, plugin): del self._plugins[plugin.name.lower()] - def plugins(self, **kwargs): + def plugins(self, plugin_type=None, is_activated=True, **kwargs): """ Return the plugins registered for a given application. Filtered by criteria """ - return list(plugins.pfilter(self._plugins, **kwargs)) + return list(plugins.pfilter(self._plugins, plugin_type=plugin_type, + is_activated=is_activated, **kwargs)) def get_plugin(self, name, default=None): if name == 'default': return self.default_plugin - plugin = name.lower() - if plugin in self._plugins: - return self._plugins[plugin] + elif name == 'conversion': + return None - results = self.plugins(id='endpoint:plugins/{}'.format(name)) + if name.lower() in self._plugins: + return self._plugins[name.lower()] - if not results: - return Error(message="Plugin not found", status=404) - return results[0] + results = self.plugins(id='endpoint:plugins/{}'.format(name.lower()), + plugin_type=None) + if results: + return results[0] + + results = self.plugins(id=name, + plugin_type=None) + if results: + return results[0] + + msg = ("Plugin not found: '{}'\n" + "Make sure it is ACTIVATED\n" + "Valid algorithms: {}").format(name, + self._plugins.keys()) + raise Error(message=msg, status=404) + + def get_plugins(self, name): + try: + name = name.split(',') + except AttributeError: + pass # Assume it is a tuple or a list + return tuple(self.get_plugin(n) for n in name) @property def analysis_plugins(self): - """ Return only the analysis plugins """ - return self.plugins(plugin_type='analysisPlugin') + """ Return only the analysis plugins that are active""" + return self.plugins(plugin_type='analysisPlugin', is_activated=True) def add_folder(self, folder, from_root=False): """ Find plugins in this folder and add them to this instance """ @@ -113,38 +133,24 @@ class Senpy(object): else: raise AttributeError("Not a folder or does not exist: %s", folder) - def _get_plugins(self, request): - '''Get a list of plugins that should be run for a specific request''' - if not self.analysis_plugins: - raise Error( - status=404, - message=("No plugins found." - " Please install one.")) - algos = request.parameters.get('algorithm', None) - if not algos: - if self.default_plugin: - algos = [self.default_plugin.name, ] - else: - raise Error( - status=404, - message="No default plugin found, and None provided") - - plugins = list() - for algo in algos: - algo = algo.lower() - if algo == 'conversion': - continue # Allow 'conversion' as a virtual plugin, which does nothing - if algo not in self._plugins: - msg = ("The algorithm '{}' is not valid\n" - "Valid algorithms: {}").format(algo, - self._plugins.keys()) - logger.debug(msg) - raise Error(status=404, message=msg) - plugins.append(self._plugins[algo]) - - return plugins - - def _process(self, req, parameters, pending, done=None): + # def check_analysis_request(self, analysis): + # '''Check if the analysis request can be fulfilled''' + # if not self.plugins(): + # raise Error( + # status=404, + # message=("No plugins found." + # " Please install one.")) + # for a in analysis: + # algo = a.algorithm + # if algo == 'default' and not self.default_plugin: + # raise Error( + # status=404, + # message="No default plugin found, and None provided") + # else: + # self.get_plugin(algo) + + + def _process(self, req, pending, done=None): """ Recursively process the entries with the first plugin in the list, and pass the results to the rest of the plugins. @@ -153,27 +159,32 @@ class Senpy(object): if not pending: return req - plugin = pending[0] - req.parameters = parameters[0] - results = plugin.process(req, conversions_applied=done) - if plugin not in results.analysis: - results.analysis.append(plugin) - return self._process(results, parameters[1:], pending[1:], done) + analysis = pending[0] + results = analysis.run(req) + results.analysis.append(analysis) + done += analysis + return self._process(results, pending[1:], done) def install_deps(self): plugins.install_deps(*self.plugins()) - def analyse(self, request): + def analyse(self, request, analysis=None): """ Main method that analyses a request, either from CLI or HTTP. It takes a processed request, provided by the user, as returned by api.parse_call(). """ - + if not self.plugins(): + raise Error( + status=404, + message=("No plugins found." + " Please install one.")) + if analysis is None: + params = str(request) + plugins = self.get_plugins(request.parameters['algorithm']) + analysis = api.parse_analysis(request.parameters, plugins) logger.debug("analysing request: {}".format(request)) - plugins = self._get_plugins(request) - parameters = api.parse_extra_params(request.parameters, plugins) - results = self._process(request, parameters, plugins) + results = self._process(request, analysis) logger.debug("Got analysis result: {}".format(results)) results = self.postprocess(results) logger.debug("Returning post-processed result: {}".format(results)) @@ -189,7 +200,10 @@ class Senpy(object): """ plugins = resp.analysis - params = resp.parameters + if 'parameters' not in resp: + return resp + + params = resp['parameters'] toModel = params.get('emotionModel', None) if not toModel: return resp @@ -290,7 +304,10 @@ class Senpy(object): results = AggregatedEvaluation() results.parameters = params datasets = self._get_datasets(results) - plugins = self._get_plugins(results) + plugins = [] + for plugname in params.algorithm: + plugins = self.get_plugin(plugname) + for eval in plugins.evaluate(plugins, datasets): results.evaluations.append(eval) if 'with_parameters' not in results.parameters: diff --git a/senpy/meta.py b/senpy/meta.py index 3b2143a..445513e 100644 --- a/senpy/meta.py +++ b/senpy/meta.py @@ -85,7 +85,8 @@ class BaseMeta(ABCMeta): schema = json.load(f) resolver = jsonschema.RefResolver(schema_path, schema) - attrs['@type'] = "".join((name[0].lower(), name[1:])) + if '@type' not in attrs: + attrs['@type'] = "".join((name[0].lower(), name[1:])) attrs['_schema_file'] = schema_file attrs['schema'] = schema attrs['_validator'] = jsonschema.Draft4Validator(schema, resolver=resolver) @@ -244,10 +245,10 @@ class CustomDict(MutableMapping, object): return key[0] == '_' def __str__(self): - return str(self.serializable()) + return json.dumps(self.serializable(), sort_keys=True, indent=4) def __repr__(self): - return str(self.serializable()) + return json.dumps(self.serializable(), sort_keys=True, indent=4) _Alias = namedtuple('Alias', 'indict') diff --git a/senpy/models.py b/senpy/models.py index 6000be1..d1b2f19 100644 --- a/senpy/models.py +++ b/senpy/models.py @@ -121,11 +121,11 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)): ''' - schema_file = DEFINITIONS_FILE + # schema_file = DEFINITIONS_FILE _context = base_context["@context"] def __init__(self, *args, **kwargs): - auto_id = kwargs.pop('_auto_id', True) + auto_id = kwargs.pop('_auto_id', False) super(BaseModel, self).__init__(*args, **kwargs) @@ -133,7 +133,7 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)): self.id if '@type' not in self: - logger.warn('Created an instance of an unknown model') + logger.warning('Created an instance of an unknown model') @property def id(self): @@ -325,7 +325,6 @@ def _add_class_from_schema(*args, **kwargs): for i in [ 'aggregatedEvaluation', - 'analysis', 'dataset', 'datasets', 'emotion', @@ -339,7 +338,7 @@ for i in [ 'entity', 'help', 'metric', - 'plugin', + 'parameter', 'plugins', 'response', 'results', @@ -349,3 +348,55 @@ for i in [ ]: _add_class_from_schema(i) + + +class Analysis(BaseModel): + schema = 'analysis' + + parameters = alias('prov:used') + + @property + def params(self): + outdict = {} + outdict['algorithm'] = self.algorithm + for param in self.parameters: + outdict[param['name']] = param['value'] + return outdict + + @params.setter + def params(self, value): + for k, v in value.items(): + for param in self.parameters: + if param.name == k: + param.value = v + break + else: + self.parameters.append(Parameter(name=k, value=v)) + + @property + def algorithm(self): + return self['prov:wasAssociatedWith'] + + @property + def plugin(self): + return self._plugin + + @plugin.setter + def plugin(self, value): + self._plugin = value + self['prov:wasAssociatedWith'] = value.id + + def run(self, request): + return self.plugin.process(request, self.params) + + +class Plugin(BaseModel): + schema = 'plugin' + + def activity(self, parameters): + '''Generate a prov:Activity from this plugin and the ''' + a = Analysis() + a.plugin = self + a.params = parameters + return a + diff --git a/senpy/plugins/__init__.py b/senpy/plugins/__init__.py index 12d9da5..361509a 100644 --- a/senpy/plugins/__init__.py +++ b/senpy/plugins/__init__.py @@ -132,12 +132,12 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)): def deactivate(self): pass - def process(self, request, **kwargs): + def process(self, request, parameters, **kwargs): """ An implemented plugin should override this method. Here, we assume that a process_entries method exists.""" newentries = list( - self.process_entries(request.entries, request.parameters)) + self.process_entries(request.entries, parameters)) request.entries = newentries return request @@ -194,13 +194,13 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)): try: request = models.Response() - request.parameters = api.parse_params(given_parameters, + parameters = api.parse_params(given_parameters, self.extra_params) request.entries = [ entry, ] - method = partial(self.process, request) + method = partial(self.process, request, parameters) if mock: res = method() @@ -249,14 +249,14 @@ class Analysis(Plugin): ''' def analyse(self, request, parameters): - return super(Analysis, self).process(request) + return super(Analysis, self).process(request, parameters) def analyse_entries(self, entries, parameters): for i in super(Analysis, self).process_entries(entries, parameters): yield i - def process(self, request, **kwargs): - return self.analyse(request, request.parameters) + def process(self, request, parameters, **kwargs): + return self.analyse(request, parameters) def process_entries(self, entries, parameters): for i in self.analyse_entries(entries, parameters): @@ -279,12 +279,12 @@ class Conversion(Plugin): e.g. a conversion of emotion models, or normalization of sentiment values. ''' - def process(self, response, plugins=None, **kwargs): + def process(self, response, parameters, plugins=None, **kwargs): plugins = plugins or [] newentries = [] for entry in response.entries: newentries.append( - self.convert_entry(entry, response.parameters, plugins)) + self.convert_entry(entry, parameters, plugins)) response.entries = newentries return response diff --git a/senpy/schemas/analysis.json b/senpy/schemas/analysis.json index 19eac15..df4c42a 100644 --- a/senpy/schemas/analysis.json +++ b/senpy/schemas/analysis.json @@ -9,7 +9,20 @@ "@type": { "type": "string", "description": "Type of the analysis. e.g. marl:SentimentAnalysis" + }, + "prov:wasAssociatedWith": { + "@type": "string", + "description": "Algorithm/plugin that was used" + }, + "prov:used": { + "description": "Parameters of the algorithm", + "@type": "array", + "default": [], + "type": "array", + "items": { + "$ref": "parameter.json" + } } }, - "required": ["@id", "@type"] + "required": ["@type", "prov:wasAssociatedWith"] } diff --git a/senpy/schemas/context.jsonld b/senpy/schemas/context.jsonld index 6879e4a..bebc6f4 100644 --- a/senpy/schemas/context.jsonld +++ b/senpy/schemas/context.jsonld @@ -41,7 +41,7 @@ "@container": "@set" }, "analysis": { - "@id": "AnalysisInvolved", + "@id": "prov:wasInformedBy", "@type": "@id", "@container": "@set" }, diff --git a/senpy/schemas/emotionSet.json b/senpy/schemas/emotionSet.json index b2ae5cc..6e8be0b 100644 --- a/senpy/schemas/emotionSet.json +++ b/senpy/schemas/emotionSet.json @@ -20,5 +20,5 @@ "description": "The ID of the analysis that generated this Emotion. The full object should be included in the \"analysis\" property of the root object" } }, - "required": ["@id", "prov:wasGeneratedBy", "onyx:hasEmotion"] + "required": ["prov:wasGeneratedBy", "onyx:hasEmotion"] } diff --git a/senpy/schemas/entry.json b/senpy/schemas/entry.json index c2ac203..406f69a 100644 --- a/senpy/schemas/entry.json +++ b/senpy/schemas/entry.json @@ -35,5 +35,5 @@ "default": [] } }, - "required": ["@id", "nif:isString"] + "required": ["nif:isString"] } diff --git a/senpy/schemas/parameter.json b/senpy/schemas/parameter.json new file mode 100644 index 0000000..328a1a7 --- /dev/null +++ b/senpy/schemas/parameter.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "Parameters for a senpy analysis", + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name of the parameter" + }, + "prov:value": { + "@type": "any", + "description": "Value of the parameter" + } + }, + "required": ["name", "prov:value"] +} diff --git a/senpy/schemas/results.json b/senpy/schemas/results.json index 06ad88e..48cf335 100644 --- a/senpy/schemas/results.json +++ b/senpy/schemas/results.json @@ -21,13 +21,7 @@ "default": [], "type": "array", "items": { - "anyOf": [ - { - "$ref": "analysis.json" - },{ - "type": "string" - } - ] + "$ref": "analysis.json" } }, "entries": { diff --git a/senpy/schemas/sentiment.json b/senpy/schemas/sentiment.json index 6d2b0eb..b7875f3 100644 --- a/senpy/schemas/sentiment.json +++ b/senpy/schemas/sentiment.json @@ -19,5 +19,5 @@ "description": "The ID of the analysis that generated this Sentiment. The full object should be included in the \"analysis\" property of the root object" } }, - "required": ["@id", "prov:wasGeneratedBy"] + "required": ["prov:wasGeneratedBy"] } diff --git a/setup.cfg b/setup.cfg index cdf2f26..3ac3a0f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,6 +12,8 @@ max-line-length = 100 universal=1 [tool:pytest] addopts = --cov=senpy --cov-report term-missing - +filterwarnings = + error + ignore:the matrix subclass:PendingDeprecationWarning [coverage:report] -omit = senpy/__main__.py \ No newline at end of file +omit = senpy/__main__.py diff --git a/tests/test_api.py b/tests/test_api.py index 6b8f64f..5f33ecf 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -3,7 +3,7 @@ import logging logger = logging.getLogger(__name__) from unittest import TestCase -from senpy.api import (boolean, parse_params, get_extra_params, parse_extra_params, +from senpy.api import (boolean, parse_params, get_extra_params, parse_analysis, API_PARAMS, NIF_PARAMS, WEB_PARAMS) from senpy.models import Error, Plugin @@ -91,7 +91,7 @@ class APITest(TestCase): assert 'input' in p assert p['input'] == 'Aloha my friend' - def test_parse_extra_params(self): + def test_parse_analysis(self): '''The API should parse user parameters and return them in a format that plugins can use''' plugins = [ Plugin({ @@ -161,10 +161,11 @@ class APITest(TestCase): } ] - p = parse_extra_params(call, plugins) + p = parse_analysis(call, plugins) for i, arg in enumerate(expected): + params = p[i].params for k, v in arg.items(): - assert p[i][k] == v + assert params[k] == v def test_get_extra_params(self): '''The API should return the list of valid parameters for a set of plugins''' @@ -216,13 +217,11 @@ class APITest(TestCase): ] expected = { - # Each plugin's parameters - '0.param0': plugins[0]['extra_params']['param0'], - '0.param1': plugins[0]['extra_params']['param1'], - '0.param2': plugins[0]['extra_params']['param2'], - '1.param0': plugins[1]['extra_params']['param0'], - '1.param1': plugins[1]['extra_params']['param1'], - '1.param3': plugins[1]['extra_params']['param3'], + # Overlapping parameters + 'plugin1.param0': plugins[0]['extra_params']['param0'], + 'plugin1.param1': plugins[0]['extra_params']['param1'], + 'plugin2.param0': plugins[1]['extra_params']['param0'], + 'plugin2.param1': plugins[1]['extra_params']['param1'], # Non-overlapping parameters 'param2': plugins[0]['extra_params']['param2'], diff --git a/tests/test_blueprints.py b/tests/test_blueprints.py index 633c443..0da44a8 100644 --- a/tests/test_blueprints.py +++ b/tests/test_blueprints.py @@ -26,8 +26,7 @@ class BlueprintsTest(TestCase): cls.senpy.init_app(cls.app) cls.dir = os.path.join(os.path.dirname(__file__), "..") cls.senpy.add_folder(cls.dir) - cls.senpy.activate_plugin("Dummy", sync=True) - cls.senpy.activate_plugin("DummyRequired", sync=True) + cls.senpy.activate_all() cls.senpy.default_plugin = 'Dummy' def setUp(self): @@ -139,16 +138,27 @@ class BlueprintsTest(TestCase): # Calling dummy twice, should return the same string self.assertCode(resp, 200) js = parse_resp(resp) - assert len(js['analysis']) == 1 + assert len(js['analysis']) == 2 assert js['entries'][0]['nif:isString'] == 'My aloha mohame' resp = self.client.get("/api/Dummy+Dummy?i=My aloha mohame") # Same with pluses instead of slashes self.assertCode(resp, 200) js = parse_resp(resp) - assert len(js['analysis']) == 1 + assert len(js['analysis']) == 2 assert js['entries'][0]['nif:isString'] == 'My aloha mohame' + def test_analysis_chain_required(self): + """ + If a parameter is required and duplicated (because two plugins require it), specifying + it once should suffice + """ + resp = self.client.get("/api/DummyRequired/DummyRequired?i=My aloha mohame&example=a") + js = parse_resp(resp) + assert len(js['analysis']) == 2 + assert js['entries'][0]['nif:isString'] == 'My aloha mohame' + assert js['entries'][0]['reversed'] == 2 + def test_requirements_chain_help(self): '''The extra parameters of each plugin should be merged if they are in a chain ''' resp = self.client.get("/api/split/DummyRequired?help=true") @@ -157,6 +167,7 @@ class BlueprintsTest(TestCase): assert 'valid_parameters' in js vp = js['valid_parameters'] assert 'example' in vp + assert 'delimiter' in vp def test_requirements_chain_repeat_help(self): ''' @@ -168,10 +179,14 @@ class BlueprintsTest(TestCase): js = parse_resp(resp) assert 'valid_parameters' in js vp = js['valid_parameters'] - assert '0.delimiter' in vp - assert '1.delimiter' in vp assert 'delimiter' in vp + resp = self.client.get("/api/split/split?help=true&verbose=false") + js = parse_resp(resp) + vp = js['valid_parameters'] + assert len(vp.keys()) == 1 + + def test_requirements_chain(self): """ It should be possible to specify different parameters for each step in the chain. diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 84968c1..3181e58 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -11,14 +11,15 @@ except ImportError: from functools import partial from senpy.extensions import Senpy from senpy import plugins -from senpy.models import Error, Results, Entry, EmotionSet, Emotion, Plugin +from senpy.models import Analysis, Error, Results, Entry, EmotionSet, Emotion, Plugin from senpy import api from flask import Flask from unittest import TestCase def analyse(instance, **kwargs): - request = api.parse_call(kwargs) + basic = api.parse_params(kwargs, api.API_PARAMS) + request = api.parse_call(basic) return instance.analyse(request) @@ -49,9 +50,9 @@ class ExtensionsTest(TestCase): '''Should be able to add and delete new plugins. ''' new = plugins.Analysis(name='new', description='new', version=0) self.senpy.add_plugin(new) - assert new in self.senpy.plugins() + assert new in self.senpy.plugins(is_activated=False) self.senpy.delete_plugin(new) - assert new not in self.senpy.plugins() + assert new not in self.senpy.plugins(is_activated=False) def test_adding_folder(self): """ It should be possible for senpy to look for plugins in more folders. """ @@ -60,7 +61,7 @@ class ExtensionsTest(TestCase): default_plugins=False) assert not senpy.analysis_plugins senpy.add_folder(self.examples_dir) - assert senpy.analysis_plugins + assert senpy.plugins(plugin_type=plugins.AnalysisPlugin, is_activated=False) self.assertRaises(AttributeError, senpy.add_folder, 'DOES NOT EXIST') def test_installing(self): @@ -121,8 +122,8 @@ class ExtensionsTest(TestCase): # Leaf (defaultdict with __setattr__ and __getattr__. r1 = analyse(self.senpy, algorithm="Dummy", input="tupni", output="tuptuo") r2 = analyse(self.senpy, input="tupni", output="tuptuo") - assert r1.analysis[0].id == "endpoint:plugins/Dummy_0.1" - assert r2.analysis[0].id == "endpoint:plugins/Dummy_0.1" + assert r1.analysis[0].algorithm == "endpoint:plugins/Dummy_0.1" + assert r2.analysis[0].algorithm == "endpoint:plugins/Dummy_0.1" assert r1.entries[0]['nif:isString'] == 'input' def test_analyse_empty(self): @@ -130,7 +131,7 @@ class ExtensionsTest(TestCase): senpy = Senpy(plugin_folder=None, app=self.app, default_plugins=False) - self.assertRaises(Error, senpy.analyse, Results()) + self.assertRaises(Error, senpy.analyse, Results(), []) def test_analyse_wrong(self): """ Trying to analyse with a non-existent plugin should raise an error.""" @@ -156,29 +157,32 @@ class ExtensionsTest(TestCase): r2 = analyse(self.senpy, input="tupni", output="tuptuo") - assert r1.analysis[0].id == "endpoint:plugins/Dummy_0.1" - assert r2.analysis[0].id == "endpoint:plugins/Dummy_0.1" + assert r1.analysis[0].algorithm == "endpoint:plugins/Dummy_0.1" + assert r2.analysis[0].algorithm == "endpoint:plugins/Dummy_0.1" assert r1.entries[0]['nif:isString'] == 'input' def test_analyse_error(self): - mm = mock.MagicMock() - mm.id = 'magic_mock' - mm.name = 'mock' - mm.is_activated = True - mm.process.side_effect = Error('error in analysis', status=500) - self.senpy.add_plugin(mm) + class ErrorPlugin(plugins.Analysis): + author = 'nobody' + version = 0 + ex = Error() + + def process(self, *args, **kwargs): + raise self.ex + + m = ErrorPlugin(ex=Error('error in analysis', status=500)) + self.senpy.add_plugin(m) try: - analyse(self.senpy, input='nothing', algorithm='MOCK') + analyse(self.senpy, input='nothing', algorithm='ErrorPlugin') assert False except Error as ex: assert 'error in analysis' in ex['message'] assert ex['status'] == 500 - ex = Exception('generic exception on analysis') - mm.process.side_effect = ex + m.ex = Exception('generic exception on analysis') try: - analyse(self.senpy, input='nothing', algorithm='MOCK') + analyse(self.senpy, input='nothing', algorithm='ErrorPlugin') assert False except Exception as ex: assert 'generic exception on analysis' in str(ex) @@ -194,7 +198,7 @@ class ExtensionsTest(TestCase): def test_load_default_plugins(self): senpy = Senpy(plugin_folder=self.examples_dir, default_plugins=True) - assert len(senpy.plugins()) > 1 + assert len(senpy.plugins(is_activated=False)) > 1 def test_convert_emotions(self): self.senpy.activate_all(sync=True) diff --git a/tests/test_models.py b/tests/test_models.py index 0f8d170..13f64be 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -5,7 +5,8 @@ import jsonschema import json import rdflib from unittest import TestCase -from senpy.models import (Emotion, +from senpy.models import (Analysis, + Emotion, EmotionAnalysis, EmotionSet, Entry, @@ -61,7 +62,7 @@ class ModelsTest(TestCase): def test_id(self): """ Adding the id after creation should overwrite the automatic ID """ - r = Entry() + r = Entry(_auto_id=True) j = r.jsonld() assert '@id' in j r.id = "test" @@ -189,6 +190,19 @@ class ModelsTest(TestCase): assert isinstance(js['plugins'], list) assert js['plugins'][0]['@type'] == 'sentimentPlugin' + def test_parameters(self): + '''An Analysis should contain the algorithm and the list of parameters to be used''' + a = Analysis() + a.params = {'param1': 1, 'param2': 2} + assert len(a.parameters) == 2 + for param in a.parameters: + if param.name == 'param1': + assert param.value == 1 + elif param.name == 'param2': + assert param.value == 2 + else: + raise Exception('Unknown value %s' % param) + def test_from_string(self): results = { '@type': 'results',