From 3e2b8baeb272e036c778108cbc309d35eea0b114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Sat, 6 Jan 2018 18:51:16 +0100 Subject: [PATCH] Last batch of big changes * Add Box plugin (i.e. black box) * Add SentimentBox, EmotionBox and MappingMixin * Refactored CustomDict --- example-plugins/README.md | 18 +- example-plugins/basic.py | 8 +- example-plugins/basic_analyse_entry_plugin.py | 47 +++++ example-plugins/basic_box_plugin.py | 41 ++++ example-plugins/basic_plugin.py | 18 +- example-plugins/configurable_plugin.py | 8 +- example-plugins/parameterized_plugin.py | 2 +- senpy/cli.py | 4 +- senpy/meta.py | 185 +++++++++++++++--- senpy/models.py | 124 +++--------- senpy/plugins/__init__.py | 119 +++++++++-- senpy/plugins/example/rand_plugin.py | 2 +- senpy/utils.py | 9 + tests/test_models.py | 8 +- tests/test_plugins.py | 69 +++++++ 15 files changed, 498 insertions(+), 164 deletions(-) create mode 100644 example-plugins/basic_analyse_entry_plugin.py create mode 100644 example-plugins/basic_box_plugin.py diff --git a/example-plugins/README.md b/example-plugins/README.md index 5316686..4fef067 100644 --- a/example-plugins/README.md +++ b/example-plugins/README.md @@ -1,9 +1,19 @@ This is a collection of plugins that exemplify certain aspects of plugin development with senpy. -In ascending order of complexity, there are: -* Basic: a very basic analysis that does sentiment analysis based on emojis. -* Configurable: a version of `basic` with a configurable map of emojis for each sentiment. -* Parameterized: like `basic_info`, but users set the map in each query (via `extra_parameters`). +The first series of plugins the `basic` ones. +Their starting point is a classification function defined in `basic.py`. +They all include testing and running them as a script will run all tests. +In ascending order of customization, the plugins are: + +* Basic is the simplest plugin of all. It leverages the `SentimentBox` Plugin class to create a plugin out of a classification method, and `MappingMixin` to convert the labels from (`pos`, `neg`) to (`marl:Positive`, `marl:Negative` +* Basic_box is just like the previous one, but replaces the mixin with a custom function. +* Basic_configurable is a version of `basic` with a configurable map of emojis for each sentiment. +* Basic_parameterized like `basic_info`, but users set the map in each query (via `extra_parameters`). +* Basic_analyse\_entry uses the more general `analyse_entry` method and adds the annotations individually. + + +In rest of the plugins show advanced topics: + * mynoop: shows how to add a definition file with external requirements for a plugin. Doing this with a python-only module would require moving all imports of the requirements to their functions, which is considered bad practice. * Async: a barebones example of training a plugin and analyzing data in parallel. diff --git a/example-plugins/basic.py b/example-plugins/basic.py index db67b35..39f23d6 100644 --- a/example-plugins/basic.py +++ b/example-plugins/basic.py @@ -2,13 +2,13 @@ # coding: utf-8 emoticons = { - 'marl:Positive': [':)', ':]', '=)', ':D'], - 'marl:Negative': [':(', ':[', '=('] + 'pos': [':)', ':]', '=)', ':D'], + 'neg': [':(', ':[', '=('] } emojis = { - 'marl:Positive': ['😁', '😂', '😃', '😄', '😆', '😅', '😄' '😍'], - 'marl:Negative': ['😢', '😡', '😠', '😞', '😖', '😔', '😓', '😒'] + 'pos': ['😁', '😂', '😃', '😄', '😆', '😅', '😄' '😍'], + 'neg': ['😢', '😡', '😠', '😞', '😖', '😔', '😓', '😒'] } diff --git a/example-plugins/basic_analyse_entry_plugin.py b/example-plugins/basic_analyse_entry_plugin.py new file mode 100644 index 0000000..8f5a4c3 --- /dev/null +++ b/example-plugins/basic_analyse_entry_plugin.py @@ -0,0 +1,47 @@ +#!/usr/local/bin/python +# coding: utf-8 + +from senpy import easy_test, models, plugins + +import basic + + +class BasicAnalyseEntry(plugins.SentimentPlugin): + '''Equivalent to Basic, implementing the analyse_entry method''' + + author = '@balkian' + version = '0.1' + + mappings = { + 'pos': 'marl:Positive', + 'neg': 'marl:Negative', + 'default': 'marl:Neutral' + } + + def analyse_entry(self, entry, params): + polarity = basic.get_polarity(entry.text) + + polarity = self.mappings.get(polarity, self.mappings['default']) + + s = models.Sentiment(marl__hasPolarity=polarity) + s.prov(self) + entry.sentiments.append(s) + yield entry + + test_cases = [{ + 'input': 'Hello :)', + 'polarity': 'marl:Positive' + }, { + 'input': 'So sad :(', + 'polarity': 'marl:Negative' + }, { + 'input': 'Yay! Emojis 😁', + 'polarity': 'marl:Positive' + }, { + 'input': 'But no emoticons 😢', + 'polarity': 'marl:Negative' + }] + + +if __name__ == '__main__': + easy_test() diff --git a/example-plugins/basic_box_plugin.py b/example-plugins/basic_box_plugin.py new file mode 100644 index 0000000..23e5cf6 --- /dev/null +++ b/example-plugins/basic_box_plugin.py @@ -0,0 +1,41 @@ +#!/usr/local/bin/python +# coding: utf-8 + +from senpy import easy_test, SentimentBox + +import basic + + +class BasicBox(SentimentBox): + ''' A modified version of Basic that also does converts annotations manually''' + + author = '@balkian' + version = '0.1' + + mappings = { + 'pos': 'marl:Positive', + 'neg': 'marl:Negative', + 'default': 'marl:Neutral' + } + + def box(self, input, **kwargs): + output = basic.get_polarity(input) + return self.mappings.get(output, self.mappings['default']) + + test_cases = [{ + 'input': 'Hello :)', + 'polarity': 'marl:Positive' + }, { + 'input': 'So sad :(', + 'polarity': 'marl:Negative' + }, { + 'input': 'Yay! Emojis 😁', + 'polarity': 'marl:Positive' + }, { + 'input': 'But no emoticons 😢', + 'polarity': 'marl:Negative' + }] + + +if __name__ == '__main__': + easy_test() diff --git a/example-plugins/basic_plugin.py b/example-plugins/basic_plugin.py index a760e73..b2205a0 100644 --- a/example-plugins/basic_plugin.py +++ b/example-plugins/basic_plugin.py @@ -1,25 +1,25 @@ #!/usr/local/bin/python # coding: utf-8 -from senpy import easy_test, models, plugins +from senpy import easy_test, SentimentBox, MappingMixin import basic -class Basic(plugins.SentimentPlugin): +class Basic(MappingMixin, SentimentBox): '''Provides sentiment annotation using a lexicon''' author = '@balkian' version = '0.1' - def analyse_entry(self, entry, params): + mappings = { + 'pos': 'marl:Positive', + 'neg': 'marl:Negative', + 'default': 'marl:Neutral' + } - polarity = basic.get_polarity(entry.text) - - s = models.Sentiment(marl__hasPolarity=polarity) - s.prov(self) - entry.sentiments.append(s) - yield entry + def box(self, input, **kwargs): + return basic.get_polarity(input) test_cases = [{ 'input': 'Hello :)', diff --git a/example-plugins/configurable_plugin.py b/example-plugins/configurable_plugin.py index 688150e..73af6ee 100644 --- a/example-plugins/configurable_plugin.py +++ b/example-plugins/configurable_plugin.py @@ -14,8 +14,12 @@ class Dictionary(plugins.SentimentPlugin): dictionaries = [basic.emojis, basic.emoticons] + mappings = {'pos': 'marl:Positive', 'neg': 'marl:Negative'} + def analyse_entry(self, entry, params): polarity = basic.get_polarity(entry.text, self.dictionaries) + if polarity in self.mappings: + polarity = self.mappings[polarity] s = models.Sentiment(marl__hasPolarity=polarity) s.prov(self) @@ -80,14 +84,14 @@ class Salutes(Dictionary): '''Sentiment annotation with a custom lexicon, for illustration purposes''' dictionaries = [{ 'marl:Positive': ['Hello', '!'], - 'marl:Negative': ['sad', ] + 'marl:Negative': ['Good bye', ] }] test_cases = [{ 'input': 'Hello :)', 'polarity': 'marl:Positive' }, { - 'input': 'So sad :(', + 'input': 'Good bye :(', 'polarity': 'marl:Negative' }, { 'input': 'Yay! Emojis 😁', diff --git a/example-plugins/parameterized_plugin.py b/example-plugins/parameterized_plugin.py index 856ead0..8a5798c 100644 --- a/example-plugins/parameterized_plugin.py +++ b/example-plugins/parameterized_plugin.py @@ -7,8 +7,8 @@ import basic class ParameterizedDictionary(plugins.SentimentPlugin): + '''This is a basic self-contained plugin''' - description = 'This is a basic self-contained plugin' author = '@balkian' version = '0.2' diff --git a/senpy/cli.py b/senpy/cli.py index 442540b..9be9325 100644 --- a/senpy/cli.py +++ b/senpy/cli.py @@ -46,9 +46,9 @@ def main(): ''' try: res = main_function(sys.argv[1:]) - print(res.to_JSON()) + print(res.serialize()) except Error as err: - print(err.to_JSON()) + print(err.serialize()) sys.exit(2) diff --git a/senpy/meta.py b/senpy/meta.py index 59fc47f..3b2143a 100644 --- a/senpy/meta.py +++ b/senpy/meta.py @@ -8,6 +8,7 @@ import inspect import copy from abc import ABCMeta +from collections import MutableMapping, namedtuple class BaseMeta(ABCMeta): @@ -31,24 +32,31 @@ class BaseMeta(ABCMeta): _subtypes = {} def __new__(mcs, name, bases, attrs, **kwargs): - defaults = {} register_afterwards = False + defaults = {} attrs = mcs.expand_with_schema(name, attrs) if 'schema' in attrs: register_afterwards = True - defaults = mcs.get_defaults(attrs['schema']) - for b in bases: - if hasattr(b, '_defaults'): - defaults.update(b._defaults) - info, attrs = mcs.split_attrs(attrs) - defaults.update(info) - attrs['_defaults'] = defaults + for base in bases: + if hasattr(base, '_defaults'): + defaults.update(getattr(base, '_defaults')) + + info, rest = mcs.split_attrs(attrs) + + for i in list(info.keys()): + if isinstance(info[i], _Alias): + fget, fset, fdel = make_property(info[i].indict) + rest[i] = property(fget=fget, fset=fset, fdel=fdel) + else: + defaults[i] = info[i] + + rest['_defaults'] = defaults - cls = super(BaseMeta, mcs).__new__(mcs, name, tuple(bases), attrs) + cls = super(BaseMeta, mcs).__new__(mcs, name, tuple(bases), rest) if register_afterwards: - mcs.register(cls, cls._defaults['@type']) + mcs.register(cls, defaults['@type']) return cls @classmethod @@ -81,17 +89,26 @@ class BaseMeta(ABCMeta): attrs['_schema_file'] = schema_file attrs['schema'] = schema attrs['_validator'] = jsonschema.Draft4Validator(schema, resolver=resolver) + + schema_defaults = BaseMeta.get_defaults(attrs['schema']) + attrs.update(schema_defaults) + return attrs @staticmethod - def is_attr(k, v): - return (not(inspect.isroutine(v) or - inspect.ismethod(v) or - inspect.ismodule(v) or - isinstance(v, property)) and - k[0] != '_' and - k != 'schema' and - k != 'data') + def is_func(v): + return inspect.isroutine(v) or inspect.ismethod(v) or \ + inspect.ismodule(v) or isinstance(v, property) + + @staticmethod + def is_internal(k): + return k[0] == '_' or k == 'schema' or k == 'data' + + @staticmethod + def get_key(key): + if key[0] != '_': + key = key.replace("__", ":", 1) + return key @staticmethod def split_attrs(attrs): @@ -102,15 +119,13 @@ class BaseMeta(ABCMeta): e.g.: ''' isattr = {} - notattr = {} + rest = {} for key, value in attrs.items(): - if BaseMeta.is_attr(key, value): - if key[0] != '_': - key = key.replace("__", ":", 1) - isattr[key] = copy.deepcopy(value) + if not (BaseMeta.is_internal(key)) and (not BaseMeta.is_func(value)): + isattr[key] = value else: - notattr[key] = value - return isattr, notattr + rest[key] = value + return isattr, rest @staticmethod def get_defaults(schema): @@ -120,5 +135,123 @@ class BaseMeta(ABCMeta): ] + schema.get('allOf', []): for k, v in obj.get('properties', {}).items(): if 'default' in v and k not in temp: - temp[k] = copy.deepcopy(v['default']) + temp[k] = v['default'] return temp + + +def make_property(key): + + def fget(self): + return self[key] + + def fdel(self): + del self[key] + + def fset(self, value): + self[key] = value + + return fget, fset, fdel + + +class CustomDict(MutableMapping, object): + ''' + A dictionary whose elements can also be accessed as attributes. Since some + characters are not valid in the dot-notation, the attribute names also + converted. e.g.: + + > d = CustomDict() + > d.key = d['ns:name'] = 1 + > d.key == d['key'] + True + > d.ns__name == d['ns:name'] + ''' + + _defaults = {} + _map_attr_key = {'id': '@id'} + + def __init__(self, *args, **kwargs): + super(CustomDict, self).__init__() + for k, v in self._defaults.items(): + self[k] = copy.copy(v) + for arg in args: + self.update(arg) + for k, v in kwargs.items(): + self[self._attr_to_key(k)] = v + return self + + def serializable(self): + def ser_or_down(item): + if hasattr(item, 'serializable'): + return item.serializable() + elif isinstance(item, dict): + temp = dict() + for kp in item: + vp = item[kp] + temp[kp] = ser_or_down(vp) + return temp + elif isinstance(item, list) or isinstance(item, set): + return list(ser_or_down(i) for i in item) + else: + return item + + return ser_or_down(self.as_dict()) + + def __getitem__(self, key): + key = self._key_to_attr(key) + return self.__dict__[key] + + def __setitem__(self, key, value): + '''Do not insert data directly, there might be a property in that key. ''' + key = self._key_to_attr(key) + return setattr(self, key, value) + + def as_dict(self): + return {self._attr_to_key(k): v for k, v in self.__dict__.items() + if not self._internal_key(k)} + + def __iter__(self): + return (k for k in self.__dict__ if not self._internal_key(k)) + + def __len__(self): + return len(self.__dict__) + + def __delitem__(self, key): + del self.__dict__[key] + + def update(self, other): + for k, v in other.items(): + self[k] = v + + def _attr_to_key(self, key): + key = key.replace("__", ":", 1) + key = self._map_attr_key.get(key, key) + return key + + def _key_to_attr(self, key): + if self._internal_key(key): + return key + key = key.replace(":", "__", 1) + return key + + def __getattr__(self, key): + try: + return self.__dict__[self._attr_to_key(key)] + except KeyError: + raise AttributeError + + @staticmethod + def _internal_key(key): + return key[0] == '_' + + def __str__(self): + return str(self.serializable()) + + def __repr__(self): + return str(self.serializable()) + + +_Alias = namedtuple('Alias', 'indict') + + +def alias(key): + return _Alias(key) diff --git a/senpy/models.py b/senpy/models.py index d845a8a..195693f 100644 --- a/senpy/models.py +++ b/senpy/models.py @@ -17,8 +17,6 @@ import copy import json import os import jsonref -from collections import UserDict - from flask import Response as FlaskResponse from pyld import jsonld @@ -30,7 +28,7 @@ logger = logging.getLogger(__name__) from rdflib import Graph -from .meta import BaseMeta +from .meta import BaseMeta, CustomDict, alias DEFINITIONS_FILE = 'definitions.json' CONTEXT_PATH = os.path.join( @@ -81,67 +79,6 @@ def register(rsubclass, rtype=None): BaseMeta.register(rsubclass, rtype) -class CustomDict(UserDict, object): - ''' - A dictionary whose elements can also be accessed as attributes. Since some - characters are not valid in the dot-notation, the attribute names also - converted. e.g.: - - > d = CustomDict() - > d.key = d['ns:name'] = 1 - > d.key == d['key'] - True - > d.ns__name == d['ns:name'] - ''' - - _defaults = [] - - def __init__(self, *args, **kwargs): - temp = copy.deepcopy(self._defaults) - for arg in args: - temp.update(copy.deepcopy(arg)) - for k, v in kwargs.items(): - temp[self._get_key(k)] = v - - super(CustomDict, self).__init__(temp) - - @staticmethod - def _get_key(key): - if key is 'id': - key = '@id' - key = key.replace("__", ":", 1) - return key - - @staticmethod - def _internal_key(key): - return key[0] == '_' or key == 'data' - - def __getattr__(self, key): - ''' - __getattr__ only gets called when the attribute could not be found - in the __dict__. So we only need to look for the the element in the - dictionary, or raise an Exception. - ''' - mkey = self._get_key(key) - if not self._internal_key(key) and mkey in self: - return self[mkey] - raise AttributeError(key) - - def __setattr__(self, key, value): - # Work as usual for internal properties or already existing - # properties - if self._internal_key(key) or key in self.__dict__: - return super(CustomDict, self).__setattr__(key, value) - key = self._get_key(key) - return self.__setitem__(self._get_key(key), value) - - def __delattr__(self, key): - if self._internal_key(key): - return object.__delattr__(self, key) - key = self._get_key(key) - self.__delitem__(self._get_key(key)) - - class BaseModel(with_metaclass(BaseMeta, CustomDict)): ''' Entities of the base model are a special kind of dictionary that emulates @@ -185,14 +122,25 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)): def __init__(self, *args, **kwargs): auto_id = kwargs.pop('_auto_id', True) + super(BaseModel, self).__init__(*args, **kwargs) - if '@id' not in self and auto_id: - self.id = ':{}_{}'.format(type(self).__name__, time.time()) + if auto_id: + self.id if '@type' not in self: logger.warn('Created an instance of an unknown model') + @property + def id(self): + if '@id' not in self: + self['@id'] = ':{}_{}'.format(type(self).__name__, time.time()) + return self['@id'] + + @id.setter + def id(self, value): + self['@id'] = value + def flask(self, in_headers=True, headers=None, @@ -246,23 +194,6 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)): else: return content - def serializable(self): - def ser_or_down(item): - if hasattr(item, 'serializable'): - return item.serializable() - elif isinstance(item, dict): - temp = dict() - for kp in item: - vp = item[kp] - temp[kp] = ser_or_down(vp) - return temp - elif isinstance(item, list) or isinstance(item, set): - return list(ser_or_down(i) for i in item) - else: - return item - - return ser_or_down(self.data) - def jsonld(self, with_context=False, context_uri=None, @@ -288,10 +219,6 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)): del result['@context'] return result - def to_JSON(self, *args, **kwargs): - js = json.dumps(self.jsonld(*args, **kwargs), indent=4, sort_keys=True) - return js - def validate(self, obj=None): if not obj: obj = self @@ -299,9 +226,6 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)): obj = obj.jsonld() self._validator.validate(obj) - def __str__(self): - return str(self.serialize()) - def prov(self, another): self['prov:wasGeneratedBy'] = another.id @@ -329,7 +253,7 @@ def from_dict(indict, cls=None): for ix, v2 in enumerate(v): if isinstance(v2, dict): v[ix] = from_dict(v2) - outdict[k] = copy.deepcopy(v) + outdict[k] = copy.copy(v) return cls(**outdict) @@ -342,22 +266,23 @@ def from_json(injson): return from_dict(indict) -class Entry(BaseModel, Exception): +class Entry(BaseModel): schema = 'entry' - @property - def text(self): - return self['nif:isString'] + text = alias('nif:isString') + + +class Sentiment(BaseModel): + schema = 'sentiment' - @text.setter - def text(self, value): - self['nif:isString'] = value + polarity = alias('marl:hasPolarity') + polarityValue = alias('marl:hasPolarityValue') class Error(BaseModel, Exception): schema = 'error' - def __init__(self, message, *args, **kwargs): + def __init__(self, message='Generic senpy exception', *args, **kwargs): Exception.__init__(self, message) super(Error, self).__init__(*args, **kwargs) self.message = message @@ -407,7 +332,6 @@ for i in [ 'plugins', 'response', 'results', - 'sentiment', 'sentimentPlugin', 'suggestion', ]: diff --git a/senpy/plugins/__init__.py b/senpy/plugins/__init__.py index 3983590..07b50e1 100644 --- a/senpy/plugins/__init__.py +++ b/senpy/plugins/__init__.py @@ -1,5 +1,7 @@ from future import standard_library standard_library.install_aliases() + + from future.utils import with_metaclass import os.path @@ -120,21 +122,20 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)): entry = models.Entry(case['entry']) given_parameters = case.get('params', case.get('parameters', {})) expected = case['expected'] + should_fail = case.get('should_fail', False) try: params = api.parse_params(given_parameters, self.extra_params) - res = list(self.analyse_entry(entry, params)) + res = list(self.analyse_entries([entry, ], params)) + + if not isinstance(expected, list): + expected = [expected] + utils.check_template(res, expected) + for r in res: + r.validate() except models.Error: - if not expected: + if should_fail: return - raise - if not expected: - raise Exception('This test should have raised an exception.') - - if not isinstance(expected, list): - expected = [expected] - utils.check_template(res, expected) - for r in res: - r.validate() + assert not should_fail def open(self, fpath, *args, **kwargs): if not os.path.isabs(fpath): @@ -241,6 +242,92 @@ class EmotionConversion(Conversion): EmotionConversionPlugin = EmotionConversion +class Box(AnalysisPlugin): + ''' + Black box plugins delegate analysis to a function. + The flow is like so: + + .. code-block:: + + entry --> input() --> box() --> output() --> entry' + + + In other words: their ``input`` method convers a query (entry and a set of parameters) into + the input to the box method. The ``output`` method convers the results given by the box into + an entry that senpy can handle. + ''' + + def input(self, entry, params=None): + '''Transforms a query (entry+param) into an input for the black box''' + return entry + + def output(self, output, entry=None, params=None): + '''Transforms the results of the black box into an entry''' + return output + + def box(self): + raise NotImplementedError('You should define the behavior of this plugin') + + def analyse_entries(self, entries, params): + for entry in entries: + input = self.input(entry=entry, params=params) + results = self.box(input=input, params=params) + yield self.output(output=results, entry=entry, params=params) + + +class TextBox(Box): + '''A black box plugin that takes only text as input''' + + def input(self, entry, params): + entry = super(TextBox, self).input(entry, params) + return entry['nif:isString'] + + +class SentimentBox(TextBox, SentimentPlugin): + ''' + A box plugin where the output is only a polarity label or a tuple (polarity, polarityValue) + ''' + + def output(self, output, entry, **kwargs): + s = models.Sentiment() + try: + label, value = output + except ValueError: + label, value = output, None + s.prov(self) + s.polarity = label + if value is not None: + s.polarityValue = value + entry.sentiments.append(s) + return entry + + +class EmotionBox(TextBox, EmotionPlugin): + ''' + A box plugin where the output is only an a tuple of emotion labels + ''' + + def output(self, output, entry, **kwargs): + if not isinstance(output, list): + output = [output] + s = models.EmotionSet() + entry.emotions.append(s) + for label in output: + e = models.Emotion(onyx__hasEmotionCategory=label) + s.append(e) + return entry + + +class MappingMixin(object): + + def output(self, output, entry, params): + output = self.mappings.get(output, + self.mappings.get('default', output)) + return super(MappingMixin, self).output(output=output, + entry=entry, + params=params) + + class ShelfMixin(object): @property def sh(self): @@ -269,9 +356,13 @@ class ShelfMixin(object): @property def shelf_file(self): - if 'shelf_file' not in self or not self['shelf_file']: - self.shelf_file = os.path.join(self.data_folder, self.name + '.p') - return self['shelf_file'] + if not hasattr(self, '_shelf_file') or not self._shelf_file: + self._shelf_file = os.path.join(self.data_folder, self.name + '.p') + return self._shelf_file + + @shelf_file.setter + def shelf_file(self, value): + self._shelf_file = value def save(self): logger.debug('saving pickle') diff --git a/senpy/plugins/example/rand_plugin.py b/senpy/plugins/example/rand_plugin.py index 37344ff..45b3166 100644 --- a/senpy/plugins/example/rand_plugin.py +++ b/senpy/plugins/example/rand_plugin.py @@ -27,7 +27,7 @@ class Rand(SentimentPlugin): '''Run several random analyses.''' params = dict() results = list() - for i in range(20): + for i in range(50): res = next(self.analyse_entry(Entry(nif__isString="Hello"), params)) res.validate() diff --git a/senpy/utils.py b/senpy/utils.py index 118b7ba..1e8c82a 100644 --- a/senpy/utils.py +++ b/senpy/utils.py @@ -36,6 +36,15 @@ def check_template(indict, template): pprint.pformat(template))) +def convert_dictionary(original, mappings): + result = {} + for key, value in original.items(): + if key in mappings: + key = mappings[key] + result[key] = value + return result + + def easy_load(app=None, plugin_list=None, plugin_folder=None, **kwargs): ''' Run a server with a specific plugin. diff --git a/tests/test_models.py b/tests/test_models.py index 380a559..abdc8f0 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -15,7 +15,8 @@ from senpy.models import (Emotion, SentimentPlugin, Plugins, from_string, - from_dict) + from_dict, + subtypes) from senpy import plugins from pprint import pprint @@ -134,6 +135,11 @@ class ModelsTest(TestCase): s = str(r) assert "_testing" not in s + def test_serialize(self): + for k, v in subtypes().items(): + e = v() + e.serialize() + def test_turtle(self): """Any model should be serializable as a turtle file""" ana = EmotionAnalysis() diff --git a/tests/test_plugins.py b/tests/test_plugins.py index cce80a1..cb080c5 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -13,6 +13,10 @@ from senpy.plugins.conversion.emotion.centroids import CentroidConversion class ShelfDummyPlugin(plugins.SentimentPlugin, plugins.ShelfMixin): '''Dummy plugin for tests.''' + name = 'Shelf' + version = 0 + author = 'the senpy community' + def activate(self, *args, **kwargs): if 'counter' not in self.sh: self.sh['counter'] = 0 @@ -41,6 +45,16 @@ class PluginsTest(TestCase): self.shelf_dir = tempfile.mkdtemp() self.shelf_file = os.path.join(self.shelf_dir, "shelf") + def test_serialize(self): + '''A plugin should be serializable and de-serializable''' + dummy = ShelfDummyPlugin() + dummy.serialize() + + def test_jsonld(self): + '''A plugin should be serializable and de-serializable''' + dummy = ShelfDummyPlugin() + dummy.jsonld() + def test_shelf_file(self): a = ShelfDummyPlugin( info={'name': 'default_shelve_file', @@ -187,6 +201,61 @@ class PluginsTest(TestCase): }) assert 'example' in a.extra_params + def test_box(self): + + class MyBox(plugins.Box): + ''' Vague description''' + + author = 'me' + version = 0 + + def input(self, entry, **kwargs): + return entry.text + + def box(self, input, **kwargs): + return 'SIGN' in input + + def output(self, output, entry, **kwargs): + if output: + entry.myAnnotation = 'DETECTED' + return entry + + test_cases = [ + { + 'input': "nothing here", + 'expected': {'myAnnotation': 'DETECTED'}, + 'should_fail': True + }, { + 'input': "SIGN", + 'expected': {'myAnnotation': 'DETECTED'} + }] + + MyBox().test() + + def test_sentimentbox(self): + + class SentimentBox(plugins.MappingMixin, plugins.SentimentBox): + ''' Vague description''' + + author = 'me' + version = 0 + + mappings = {'happy': 'marl:Positive', 'sad': 'marl:Negative'} + + def box(self, input, **kwargs): + return 'happy' if ':)' in input else 'sad' + + test_cases = [ + { + 'input': 'a happy face :)', + 'polarity': 'marl:Positive' + }, { + 'input': "Nothing", + 'polarity': 'marl:Negative' + }] + + SentimentBox().test() + def test_conversion_centroids(self): info = { "name": "CentroidTest",