Macro commit

* Fixed Options for extra_params in UI
* Enhanced meta-programming for models
* Plugins can be imported from a python file if they're named
`senpy_<whatever>.py>` (no need for `.senpy` anymore!)
* Add docstings and tests to most plugins
* Read plugin description from the docstring
* Refactor code to get rid of unnecessary `.senpy`s
* Load models, plugins and utils into the main namespace (see __init__.py)
* Enhanced plugin development/experience with utils (easy_test, easy_serve)
* Fix bug in check_template that wouldn't check objects
* Make model defaults a private variable
* Add option to list loaded plugins in CLI
* Update docs

example-plugins/README.md

@@ -0,0 +1,13 @@
+This is a collection of plugins that exemplify certain aspects of plugin development with senpy.
+In ascending order of complexity, there are:
+
+* Basic: a very basic analysis that does sentiment analysis based on emojis.
+* Configurable: a version of `basic` with a configurable map of emojis for each sentiment.
+* Parameterized: like `basic_info`, but users set the map in each query (via `extra_parameters`).
+* mynoop: shows how to add a definition file with external requirements for a plugin. Doing this with a python-only module would require moving all imports of the requirements to their functions, which is considered bad practice.
+* Async: a barebones example of training a plugin and analyzing data in parallel.
+
+All of the plugins in this folder include a set of test cases and they are periodically tested with the latest version of senpy.
+
+Additioanlly, for an example of stand-alone plugin that can be tested and deployed with docker, take a look at: lab.cluster.gsi.dit.upm.es/senpy/plugin-example
+ bbm

example-plugins/async_plugin.py

@@ -1,4 +1,4 @@
-from senpy.plugins import AnalysisPlugin
+from senpy import AnalysisPlugin
 
 import multiprocessing
 
@@ -7,10 +7,15 @@ def _train(process_number):
     return process_number
 
 
-class AsyncPlugin(AnalysisPlugin):
+class Async(AnalysisPlugin):
+    '''An example of an asynchronous module'''
+    author = '@balkian'
+    version = '0.2'
+    async = True
+
     def _do_async(self, num_processes):
         pool = multiprocessing.Pool(processes=num_processes)
-        values = pool.map(_train, range(num_processes))
+        values = sorted(pool.map(_train, range(num_processes)))
 
         return values
 
@@ -22,5 +27,11 @@ class AsyncPlugin(AnalysisPlugin):
         entry.async_values = values
         yield entry
 
-    def test(self):
-        pass
+    test_cases = [
+        {
+            'input': 'any',
+            'expected': {
+                'async_values': [0, 1]
+            }
+        }
+    ]

example-plugins/async_plugin/asyncplugin.senpy

@@ -1,8 +0,0 @@
----
-name: Async
-module: asyncplugin
-description: I am async
-author: "@balkian"
-version: '0.1'
-async: true
-extra_params: {}

example-plugins/basic.py

@@ -0,0 +1,23 @@
+#!/usr/local/bin/python
+# coding: utf-8
+
+emoticons = {
+    'marl:Positive': [':)', ':]', '=)', ':D'],
+    'marl:Negative': [':(', ':[', '=(']
+}
+
+emojis = {
+    'marl:Positive': ['😁', '😂', '😃', '😄', '😆', '😅', '😄' '😍'],
+    'marl:Negative': ['😢', '😡', '😠', '😞', '😖', '😔', '😓', '😒']
+}
+
+
+def get_polarity(text, dictionaries=[emoticons, emojis]):
+    polarity = 'marl:Neutral'
+    for dictionary in dictionaries:
+        for label, values in dictionary.items():
+            for emoticon in values:
+                if emoticon and emoticon in text:
+                    polarity = label
+                    break
+    return polarity

example-plugins/basic_plugin.py

@@ -0,0 +1,40 @@
+#!/usr/local/bin/python
+# coding: utf-8
+
+from senpy import easy_test, models, plugins
+
+import basic
+
+
+class Basic(plugins.SentimentPlugin):
+    '''Provides sentiment annotation using a lexicon'''
+
+    author = '@balkian'
+    version = '0.1'
+
+    def analyse_entry(self, entry, params):
+
+        polarity = basic.get_polarity(entry.text)
+
+        s = models.Sentiment(marl__hasPolarity=polarity)
+        s.prov(self)
+        entry.sentiments.append(s)
+        yield entry
+
+    test_cases = [{
+        'input': 'Hello :)',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'So sad :(',
+        'polarity': 'marl:Negative'
+    }, {
+        'input': 'Yay! Emojis  😁',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'But no emoticons 😢',
+        'polarity': 'marl:Negative'
+    }]
+
+
+if __name__ == '__main__':
+    easy_test()

example-plugins/configurable_plugin.py

@@ -0,0 +1,102 @@
+#!/usr/local/bin/python
+# coding: utf-8
+
+from senpy import easy_test, models, plugins
+
+import basic
+
+
+class Dictionary(plugins.SentimentPlugin):
+    '''Sentiment annotation using a configurable lexicon'''
+
+    author = '@balkian'
+    version = '0.2'
+
+    dictionaries = [basic.emojis, basic.emoticons]
+
+    def analyse_entry(self, entry, params):
+        polarity = basic.get_polarity(entry.text, self.dictionaries)
+
+        s = models.Sentiment(marl__hasPolarity=polarity)
+        s.prov(self)
+        entry.sentiments.append(s)
+        yield entry
+
+    test_cases = [{
+        'input': 'Hello :)',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'So sad :(',
+        'polarity': 'marl:Negative'
+    }, {
+        'input': 'Yay! Emojis  😁',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'But no emoticons 😢',
+        'polarity': 'marl:Negative'
+    }]
+
+
+class EmojiOnly(Dictionary):
+    '''Sentiment annotation with a basic lexicon of emojis'''
+    description = 'A plugin'
+    dictionaries = [basic.emojis]
+
+    test_cases = [{
+        'input': 'Hello :)',
+        'polarity': 'marl:Neutral'
+    }, {
+        'input': 'So sad :(',
+        'polarity': 'marl:Neutral'
+    }, {
+        'input': 'Yay! Emojis  😁',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'But no emoticons 😢',
+        'polarity': 'marl:Negative'
+    }]
+
+
+class EmoticonsOnly(Dictionary):
+    '''Sentiment annotation with a basic lexicon of emoticons'''
+    dictionaries = [basic.emoticons]
+
+    test_cases = [{
+        'input': 'Hello :)',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'So sad :(',
+        'polarity': 'marl:Negative'
+    }, {
+        'input': 'Yay! Emojis  😁',
+        'polarity': 'marl:Neutral'
+    }, {
+        'input': 'But no emoticons 😢',
+        'polarity': 'marl:Neutral'
+    }]
+
+
+class Salutes(Dictionary):
+    '''Sentiment annotation with a custom lexicon, for illustration purposes'''
+    dictionaries = [{
+        'marl:Positive': ['Hello', '!'],
+        'marl:Negative': ['sad', ]
+    }]
+
+    test_cases = [{
+        'input': 'Hello :)',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'So sad :(',
+        'polarity': 'marl:Negative'
+    }, {
+        'input': 'Yay! Emojis  😁',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'But no emoticons 😢',
+        'polarity': 'marl:Neutral'
+    }]
+
+
+if __name__ == '__main__':
+    easy_test()

example-plugins/dummy_plugin.py

@@ -0,0 +1,25 @@
+from senpy import AnalysisPlugin, easy
+
+
+class Dummy(AnalysisPlugin):
+    '''This is a dummy self-contained plugin'''
+    author = '@balkian'
+    version = '0.1'
+
+    def analyse_entry(self, entry, params):
+        entry['nif:isString'] = entry['nif:isString'][::-1]
+        entry.reversed = entry.get('reversed', 0) + 1
+        yield entry
+
+    test_cases = [{
+        'entry': {
+            'nif:isString': 'Hello',
+        },
+        'expected': {
+            'nif:isString': 'olleH'
+        }
+    }]
+
+
+if __name__ == '__main__':
+    easy()

example-plugins/dummy_plugin/dummy.py

@@ -1,11 +0,0 @@
-from senpy.plugins import SentimentPlugin
-
-
-class DummyPlugin(SentimentPlugin):
-    def analyse_entry(self, entry, params):
-        entry['nif:isString'] = entry['nif:isString'][::-1]
-        entry.reversed = entry.get('reversed', 0) + 1
-        yield entry
-
-    def test(self):
-        pass

example-plugins/dummy_plugin/dummy.senpy

@@ -1,15 +0,0 @@
-{
-    "name": "Dummy",
-    "module": "dummy",
-    "description": "I am dummy",
-    "author": "@balkian",
-    "version": "0.1",
-    "extra_params": {
-        "example": {
-            "@id": "example_parameter",
-            "aliases": ["example", "ex"],
-            "required": false,
-            "default": 0
-        }
-    }
-}

example-plugins/dummy_plugin/dummy_noinfo.py

@@ -1,27 +0,0 @@
-from senpy.plugins import SentimentPlugin
-
-
-class DummyNoInfo(SentimentPlugin):
-
-    description = 'This is a dummy self-contained plugin'
-    author = '@balkian'
-    version = '0.1'
-
-    def analyse_entry(self, entry, params):
-        entry['nif:isString'] = entry['nif:isString'][::-1]
-        entry.reversed = entry.get('reversed', 0) + 1
-        yield entry
-
-    test_cases = [{
-        "entry": {
-            "nif:isString": "Hello world!"
-        },
-        "expected": [{
-            "nif:isString": "!dlrow olleH"
-        }]
-    }]
-
-
-if __name__ == '__main__':
-    d = DummyNoInfo()
-    d.test()

example-plugins/dummy_plugin/dummy_noinfo.senpy

@@ -1,2 +0,0 @@
-name: DummyNoInfo
-module: dummy_noinfo

example-plugins/dummy_plugin/dummy_required.senpy

@@ -1,14 +0,0 @@
-{
-    "name": "DummyRequired",
-    "module": "dummy",
-    "description": "I am dummy",
-    "author": "@balkian",
-    "version": "0.1",
-    "extra_params": {
-        "example": {
-            "@id": "example_parameter",
-            "aliases": ["example", "ex"],
-            "required": true
-        }
-    }
-}

example-plugins/dummy_required_plugin.py

@@ -0,0 +1,40 @@
+from senpy import AnalysisPlugin, easy
+
+
+class DummyRequired(AnalysisPlugin):
+    '''This is a dummy self-contained plugin'''
+    author = '@balkian'
+    version = '0.1'
+    extra_params = {
+        'example': {
+            'description': 'An example parameter',
+            'required': True,
+            'options': ['a', 'b']
+        }
+    }
+
+    def analyse_entry(self, entry, params):
+        entry['nif:isString'] = entry['nif:isString'][::-1]
+        entry.reversed = entry.get('reversed', 0) + 1
+        yield entry
+
+    test_cases = [{
+        'entry': {
+            'nif:isString': 'Hello',
+        },
+        'expected': None
+    }, {
+        'entry': {
+            'nif:isString': 'Hello',
+        },
+        'params': {
+            'example': 'a'
+        },
+        'expected': {
+            'nif:isString': 'olleH'
+        }
+    }]
+
+
+if __name__ == '__main__':
+    easy()

example-plugins/mynoop.py

@@ -0,0 +1,24 @@
+import noop
+from senpy.plugins import SentimentPlugin
+
+
+class NoOp(SentimentPlugin):
+    '''This plugin does nothing. Literally nothing.'''
+
+    version = 0
+
+    def analyse_entry(self, entry, *args, **kwargs):
+        yield entry
+
+    def test(self):
+        print(dir(noop))
+        super(NoOp, self).test()
+
+    test_cases = [{
+        'entry': {
+            'nif:isString': 'hello'
+        },
+        'expected': {
+            'nif:isString': 'hello'
+        }
+    }]

example-plugins/mynoop.senpy

@@ -0,0 +1,3 @@
+module: mynoop
+requirements:
+ - noop

example-plugins/noop/noop_plugin.py

@@ -1,5 +0,0 @@
-from senpy.plugins import SentimentPlugin
-
-
-class NoOp(SentimentPlugin):
-    import noop

example-plugins/parameterized_plugin.py

@@ -0,0 +1,63 @@
+#!/usr/local/bin/python
+# coding: utf-8
+
+from senpy import easy_test, models, plugins
+
+import basic
+
+
+class ParameterizedDictionary(plugins.SentimentPlugin):
+
+    description = 'This is a basic self-contained plugin'
+    author = '@balkian'
+    version = '0.2'
+
+    extra_params = {
+        'positive-words': {
+            'description': 'Comma-separated list of words that are considered positive',
+            'aliases': ['positive'],
+            'required': True
+        },
+        'negative-words': {
+            'description': 'Comma-separated list of words that are considered negative',
+            'aliases': ['negative'],
+            'required': False
+        }
+    }
+
+    def analyse_entry(self, entry, params):
+        positive_words = params['positive-words'].split(',')
+        negative_words = params['negative-words'].split(',')
+        dictionary = {
+            'marl:Positive': positive_words,
+            'marl:Negative': negative_words,
+        }
+        polarity = basic.get_polarity(entry.text, [dictionary])
+
+        s = models.Sentiment(marl__hasPolarity=polarity)
+        s.prov(self)
+        entry.sentiments.append(s)
+        yield entry
+
+    test_cases = [
+        {
+            'input': 'Hello :)',
+            'polarity': 'marl:Positive',
+            'parameters': {
+                'positive': "Hello,:)",
+                'negative': "sad,:()"
+            }
+        },
+        {
+            'input': 'Hello :)',
+            'polarity': 'marl:Negative',
+            'parameters': {
+                'positive': "",
+                'negative': "Hello"
+            }
+        }
+    ]
+
+
+if __name__ == '__main__':
+    easy_test()

example-plugins/sleep_plugin.py

@@ -0,0 +1,27 @@
+from senpy.plugins import AnalysisPlugin
+from time import sleep
+
+
+class Sleep(AnalysisPlugin):
+    '''Dummy plugin to test async'''
+    author = "@balkian"
+    version = "0.2"
+    timeout = 0.05
+    extra_params = {
+        "timeout": {
+            "@id": "timeout_sleep",
+            "aliases": ["timeout", "to"],
+            "required": False,
+            "default": 0
+        }
+    }
+
+    def activate(self, *args, **kwargs):
+        sleep(self.timeout)
+
+    def analyse_entry(self, entry, params):
+        sleep(float(params.get("timeout", self.timeout)))
+        yield entry
+
+    def test(self):
+        pass

example-plugins/sleep_plugin/sleep.py

@@ -1,14 +0,0 @@
-from senpy.plugins import AnalysisPlugin
-from time import sleep
-
-
-class SleepPlugin(AnalysisPlugin):
-    def activate(self, *args, **kwargs):
-        sleep(self.timeout)
-
-    def analyse_entry(self, entry, params):
-        sleep(float(params.get("timeout", self.timeout)))
-        yield entry
-
-    def test(self):
-        pass

example-plugins/sleep_plugin/sleep.senpy

@@ -1,16 +0,0 @@
-{
-    "name": "Sleep",
-    "module": "sleep",
-    "description": "I am dummy",
-    "author": "@balkian",
-    "version": "0.1",
-    "timeout": 0.05,
-    "extra_params": {
-        "timeout": {
-            "@id": "timeout_sleep",
-            "aliases": ["timeout", "to"],
-            "required": false,
-            "default": 0
-        }
-     }
-}