diff --git a/docs/plugins-definition.rst b/docs/plugins-definition.rst new file mode 100644 index 0000000..706195f --- /dev/null +++ b/docs/plugins-definition.rst @@ -0,0 +1,113 @@ +Advanced plugin definition +-------------------------- +In addition to finding plugins defined in source code files, senpy can also load a special type of definition file (`.senpy` files). +This used to be the only mechanism for loading in earlier versions of senpy. + +The definition file contains basic information + +Lastly, it is also possible to add new plugins programmatically. + +.. contents:: :local: + +What is a plugin? +================= + +A plugin is a program that, given a text, will add annotations to it. +In practice, a plugin consists of at least two files: + +- Definition file: a `.senpy` file that describes the plugin (e.g. what input parameters it accepts, what emotion model it uses). +- Python module: the actual code that will add annotations to each input. + +This separation allows us to deploy plugins that use the same code but employ different parameters. +For instance, one could use the same classifier and processing in several plugins, but train with different datasets. +This scenario is particularly useful for evaluation purposes. + +The only limitation is that the name of each plugin needs to be unique. + +Definition files +================ + +The definition file complements and overrides the attributes provided by the plugin. +It can be written in YAML or JSON. +The most important attributes are: + +* **name**: unique name that senpy will use internally to identify the plugin. +* **module**: indicates the module that contains the plugin code, which will be automatically loaded by senpy. +* **version** +* extra_params: to add parameters to the senpy API when this plugin is requested. Those parameters may be required, and have aliased names. For instance: + + .. code:: yaml + + extra_params: + hello_param: + aliases: # required + - hello_param + - hello + required: true + default: Hi you + values: + - Hi you + - Hello y'all + - Howdy + +A complete example: + +.. code:: yaml + + name: + module: + version: 0.1 + +And the json equivalent: + +.. code:: json + + { + "name": "", + "module": "", + "version": "0.1" + } + + +Example plugin with a definition file +===================================== + +In this section, we will implement a basic sentiment analysis plugin. +To determine the polarity of each entry, the plugin will compare the length of the string to a threshold. +This threshold will be included in the definition file. + +The definition file would look like this: + +.. code:: yaml + + name: helloworld + module: helloworld + version: 0.0 + threshold: 10 + description: Hello World + +Now, in a file named ``helloworld.py``: + +.. code:: python + + #!/bin/env python + #helloworld.py + + from senpy import AnalysisPlugin + from senpy import Sentiment + + + class HelloWorld(AnalysisPlugin): + + def analyse_entry(entry, params): + '''Basically do nothing with each entry''' + + sentiment = Sentiment() + if len(entry.text) < self.threshold: + sentiment['marl:hasPolarity'] = 'marl:Positive' + else: + sentiment['marl:hasPolarity'] = 'marl:Negative' + entry.sentiments.append(sentiment) + yield entry + +The complete code of the example plugin is available `here `__. diff --git a/docs/plugins.rst b/docs/plugins.rst index e5cf91b..e56ecec 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -1,6 +1,8 @@ Developing new plugins ---------------------- -This document describes how to develop a new analysis plugin. For an example of conversion plugins, see :doc:`conversion`. +This document contains the minimum to get you started with developing new analysis plugin. +For an example of conversion plugins, see :doc:`conversion`. +For a description of definition files, see :doc:`plugins-definition`. A more step-by-step tutorial with slides is available `here `__ @@ -9,83 +11,29 @@ A more step-by-step tutorial with slides is available `here - module: - version: 0.1 - -And the json equivalent: - -.. code:: json - - { - "name": "", - "module": "", - "version": "0.1" - } - - -Plugins Code -============ - -The basic methods in a plugin are: +What is an entry? +================= -* __init__ -* activate: used to load memory-hungry resources -* deactivate: used to free up resources -* analyse_entry: called in every user requests. It takes two parameters: ``Entry``, the entry object, and ``params``, the parameters supplied by the user. It should yield one or more ``Entry`` objects. +Entries are objects that can be annotated. +In general, they will be a piece of text. +By default, entries are `NIF contexts `_ represented in JSON-LD format. +It is a dictionary/JSON object that looks like this: -Plugins are loaded asynchronously, so don't worry if the activate method takes too long. The plugin will be marked as activated once it is finished executing the method. + .. code:: python -Entries -======= + { + "@id": "", + "nif:isString": "input text", + "sentiments": [ { + ... + } + ], + ... + } -Entries are objects that can be annotated. -By default, entries are `NIF contexts `_ represented in JSON-LD format. Annotations are added to the object like this: .. code:: python @@ -100,96 +48,111 @@ The value may be any valid JSON-LD dictionary. For simplicity, senpy includes a series of models by default in the ``senpy.models`` module. -Example plugin -============== +What are annotations? +===================== +They are objects just like entries. +Senpy ships with several default annotations, including: ``Sentiment``, ``Emotion``, ``EmotionSet``...jk bb -In this section, we will implement a basic sentiment analysis plugin. -To determine the polarity of each entry, the plugin will compare the length of the string to a threshold. -This threshold will be included in the definition file. -The definition file would look like this: +What's a plugin made of? +======================== -.. code:: yaml +When receiving a query, senpy selects what plugin or plugins should process each entry, and in what order. +It also makes sure the every entry and the parameters provided by the user meet the plugin requirements. - name: helloworld - module: helloworld - version: 0.0 - threshold: 10 - description: Hello World +Hence, two parts are necessary: 1) the code that will process the entry, and 2) some attributes and metadata that will tell senpy how to interact with the plugin. -Now, in a file named ``helloworld.py``: +In practice, this is what a plugin looks like, tests included: -.. code:: python - #!/bin/env python - #helloworld.py +.. literalinclude:: ../senpy/plugins/example/rand_plugin.py + :emphasize-lines: 5-11 + :language: python - from senpy.plugins import AnalysisPlugin - from senpy.models import Sentiment +The lines highlighted contain some information about the plugin. +In particular, the following information is mandatory: - class HelloWorld(AnalysisPlugin): +* A unique name for the class. In our example, Rand. +* The subclass/type of plugin. This is typically either `SentimentPlugin` or `EmotionPlugin`. However, new types of plugin can be created for different annotations. The only requirement is that these new types inherit from `senpy.Analysis` +* A description of the plugin. This can be done simply by adding a doc to the class. +* A version, which should get updated. +* An author name. - def analyse_entry(entry, params): - '''Basically do nothing with each entry''' - sentiment = Sentiment() - if len(entry.text) < self.threshold: - sentiment['marl:hasPolarity'] = 'marl:Positive' - else: - sentiment['marl:hasPolarity'] = 'marl:Negative' - entry.sentiments.append(sentiment) - yield entry +Plugins Code +============ -The complete code of the example plugin is available `here `__. +The basic methods in a plugin are: -Loading data and files -====================== +* analyse_entry: called in every user requests. It takes two parameters: ``Entry``, the entry object, and ``params``, the parameters supplied by the user. It should yield one or more ``Entry`` objects. +* activate: used to load memory-hungry resources. For instance, to train a classifier. +* deactivate: used to free up resources when the plugin is no longer needed. -Most plugins will need access to files (dictionaries, lexicons, etc.). -It is good practice to specify the paths of these files in the plugin configuration, so the same code can be reused with different resources. +Plugins are loaded asynchronously, so don't worry if the activate method takes too long. The plugin will be marked as activated once it is finished executing the method. -.. code:: yaml +How does senpy find modules? +============================ - name: dictworld - module: dictworld - dictionary_path: - -The path can be either absolute, or relative. +Senpy looks for files of two types: + +* Python files of the form `senpy_.py` or `_plugin.py`. In these files, it will look for: 1) Instances that inherit from `senpy.Plugin`, or subclasses of `senpy.Plugin` that can be initialized without a configuration file. i.e. classes that contain all the required attributes for a plugin. +* Plugin definition files (see :doc:`advanced-plugins`) -From absolute paths -??????????????????? +Defining additional parameters +============================== -Absolute paths (such as ``/data/dictionary.csv`` are straightfoward: +Your plugin may ask for additional parameters from the users of the service by using the attribute ``extra_params`` in your plugin definition. +It takes a dictionary, where the keys are the name of the argument/parameter, and the value has the following fields: + +* aliases: the different names which can be used in the request to use the parameter. +* required: if set to true, users need to provide this parameter unless a default is set. +* options: the different acceptable values of the parameter (i.e. an enum). If set, the value provided must match one of the options. +* default: the default value of the parameter, if none is provided in the request. .. code:: python - with open(os.path.join(self.dictionary_path) as f: - ... + "extra_params":{ + "language": { + "aliases": ["language", "lang", "l"], + "required": True, + "options": ["es", "en"], + "default": "es" + } + } -From relative paths -??????????????????? -Since plugins are loading dynamically, relative paths will refer to the current working directory. -Instead, what you usually want is to load files *relative to the plugin source folder*, like so: -:: +Loading data and files +====================== + +Most plugins will need access to files (dictionaries, lexicons, etc.). +These files are usually heavy or under a license that does not allow redistribution. +For this reason, senpy has a `data_folder` that is separated from the source files. +The location of this folder is controlled programmatically or by setting the `SENPY_DATA` environment variable. - . - .. - plugin.senpy - plugin.py - dictionary.csv +Plugins have a convenience function `self.open` which will automatically prepend the data folder to relative paths: -For this, we need to first get the path of your source folder first, like so: .. code:: python - import os - root = os.path.realpath(__file__) - with open(os.path.join(root, self.dictionary_path) as f: - ... + import os + + + class PluginWithResources(AnalysisPlugin): + file_in_data = + file_in_sources = + + def activate(self): + with self.open(self.file_in_data) as f: + self._classifier = train_from_file(f) + file_in_source = os.path.join(self.get_folder(), self.file_in_sources) + with self.open(file_in_source) as f: + pass + + +It is good practice to specify the paths of these files in the plugin configuration, so the same code can be reused with different resources. Docker image @@ -199,8 +162,17 @@ Add the following dockerfile to your project to generate a docker image with you .. code:: dockerfile - FROM gsiupm/senpy:0.8.8 + FROM gsiupm/senpy + +Once you make sure your plugin works with a specific version of senpy, modify that file to make sure your build will work even if senpy gets updated. +e.g.: + + +.. code:: dockerfile + + FROM gsiupm/senpy:1.0.1 + This will copy your source folder to the image, and install all dependencies. Now, to build an image: @@ -215,7 +187,7 @@ And you can run it with: docker run -p 5000:5000 gsiupm/exampleplugin -If the plugin non-source files (:ref:`loading data and files`), the recommended way is to use absolute paths. +If the plugin uses non-source files (:ref:`loading data and files`), the recommended way is to use `SENPY_DATA` folder. Data can then be mounted in the container or added to the image. The former is recommended for open source plugins with licensed resources, whereas the latter is the most convenient and can be used for private images. @@ -229,7 +201,7 @@ Adding data to the image: .. code:: dockerfile - FROM gsiupm/senpy:0.8.8 + FROM gsiupm/senpy:1.0.1 COPY data / F.A.Q. @@ -245,7 +217,7 @@ Why does the analyse function yield instead of return? ?????????????????????????????????????????????????????? This is so that plugins may add new entries to the response or filter some of them. -For instance, a `context detection` plugin may add a new entry for each context in the original entry. +For instance, a chunker may split one entry into several. On the other hand, a conversion plugin may leave out those entries that do not contain relevant information. @@ -275,11 +247,13 @@ Training a classifier can be time time consuming. To avoid running the training def deactivate(self): self.close() -You can specify a 'shelf_file' in your .senpy file. By default the ShelfMixin creates a file based on the plugin name and stores it in that plugin's folder. + +By default the ShelfMixin creates a file based on the plugin name and stores it in that plugin's folder. +However, you can manually specify a 'shelf_file' in your .senpy file. Shelves may get corrupted if the plugin exists unexpectedly. A corrupt shelf prevents the plugin from loading. -If you do not care about the pickle, you can force your plugin to remove the corrupted file and load anyway, set the 'force_shelf' to True in your .senpy file. +If you do not care about the data in the shelf, you can force your plugin to remove the corrupted file and load anyway, set the 'force_shelf' to True in your plugin and start it again. How can I turn an external service into a plugin? ????????????????????????????????????????????????? @@ -313,50 +287,11 @@ This example ilustrate how to implement a plugin that accesses the Sentiment140 prefix=p, marl__hasPolarity=polarity, marl__polarityValue=polarity_value) - sentiment.prov__wasGeneratedBy = self.id + sentiment.prov(self) entry.sentiments.append(sentiment) yield entry -Can my plugin require additional parameters from the user? -?????????????????????????????????????????????????????????? - -You can add extra parameters in the definition file under the attribute ``extra_params``. -It takes a dictionary, where the keys are the name of the argument/parameter, and the value has the following fields: - -* aliases: the different names which can be used in the request to use the parameter. -* required: if set to true, users need to provide this parameter unless a default is set. -* options: the different acceptable values of the parameter (i.e. an enum). If set, the value provided must match one of the options. -* default: the default value of the parameter, if none is provided in the request. - -.. code:: python - - extra_params - language: - aliases: - - language - - lang - - l - required: true, - options: - - es - - en - default: es - -This example shows how to introduce a parameter associated with language. -The extraction of this paremeter is used in the analyse method of the Plugin interface. - -.. code:: python - - lang = params.get("language") - -Where can I set up variables for using them in my plugin? -????????????????????????????????????????????????????????? - -You can add these variables in the definition file with the structure of attribute-value pairs. - -Every field added to the definition file is available to the plugin instance. - Can I activate a DEBUG mode for my plugin? ??????????????????????????????????????????? @@ -371,7 +306,7 @@ Additionally, with the ``--pdb`` option you will be dropped into a pdb post mort .. code:: bash - senpy --pdb + python -m pdb yourplugin.py Where can I find more code examples? ???????????????????????????????????? diff --git a/docs/server.rst b/docs/server.rst index 98f4f3c..0255c0d 100644 --- a/docs/server.rst +++ b/docs/server.rst @@ -7,21 +7,29 @@ The senpy server is launched via the `senpy` command: usage: senpy [-h] [--level logging_level] [--debug] [--default-plugins] [--host HOST] [--port PORT] [--plugins-folder PLUGINS_FOLDER] - [--only-install] + [--only-install] [--only-list] [--data-folder DATA_FOLDER] + [--threaded] [--version] Run a Senpy server optional arguments: - -h, --help show this help message and exit - --level logging_level, -l logging_level + -h, --help show this help message and exit + --level logging_level, -l logging_level Logging level - --debug, -d Run the application in debug mode - --default-plugins Load the default plugins - --host HOST Use 0.0.0.0 to accept requests from any host. - --port PORT, -p PORT Port to listen on. - --plugins-folder PLUGINS_FOLDER, -f PLUGINS_FOLDER + --debug, -d Run the application in debug mode + --default-plugins Load the default plugins + --host HOST Use 0.0.0.0 to accept requests from any host. + --port PORT, -p PORT Port to listen on. + --plugins-folder PLUGINS_FOLDER, -f PLUGINS_FOLDER Where to look for plugins. - --only-install, -i Do not run a server, only install plugin dependencies + --only-install, -i Do not run a server, only install plugin dependencies + --only-list, --list Do not run a server, only list plugins found + --data-folder DATA_FOLDER, --data DATA_FOLDER + Where to look for data. It be set with the SENPY_DATA + environment variable as well. + --threaded Run a threaded server + --version, -v Output the senpy version and exit + When launched, the server will recursively look for plugins in the specified plugins folder (the current working directory by default). diff --git a/example-plugins/README.md b/example-plugins/README.md new file mode 100644 index 0000000..5316686 --- /dev/null +++ b/example-plugins/README.md @@ -0,0 +1,13 @@ +This is a collection of plugins that exemplify certain aspects of plugin development with senpy. +In ascending order of complexity, there are: + +* Basic: a very basic analysis that does sentiment analysis based on emojis. +* Configurable: a version of `basic` with a configurable map of emojis for each sentiment. +* Parameterized: like `basic_info`, but users set the map in each query (via `extra_parameters`). +* mynoop: shows how to add a definition file with external requirements for a plugin. Doing this with a python-only module would require moving all imports of the requirements to their functions, which is considered bad practice. +* Async: a barebones example of training a plugin and analyzing data in parallel. + +All of the plugins in this folder include a set of test cases and they are periodically tested with the latest version of senpy. + +Additioanlly, for an example of stand-alone plugin that can be tested and deployed with docker, take a look at: lab.cluster.gsi.dit.upm.es/senpy/plugin-example + bbm diff --git a/example-plugins/async_plugin/asyncplugin.py b/example-plugins/async_plugin.py similarity index 51% rename from example-plugins/async_plugin/asyncplugin.py rename to example-plugins/async_plugin.py index a37f2cb..e274425 100644 --- a/example-plugins/async_plugin/asyncplugin.py +++ b/example-plugins/async_plugin.py @@ -1,4 +1,4 @@ -from senpy.plugins import AnalysisPlugin +from senpy import AnalysisPlugin import multiprocessing @@ -7,10 +7,15 @@ def _train(process_number): return process_number -class AsyncPlugin(AnalysisPlugin): +class Async(AnalysisPlugin): + '''An example of an asynchronous module''' + author = '@balkian' + version = '0.2' + async = True + def _do_async(self, num_processes): pool = multiprocessing.Pool(processes=num_processes) - values = pool.map(_train, range(num_processes)) + values = sorted(pool.map(_train, range(num_processes))) return values @@ -22,5 +27,11 @@ class AsyncPlugin(AnalysisPlugin): entry.async_values = values yield entry - def test(self): - pass + test_cases = [ + { + 'input': 'any', + 'expected': { + 'async_values': [0, 1] + } + } + ] diff --git a/example-plugins/async_plugin/asyncplugin.senpy b/example-plugins/async_plugin/asyncplugin.senpy deleted file mode 100644 index 8c71849..0000000 --- a/example-plugins/async_plugin/asyncplugin.senpy +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: Async -module: asyncplugin -description: I am async -author: "@balkian" -version: '0.1' -async: true -extra_params: {} \ No newline at end of file diff --git a/example-plugins/basic.py b/example-plugins/basic.py new file mode 100644 index 0000000..db67b35 --- /dev/null +++ b/example-plugins/basic.py @@ -0,0 +1,23 @@ +#!/usr/local/bin/python +# coding: utf-8 + +emoticons = { + 'marl:Positive': [':)', ':]', '=)', ':D'], + 'marl:Negative': [':(', ':[', '=('] +} + +emojis = { + 'marl:Positive': ['😁', '😂', '😃', '😄', '😆', '😅', '😄' '😍'], + 'marl:Negative': ['😢', '😡', '😠', '😞', '😖', '😔', '😓', '😒'] +} + + +def get_polarity(text, dictionaries=[emoticons, emojis]): + polarity = 'marl:Neutral' + for dictionary in dictionaries: + for label, values in dictionary.items(): + for emoticon in values: + if emoticon and emoticon in text: + polarity = label + break + return polarity diff --git a/example-plugins/basic_plugin.py b/example-plugins/basic_plugin.py new file mode 100644 index 0000000..a760e73 --- /dev/null +++ b/example-plugins/basic_plugin.py @@ -0,0 +1,40 @@ +#!/usr/local/bin/python +# coding: utf-8 + +from senpy import easy_test, models, plugins + +import basic + + +class Basic(plugins.SentimentPlugin): + '''Provides sentiment annotation using a lexicon''' + + author = '@balkian' + version = '0.1' + + def analyse_entry(self, entry, params): + + polarity = basic.get_polarity(entry.text) + + s = models.Sentiment(marl__hasPolarity=polarity) + s.prov(self) + entry.sentiments.append(s) + yield entry + + test_cases = [{ + 'input': 'Hello :)', + 'polarity': 'marl:Positive' + }, { + 'input': 'So sad :(', + 'polarity': 'marl:Negative' + }, { + 'input': 'Yay! Emojis 😁', + 'polarity': 'marl:Positive' + }, { + 'input': 'But no emoticons 😢', + 'polarity': 'marl:Negative' + }] + + +if __name__ == '__main__': + easy_test() diff --git a/example-plugins/configurable_plugin.py b/example-plugins/configurable_plugin.py new file mode 100644 index 0000000..688150e --- /dev/null +++ b/example-plugins/configurable_plugin.py @@ -0,0 +1,102 @@ +#!/usr/local/bin/python +# coding: utf-8 + +from senpy import easy_test, models, plugins + +import basic + + +class Dictionary(plugins.SentimentPlugin): + '''Sentiment annotation using a configurable lexicon''' + + author = '@balkian' + version = '0.2' + + dictionaries = [basic.emojis, basic.emoticons] + + def analyse_entry(self, entry, params): + polarity = basic.get_polarity(entry.text, self.dictionaries) + + s = models.Sentiment(marl__hasPolarity=polarity) + s.prov(self) + entry.sentiments.append(s) + yield entry + + test_cases = [{ + 'input': 'Hello :)', + 'polarity': 'marl:Positive' + }, { + 'input': 'So sad :(', + 'polarity': 'marl:Negative' + }, { + 'input': 'Yay! Emojis 😁', + 'polarity': 'marl:Positive' + }, { + 'input': 'But no emoticons 😢', + 'polarity': 'marl:Negative' + }] + + +class EmojiOnly(Dictionary): + '''Sentiment annotation with a basic lexicon of emojis''' + description = 'A plugin' + dictionaries = [basic.emojis] + + test_cases = [{ + 'input': 'Hello :)', + 'polarity': 'marl:Neutral' + }, { + 'input': 'So sad :(', + 'polarity': 'marl:Neutral' + }, { + 'input': 'Yay! Emojis 😁', + 'polarity': 'marl:Positive' + }, { + 'input': 'But no emoticons 😢', + 'polarity': 'marl:Negative' + }] + + +class EmoticonsOnly(Dictionary): + '''Sentiment annotation with a basic lexicon of emoticons''' + dictionaries = [basic.emoticons] + + test_cases = [{ + 'input': 'Hello :)', + 'polarity': 'marl:Positive' + }, { + 'input': 'So sad :(', + 'polarity': 'marl:Negative' + }, { + 'input': 'Yay! Emojis 😁', + 'polarity': 'marl:Neutral' + }, { + 'input': 'But no emoticons 😢', + 'polarity': 'marl:Neutral' + }] + + +class Salutes(Dictionary): + '''Sentiment annotation with a custom lexicon, for illustration purposes''' + dictionaries = [{ + 'marl:Positive': ['Hello', '!'], + 'marl:Negative': ['sad', ] + }] + + test_cases = [{ + 'input': 'Hello :)', + 'polarity': 'marl:Positive' + }, { + 'input': 'So sad :(', + 'polarity': 'marl:Negative' + }, { + 'input': 'Yay! Emojis 😁', + 'polarity': 'marl:Positive' + }, { + 'input': 'But no emoticons 😢', + 'polarity': 'marl:Neutral' + }] + + +if __name__ == '__main__': + easy_test() diff --git a/example-plugins/dummy_plugin.py b/example-plugins/dummy_plugin.py new file mode 100644 index 0000000..7ee2754 --- /dev/null +++ b/example-plugins/dummy_plugin.py @@ -0,0 +1,25 @@ +from senpy import AnalysisPlugin, easy + + +class Dummy(AnalysisPlugin): + '''This is a dummy self-contained plugin''' + author = '@balkian' + version = '0.1' + + def analyse_entry(self, entry, params): + entry['nif:isString'] = entry['nif:isString'][::-1] + entry.reversed = entry.get('reversed', 0) + 1 + yield entry + + test_cases = [{ + 'entry': { + 'nif:isString': 'Hello', + }, + 'expected': { + 'nif:isString': 'olleH' + } + }] + + +if __name__ == '__main__': + easy() diff --git a/example-plugins/dummy_plugin/dummy.py b/example-plugins/dummy_plugin/dummy.py deleted file mode 100644 index 8dd987f..0000000 --- a/example-plugins/dummy_plugin/dummy.py +++ /dev/null @@ -1,11 +0,0 @@ -from senpy.plugins import SentimentPlugin - - -class DummyPlugin(SentimentPlugin): - def analyse_entry(self, entry, params): - entry['nif:isString'] = entry['nif:isString'][::-1] - entry.reversed = entry.get('reversed', 0) + 1 - yield entry - - def test(self): - pass diff --git a/example-plugins/dummy_plugin/dummy.senpy b/example-plugins/dummy_plugin/dummy.senpy deleted file mode 100644 index ea0c405..0000000 --- a/example-plugins/dummy_plugin/dummy.senpy +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "Dummy", - "module": "dummy", - "description": "I am dummy", - "author": "@balkian", - "version": "0.1", - "extra_params": { - "example": { - "@id": "example_parameter", - "aliases": ["example", "ex"], - "required": false, - "default": 0 - } - } -} diff --git a/example-plugins/dummy_plugin/dummy_noinfo.py b/example-plugins/dummy_plugin/dummy_noinfo.py deleted file mode 100644 index 5a46c95..0000000 --- a/example-plugins/dummy_plugin/dummy_noinfo.py +++ /dev/null @@ -1,27 +0,0 @@ -from senpy.plugins import SentimentPlugin - - -class DummyNoInfo(SentimentPlugin): - - description = 'This is a dummy self-contained plugin' - author = '@balkian' - version = '0.1' - - def analyse_entry(self, entry, params): - entry['nif:isString'] = entry['nif:isString'][::-1] - entry.reversed = entry.get('reversed', 0) + 1 - yield entry - - test_cases = [{ - "entry": { - "nif:isString": "Hello world!" - }, - "expected": [{ - "nif:isString": "!dlrow olleH" - }] - }] - - -if __name__ == '__main__': - d = DummyNoInfo() - d.test() diff --git a/example-plugins/dummy_plugin/dummy_noinfo.senpy b/example-plugins/dummy_plugin/dummy_noinfo.senpy deleted file mode 100644 index da4e83e..0000000 --- a/example-plugins/dummy_plugin/dummy_noinfo.senpy +++ /dev/null @@ -1,2 +0,0 @@ -name: DummyNoInfo -module: dummy_noinfo diff --git a/example-plugins/dummy_plugin/dummy_required.senpy b/example-plugins/dummy_plugin/dummy_required.senpy deleted file mode 100644 index 3e361f6..0000000 --- a/example-plugins/dummy_plugin/dummy_required.senpy +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "DummyRequired", - "module": "dummy", - "description": "I am dummy", - "author": "@balkian", - "version": "0.1", - "extra_params": { - "example": { - "@id": "example_parameter", - "aliases": ["example", "ex"], - "required": true - } - } -} diff --git a/example-plugins/dummy_required_plugin.py b/example-plugins/dummy_required_plugin.py new file mode 100644 index 0000000..bc61d38 --- /dev/null +++ b/example-plugins/dummy_required_plugin.py @@ -0,0 +1,40 @@ +from senpy import AnalysisPlugin, easy + + +class DummyRequired(AnalysisPlugin): + '''This is a dummy self-contained plugin''' + author = '@balkian' + version = '0.1' + extra_params = { + 'example': { + 'description': 'An example parameter', + 'required': True, + 'options': ['a', 'b'] + } + } + + def analyse_entry(self, entry, params): + entry['nif:isString'] = entry['nif:isString'][::-1] + entry.reversed = entry.get('reversed', 0) + 1 + yield entry + + test_cases = [{ + 'entry': { + 'nif:isString': 'Hello', + }, + 'expected': None + }, { + 'entry': { + 'nif:isString': 'Hello', + }, + 'params': { + 'example': 'a' + }, + 'expected': { + 'nif:isString': 'olleH' + } + }] + + +if __name__ == '__main__': + easy() diff --git a/example-plugins/mynoop.py b/example-plugins/mynoop.py new file mode 100644 index 0000000..98b05b7 --- /dev/null +++ b/example-plugins/mynoop.py @@ -0,0 +1,24 @@ +import noop +from senpy.plugins import SentimentPlugin + + +class NoOp(SentimentPlugin): + '''This plugin does nothing. Literally nothing.''' + + version = 0 + + def analyse_entry(self, entry, *args, **kwargs): + yield entry + + def test(self): + print(dir(noop)) + super(NoOp, self).test() + + test_cases = [{ + 'entry': { + 'nif:isString': 'hello' + }, + 'expected': { + 'nif:isString': 'hello' + } + }] diff --git a/example-plugins/mynoop.senpy b/example-plugins/mynoop.senpy new file mode 100644 index 0000000..2021b40 --- /dev/null +++ b/example-plugins/mynoop.senpy @@ -0,0 +1,3 @@ +module: mynoop +requirements: + - noop \ No newline at end of file diff --git a/example-plugins/noop/noop_plugin.py b/example-plugins/noop/noop_plugin.py deleted file mode 100644 index 8fd9b56..0000000 --- a/example-plugins/noop/noop_plugin.py +++ /dev/null @@ -1,5 +0,0 @@ -from senpy.plugins import SentimentPlugin - - -class NoOp(SentimentPlugin): - import noop diff --git a/example-plugins/parameterized_plugin.py b/example-plugins/parameterized_plugin.py new file mode 100644 index 0000000..856ead0 --- /dev/null +++ b/example-plugins/parameterized_plugin.py @@ -0,0 +1,63 @@ +#!/usr/local/bin/python +# coding: utf-8 + +from senpy import easy_test, models, plugins + +import basic + + +class ParameterizedDictionary(plugins.SentimentPlugin): + + description = 'This is a basic self-contained plugin' + author = '@balkian' + version = '0.2' + + extra_params = { + 'positive-words': { + 'description': 'Comma-separated list of words that are considered positive', + 'aliases': ['positive'], + 'required': True + }, + 'negative-words': { + 'description': 'Comma-separated list of words that are considered negative', + 'aliases': ['negative'], + 'required': False + } + } + + def analyse_entry(self, entry, params): + positive_words = params['positive-words'].split(',') + negative_words = params['negative-words'].split(',') + dictionary = { + 'marl:Positive': positive_words, + 'marl:Negative': negative_words, + } + polarity = basic.get_polarity(entry.text, [dictionary]) + + s = models.Sentiment(marl__hasPolarity=polarity) + s.prov(self) + entry.sentiments.append(s) + yield entry + + test_cases = [ + { + 'input': 'Hello :)', + 'polarity': 'marl:Positive', + 'parameters': { + 'positive': "Hello,:)", + 'negative': "sad,:()" + } + }, + { + 'input': 'Hello :)', + 'polarity': 'marl:Negative', + 'parameters': { + 'positive': "", + 'negative': "Hello" + } + } + ] + + +if __name__ == '__main__': + easy_test() diff --git a/example-plugins/sleep_plugin.py b/example-plugins/sleep_plugin.py new file mode 100644 index 0000000..95d4381 --- /dev/null +++ b/example-plugins/sleep_plugin.py @@ -0,0 +1,27 @@ +from senpy.plugins import AnalysisPlugin +from time import sleep + + +class Sleep(AnalysisPlugin): + '''Dummy plugin to test async''' + author = "@balkian" + version = "0.2" + timeout = 0.05 + extra_params = { + "timeout": { + "@id": "timeout_sleep", + "aliases": ["timeout", "to"], + "required": False, + "default": 0 + } + } + + def activate(self, *args, **kwargs): + sleep(self.timeout) + + def analyse_entry(self, entry, params): + sleep(float(params.get("timeout", self.timeout))) + yield entry + + def test(self): + pass diff --git a/example-plugins/sleep_plugin/sleep.py b/example-plugins/sleep_plugin/sleep.py deleted file mode 100644 index 770dd3b..0000000 --- a/example-plugins/sleep_plugin/sleep.py +++ /dev/null @@ -1,14 +0,0 @@ -from senpy.plugins import AnalysisPlugin -from time import sleep - - -class SleepPlugin(AnalysisPlugin): - def activate(self, *args, **kwargs): - sleep(self.timeout) - - def analyse_entry(self, entry, params): - sleep(float(params.get("timeout", self.timeout))) - yield entry - - def test(self): - pass diff --git a/example-plugins/sleep_plugin/sleep.senpy b/example-plugins/sleep_plugin/sleep.senpy deleted file mode 100644 index 166f234..0000000 --- a/example-plugins/sleep_plugin/sleep.senpy +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "Sleep", - "module": "sleep", - "description": "I am dummy", - "author": "@balkian", - "version": "0.1", - "timeout": 0.05, - "extra_params": { - "timeout": { - "@id": "timeout_sleep", - "aliases": ["timeout", "to"], - "required": false, - "default": 0 - } - } -} diff --git a/senpy/__init__.py b/senpy/__init__.py index 49ea183..3270987 100644 --- a/senpy/__init__.py +++ b/senpy/__init__.py @@ -28,4 +28,10 @@ logger = logging.getLogger(__name__) logger.info('Using senpy version: {}'.format(__version__)) +from .utils import easy, easy_load, easy_test # noqa: F401 + +from .models import * # noqa: F401,F403 +from .plugins import * # noqa: F401,F403 +from .extensions import * # noqa: F401,F403 + __all__ = ['api', 'blueprints', 'cli', 'extensions', 'models', 'plugins'] diff --git a/senpy/__main__.py b/senpy/__main__.py index 4c05d1a..be5b31d 100644 --- a/senpy/__main__.py +++ b/senpy/__main__.py @@ -39,7 +39,7 @@ def main(): '-l', metavar='logging_level', type=str, - default="INFO", + default="ERROR", help='Logging level') parser.add_argument( '--debug', @@ -75,6 +75,12 @@ def main(): action='store_true', default=False, help='Do not run a server, only install plugin dependencies') + parser.add_argument( + '--only-list', + '--list', + action='store_true', + default=False, + help='Do not run a server, only list plugins found') parser.add_argument( '--data-folder', '--data', @@ -97,7 +103,6 @@ def main(): print('Senpy version {}'.format(senpy.__version__)) print(sys.version) exit(1) - logging.basicConfig() rl = logging.getLogger() rl.setLevel(getattr(logging, args.level)) app = Flask(__name__) @@ -105,6 +110,14 @@ def main(): sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins, data_folder=args.data_folder) + if args.only_list: + plugins = sp.plugins() + maxwidth = max(len(x.id) for x in plugins) + for plugin in plugins: + import inspect + fpath = inspect.getfile(plugin.__class__) + print('{: <{width}} @ {}'.format(plugin.id, fpath, width=maxwidth)) + return sp.install_deps() if args.only_install: return diff --git a/senpy/api.py b/senpy/api.py index dee9856..358bb66 100644 --- a/senpy/api.py +++ b/senpy/api.py @@ -99,7 +99,7 @@ NIF_PARAMS = { "aliases": ["f"], "required": False, "default": "text", - "options": ["turtle", "text", "json-ld"], + "options": ["text", "json-ld"], }, "language": { "@id": "language", @@ -130,13 +130,11 @@ def parse_params(indict, *specs): wrong_params = {} for spec in specs: for param, options in iteritems(spec): - if param[0] == "@": # Exclude json-ld properties - continue for alias in options.get("aliases", []): # Replace each alias with the correct name of the parameter if alias in indict and alias is not param: outdict[param] = indict[alias] - del indict[alias] + del outdict[alias] continue if param not in outdict: if "default" in options: @@ -154,10 +152,9 @@ def parse_params(indict, *specs): logger.debug("Error parsing: %s", wrong_params) message = Error( status=400, - message="Missing or invalid parameters", + message='Missing or invalid parameters', parameters=outdict, - errors={param: error - for param, error in iteritems(wrong_params)}) + errors=wrong_params) raise message if 'algorithm' in outdict and not isinstance(outdict['algorithm'], list): outdict['algorithm'] = outdict['algorithm'].split(',') @@ -182,7 +179,7 @@ def parse_call(params): results.entries.append(entry) elif params['informat'] == 'json-ld': results = from_string(params['input'], cls=Results) - else: + else: # pragma: no cover raise NotImplementedError('Informat {} is not implemented'.format(params['informat'])) results.parameters = params return results diff --git a/senpy/blueprints.py b/senpy/blueprints.py index 7943e16..6a4fbe0 100644 --- a/senpy/blueprints.py +++ b/senpy/blueprints.py @@ -31,7 +31,7 @@ import json logger = logging.getLogger(__name__) api_blueprint = Blueprint("api", __name__) -demo_blueprint = Blueprint("demo", __name__) +demo_blueprint = Blueprint("demo", __name__, template_folder='templates') ns_blueprint = Blueprint("ns", __name__) @@ -83,12 +83,11 @@ def basic_api(f): @wraps(f) def decorated_function(*args, **kwargs): raw_params = get_params(request) + logger.info('Getting request: {}'.format(raw_params)) headers = {'X-ORIGINAL-PARAMS': json.dumps(raw_params)} params = default_params try: - print('Getting request:') - print(request) params = api.parse_params(raw_params, api.WEB_PARAMS, api.API_PARAMS) if hasattr(request, 'parameters'): request.parameters.update(params) @@ -108,10 +107,9 @@ def basic_api(f): logger.error(ex) if 'parameters' in response and not params['with_parameters']: - print(response) - print(response.data) del response.parameters + logger.info('Response: {}'.format(response)) return response.flask( in_headers=params['inHeaders'], headers=headers, @@ -142,8 +140,8 @@ def plugins(): sp = current_app.senpy params = api.parse_params(request.parameters, api.PLUGINS_PARAMS) ptype = params.get('plugin_type') - plugins = sp.filter_plugins(plugin_type=ptype) - dic = Plugins(plugins=list(plugins.values())) + plugins = list(sp.plugins(plugin_type=ptype)) + dic = Plugins(plugins=plugins) return dic @@ -151,12 +149,4 @@ def plugins(): @basic_api def plugin(plugin=None): sp = current_app.senpy - if plugin == 'default' and sp.default_plugin: - return sp.default_plugin - plugins = sp.filter_plugins( - id='plugins/{}'.format(plugin)) or sp.filter_plugins(name=plugin) - if plugins: - response = list(plugins.values())[0] - else: - return Error(message="Plugin not found", status=404) - return response + return sp.get_plugin(plugin) diff --git a/senpy/cli.py b/senpy/cli.py index 28b1a5e..442540b 100644 --- a/senpy/cli.py +++ b/senpy/cli.py @@ -28,11 +28,15 @@ def main_function(argv): api.API_PARAMS, api.NIF_PARAMS) plugin_folder = params['plugin_folder'] - sp = Senpy(default_plugins=False, plugin_folder=plugin_folder) + default_plugins = params.get('default-plugins', False) + sp = Senpy(default_plugins=default_plugins, plugin_folder=plugin_folder) request = api.parse_call(params) - algos = request.parameters.get('algorithm', sp.plugins.keys()) - for algo in algos: - sp.activate_plugin(algo) + algos = request.parameters.get('algorithm', None) + if algos: + for algo in algos: + sp.activate_plugin(algo) + else: + sp.activate_all() res = sp.analyse(request) return res diff --git a/senpy/extensions.py b/senpy/extensions.py index 61a0c3a..facdc1c 100644 --- a/senpy/extensions.py +++ b/senpy/extensions.py @@ -6,13 +6,12 @@ from future import standard_library standard_library.install_aliases() from . import plugins, api -from .plugins import SenpyPlugin +from .plugins import Plugin from .models import Error from .blueprints import api_blueprint, demo_blueprint, ns_blueprint from threading import Thread from functools import partial - import os import copy import errno @@ -30,31 +29,29 @@ class Senpy(object): plugin_folder=".", data_folder=None, default_plugins=False): - self.app = app - self._search_folders = set() - self._plugin_list = [] - self._outdated = True + + default_data = os.path.join(os.getcwd(), 'senpy_data') + self.data_folder = data_folder or os.environ.get('SENPY_DATA', default_data) + try: + os.makedirs(self.data_folder) + except OSError as e: + if e.errno == errno.EEXIST: + logger.debug('Data folder exists: {}'.format(self.data_folder)) + else: # pragma: no cover + raise + self._default = None + self._plugins = {} + if plugin_folder: + self.add_folder(plugin_folder) - self.add_folder(plugin_folder) if default_plugins: self.add_folder('plugins', from_root=True) else: # Add only conversion plugins self.add_folder(os.path.join('plugins', 'conversion'), from_root=True) - - self.data_folder = data_folder or os.environ.get('SENPY_DATA', - os.path.join(os.getcwd(), - 'senpy_data')) - try: - os.makedirs(self.data_folder) - except OSError as e: - if e.errno == errno.EEXIST: - print('Directory not created.') - else: - raise - + self.app = app if app is not None: self.init_app(app) @@ -69,21 +66,52 @@ class Senpy(object): # otherwise fall back to the request context if hasattr(app, 'teardown_appcontext'): app.teardown_appcontext(self.teardown) - else: + else: # pragma: no cover app.teardown_request(self.teardown) app.register_blueprint(api_blueprint, url_prefix="/api") app.register_blueprint(ns_blueprint, url_prefix="/ns") app.register_blueprint(demo_blueprint, url_prefix="/") + def add_plugin(self, plugin): + self._plugins[plugin.name.lower()] = plugin + + def delete_plugin(self, plugin): + del self._plugins[plugin.name.lower()] + + def plugins(self, **kwargs): + """ Return the plugins registered for a given application. Filtered by criteria """ + return list(plugins.pfilter(self._plugins, **kwargs)) + + def get_plugin(self, name, default=None): + if name == 'default': + return self.default_plugin + plugin = name.lower() + if plugin in self._plugins: + return self._plugins[plugin] + + results = self.plugins(id='plugins/{}'.format(name)) + + if not results: + return Error(message="Plugin not found", status=404) + return results[0] + + @property + def analysis_plugins(self): + """ Return only the analysis plugins """ + return self.plugins(plugin_type='analysisPlugin') + def add_folder(self, folder, from_root=False): + """ Find plugins in this folder and add them to this instance """ if from_root: folder = os.path.join(os.path.dirname(__file__), folder) logger.debug("Adding folder: %s", folder) if os.path.isdir(folder): - self._search_folders.add(folder) - self._outdated = True + new_plugins = plugins.from_folder([folder], + data_folder=self.data_folder) + for plugin in new_plugins: + self.add_plugin(plugin) else: - raise AttributeError("Not a folder: %s", folder) + raise AttributeError("Not a folder or does not exist: %s", folder) def _get_plugins(self, request): if not self.analysis_plugins: @@ -102,14 +130,16 @@ class Senpy(object): plugins = list() for algo in algos: - if algo not in self.plugins: - logger.debug(("The algorithm '{}' is not valid\n" - "Valid algorithms: {}").format(algo, - self.plugins.keys())) + algo = algo.lower() + if algo not in self._plugins: + msg = ("The algorithm '{}' is not valid\n" + "Valid algorithms: {}").format(algo, + self._plugins.keys()) + logger.debug(msg) raise Error( status=404, - message="The algorithm '{}' is not valid".format(algo)) - plugins.append(self.plugins[algo]) + message=msg) + plugins.append(self._plugins[algo]) return plugins def _process_entries(self, entries, req, plugins): @@ -131,7 +161,7 @@ class Senpy(object): yield i def install_deps(self): - for plugin in self.filter_plugins(is_activated=True): + for plugin in self.plugins(is_activated=True): plugins.install_deps(plugin) def analyse(self, request): @@ -149,8 +179,6 @@ class Senpy(object): for i in self._process_entries(entries, results, plugins): results.entries.append(i) self.convert_emotions(results) - if 'with_parameters' not in results.parameters: - del results.parameters logger.debug("Returning analysis result: {}".format(results)) except (Error, Exception) as ex: if not isinstance(ex, Error): @@ -163,14 +191,13 @@ class Senpy(object): return results def _conversion_candidates(self, fromModel, toModel): - candidates = self.filter_plugins(plugin_type='emotionConversionPlugin') - for name, candidate in candidates.items(): + candidates = self.plugins(plugin_type='emotionConversionPlugin') + for candidate in candidates: for pair in candidate.onyx__doesConversion: logging.debug(pair) if pair['onyx:conversionFrom'] == fromModel \ and pair['onyx:conversionTo'] == toModel: - # logging.debug('Found candidate: {}'.format(candidate)) yield candidate def convert_emotions(self, resp): @@ -197,7 +224,8 @@ class Senpy(object): logger.debug('Analysis plugin {} uses model: {}'.format(plugin.id, fromModel)) except StopIteration: e = Error(('No conversion plugin found for: ' - '{} -> {}'.format(fromModel, toModel))) + '{} -> {}'.format(fromModel, toModel)), + status=404) e.original_response = resp e.parameters = params raise e @@ -223,36 +251,40 @@ class Senpy(object): @property def default_plugin(self): - candidate = self._default - if not candidate: - candidates = self.filter_plugins(plugin_type='analysisPlugin', - is_activated=True) + if not self._default or not self._default.is_activated: + candidates = self.plugins(plugin_type='analysisPlugin', + is_activated=True) if len(candidates) > 0: - candidate = list(candidates.values())[0] - logger.debug("Default: {}".format(candidate)) - return candidate + self._default = candidates[0] + else: + self._default = None + logger.debug("Default: {}".format(self._default)) + return self._default @default_plugin.setter def default_plugin(self, value): - if isinstance(value, SenpyPlugin): + if isinstance(value, Plugin): + if not value.is_activated: + raise AttributeError('The default plugin has to be activated.') self._default = value + else: - self._default = self.plugins[value] + self._default = self._plugins[value.lower()] def activate_all(self, sync=True): ps = [] - for plug in self.plugins.keys(): + for plug in self._plugins.keys(): ps.append(self.activate_plugin(plug, sync=sync)) return ps def deactivate_all(self, sync=True): ps = [] - for plug in self.plugins.keys(): + for plug in self._plugins.keys(): ps.append(self.deactivate_plugin(plug, sync=sync)) return ps def _set_active(self, plugin, active=True, *args, **kwargs): - ''' We're using a variable in the plugin itself to activate/deactive plugins.\ + ''' We're using a variable in the plugin itself to activate/deactivate plugins.\ Note that plugins may activate themselves by setting this variable. ''' plugin.is_activated = active @@ -269,11 +301,11 @@ class Senpy(object): self._set_active(plugin, success) def activate_plugin(self, plugin_name, sync=True): - try: - plugin = self.plugins[plugin_name] - except KeyError: + plugin_name = plugin_name.lower() + if plugin_name not in self._plugins: raise Error( message="Plugin not found: {}".format(plugin_name), status=404) + plugin = self._plugins[plugin_name] logger.info("Activating plugin: {}".format(plugin.name)) @@ -292,11 +324,11 @@ class Senpy(object): logger.info("Plugin deactivated: {}".format(plugin.name)) def deactivate_plugin(self, plugin_name, sync=True): - try: - plugin = self.plugins[plugin_name] - except KeyError: + plugin_name = plugin_name.lower() + if plugin_name not in self._plugins: raise Error( message="Plugin not found: {}".format(plugin_name), status=404) + plugin = self._plugins[plugin_name] self._set_active(plugin, False) @@ -309,20 +341,3 @@ class Senpy(object): def teardown(self, exception): pass - - @property - def plugins(self): - """ Return the plugins registered for a given application. """ - if self._outdated: - self._plugin_list = plugins.load_plugins(self._search_folders, - data_folder=self.data_folder) - self._outdated = False - return self._plugin_list - - def filter_plugins(self, **kwargs): - return plugins.pfilter(self.plugins, **kwargs) - - @property - def analysis_plugins(self): - """ Return only the analysis plugins """ - return self.filter_plugins(plugin_type='analysisPlugin') diff --git a/senpy/meta.py b/senpy/meta.py new file mode 100644 index 0000000..59fc47f --- /dev/null +++ b/senpy/meta.py @@ -0,0 +1,124 @@ +''' +Meta-programming for the models. +''' +import os +import json +import jsonschema +import inspect +import copy + +from abc import ABCMeta + + +class BaseMeta(ABCMeta): + ''' + Metaclass for models. It extracts the default values for the fields in + the model. + + For instance, instances of the following class wouldn't need to mark + their version or description on initialization: + + .. code-block:: python + + class MyPlugin(Plugin): + version=0.3 + description='A dull plugin' + + + Note that these operations could be included in the __init__ of the + class, but it would be very inefficient. + ''' + _subtypes = {} + + def __new__(mcs, name, bases, attrs, **kwargs): + defaults = {} + register_afterwards = False + + attrs = mcs.expand_with_schema(name, attrs) + if 'schema' in attrs: + register_afterwards = True + defaults = mcs.get_defaults(attrs['schema']) + for b in bases: + if hasattr(b, '_defaults'): + defaults.update(b._defaults) + info, attrs = mcs.split_attrs(attrs) + defaults.update(info) + attrs['_defaults'] = defaults + + cls = super(BaseMeta, mcs).__new__(mcs, name, tuple(bases), attrs) + + if register_afterwards: + mcs.register(cls, cls._defaults['@type']) + return cls + + @classmethod + def register(mcs, rsubclass, rtype=None): + mcs._subtypes[rtype or rsubclass.__name__] = rsubclass + + @staticmethod + def expand_with_schema(name, attrs): + if 'schema' in attrs: # Schema specified by name + schema_file = '{}.json'.format(attrs['schema']) + elif 'schema_file' in attrs: + schema_file = attrs['schema_file'] + del attrs['schema_file'] + else: + return attrs + + if '/' not in 'schema_file': + thisdir = os.path.dirname(os.path.realpath(__file__)) + schema_file = os.path.join(thisdir, + 'schemas', + schema_file) + + schema_path = 'file://' + schema_file + + with open(schema_file) as f: + schema = json.load(f) + + resolver = jsonschema.RefResolver(schema_path, schema) + attrs['@type'] = "".join((name[0].lower(), name[1:])) + attrs['_schema_file'] = schema_file + attrs['schema'] = schema + attrs['_validator'] = jsonschema.Draft4Validator(schema, resolver=resolver) + return attrs + + @staticmethod + def is_attr(k, v): + return (not(inspect.isroutine(v) or + inspect.ismethod(v) or + inspect.ismodule(v) or + isinstance(v, property)) and + k[0] != '_' and + k != 'schema' and + k != 'data') + + @staticmethod + def split_attrs(attrs): + ''' + Extract the attributes of the class. + + This allows adding default values in the class definition. + e.g.: + ''' + isattr = {} + notattr = {} + for key, value in attrs.items(): + if BaseMeta.is_attr(key, value): + if key[0] != '_': + key = key.replace("__", ":", 1) + isattr[key] = copy.deepcopy(value) + else: + notattr[key] = value + return isattr, notattr + + @staticmethod + def get_defaults(schema): + temp = {} + for obj in [ + schema, + ] + schema.get('allOf', []): + for k, v in obj.get('properties', {}).items(): + if 'default' in v and k not in temp: + temp[k] = copy.deepcopy(v['default']) + return temp diff --git a/senpy/models.py b/senpy/models.py index 44f4d45..d845a8a 100644 --- a/senpy/models.py +++ b/senpy/models.py @@ -17,20 +17,21 @@ import copy import json import os import jsonref -import jsonschema -import inspect from collections import UserDict -from abc import ABCMeta from flask import Response as FlaskResponse from pyld import jsonld -from rdflib import Graph - import logging +logging.getLogger('rdflib').setLevel(logging.WARN) logger = logging.getLogger(__name__) +from rdflib import Graph + + +from .meta import BaseMeta + DEFINITIONS_FILE = 'definitions.json' CONTEXT_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld') @@ -52,94 +53,32 @@ def read_schema(schema_file, absolute=False): return jsonref.load(f, base_uri=schema_uri) -base_schema = read_schema(DEFINITIONS_FILE) - - -class Context(dict): - @staticmethod - def load(context): - logging.debug('Loading context: {}'.format(context)) - if not context: +def load_context(context): + logging.debug('Loading context: {}'.format(context)) + if not context: + return context + elif isinstance(context, list): + contexts = [] + for c in context: + contexts.append(load_context(c)) + return contexts + elif isinstance(context, dict): + return dict(context) + elif isinstance(context, basestring): + try: + with open(context) as f: + return dict(json.loads(f.read())) + except IOError: return context - elif isinstance(context, list): - contexts = [] - for c in context: - contexts.append(Context.load(c)) - return contexts - elif isinstance(context, dict): - return Context(context) - elif isinstance(context, basestring): - try: - with open(context) as f: - return Context(json.loads(f.read())) - except IOError: - return context - else: - raise AttributeError('Please, provide a valid context') - - -base_context = Context.load(CONTEXT_PATH) - - -class BaseMeta(ABCMeta): - ''' - Metaclass for models. It extracts the default values for the fields in - the model. - - For instance, instances of the following class wouldn't need to mark - their version or description on initialization: - - .. code-block:: python - - class MyPlugin(Plugin): - version=0.3 - description='A dull plugin' - - - Note that these operations could be included in the __init__ of the - class, but it would be very inefficient. - ''' - def __new__(mcs, name, bases, attrs, **kwargs): - defaults = {} - if 'schema' in attrs: - defaults = mcs.get_defaults(attrs['schema']) - for b in bases: - if hasattr(b, 'defaults'): - defaults.update(b.defaults) - info = mcs.attrs_to_dict(attrs) - defaults.update(info) - attrs['defaults'] = defaults - return super(BaseMeta, mcs).__new__(mcs, name, bases, attrs) + else: + raise AttributeError('Please, provide a valid context') - @staticmethod - def attrs_to_dict(attrs): - ''' - Extract the attributes of the class. - This allows adding default values in the class definition. - e.g.: - ''' - def is_attr(k, v): - return (not(inspect.isroutine(v) or - inspect.ismethod(v) or - inspect.ismodule(v) or - isinstance(v, property)) and - k[0] != '_' and - k != 'schema' and - k != 'data') +base_context = load_context(CONTEXT_PATH) - return {key: copy.deepcopy(value) for key, value in attrs.items() if is_attr(key, value)} - @staticmethod - def get_defaults(schema): - temp = {} - for obj in [ - schema, - ] + schema.get('allOf', []): - for k, v in obj.get('properties', {}).items(): - if 'default' in v and k not in temp: - temp[k] = copy.deepcopy(v['default']) - return temp +def register(rsubclass, rtype=None): + BaseMeta.register(rsubclass, rtype) class CustomDict(UserDict, object): @@ -155,10 +94,10 @@ class CustomDict(UserDict, object): > d.ns__name == d['ns:name'] ''' - defaults = [] + _defaults = [] def __init__(self, *args, **kwargs): - temp = copy.deepcopy(self.defaults) + temp = copy.deepcopy(self._defaults) for arg in args: temp.update(copy.deepcopy(arg)) for k, v in kwargs.items(): @@ -210,13 +149,38 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)): For convenience, the values can also be accessed as attributes (a la Javascript). e.g.: - > myobject.key == myobject['key'] + >>> myobject.key == myobject['key'] + True + >>> myobject.ns__name == myobject['ns:name'] + True + + Additionally, subclasses of this class can specify default values for their + instances. These defaults are inherited by subclasses. e.g.: + + >>> class NewModel(BaseModel): + ... mydefault = 5 + >>> n1 = NewModel() + >>> n1['mydefault'] == 5 + True + >>> n1.mydefault = 3 + >>> n1['mydefault'] = 3 True - > myobject.ns__name == myobject['ns:name'] + >>> n2 = NewModel() + >>> n2 == 5 True + >>> class SubModel(NewModel): + pass + >>> subn = SubModel() + >>> subn.mydefault == 5 + True + + Lastly, every subclass that also specifies a schema will get registered, so it + is possible to deserialize JSON and get the right type. + i.e. to recover an instance of the original class from a plain JSON. + ''' - schema = base_schema + schema_file = DEFINITIONS_FILE _context = base_context["@context"] def __init__(self, *args, **kwargs): @@ -300,7 +264,7 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)): return ser_or_down(self.data) def jsonld(self, - with_context=True, + with_context=False, context_uri=None, prefix=None, expanded=False): @@ -338,54 +302,22 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)): def __str__(self): return str(self.serialize()) - -_subtypes = {} + def prov(self, another): + self['prov:wasGeneratedBy'] = another.id -def register(rsubclass, rtype=None): - _subtypes[rtype or rsubclass.__name__] = rsubclass - - -def from_schema(name, schema=None, schema_file=None, base_classes=None): - base_classes = base_classes or [] - base_classes.append(BaseModel) - schema_file = schema_file or '{}.json'.format(name) - class_name = '{}{}'.format(name[0].upper(), name[1:]) - if '/' not in 'schema_file': - thisdir = os.path.dirname(os.path.realpath(__file__)) - schema_file = os.path.join(thisdir, - 'schemas', - schema_file) - - schema_path = 'file://' + schema_file - - with open(schema_file) as f: - schema = json.load(f) - - dct = {} - - resolver = jsonschema.RefResolver(schema_path, schema) - dct['@type'] = name - dct['_schema_file'] = schema_file - dct['schema'] = schema - dct['_validator'] = jsonschema.Draft4Validator(schema, resolver=resolver) - - newclass = type(class_name, tuple(base_classes), dct) - - register(newclass, name) - return newclass +def subtypes(): + return BaseMeta._subtypes def from_dict(indict, cls=None): if not cls: target = indict.get('@type', None) + cls = BaseModel try: - if target and target in _subtypes: - cls = _subtypes[target] - else: - cls = BaseModel - except Exception: - cls = BaseModel + cls = subtypes()[target] + except KeyError: + pass outdict = dict() for k, v in indict.items(): if k == '@context': @@ -410,8 +342,53 @@ def from_json(injson): return from_dict(indict) -def _add_from_schema(*args, **kwargs): - generatedClass = from_schema(*args, **kwargs) +class Entry(BaseModel, Exception): + schema = 'entry' + + @property + def text(self): + return self['nif:isString'] + + @text.setter + def text(self, value): + self['nif:isString'] = value + + +class Error(BaseModel, Exception): + schema = 'error' + + def __init__(self, message, *args, **kwargs): + Exception.__init__(self, message) + super(Error, self).__init__(*args, **kwargs) + self.message = message + + def __str__(self): + if not hasattr(self, 'errors'): + return self.message + return '{}:\n\t{}'.format(self.message, self.errors) + + def __hash__(self): + return Exception.__hash__(self) + + +# Add the remaining schemas programmatically + +def _class_from_schema(name, schema=None, schema_file=None, base_classes=None): + base_classes = base_classes or [] + base_classes.append(BaseModel) + attrs = {} + if schema: + attrs['schema'] = schema + elif schema_file: + attrs['schema_file'] = schema_file + else: + attrs['schema'] = name + name = "".join((name[0].upper(), name[1:])) + return BaseMeta(name, base_classes, attrs) + + +def _add_class_from_schema(*args, **kwargs): + generatedClass = _class_from_schema(*args, **kwargs) globals()[generatedClass.__name__] = generatedClass del generatedClass @@ -425,7 +402,6 @@ for i in [ 'emotionModel', 'emotionPlugin', 'emotionSet', - 'entry', 'help', 'plugin', 'plugins', @@ -435,19 +411,4 @@ for i in [ 'sentimentPlugin', 'suggestion', ]: - _add_from_schema(i) - -_ErrorModel = from_schema('error') - - -class Error(_ErrorModel, Exception): - def __init__(self, message, *args, **kwargs): - Exception.__init__(self, message) - super(Error, self).__init__(*args, **kwargs) - self.message = message - - def __hash__(self): - return Exception.__hash__(self) - - -register(Error, 'error') + _add_class_from_schema(i) diff --git a/senpy/plugins/__init__.py b/senpy/plugins/__init__.py index 135cd77..3983590 100644 --- a/senpy/plugins/__init__.py +++ b/senpy/plugins/__init__.py @@ -4,11 +4,12 @@ from future.utils import with_metaclass import os.path import os +import re import pickle import logging import copy +import pprint -import fnmatch import inspect import sys import subprocess @@ -30,12 +31,25 @@ class PluginMeta(models.BaseMeta): if hasattr(bases[0], 'plugin_type'): plugin_type += bases[0].plugin_type plugin_type.append(name) + alias = attrs.get('name', name) attrs['plugin_type'] = plugin_type + attrs['name'] = alias + if 'description' not in attrs: + doc = attrs.get('__doc__', None) + if not doc: + raise Exception(('Please, add a description or ' + 'documentation to class {}').format(name)) + attrs['description'] = doc + attrs['name'] = alias cls = super(PluginMeta, mcs).__new__(mcs, name, bases, attrs) - if name in mcs._classes: - raise Exception(('The type of plugin {} already exists. ' - 'Please, choose a different name').format(name)) - mcs._classes[name] = cls + + if alias in mcs._classes: + if os.environ.get('SENPY_TESTING', ""): + raise Exception(('The type of plugin {} already exists. ' + 'Please, choose a different name').format(name)) + else: + logger.warn('Overloading plugin class: {}'.format(alias)) + mcs._classes[alias] = cls return cls @classmethod @@ -44,6 +58,17 @@ class PluginMeta(models.BaseMeta): class Plugin(with_metaclass(PluginMeta, models.Plugin)): + ''' + Base class for all plugins in senpy. + A plugin must provide at least these attributes: + + - version + - description (or docstring) + - author + + Additionally, they may provide a URL (url) of a repository or website. + + ''' def __init__(self, info=None, data_folder=None, **kwargs): """ @@ -54,16 +79,19 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)): super(Plugin, self).__init__(**kwargs) if info: self.update(info) - if not self.validate(): - raise models.Error(message=("You need to provide configuration" - "information for the plugin.")) + self.validate() self.id = 'plugins/{}_{}'.format(self['name'], self['version']) self.is_activated = False self._lock = threading.Lock() self.data_folder = data_folder or os.getcwd() def validate(self): - return all(x in self for x in ('name', 'description', 'version')) + missing = [] + for x in ['name', 'description', 'version']: + if x not in self: + missing.append(x) + if missing: + raise models.Error('Missing configuration parameters: {}'.format(missing)) def get_folder(self): return os.path.dirname(inspect.getfile(self.__class__)) @@ -74,48 +102,61 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)): def deactivate(self): pass - def test(self): - if not hasattr(self, 'test_cases'): - raise AttributeError(('Plugin {} [{}] does not have any defined ' - 'test cases').format(self.id, inspect.getfile(self.__class__))) - for case in self.test_cases: - entry = models.Entry(case['entry']) - given_parameters = case.get('params', {}) - params = api.parse_params(given_parameters, self.extra_params) - fails = case.get('fails', False) + def test(self, test_cases=None): + if not test_cases: + if not hasattr(self, 'test_cases'): + raise AttributeError(('Plugin {} [{}] does not have any defined ' + 'test cases').format(self.id, + inspect.getfile(self.__class__))) + test_cases = self.test_cases + for case in test_cases: try: - res = list(self.analyse_entry(entry, params)) - except models.Error: - if fails: - continue + self.test_case(case) + except Exception as ex: + logger.warn('Test case failed:\n{}'.format(pprint.pformat(case))) raise - if fails: - raise Exception('This test should have raised an exception.') - exp = case['expected'] - if not isinstance(exp, list): - exp = [exp] - utils.check_template(res, exp) - for r in res: - r.validate() + def test_case(self, case): + entry = models.Entry(case['entry']) + given_parameters = case.get('params', case.get('parameters', {})) + expected = case['expected'] + try: + params = api.parse_params(given_parameters, self.extra_params) + res = list(self.analyse_entry(entry, params)) + except models.Error: + if not expected: + return + raise + if not expected: + raise Exception('This test should have raised an exception.') + + if not isinstance(expected, list): + expected = [expected] + utils.check_template(res, expected) + for r in res: + r.validate() def open(self, fpath, *args, **kwargs): if not os.path.isabs(fpath): fpath = os.path.join(self.data_folder, fpath) return open(fpath, *args, **kwargs) - def serve(self, **kwargs): - utils.serve(plugin=self, **kwargs) + def serve(self, debug=True, **kwargs): + utils.easy(plugin_list=[self, ], plugin_folder=None, debug=debug, **kwargs) +# For backwards compatibility SenpyPlugin = Plugin -class AnalysisPlugin(Plugin): +class Analysis(Plugin): + ''' + A subclass of Plugin that analyses text and provides an annotation. + ''' def analyse(self, *args, **kwargs): raise NotImplementedError( - 'Your method should implement either analyse or analyse_entry') + 'Your plugin should implement either analyse or analyse_entry') def analyse_entry(self, entry, parameters): """ An implemented plugin should override this method. @@ -134,28 +175,72 @@ class AnalysisPlugin(Plugin): def analyse_entries(self, entries, parameters): for entry in entries: logger.debug('Analysing entry with plugin {}: {}'.format(self, entry)) - for result in self.analyse_entry(entry, parameters): - yield result + results = self.analyse_entry(entry, parameters) + if inspect.isgenerator(results): + for result in results: + yield result + else: + yield results + + def test_case(self, case): + if 'entry' not in case and 'input' in case: + entry = models.Entry(_auto_id=False) + entry.nif__isString = case['input'] + case['entry'] = entry + super(Analysis, self).test_case(case) -class ConversionPlugin(Plugin): +AnalysisPlugin = Analysis + + +class Conversion(Plugin): + ''' + A subclass of Plugins that convert between different annotation models. + e.g. a conversion of emotion models, or normalization of sentiment values. + ''' pass -class SentimentPlugin(AnalysisPlugin, models.SentimentPlugin): +ConversionPlugin = Conversion + + +class SentimentPlugin(Analysis, models.SentimentPlugin): + ''' + Sentiment plugins provide sentiment annotation (using Marl) + ''' minPolarityValue = 0 maxPolarityValue = 1 - -class EmotionPlugin(AnalysisPlugin, models.EmotionPlugin): + def test_case(self, case): + expected = case.get('expected', {}) + if 'polarity' in case: + s = models.Sentiment(_auto_id=False) + s.marl__hasPolarity = case['polarity'] + if 'sentiments' not in expected: + expected['sentiments'] = [] + expected['sentiments'].append(s) + case['expected'] = expected + super(SentimentPlugin, self).test_case(case) + + +class EmotionPlugin(Analysis, models.EmotionPlugin): + ''' + Emotion plugins provide emotion annotation (using Onyx) + ''' minEmotionValue = 0 maxEmotionValue = 1 -class EmotionConversionPlugin(ConversionPlugin): +class EmotionConversion(Conversion): + ''' + A subclass of Conversion that converts emotion annotations using different models + ''' pass +EmotionConversionPlugin = EmotionConversion + + class ShelfMixin(object): @property def sh(self): @@ -201,7 +286,7 @@ def pfilter(plugins, **kwargs): plugins = plugins.plugins elif isinstance(plugins, dict): plugins = plugins.values() - ptype = kwargs.pop('plugin_type', AnalysisPlugin) + ptype = kwargs.pop('plugin_type', Plugin) logger.debug('#' * 100) logger.debug('ptype {}'.format(ptype)) if ptype: @@ -228,11 +313,7 @@ def pfilter(plugins, **kwargs): if kwargs: candidates = filter(matches, candidates) - return {p.name: p for p in candidates} - - -def validate_info(info): - return all(x in info for x in ('name',)) + return candidates def load_module(name, root=None): @@ -271,66 +352,109 @@ def install_deps(*plugins): return installed -def get_plugin_class(module): - candidate = None - for _, obj in inspect.getmembers(module): - if inspect.isclass(obj) and inspect.getmodule(obj) == module: - logger.debug(("Found plugin class:" - " {}@{}").format(obj, inspect.getmodule(obj))) - candidate = obj - break - return candidate +is_plugin_file = re.compile(r'.*\.senpy$|senpy_[a-zA-Z0-9_]+\.py$|[a-zA-Z0-9_]+_plugin.py$') -def load_plugin_from_info(info, root=None, validator=validate_info, install=True, *args, **kwargs): - if not root and '_path' in info: - root = os.path.dirname(info['_path']) - if not validator(info): +def find_plugins(folders): + for search_folder in folders: + for root, dirnames, filenames in os.walk(search_folder): + # Do not look for plugins in hidden or special folders + dirnames[:] = [d for d in dirnames if d[0] not in ['.', '_']] + for filename in filter(is_plugin_file.match, filenames): + fpath = os.path.join(root, filename) + yield fpath + + +def from_path(fpath, **kwargs): + logger.debug("Loading plugin from {}".format(fpath)) + if fpath.endswith('.py'): + # We asume root is the dir of the file, and module is the name of the file + root = os.path.dirname(fpath) + module = os.path.basename(fpath)[:-3] + for instance in _from_module_name(module=module, root=root, **kwargs): + yield instance + else: + info = parse_plugin_info(fpath) + yield from_info(info, **kwargs) + + +def from_folder(folders, loader=from_path, **kwargs): + plugins = [] + for fpath in find_plugins(folders): + for plugin in loader(fpath, **kwargs): + plugins.append(plugin) + return plugins + + +def from_info(info, root=None, **kwargs): + if any(x not in info for x in ('module',)): raise ValueError('Plugin info is not valid: {}'.format(info)) module = info["module"] - try: - tmp = load_module(module, root) - except ImportError: - if not install: - raise - install_deps(info) - tmp = load_module(module, root) - cls = None - if '@type' not in info: - cls = get_plugin_class(tmp) - else: - cls = PluginMeta.from_type(info['@type']) - if not cls: - raise Exception("No valid plugin for: {}".format(module)) - return cls(info=info, *args, **kwargs) + if not root and '_path' in info: + root = os.path.dirname(info['_path']) + + return one_from_module(module, root=root, info=info, **kwargs) def parse_plugin_info(fpath): - logger.debug("Loading plugin: {}".format(fpath)) + logger.debug("Parsing plugin info: {}".format(fpath)) with open(fpath, 'r') as f: info = yaml.load(f) info['_path'] = fpath - name = info['name'] - return name, info + return info -def load_plugin(fpath, *args, **kwargs): - name, info = parse_plugin_info(fpath) - logger.debug("Info: {}".format(info)) - plugin = load_plugin_from_info(info, *args, **kwargs) - return name, plugin +def from_module(module, **kwargs): + if inspect.ismodule(module): + res = _from_loaded_module(module, **kwargs) + else: + res = _from_module_name(module, **kwargs) + for p in res: + yield p -def load_plugins(folders, loader=load_plugin, *args, **kwargs): - plugins = {} - for search_folder in folders: - for root, dirnames, filenames in os.walk(search_folder): - # Do not look for plugins in hidden or special folders - dirnames[:] = [d for d in dirnames if d[0] not in ['.', '_']] - for filename in fnmatch.filter(filenames, '*.senpy'): - fpath = os.path.join(root, filename) - name, plugin = loader(fpath, *args, **kwargs) - if plugin and name: - plugins[name] = plugin - return plugins + +def one_from_module(module, root, info, **kwargs): + if '@type' in info: + cls = PluginMeta.from_type(info['@type']) + return cls(info=info, **kwargs) + instance = next(from_module(module=module, root=root, info=info, **kwargs), None) + if not instance: + raise Exception("No valid plugin for: {}".format(module)) + return instance + + +def _classes_in_module(module): + for _, obj in inspect.getmembers(module): + if inspect.isclass(obj) and inspect.getmodule(obj) == module: + logger.debug(("Found plugin class:" + " {}@{}").format(obj, inspect.getmodule(obj))) + yield obj + + +def _instances_in_module(module): + for _, obj in inspect.getmembers(module): + if isinstance(obj, Plugin) and inspect.getmodule(obj) == module: + logger.debug(("Found plugin instance:" + " {}@{}").format(obj, inspect.getmodule(obj))) + yield obj + + +def _from_module_name(module, root, info=None, install=True, **kwargs): + try: + module = load_module(module, root) + except ImportError: + if not install or not info: + raise + install_deps(info) + module = load_module(module, root) + for plugin in _from_loaded_module(module=module, root=root, info=info, **kwargs): + yield plugin + + +def _from_loaded_module(module, info=None, **kwargs): + for cls in _classes_in_module(module): + yield cls(info=info, **kwargs) + for instance in _instances_in_module(module): + yield instance diff --git a/senpy/plugins/conversion/emotion/centroids.py b/senpy/plugins/conversion/emotion/centroids.py index 6222152..eae28d6 100644 --- a/senpy/plugins/conversion/emotion/centroids.py +++ b/senpy/plugins/conversion/emotion/centroids.py @@ -6,6 +6,11 @@ logger = logging.getLogger(__name__) class CentroidConversion(EmotionConversionPlugin): + ''' + This plugin converts emotion annotations from a dimensional model to a + categorical one, and vice versa. The centroids used in the conversion + are configurable and appear in the semantic description of the plugin. + ''' def __init__(self, info, *args, **kwargs): if 'centroids' not in info: raise Error('Centroid conversion plugins should provide ' diff --git a/senpy/plugins/example/emoRand/emoRand.py b/senpy/plugins/example/emoRand/emoRand.py index 63c2e56..25327c4 100644 --- a/senpy/plugins/example/emoRand/emoRand.py +++ b/senpy/plugins/example/emoRand/emoRand.py @@ -4,7 +4,15 @@ from senpy.plugins import EmotionPlugin from senpy.models import EmotionSet, Emotion, Entry -class EmoRandPlugin(EmotionPlugin): +class EmoRand(EmotionPlugin): + name = "emoRand" + description = 'A sample plugin that returns a random emotion annotation' + author = '@balkian' + version = '0.1' + url = "https://github.com/gsi-upm/senpy-plugins-community" + requirements = {} + onyx__usesEmotionModel = "emoml:big6" + def analyse_entry(self, entry, params): category = "emoml:big6happiness" number = max(-1, min(1, random.gauss(0, 0.5))) diff --git a/senpy/plugins/example/emoRand/emoRand.senpy b/senpy/plugins/example/emoRand/emoRand.senpy deleted file mode 100644 index a3ffae8..0000000 --- a/senpy/plugins/example/emoRand/emoRand.senpy +++ /dev/null @@ -1,9 +0,0 @@ ---- -name: emoRand -module: emoRand -description: A sample plugin that returns a random emotion annotation -author: "@balkian" -version: '0.1' -url: "https://github.com/gsi-upm/senpy-plugins-community" -requirements: {} -onyx:usesEmotionModel: "emoml:big6" \ No newline at end of file diff --git a/senpy/plugins/example/emorand_plugin.py b/senpy/plugins/example/emorand_plugin.py new file mode 100644 index 0000000..c65d47d --- /dev/null +++ b/senpy/plugins/example/emorand_plugin.py @@ -0,0 +1,32 @@ +import random + +from senpy.plugins import EmotionPlugin +from senpy.models import EmotionSet, Emotion, Entry + + +class EmoRand(EmotionPlugin): + '''A sample plugin that returns a random emotion annotation''' + author = '@balkian' + version = '0.1' + url = "https://github.com/gsi-upm/senpy-plugins-community" + onyx__usesEmotionModel = "emoml:big6" + + def analyse_entry(self, entry, params): + category = "emoml:big6happiness" + number = max(-1, min(1, random.gauss(0, 0.5))) + if number > 0: + category = "emoml:big6anger" + emotionSet = EmotionSet() + emotion = Emotion({"onyx:hasEmotionCategory": category}) + emotionSet.onyx__hasEmotion.append(emotion) + emotionSet.prov__wasGeneratedBy = self.id + entry.emotions.append(emotionSet) + yield entry + + def test(self): + params = dict() + results = list() + for i in range(100): + res = next(self.analyse_entry(Entry(nif__isString="Hello"), params)) + res.validate() + results.append(res.emotions[0]['onyx:hasEmotion'][0]['onyx:hasEmotionCategory']) diff --git a/senpy/plugins/example/rand/rand.senpy b/senpy/plugins/example/rand/rand.senpy deleted file mode 100644 index b7ee693..0000000 --- a/senpy/plugins/example/rand/rand.senpy +++ /dev/null @@ -1,10 +0,0 @@ ---- -name: rand -module: rand -description: A sample plugin that returns a random sentiment annotation -author: "@balkian" -version: '0.1' -url: "https://github.com/gsi-upm/senpy-plugins-community" -requirements: {} -marl:maxPolarityValue: '1' -marl:minPolarityValue: "-1" diff --git a/senpy/plugins/example/rand/rand.py b/senpy/plugins/example/rand_plugin.py similarity index 52% rename from senpy/plugins/example/rand/rand.py rename to senpy/plugins/example/rand_plugin.py index 645e63b..37344ff 100644 --- a/senpy/plugins/example/rand/rand.py +++ b/senpy/plugins/example/rand_plugin.py @@ -1,33 +1,35 @@ import random +from senpy import SentimentPlugin, Sentiment, Entry -from senpy.plugins import SentimentPlugin -from senpy.models import Sentiment, Entry +class Rand(SentimentPlugin): + '''A sample plugin that returns a random sentiment annotation''' + author = "@balkian" + version = '0.1' + url = "https://github.com/gsi-upm/senpy-plugins-community" + marl__maxPolarityValue = '1' + marl__minPolarityValue = "-1" -class RandPlugin(SentimentPlugin): def analyse_entry(self, entry, params): - lang = params.get("language", "auto") - polarity_value = max(-1, min(1, random.gauss(0.2, 0.2))) polarity = "marl:Neutral" if polarity_value > 0: polarity = "marl:Positive" elif polarity_value < 0: polarity = "marl:Negative" - sentiment = Sentiment({ - "marl:hasPolarity": polarity, - "marl:polarityValue": polarity_value - }) - sentiment["prov:wasGeneratedBy"] = self.id + sentiment = Sentiment(marl__hasPolarity=polarity, + marl__polarityValue=polarity_value) + sentiment.prov(self) entry.sentiments.append(sentiment) - entry.language = lang yield entry def test(self): + '''Run several random analyses.''' params = dict() results = list() - for i in range(100): - res = next(self.analyse_entry(Entry(nif__isString="Hello"), params)) + for i in range(20): + res = next(self.analyse_entry(Entry(nif__isString="Hello"), + params)) res.validate() results.append(res.sentiments[0]['marl:hasPolarity']) assert 'marl:Positive' in results diff --git a/senpy/plugins/misc/split.py b/senpy/plugins/misc/split.py index edf0e40..c7cea73 100644 --- a/senpy/plugins/misc/split.py +++ b/senpy/plugins/misc/split.py @@ -6,6 +6,7 @@ import nltk class SplitPlugin(AnalysisPlugin): + '''description: A sample plugin that chunks input text''' def activate(self): nltk.download('punkt') diff --git a/senpy/plugins/sentiment/sentiment140/sentiment140.py b/senpy/plugins/sentiment/sentiment140/sentiment140.py index 6a8426b..782e06d 100644 --- a/senpy/plugins/sentiment/sentiment140/sentiment140.py +++ b/senpy/plugins/sentiment/sentiment140/sentiment140.py @@ -6,6 +6,7 @@ from senpy.models import Sentiment class Sentiment140Plugin(SentimentPlugin): + '''Connects to the sentiment140 free API: http://sentiment140.com''' def analyse_entry(self, entry, params): lang = params["language"] res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson", diff --git a/senpy/static/js/main.js b/senpy/static/js/main.js index 933cb6a..be243a4 100644 --- a/senpy/static/js/main.js +++ b/senpy/static/js/main.js @@ -48,18 +48,7 @@ function get_parameters(){ for (p in plugins){ plugin = plugins[p]; if (plugin["extra_params"]){ - plugins_params[plugin["name"]]={}; - for (param in plugin["extra_params"]){ - if (typeof plugin["extra_params"][param] !="string"){ - var params = new Array(); - var alias = plugin["extra_params"][param]["aliases"][0]; - params[alias]=new Array(); - for (option in plugin["extra_params"][param]["options"]){ - params[alias].push(plugin["extra_params"][param]["options"][option]) - } - plugins_params[plugin["name"]][alias] = (params[alias]) - } - } + plugins_params[plugin["name"]] = plugin["extra_params"]; } } } @@ -175,13 +164,13 @@ function params_div(params){ param = params[pname]; html+='
'; html += '
' - html+= '' + html+= '' if (param.options){ opts = param.options; if(param.options.length == 1 && param.options[0] == 'boolean') { opts = [true, false]; } - html+= '" var defaultopt = param.default; for (option in opts){ isselected = ""; @@ -198,7 +187,7 @@ function params_div(params){ if(param.default != undefined){ default_value = param.default; }; - html +=''; + html +=''; } html+='
'; html+='
'; diff --git a/senpy/test.py b/senpy/test.py index b5a5c68..11b36fd 100644 --- a/senpy/test.py +++ b/senpy/test.py @@ -10,16 +10,6 @@ from contextlib import contextmanager from .models import BaseModel -class Call(dict): - def __init__(self, obj): - self.obj = obj.serialize() - self.status_code = 200 - self.content = self.json() - - def json(self): - return json.loads(self.obj) - - @contextmanager def patch_requests(value, code=200): success = MagicMock() @@ -31,10 +21,7 @@ def patch_requests(value, code=200): success.data.return_value = data success.status_code = code - if hasattr(value, 'jsonld'): - success.content = value.jsonld() - else: - success.content = json.dumps(value) + success.content = json.dumps(value) method_mocker = MagicMock() method_mocker.return_value = success with patch.multiple('requests', request=method_mocker, diff --git a/senpy/utils.py b/senpy/utils.py index e869858..118b7ba 100644 --- a/senpy/utils.py +++ b/senpy/utils.py @@ -1,5 +1,9 @@ -from . import models +from . import models, __version__ from collections import MutableMapping +import pprint + +import logging +logger = logging.getLogger(__name__) # MutableMapping should be enough, but it causes problems with py2 DICTCLASSES = (MutableMapping, dict, models.BaseModel) @@ -9,40 +13,78 @@ def check_template(indict, template): if isinstance(template, DICTCLASSES) and isinstance(indict, DICTCLASSES): for k, v in template.items(): if k not in indict: - return '{} not in {}'.format(k, indict) + raise models.Error('{} not in {}'.format(k, indict)) check_template(indict[k], v) elif isinstance(template, list) and isinstance(indict, list): for e in template: - found = False for i in indict: try: check_template(i, e) - found = True + break except models.Error as ex: # raise continue - if not found: - raise models.Error('{} not found in {}'.format(e, indict)) + else: + raise models.Error(('Element not found.' + '\nExpected: {}\nIn: {}').format(pprint.pformat(e), + pprint.pformat(indict))) else: if indict != template: - raise models.Error('{} and {} are different'.format(indict, template)) + raise models.Error(('Differences found.\n' + '\tExpected: {}\n' + '\tFound: {}').format(pprint.pformat(indict), + pprint.pformat(template))) -def easy(app=None, plugin=None, host='0.0.0.0', port=5000, **kwargs): +def easy_load(app=None, plugin_list=None, plugin_folder=None, **kwargs): ''' Run a server with a specific plugin. ''' from flask import Flask - from senpy.extensions import Senpy + from .extensions import Senpy if not app: app = Flask(__name__) - sp = Senpy(app) - if plugin: + sp = Senpy(app, plugin_folder=plugin_folder, **kwargs) + if not plugin_list: + from . import plugins + import __main__ + plugin_list = plugins.from_module(__main__) + for plugin in plugin_list: sp.add_plugin(plugin) sp.install_deps() + sp.activate_all() + return sp, app + + +def easy_test(plugin_list=None): + logger.setLevel(logging.DEBUG) + logging.getLogger().setLevel(logging.INFO) + if not plugin_list: + from . import plugins + import __main__ + plugin_list = plugins.from_module(__main__) + for plug in plugin_list: + plug.test() + logger.info('All tests passed!') + + +def easy(host='0.0.0.0', port=5000, debug=True, **kwargs): + ''' + Run a server with a specific plugin. + ''' + logging.getLogger().setLevel(logging.DEBUG) + logging.getLogger('senpy').setLevel(logging.INFO) + sp, app = easy_load(**kwargs) + easy_test(sp.plugins()) + app.debug = debug + import time + logger.info(time.time()) + logger.info('Senpy version {}'.format(__version__)) + logger.info('Server running on port %s:%d. Ctrl+C to quit' % (host, + port)) + app.debug = debug app.run(host, port, - debug=app.debug, - **kwargs) + debug=app.debug) diff --git a/tests/test_api.py b/tests/test_api.py index ec5b79d..96bd9e3 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -19,11 +19,8 @@ class APITest(TestCase): def test_basic(self): a = {} - try: - parse_params(a, NIF_PARAMS) - raise AssertionError() - except Error: - pass + self.assertRaises(Error, parse_params, a) + self.assertRaises(Error, parse_params, a, NIF_PARAMS) a = {'input': 'hello'} p = parse_params(a, NIF_PARAMS) assert 'input' in p @@ -39,11 +36,7 @@ class APITest(TestCase): 'required': True } } - try: - parse_params(query, plug_params) - raise AssertionError() - except Error: - pass + self.assertRaises(Error, parse_params, plug_params) query['hello'] = 'world' p = parse_params(query, plug_params) assert 'hello' in p @@ -53,7 +46,6 @@ class APITest(TestCase): query['hiya'] = 'dlrow' p = parse_params(query, plug_params) assert 'hello' in p - assert 'hiya' in p assert p['hello'] == 'dlrow' def test_default(self): diff --git a/tests/test_blueprints.py b/tests/test_blueprints.py index db6d78c..1698bc8 100644 --- a/tests/test_blueprints.py +++ b/tests/test_blueprints.py @@ -23,7 +23,7 @@ class BlueprintsTest(TestCase): cls.app = Flask("test_extensions") cls.app.debug = False cls.client = cls.app.test_client() - cls.senpy = Senpy() + cls.senpy = Senpy(default_plugins=True) cls.senpy.init_app(cls.app) cls.dir = os.path.join(os.path.dirname(__file__), "..") cls.senpy.add_folder(cls.dir) @@ -34,11 +34,14 @@ class BlueprintsTest(TestCase): def assertCode(self, resp, code): self.assertEqual(resp.status_code, code) + def test_playground(self): + resp = self.client.get("/") + assert "main.js" in resp.data.decode() + def test_home(self): """ Calling with no arguments should ask the user for more arguments """ - self.app.debug = False resp = self.client.get("/api/") self.assertCode(resp, 400) js = parse_resp(resp) @@ -84,6 +87,10 @@ class BlueprintsTest(TestCase): js = parse_resp(resp) logging.debug("Got response: %s", js) assert isinstance(js, models.Error) + resp = self.client.get("/api/?i=My aloha mohame&algo=DummyRequired&example=notvalid") + self.assertCode(resp, 400) + resp = self.client.get("/api/?i=My aloha mohame&algo=DummyRequired&example=a") + self.assertCode(resp, 200) def test_error(self): """ @@ -155,8 +162,7 @@ class BlueprintsTest(TestCase): def test_schema(self): resp = self.client.get("/api/schemas/definitions.json") self.assertCode(resp, 200) - js = parse_resp(resp) - assert "$schema" in js + assert "$schema" in resp.data.decode() def test_help(self): resp = self.client.get("/api/?help=true") @@ -164,3 +170,7 @@ class BlueprintsTest(TestCase): js = parse_resp(resp) assert "valid_parameters" in js assert "help" in js["valid_parameters"] + + def test_conversion(self): + resp = self.client.get("/api/?input=hello&algo=emoRand&emotionModel=DOES NOT EXIST") + self.assertCode(resp, 404) diff --git a/tests/test_cli.py b/tests/test_cli.py index 4bd059d..422ff9e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -12,7 +12,8 @@ class CLITest(TestCase): def test_basic(self): self.assertRaises(Error, partial(main_function, [])) - res = main_function(['--input', 'test', '--algo', 'rand', '--with-parameters']) + res = main_function(['--input', 'test', '--algo', 'rand', + '--with-parameters', '--default-plugins']) assert res.parameters['input'] == 'test' assert 'rand' in res.parameters['algorithm'] assert res.parameters['input'] == 'test' diff --git a/tests/test_extensions.py b/tests/test_extensions.py index d2e8060..849f5d9 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -29,6 +29,7 @@ class ExtensionsTest(TestCase): self.senpy = Senpy(plugin_folder=self.examples_dir, app=self.app, default_plugins=False) + self.senpy.deactivate_all() self.senpy.activate_plugin("Dummy", sync=True) def test_init(self): @@ -41,21 +42,37 @@ class ExtensionsTest(TestCase): def test_discovery(self): """ Discovery of plugins in given folders. """ # noinspection PyProtectedMember - assert self.examples_dir in self.senpy._search_folders - print(self.senpy.plugins) - assert "Dummy" in self.senpy.plugins + print(self.senpy.plugins()) + assert self.senpy.get_plugin("dummy") + + def test_add_delete(self): + '''Should be able to add and delete new plugins. ''' + new = plugins.Plugin(name='new', description='new', version=0) + self.senpy.add_plugin(new) + assert new in self.senpy.plugins() + self.senpy.delete_plugin(new) + assert new not in self.senpy.plugins() + + def test_adding_folder(self): + """ It should be possible for senpy to look for plugins in more folders. """ + senpy = Senpy(plugin_folder=None, + app=self.app, + default_plugins=False) + assert not senpy.analysis_plugins + senpy.add_folder(self.examples_dir) + assert senpy.analysis_plugins + self.assertRaises(AttributeError, senpy.add_folder, 'DOES NOT EXIST') def test_installing(self): """ Installing a plugin """ info = { 'name': 'TestPip', - 'module': 'noop_plugin', + 'module': 'mynoop', 'description': None, 'requirements': ['noop'], 'version': 0 } - root = os.path.join(self.examples_dir, 'noop') - module = plugins.load_plugin_from_info(info, root=root, install=True) + module = plugins.from_info(info, root=self.examples_dir, install=True) assert module.name == 'TestPip' assert module import noop @@ -64,8 +81,8 @@ class ExtensionsTest(TestCase): def test_enabling(self): """ Enabling a plugin """ self.senpy.activate_all(sync=True) - assert len(self.senpy.plugins) >= 3 - assert self.senpy.plugins["Sleep"].is_activated + assert len(self.senpy.plugins()) >= 3 + assert self.senpy.get_plugin("Sleep").is_activated def test_installing_nonexistent(self): """ Fail if the dependencies cannot be met """ @@ -82,8 +99,8 @@ class ExtensionsTest(TestCase): def test_disabling(self): """ Disabling a plugin """ self.senpy.deactivate_all(sync=True) - assert not self.senpy.plugins["Dummy"].is_activated - assert not self.senpy.plugins["Sleep"].is_activated + assert not self.senpy.get_plugin("dummy").is_activated + assert not self.senpy.get_plugin("sleep").is_activated def test_default(self): """ Default plugin should be set """ @@ -108,6 +125,17 @@ class ExtensionsTest(TestCase): assert r2.analysis[0] == "plugins/Dummy_0.1" assert r1.entries[0]['nif:isString'] == 'input' + def test_analyse_empty(self): + """ Trying to analyse when no plugins are installed should raise an error.""" + senpy = Senpy(plugin_folder=None, + app=self.app, + default_plugins=False) + self.assertRaises(Error, senpy.analyse, Results()) + + def test_analyse_wrong(self): + """ Trying to analyse with a non-existent plugin should raise an error.""" + self.assertRaises(Error, analyse, self.senpy, algorithm='DOES NOT EXIST', input='test') + def test_analyse_jsonld(self): """ Using a plugin with JSON-LD input""" js_input = '''{ @@ -135,9 +163,10 @@ class ExtensionsTest(TestCase): def test_analyse_error(self): mm = mock.MagicMock() mm.id = 'magic_mock' + mm.name = 'mock' mm.is_activated = True mm.analyse_entries.side_effect = Error('error in analysis', status=500) - self.senpy.plugins['MOCK'] = mm + self.senpy.add_plugin(mm) try: analyse(self.senpy, input='nothing', algorithm='MOCK') assert False @@ -145,29 +174,29 @@ class ExtensionsTest(TestCase): assert 'error in analysis' in ex['message'] assert ex['status'] == 500 - mm.analyse.side_effect = Exception('generic exception on analysis') - mm.analyse_entries.side_effect = Exception( - 'generic exception on analysis') + ex = Exception('generic exception on analysis') + mm.analyse.side_effect = ex + mm.analyse_entries.side_effect = ex try: analyse(self.senpy, input='nothing', algorithm='MOCK') assert False - except Error as ex: + except Exception as ex: assert 'generic exception on analysis' in ex['message'] assert ex['status'] == 500 def test_filtering(self): """ Filtering plugins """ - assert len(self.senpy.filter_plugins(name="Dummy")) > 0 - assert not len(self.senpy.filter_plugins(name="notdummy")) - assert self.senpy.filter_plugins(name="Dummy", is_activated=True) + assert len(self.senpy.plugins(name="Dummy")) > 0 + assert not len(self.senpy.plugins(name="NotDummy")) + assert self.senpy.plugins(name="Dummy", is_activated=True) self.senpy.deactivate_plugin("Dummy", sync=True) - assert not len( - self.senpy.filter_plugins(name="Dummy", is_activated=True)) + assert not len(self.senpy.plugins(name="Dummy", + is_activated=True)) def test_load_default_plugins(self): senpy = Senpy(plugin_folder=self.examples_dir, default_plugins=True) - assert len(senpy.plugins) > 1 + assert len(senpy.plugins()) > 1 def test_convert_emotions(self): self.senpy.activate_all(sync=True) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index cf52108..cce80a1 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -12,6 +12,7 @@ from senpy.plugins.conversion.emotion.centroids import CentroidConversion class ShelfDummyPlugin(plugins.SentimentPlugin, plugins.ShelfMixin): + '''Dummy plugin for tests.''' def activate(self, *args, **kwargs): if 'counter' not in self.sh: self.sh['counter'] = 0 @@ -65,7 +66,7 @@ class PluginsTest(TestCase): ('EmotionPlugin', 1)] for name, num in cases: - res = plugins.pfilter(ps.plugins, plugin_type=name) + res = list(plugins.pfilter(ps.plugins, plugin_type=name)) assert len(res) == num def test_shelf(self): @@ -240,21 +241,20 @@ class PluginsTest(TestCase): assert res["onyx:hasEmotionCategory"] == "c2" -def make_mini_test(plugin_info): +def make_mini_test(fpath): def mini_test(self): - plugin = plugins.load_plugin_from_info(plugin_info, install=True) - plugin.test() + for plugin in plugins.from_path(fpath, install=True): + plugin.test() return mini_test def _add_tests(): root = os.path.join(os.path.dirname(__file__), '..') print(root) - plugs = plugins.load_plugins([root, ], loader=plugins.parse_plugin_info) - for k, v in plugs.items(): + for fpath in plugins.find_plugins([root, ]): pass - t_method = make_mini_test(v) - t_method.__name__ = 'test_plugin_{}'.format(k) + t_method = make_mini_test(fpath) + t_method.__name__ = 'test_plugin_{}'.format(fpath) setattr(PluginsTest, t_method.__name__, t_method) del t_method