Fix for async

Should fix #11

.gitignore

@@ -4,4 +4,7 @@
 dist
 build
 README.html
-__pycache__
+__pycache__
+VERSION
+Dockerfile-*
+Dockerfile

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-image: docker:latest
+image: gsiupm/dockermake:latest
 
 
 # When using dind, it's wise to use the overlayfs driver for
@@ -6,68 +6,60 @@ image: docker:latest
 variables:
   DOCKER_DRIVER: overlay
   DOCKERFILE: Dockerfile
+  IMAGENAME: $CI_REGISTRY_IMAGE
 
 stages:
   - test
-  - images
-  - release
+  - push
+  - clean
 
 .test: &test_definition
-  variables:
-    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.eggs"
-  cache:
-    paths:
-      - .eggs/
-    key: "$CI_PROJECT_NAME"
   stage: test
   script:
-    - python setup.py test
- 
+    - make -e test-$PYTHON_VERSION
+    
 test-3.5:
   <<: *test_definition
-  image: "python:3.5"
-
-test-3.4:
-  <<: *test_definition
-  image: "python:3.4"
+  variables:
+    PYTHON_VERSION: "3.5"
 
 test-2.7:
   <<: *test_definition
-  image: "python:2.7"
+  variables:
+    PYTHON_VERSION: "2.7"
 
 
 .image: &image_definition
-  variables:
-    PYTHON_VERSION: "3.5"
+  stage: push
   before_script:
     - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
   script:
-    - docker build -f Dockerfile-3.5 . -t $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG-$PYTHON_VERSION
-    - docker push $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG-$PYTHON_VERSION
-  stage: images
+    - make -e push-$PYTHON_VERSION
   only:
     - tags
     - triggers
 
-image-3.5:
+push-3.5:
   <<: *image_definition
   variables:
     PYTHON_VERSION: "3.5"
 
-image-2.7:
+push-2.7:
   <<: *image_definition
   variables:
     PYTHON_VERSION: "2.7"
 
-image-latest:
-  stage: release
-  before_script:
-    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
-  script:
-    - docker build -f Dockerfile . -t $CI_REGISTRY_IMAGE
-    - docker tag $CI_REGISTRY_IMAGE $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG
-    - docker push $CI_REGISTRY_IMAGE
-    - docker push $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG
+push-latest:
+  <<: *image_definition
+  variables:
+    PYTHON_VERSION: latest
   only:
     - master
-    - triggers
+    - triggers
+
+clean :
+  stage: clean
+  script:
+    - make -e clean
+  only:
+    - master

.pre-commit-config.yaml

@@ -0,0 +1,5 @@
+-   repo: git://github.com/pre-commit/pre-commit-hooks
+    sha: e626cd57090d8df0be21e4df0f4e55cc3511d6ab
+    hooks:
+    -   id: flake8
+    -   id: check-json

.travis.yml

@@ -1,8 +1,13 @@
+sudo: required
+
+services:
+  - docker
+
 language: python
-python:
-  - "2.7"
-  - "3.4"
-  - "3.5"
-install: "pip install -r requirements.txt"
+
+env:
+  - PYV=2.7
+  - PYV=3.4
+  - PYV=3.5
 # run nosetests - Tests
-script: nosetests
+script: make test-$PYV

Dockerfile

@@ -1 +0,0 @@
-Dockerfile-3.5

Dockerfile-2.7

@@ -1,9 +0,0 @@
-from python:2.7
-
-WORKDIR /usr/src/app
-ADD requirements.txt /usr/src/app/
-RUN pip install -r requirements.txt
-ADD . /usr/src/app/
-RUN pip install .
-
-ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]

Dockerfile-3.4

@@ -1,9 +0,0 @@
-from python:3.4
-
-WORKDIR /usr/src/app
-ADD requirements.txt /usr/src/app/
-RUN pip install -r requirements.txt
-ADD . /usr/src/app/
-RUN pip install .
-
-ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]

Dockerfile-3.5

@@ -1,9 +0,0 @@
-from python:3.5
-
-WORKDIR /usr/src/app
-ADD requirements.txt /usr/src/app/
-RUN pip install -r requirements.txt
-ADD . /usr/src/app/
-RUN pip install .
-
-ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]

Dockerfile.deps

@@ -1,33 +0,0 @@
-from python:2.7
-
-RUN apt-get update
-RUN apt-get -y install git
-RUN mkdir -p /senpy-plugins
-
-RUN apt-get -y install python-numpy
-RUN apt-get -y install python-scipy
-RUN apt-get -y install python-sklearn
-RUN apt-get -y install python-gevent
-RUN apt-get -y install libopenblas-dev
-RUN apt-get -y install gfortran
-RUN apt-get -y install libxml2-dev libxslt1-dev python-dev
-
-#RUN pip install --upgrade pip
-
-ADD id_rsa /root/.ssh/id_rsa
-RUN chmod 700 /root/.ssh/id_rsa
-RUN echo "Host github.com\n\tStrictHostKeyChecking no\n" >> /root/.ssh/config
-
-RUN git clone https://github.com/gsi-upm/senpy /usr/src/app/
-RUN git clone git@github.com:gsi-upm/senpy-plugins-enterprise /senpy-plugins/enterprise
-RUN git clone https://github.com/gsi-upm/senpy-plugins-community /senpy-plugins/community
-
-RUN pip install /usr/src/app
-RUN pip install --no-use-wheel -r /senpy-plugins/enterprise/requirements.txt
-RUN python -m nltk.downloader stopwords
-RUN python -m nltk.downloader punkt
-RUN python -m nltk.downloader maxent_treebank_pos_tagger
-RUN python -m nltk.downloader wordnet
-
-WORKDIR /senpy-plugins
-ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]

Dockerfile.template

@@ -1,9 +1,22 @@
 from python:{{PYVERSION}}
 
-WORKDIR /usr/src/app
-ADD requirements.txt /usr/src/app/
-RUN pip install -r requirements.txt
-ADD . /usr/src/app/
-RUN pip install .
+MAINTAINER J. Fernando Sánchez <jf.sanchez@upm.es>
 
-ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]
+RUN mkdir /cache/ /senpy-plugins /data/
+
+VOLUME /data/
+
+ENV PIP_CACHE_DIR=/cache/ SENPY_DATA=/data
+
+ONBUILD COPY . /senpy-plugins/
+ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
+ONBUILD WORKDIR /senpy-plugins/
+
+
+WORKDIR /usr/src/app
+COPY test-requirements.txt requirements.txt /usr/src/app/
+RUN pip install --use-wheel -r test-requirements.txt -r requirements.txt
+COPY . /usr/src/app/
+RUN pip install --no-deps --no-index .
+
+ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

Makefile

@@ -1,19 +1,26 @@
-PYVERSIONS=3.5 3.4 2.7
+PYVERSIONS=3.5 2.7
 PYMAIN=$(firstword $(PYVERSIONS))
 NAME=senpy
 REPO=gsiupm
-VERSION=$(shell cat $(NAME)/VERSION)
-TARNAME=$(NAME)-$(subst -,.,$(VERSION)).tar.gz 
-IMAGENAME=$(REPO)/$(NAME):$(VERSION)
-TEST_COMMAND=gitlab-runner exec docker --cache-dir=/tmp/gitlabrunner --docker-volumes /tmp/gitlabrunner:/tmp/gitlabrunner --env CI_PROJECT_NAME=$(NAME)
+VERSION=$(shell git describe --tags --dirty 2>/dev/null)
+TARNAME=$(NAME)-$(VERSION).tar.gz 
+IMAGENAME=$(REPO)/$(NAME)
+IMAGEWTAG=$(IMAGENAME):$(VERSION)
+action="test-${PYMAIN}"
 
 all: build run
 
+.FORCE:
+
+version: .FORCE
+	@echo $(VERSION) > $(NAME)/VERSION
+	@echo $(VERSION)
+
 yapf:
 	yapf -i -r senpy
 	yapf -i -r tests
 
-dev:
+init:
 	pip install --user pre-commit
 	pre-commit install
 
@@ -28,20 +35,27 @@ quick_build: $(addprefix build-, $(PYMAIN))
 
 build: $(addprefix build-, $(PYVERSIONS))
 
-build-%: Dockerfile-%
-	docker build -t '$(IMAGENAME)-python$*' -f Dockerfile-$* .;
+build-%: version Dockerfile-%
+	docker build -t '$(IMAGEWTAG)-python$*' -f Dockerfile-$* .;
 
 quick_test: $(addprefix test-,$(PYMAIN))
 
-test: $(addprefix test-,$(PYVERSIONS))
+dev-%:
+	@docker start $(NAME)-dev$* || (\
+		$(MAKE) build-$*; \
+		docker run -d -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti --name $(NAME)-dev$* '$(IMAGEWTAG)-python$*'; \
+	)\
 
-debug-%:
-	(docker start $(NAME)-debug && docker attach $(NAME)-debug) || docker run -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti --name $(NAME)-debug '$(IMAGENAME)-python$*'
+	docker exec -ti $(NAME)-dev$* bash
 
-debug: debug-$(PYMAIN)
+dev: dev-$(PYMAIN)
 
-test-%:
-	$(TEST_COMMAND) test-$*
+test-all: $(addprefix test-,$(PYVERSIONS))
+
+test-%: build-%
+	docker run --rm --entrypoint /usr/local/bin/python -w /usr/src/app $(IMAGEWTAG)-python$*  setup.py test
+
+test: test-$(PYMAIN)
 
 dist/$(TARNAME):
 	docker run --rm -ti -v $$PWD:/usr/src/app/ -w /usr/src/app/ python:$(PYMAIN) python setup.py sdist;
@@ -53,23 +67,19 @@ pip_test-%: sdist
 
 pip_test: $(addprefix pip_test-,$(PYVERSIONS))
 
-upload-%: test-%
-	docker push '$(IMAGENAME)-python$*'
-
-upload: test $(addprefix upload-,$(PYVERSIONS))
-	docker tag '$(IMAGENAME)-python$(PYMAIN)' '$(IMAGENAME)'
-	docker tag '$(IMAGENAME)-python$(PYMAIN)' '$(REPO)/$(NAME)'
-	docker push '$(IMAGENAME)'
-	docker push '$(REPO)/$(NAME)'
-
 clean:
-	@docker ps -a | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[1] != "${VERSION}"){ print $$1;}}' | xargs docker rm 2>/dev/null|| true
-	@docker images | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[1] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true
-	@docker rmi $(NAME)-debug 2>/dev/null || true
+	@docker ps -a | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true
+	@docker images | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true
+	@docker rmi $(NAME)-dev 2>/dev/null || true
 
-upload_git:
+
+git_commit:
 	git commit -a
+
+git_tag:
 	git tag ${VERSION}
+
+git_push:
 	git push --tags origin master
 
 pip_upload:
@@ -77,7 +87,21 @@ pip_upload:
 
 pip_test: $(addprefix pip_test-,$(PYVERSIONS))
 
-run: build
-	docker run --rm -p 5000:5000 -ti '$(IMAGENAME)-python$(PYMAIN)'
+run-%: build-%
+	docker run --rm -p 5000:5000 -ti '$(IMAGEWTAG)-python$(PYMAIN)' --default-plugins
 
-.PHONY: test test-% build-% build test pip_test run yapf dev
+run: run-$(PYMAIN)
+
+push-latest: build-$(PYMAIN)
+	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
+	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGENAME)'
+	docker push '$(IMAGENAME)'
+	docker push '$(IMAGEWTAG)'
+
+push-%: build-%
+	docker push $(IMAGENAME):$(VERSION)-python$*
+
+ci:
+	gitlab-runner exec docker --docker-volumes /var/run/docker.sock:/var/run/docker.sock --env CI_PROJECT_NAME=$(NAME) ${action}
+
+.PHONY: test test-% test-all build-% build test pip_test run yapf push-main push-% dev ci version .FORCE

README.rst

@@ -38,9 +38,9 @@ If you want to install senpy globally, use sudo instead of the ``--user`` flag.
 
 Docker Image
 ************
-Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy --host 0.0.0.0 --default-plugins``.
+Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy --default-plugins``.
 
-To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --host 0.0.0.0 --default-plugins -f /plugins``
+To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --default-plugins -f /plugins``
 
 Usage
 -----

docs/api.rst

@@ -62,6 +62,7 @@ NIF API
    :query o outformat: one of `turtle` (default), `text`, `json-ld`
    :query p prefix: prefix for the URIs
    :query algo algorithm: algorithm/plugin to use for the analysis. For a list of options, see :http:get:`/api/plugins`. If not provided, the default plugin will be used (:http:get:`/api/plugins/default`).
+   :query algo emotionModel: desired emotion model in the results. If the requested algorithm does not use that emotion model, there are conversion plugins specifically for this. If none of the plugins match, an error will be returned, which includes the results *as is*.
 
    :reqheader Accept: the response content type depends on
                       :mailheader:`Accept` header
@@ -69,6 +70,7 @@ NIF API
                             header of request
    :statuscode 200: no error
    :statuscode 404: service not found
+   :statuscode 400: error while processing the request
 
 .. http:post:: /api
 
@@ -93,51 +95,53 @@ NIF API
         {
             "@context": {
                    ...
-            }, 
-            "sentiment140": {
-                "name": "sentiment140", 
-                "is_activated": true, 
-                "version": "0.1", 
-                "extra_params": {
-                    "@id": "extra_params_sentiment140_0.1", 
-                    "language": {
-                        "required": false, 
-                        "@id": "lang_sentiment140", 
-                        "options": [
-                            "es", 
-                            "en", 
-                            "auto"
-                        ], 
-                        "aliases": [
-                            "language", 
-                            "l"
-                        ]
-                    }
-                }, 
-                "@id": "sentiment140_0.1"
-            }, 
-            "rand": {
-                "name": "rand", 
-                "is_activated": true, 
-                "version": "0.1", 
-                "extra_params": {
-                    "@id": "extra_params_rand_0.1", 
-                    "language": {
-                        "required": false, 
-                        "@id": "lang_rand", 
-                        "options": [
-                            "es", 
-                            "en", 
-                            "auto"
-                        ], 
-                        "aliases": [
-                            "language", 
-                            "l"
-                        ]
-                    }
-                }, 
-                "@id": "rand_0.1"
-            }
+            },
+            "@type": "plugins",
+            "plugins": [
+                  {
+                    "name": "sentiment140", 
+                    "is_activated": true, 
+                    "version": "0.1", 
+                    "extra_params": {
+                        "@id": "extra_params_sentiment140_0.1", 
+                        "language": {
+                            "required": false, 
+                            "@id": "lang_sentiment140", 
+                            "options": [
+                                "es", 
+                                "en", 
+                                "auto"
+                            ], 
+                            "aliases": [
+                                "language", 
+                                "l"
+                            ]
+                        }
+                    }, 
+                    "@id": "sentiment140_0.1"
+                }, {
+                    "name": "rand", 
+                    "is_activated": true, 
+                    "version": "0.1", 
+                    "extra_params": {
+                        "@id": "extra_params_rand_0.1", 
+                        "language": {
+                            "required": false, 
+                            "@id": "lang_rand", 
+                            "options": [
+                                "es", 
+                                "en", 
+                                "auto"
+                            ], 
+                            "aliases": [
+                                "language", 
+                                "l"
+                            ]
+                        }
+                    }, 
+                    "@id": "rand_0.1"
+                }
+            ]
         }
 
 
@@ -148,7 +152,7 @@ NIF API
 
    .. sourcecode:: http
 
-      GET /api/plugins/rand HTTP/1.1
+      GET /api/plugins/rand/ HTTP/1.1
       Host: localhost
       Accept: application/json, text/javascript
 
@@ -159,6 +163,7 @@ NIF API
 
       {
           "@id": "rand_0.1",
+          "@type": "sentimentPlugin",
           "extra_params": {
               "@id": "extra_params_rand_0.1",
               "language": {
@@ -185,24 +190,3 @@ NIF API
 
    Return the information about the default plugin.
 
-.. http:get:: /api/plugins/<pluginname>/{de}activate
-
-   {De}activate a plugin.
-
-   **Example request**:
-
-   .. sourcecode:: http
-
-      GET /api/plugins/rand/deactivate HTTP/1.1
-      Host: localhost
-      Accept: application/json, text/javascript
-
-
-   **Example response**:
-
-   .. sourcecode:: http
-
-       {
-          "@context": {}, 
-          "message": "Ok"
-       }

docs/conversion.rst

@@ -0,0 +1,116 @@
+Introduction
+------------
+
+Senpy includes experimental support for emotion/sentiment conversion plugins.
+
+
+Use
+---
+
+Consider the original query: `http://127.0.0.1:5000/api/?i=hello&algo=emoRand`_
+
+The requested plugin (emoRand) returns emotions using Ekman's model (or big6 in EmotionML):
+
+.. code:: json
+
+
+          ... rest of the document ...
+          {
+            "@type": "emotionSet",
+            "onyx:hasEmotion": {
+                "@type": "emotion",
+                "onyx:hasEmotionCategory": "emoml:big6anger"
+            },
+            "prov:wasGeneratedBy": "plugins/emoRand_0.1"
+          }
+
+          
+
+To get these emotions in VAD space (FSRE dimensions in EmotionML), we'd do this:
+
+`http://127.0.0.1:5000/api/?i=hello&algo=emoRand&emotionModel=emoml:fsre-dimensions`_
+
+This call, provided there is a valid conversion plugin from Ekman's to VAD, would return something like this:
+
+.. code:: json
+
+
+          ... rest of the document ...
+          {
+            "@type": "emotionSet",
+            "onyx:hasEmotion": {
+                "@type": "emotion",
+                "onyx:hasEmotionCategory": "emoml:big6anger"
+                },
+            "prov:wasGeneratedBy": "plugins/emoRand_0.1"
+          }, {
+            "@type": "emotionSet",
+            "onyx:hasEmotion": {
+                "@type": "emotion",
+                "A": 7.22,
+                "D": 6.28,
+                "V": 8.6
+            },
+            "prov:wasGeneratedBy": "plugins/Ekman2VAD_0.1"
+
+          }
+
+
+That is called a *full* response, as it simply adds the converted emotion alongside.
+It is also possible to get the original emotion nested within the new converted emotion, using the `conversion=nested` parameter:
+
+.. code:: json
+
+
+          ... rest of the document ...
+          {
+            "@type": "emotionSet",
+            "onyx:hasEmotion": {
+                "@type": "emotion",
+                "onyx:hasEmotionCategory": "emoml:big6anger"
+                },
+            "prov:wasGeneratedBy": "plugins/emoRand_0.1"
+            "onyx:wasDerivedFrom": {
+                "@type": "emotionSet",
+                "onyx:hasEmotion": {
+                    "@type": "emotion",
+                    "A": 7.22,
+                    "D": 6.28,
+                    "V": 8.6
+                },
+                "prov:wasGeneratedBy": "plugins/Ekman2VAD_0.1"
+             }
+
+          }
+
+
+Lastly, `conversion=filtered` would only return the converted emotions.
+
+Developing a conversion plugin
+------------------------------
+
+Conversion plugins are discovered by the server just like any other plugin.
+The difference is the slightly different API, and the need to specify the `source` and `target` of the conversion.
+For instance, an emotion conversion plugin needs the following:
+
+
+.. code:: yaml
+          
+
+          ---
+          onyx:doesConversion:
+            - onyx:conversionFrom: emoml:big6
+              onyx:conversionTo: emoml:fsre-dimensions
+            - onyx:conversionFrom: emoml:fsre-dimensions
+              onyx:conversionTo: emoml:big6
+
+
+
+
+.. code:: python
+
+
+          class MyConversion(EmotionConversionPlugin):
+
+              def convert(self, emotionSet, fromModel, toModel, params):
+                  pass

docs/examples/results/example-complete.json

@@ -53,7 +53,8 @@
           "@id": "http://micro.blog/status1#char=16,77",
           "nif:beginIndex": 16,
           "nif:endIndex": 77,
-          "nif:anchorOf": "put your Windows Phone on your newest #open technology program"
+          "nif:anchorOf": "put your Windows Phone on your newest #open technology program",
+          "prov:wasGeneratedBy": "me:SgAnalysis1"
         }
       ],
       "sentiments": [

docs/examples/results/example-suggestion.json

@@ -24,7 +24,8 @@
           "@id": "http://micro.blog/status1#char=16,77",
           "nif:beginIndex": 16,
           "nif:endIndex": 77,
-          "nif:anchorOf": "put your Windows Phone on your newest #open technology program"
+          "nif:anchorOf": "put your Windows Phone on your newest #open technology program",
+          "prov:wasGeneratedBy": "me:SgAnalysis1"
         }
       ],
       "sentiments": [

docs/plugins.rst

@@ -1,47 +1,130 @@
 Developing new plugins
 ----------------------
+This document describes how to develop a new analysis plugin. For an example of conversion plugins, see :doc:`conversion`.
+
 Each plugin represents a different analysis process.There are two types of files that are needed by senpy for loading a plugin:
 
-Plugins Interface
-=======
 - Definition file, has the ".senpy" extension.
 - Code file, is a python file.
 
+This separation will allow us to deploy plugins that use the same code but employ different parameters.
+For instance, one could use the same classifier and processing in several plugins, but train with different datasets.
+This scenario is particularly useful for evaluation purposes.
+
+The only limitation is that the name of each plugin needs to be unique.
+
 Plugins Definitions
 ===================
 
-The definition file can be written in JSON or YAML, where the data representation consists on attribute-value pairs.
-The principal attributes are:
+The definition file contains all the attributes of the plugin, and can be written in YAML or JSON.
+The most important attributes are:
 
-* name: plugin name used in senpy to call the plugin.
-* module: indicates the module that will be loaded
+* **name**: unique name that senpy will use internally to identify the plugin.
+* **module**: indicates the module that contains the plugin code, which will be automatically loaded by senpy.
+* **version**
+* extra_params: used to specify parameters that the plugin accepts that are not already part of the senpy API. Those parameters may be required, and have aliased names. For instance:
 
-.. code:: python
+  .. code:: yaml
+
+            extra_params:
+                hello_param:
+                    aliases: # required
+                        - hello_param
+                        - hello
+                    required: true
+                    default: Hi you
+                    values:
+                        - Hi you
+                        - Hello y'all
+                        - Howdy
+
+  Parameter validation will fail if a required parameter without a default has not been provided, or if the definition includes a set of values and the provided one does not match one of them.
+
+
+A complete example:
+
+.. code:: yaml
+          
+          name: <Name of the plugin>
+          module: <Python file>
+          version: 0.1
+
+And the json equivalent:
+
+.. code:: json
 
           {
-            "name" : "senpyPlugin",
-            "module" : "{python code file}"
+            "name": "<Name of the plugin>",
+            "module": "<Python file>",
+            "version": "0.1"
           }
 
-.. code:: python
-          
-          name: senpyPlugin
-          module: {python code file}
 
 Plugins Code
-=================
+============
 
 The basic methods in a plugin are:
 
 * __init__
 * activate: used to load memory-hungry resources
 * deactivate: used to free up resources
-* analyse: called in every user requests. It takes in the parameters supplied by a user and should return a senpy Response.
+* analyse_entry: called in every user requests. It takes in the parameters supplied by a user and should yield one or more ``Entry`` objects.
 
 Plugins are loaded asynchronously, so don't worry if the activate method takes too long. The plugin will be marked as activated once it is finished executing the method.
 
+
+Example plugin
+==============
+
+In this section, we will implement a basic sentiment analysis plugin.
+To determine the polarity of each entry, the plugin will compare the length of the string to a threshold.
+This threshold will be included in the definition file.
+
+The definition file would look like this:
+
+.. code:: yaml
+
+          name: helloworld
+          module: helloworld
+          version: 0.0
+          threshold: 10
+
+
+Now, in a file named ``helloworld.py``:
+
+.. code:: python
+
+          #!/bin/env python
+          #helloworld.py
+
+          from senpy.plugins import SenpyPlugin
+          from senpy.models import Sentiment
+
+
+          class HelloWorld(SenpyPlugin):
+
+              def analyse_entry(entry, params):
+                  '''Basically do nothing with each entry'''
+
+                  sentiment = Sentiment()
+                  if len(entry.text) < self.threshold:
+                      sentiment['marl:hasPolarity'] = 'marl:Positive'
+                  else:
+                      sentiment['marl:hasPolarity'] = 'marl:Negative'
+                  entry.sentiments.append(sentiment)
+                  yield entry
+
+
 F.A.Q.
 ======
+Why does the analyse function yield instead of return?
+??????????????????????????????????????????????????????
+
+This is so that plugins may add new entries to the response or filter some of them.
+For instance, a `context detection` plugin may add a new entry for each context in the original entry.
+On the other hand, a conveersion plugin may leave out those entries that do not contain relevant information.
+
+
 If I'm using a classifier, where should I train it?
 ???????????????????????????????????????????????????
 
@@ -78,17 +161,17 @@ This example ilustrate how to implement the Sentiment140 service as a plugin in
 .. code:: python
 
           class Sentiment140Plugin(SentimentPlugin):
-              def analyse(self, **params):
+              def analyse_entry(self, entry, params):
+                  text = entry.text
                   lang = params.get("language", "auto")
                   res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
                                       json.dumps({"language": lang,
-                                                  "data": [{"text": params["input"]}]
+                                                  "data": [{"text": text}]
                                                   }
                                                  )
                                       )
 
                   p = params.get("prefix", None)
-                  response = Results(prefix=p)
                   polarity_value = self.maxPolarityValue*int(res.json()["data"][0]
                                                              ["polarity"]) * 0.25
                   polarity = "marl:Neutral"
@@ -98,18 +181,13 @@ This example ilustrate how to implement the Sentiment140 service as a plugin in
                   elif polarity_value < neutral_value:
                       polarity = "marl:Negative"
 
-                  entry = Entry(id="Entry0",
-                                nif__isString=params["input"])
                   sentiment = Sentiment(id="Sentiment0",
                                       prefix=p,
                                       marl__hasPolarity=polarity,
                                       marl__polarityValue=polarity_value)
                   sentiment.prov__wasGeneratedBy = self.id
-                  entry.sentiments = []
                   entry.sentiments.append(sentiment)
-                  entry.language = lang
-                  response.entries.append(entry)
-                  return response
+                  yield entry
 
 
 Where can I define extra parameters to be introduced in the request to my plugin?
@@ -143,9 +221,9 @@ The extraction of this paremeter is used in the analyse method of the Plugin int
 Where can I set up variables for using them in my plugin?
 ?????????????????????????????????????????????????????????
 
-You can add these variables in the definition file with the extracture of attribute-value pair.
+You can add these variables in the definition file with the structure of attribute-value pairs.
 
-Once you have added your variables, the next step is to extract them into the plugin. The plugin's __init__ method has a parameter called `info` where you can extract the values of the variables. This info parameter has the structure of a python dictionary.
+Every field added to the definition file is available to the plugin instance.
 
 Can I activate a DEBUG mode for my plugin?
 ???????????????????????????????????????????
@@ -154,7 +232,15 @@ You can activate the DEBUG mode by the command-line tool using the option -d.
 
 .. code:: bash
 
-   python -m senpy -d
+   senpy -d
+
+
+Additionally, with the ``--pdb`` option you will be dropped into a pdb post mortem shell if an exception is raised.
+
+.. code:: bash
+
+   senpy --pdb
+
 
 Where can I find more code examples?
 ????????????????????????????????????

docs/usage.rst

@@ -48,8 +48,8 @@ Once the server is launched, there is a basic endpoint in the server, which prov
 
 In case you want to know the different endpoints of the server, there is more information available in the NIF API section_.
 
-Video example
-=============
+CLI
+===
 
 This video shows how to use senpy through command-line tool.
 
@@ -58,16 +58,17 @@ https://asciinema.org/a/9uwef1ghkjk062cw2t4mhzpyk
 Request example in python
 =========================
 
-This example shows how to make a request to a plugin.
+This example shows how to make a request to the default plugin:
 
 .. code:: python
 
-    import requests
-    import json
+    from senpy.client import Client
 
-    r = requests.get('http://127.0.0.1:5000/api/?algo=rand&i=Testing')
-    response = r.content.decode('utf-8')
-    response_json = json.loads(response)
+    c = Client('http://127.0.0.1:5000/api/')
+    r = c.analyse('hello world')
+
+    for entry in r.entries:
+          print('{} -> {}'.format(entry.text, entry.emotions))
 
 
 

requirements.txt

@@ -1,7 +1,5 @@
 Flask>=0.10.1
-gunicorn>=19.0.0
 requests>=2.4.1
-GitPython>=0.3.2.RC1
 gevent>=1.1rc4
 PyLD>=0.6.5
 six
@@ -9,4 +7,5 @@ future
 jsonschema
 jsonref
 PyYAML
-semver
+rdflib
+rdflib-jsonld

senpy/VERSION

@@ -1 +0,0 @@
-0.7.1

senpy/__init__.py

@@ -20,21 +20,10 @@ Sentiment analysis server in Python
 from __future__ import print_function
 from .version import __version__
 
-try:
-    import semver
-    __version_info__ = semver.parse_version_info(__version__)
+import logging
 
-    if __version_info__.prerelease:
-        import logging
-        logger = logging.getLogger(__name__)
-        msg = 'WARNING: You are using a pre-release version of {} ({})'.format(
-            __name__, __version__)
-        if len(logging.root.handlers) > 0:
-            logger.info(msg)
-        else:
-            import sys
-            print(msg, file=sys.stderr)
-except ImportError:
-    print('semver not installed. Not doing version checking')
+logger = logging.getLogger(__name__)
+
+logger.info('Using senpy version: {}'.format(__version__))
 
 __all__ = ['api', 'blueprints', 'cli', 'extensions', 'models', 'plugins']

senpy/__main__.py

@@ -26,6 +26,7 @@ from gevent.wsgi import WSGIServer
 from gevent.monkey import patch_all
 import logging
 import os
+import sys
 import argparse
 import senpy
 
@@ -34,6 +35,22 @@ patch_all(thread=False)
 SERVER_PORT = os.environ.get("PORT", 5000)
 
 
+def info(type, value, tb):
+    if hasattr(sys, 'ps1') or not sys.stderr.isatty():
+        #  we are in interactive mode or we don't have a tty-like
+        #  device, so we call the default hook
+        sys.__excepthook__(type, value, tb)
+    else:
+        import traceback
+        import pdb
+        # we are NOT in interactive mode, print the exception...
+        traceback.print_exception(type, value, tb)
+        print
+        # ...then start the debugger in post-mortem mode.
+        # pdb.pm() # deprecated
+        pdb.post_mortem(tb)  # more "modern"
+
+
 def main():
     parser = argparse.ArgumentParser(description='Run a Senpy server')
     parser.add_argument(
@@ -57,7 +74,7 @@ def main():
     parser.add_argument(
         '--host',
         type=str,
-        default="127.0.0.1",
+        default="0.0.0.0",
         help='Use 0.0.0.0 to accept requests from any host.')
     parser.add_argument(
         '--port',
@@ -76,14 +93,15 @@ def main():
         '-i',
         action='store_true',
         default=False,
-        help='Do not run a server, only install plugin dependencies'
-    )
+        help='Do not run a server, only install plugin dependencies')
     args = parser.parse_args()
     logging.basicConfig()
     rl = logging.getLogger()
     rl.setLevel(getattr(logging, args.level))
     app = Flask(__name__)
     app.debug = args.debug
+    if args.debug:
+        sys.excepthook = info
     sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins)
     if args.only_install:
         sp.install_deps()

senpy/api.py

@@ -7,6 +7,31 @@ API_PARAMS = {
     "algorithm": {
         "aliases": ["algorithm", "a", "algo"],
         "required": False,
+    },
+    "outformat": {
+        "@id": "outformat",
+        "aliases": ["outformat", "o"],
+        "default": "json-ld",
+        "required": True,
+        "options": ["json-ld", "turtle"],
+    },
+    "expanded-jsonld": {
+        "@id": "expanded-jsonld",
+        "aliases": ["expanded", "expanded-jsonld"],
+        "required": True,
+        "default": 0
+    },
+    "emotionModel": {
+        "@id": "emotionModel",
+        "aliases": ["emotionModel", "emoModel"],
+        "required": False
+    },
+    "conversion": {
+        "@id": "conversion",
+        "description": "How to show the elements that have (not) been converted",
+        "required": True,
+        "options": ["filtered", "nested", "full"],
+        "default": "full"
     }
 }
 
@@ -47,13 +72,6 @@ NIF_PARAMS = {
         "default": "direct",
         "options": ["direct", "url", "file"],
     },
-    "outformat": {
-        "@id": "outformat",
-        "aliases": ["outformat", "o"],
-        "default": "json-ld",
-        "required": False,
-        "options": ["json-ld"],
-    },
     "language": {
         "@id": "language",
         "aliases": ["language", "l"],
@@ -76,12 +94,12 @@ NIF_PARAMS = {
 
 
 def parse_params(indict, spec=NIF_PARAMS):
-    outdict = {}
+    logger.debug("Parsing: {}\n{}".format(indict, spec))
+    outdict = indict.copy()
     wrong_params = {}
     for param, options in iteritems(spec):
         if param[0] != "@":  # Exclude json-ld properties
-            logger.debug("Param: %s - Options: %s", param, options)
-            for alias in options["aliases"]:
+            for alias in options.get("aliases", []):
                 if alias in indict:
                     outdict[param] = indict[alias]
             if param not in outdict:
@@ -95,8 +113,9 @@ def parse_params(indict, spec=NIF_PARAMS):
                    outdict[param] not in spec[param]["options"]:
                     wrong_params[param] = spec[param]
     if wrong_params:
+        logger.debug("Error parsing: %s", wrong_params)
         message = Error(
-            status=404,
+            status=400,
             message="Missing or invalid parameters",
             parameters=outdict,
             errors={param: error

senpy/blueprints.py

@@ -17,10 +17,11 @@
 """
 Blueprints for Senpy
 """
-from flask import (Blueprint, request, current_app,
-                   render_template, url_for, jsonify)
+from flask import (Blueprint, request, current_app, render_template, url_for,
+                   jsonify)
 from .models import Error, Response, Plugins, read_schema
-from .api import WEB_PARAMS, parse_params
+from .api import WEB_PARAMS, API_PARAMS, parse_params
+from .version import __version__
 from functools import wraps
 
 import logging
@@ -29,6 +30,7 @@ logger = logging.getLogger(__name__)
 
 api_blueprint = Blueprint("api", __name__)
 demo_blueprint = Blueprint("demo", __name__)
+ns_blueprint = Blueprint("ns", __name__)
 
 
 def get_params(req):
@@ -43,12 +45,21 @@ def get_params(req):
 
 @demo_blueprint.route('/')
 def index():
-    return render_template("index.html")
+    return render_template("index.html", version=__version__)
 
 
 @api_blueprint.route('/contexts/<entity>.jsonld')
 def context(entity="context"):
-    return jsonify({"@context": Response.context})
+    context = Response._context
+    context['@vocab'] = url_for('ns.index', _external=True)
+    return jsonify({"@context": context})
+
+
+@ns_blueprint.route('/')  # noqa: F811
+def index():
+    context = Response._context
+    context['@vocab'] = url_for('.ns', _external=True)
+    return jsonify({"@context": context})
 
 
 @api_blueprint.route('/schemas/<schema>')
@@ -62,26 +73,39 @@ def schema(schema="definitions"):
 def basic_api(f):
     @wraps(f)
     def decorated_function(*args, **kwargs):
-        print('Getting request:')
-        print(request)
         raw_params = get_params(request)
-        web_params = parse_params(raw_params, spec=WEB_PARAMS)
+        headers = {'X-ORIGINAL-PARAMS': raw_params}
+        # Get defaults
+        web_params = parse_params({}, spec=WEB_PARAMS)
+        api_params = parse_params({}, spec=API_PARAMS)
 
-        if hasattr(request, 'params'):
-            request.params.update(raw_params)
-        else:
-            request.params = raw_params
+        outformat = 'json-ld'
         try:
+            print('Getting request:')
+            print(request)
+            web_params = parse_params(raw_params, spec=WEB_PARAMS)
+            api_params = parse_params(raw_params, spec=API_PARAMS)
+            if hasattr(request, 'params'):
+                request.params.update(api_params)
+            else:
+                request.params = api_params
             response = f(*args, **kwargs)
         except Error as ex:
             response = ex
-        in_headers = web_params["inHeaders"] != "0"
-        headers = {'X-ORIGINAL-PARAMS': raw_params}
+
+        in_headers = web_params['inHeaders'] != "0"
+        expanded = api_params['expanded-jsonld']
+        outformat = api_params['outformat']
+
         return response.flask(
             in_headers=in_headers,
             headers=headers,
-            context_uri=url_for(
-                'api.context', entity=type(response).__name__, _external=True))
+            prefix=url_for('.api', _external=True),
+            context_uri=url_for('api.context',
+                                entity=type(response).__name__,
+                                _external=True),
+            outformat=outformat,
+            expanded=expanded)
 
     return decorated_function
 
@@ -102,22 +126,15 @@ def plugins():
 
 
 @api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
-@api_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
 @basic_api
-def plugin(plugin=None, action="list"):
+def plugin(plugin=None):
     sp = current_app.senpy
     if plugin == 'default' and sp.default_plugin:
-        response = sp.default_plugin
-        plugin = response.name
-    elif plugin in sp.plugins:
-        response = sp.plugins[plugin]
+        return sp.default_plugin
+    plugins = sp.filter_plugins(
+        id='plugins/{}'.format(plugin)) or sp.filter_plugins(name=plugin)
+    if plugins:
+        response = list(plugins.values())[0]
     else:
         return Error(message="Plugin not found", status=404)
-    if action == "list":
-        return response
-    method = "{}_plugin".format(action)
-    if (hasattr(sp, method)):
-        getattr(sp, method)(plugin)
-        return Response(message="Ok")
-    else:
-        return Error(message="action '{}' not allowed".format(action))
+    return response

senpy/client.py

@@ -6,7 +6,6 @@ logger = logging.getLogger(__name__)
 
 
 class Client(object):
-
     def __init__(self, endpoint):
         self.endpoint = endpoint
 
@@ -15,13 +14,10 @@ class Client(object):
 
     def request(self, path=None, method='GET', **params):
         url = '{}{}'.format(self.endpoint, path)
-        response = requests.request(method=method,
-                                    url=url,
-                                    params=params)
+        response = requests.request(method=method, url=url, params=params)
         try:
             resp = models.from_dict(response.json())
             resp.validate(resp)
-            return resp
         except Exception as ex:
             logger.error(('There seems to be a problem with the response:\n'
                           '\tURL: {url}\n'
@@ -30,16 +26,12 @@ class Client(object):
                           '#### Response:\n'
                           '\tCode: {code}'
                           '\tContent: {content}'
-                          '\n').format(error=ex,
-                                       url=url,
-                                       code=response.status_code,
-                                       content=response.content))
+                          '\n').format(
+                              error=ex,
+                              url=url,
+                              code=response.status_code,
+                              content=response.content))
             raise ex
-
-
-if __name__ == '__main__':
-    c = Client('http://senpy.cluster.gsi.dit.upm.es/api/')
-    resp = c.analyse('hello')
-    # print(resp)
-    print(resp.entries)
-    resp.validate()
+        if isinstance(resp, models.Error):
+            raise resp
+        return resp

senpy/extensions.py

@@ -1,22 +1,23 @@
 """
+Main class for Senpy.
+It orchestrates plugin (de)activation and analysis.
 """
 from future import standard_library
 standard_library.install_aliases()
 
-from .plugins import SentimentPlugin
-from .models import Error
-from .blueprints import api_blueprint, demo_blueprint
+from .plugins import SentimentPlugin, SenpyPlugin
+from .models import Error, Entry, Results
+from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
 from .api import API_PARAMS, NIF_PARAMS, parse_params
 
-from git import Repo, InvalidGitRepositoryError
-
 from threading import Thread
 
 import os
+import copy
 import fnmatch
 import inspect
 import sys
-import imp
+import importlib
 import logging
 import traceback
 import yaml
@@ -30,18 +31,21 @@ class Senpy(object):
 
     def __init__(self,
                  app=None,
-                 plugin_folder="plugins",
+                 plugin_folder=".",
                  default_plugins=False):
         self.app = app
-
         self._search_folders = set()
         self._plugin_list = []
         self._outdated = True
+        self._default = None
 
         self.add_folder(plugin_folder)
         if default_plugins:
-            base_folder = os.path.join(os.path.dirname(__file__), "plugins")
-            self.add_folder(base_folder)
+            self.add_folder('plugins', from_root=True)
+        else:
+            # Add only conversion plugins
+            self.add_folder(os.path.join('plugins', 'conversion'),
+                            from_root=True)
 
         if app is not None:
             self.init_app(app)
@@ -60,9 +64,12 @@ class Senpy(object):
         else:
             app.teardown_request(self.teardown)
         app.register_blueprint(api_blueprint, url_prefix="/api")
+        app.register_blueprint(ns_blueprint, url_prefix="/ns")
         app.register_blueprint(demo_blueprint, url_prefix="/")
 
-    def add_folder(self, folder):
+    def add_folder(self, folder, from_root=False):
+        if from_root:
+            folder = os.path.join(os.path.dirname(__file__), folder)
         logger.debug("Adding folder: %s", folder)
         if os.path.isdir(folder):
             self._search_folders.add(folder)
@@ -70,10 +77,9 @@ class Senpy(object):
         else:
             logger.debug("Not a folder: %s", folder)
 
-    def analyse(self, **params):
-        algo = None
-        logger.debug("analysing with params: {}".format(params))
+    def _find_plugin(self, params):
         api_params = parse_params(params, spec=API_PARAMS)
+        algo = None
         if "algorithm" in api_params and api_params["algorithm"]:
             algo = api_params["algorithm"]
         elif self.plugins:
@@ -97,32 +103,114 @@ class Senpy(object):
                 status=400,
                 message=("The algorithm '{}'"
                          " is not activated yet").format(algo))
-        plug = self.plugins[algo]
+        return self.plugins[algo]
+
+    def _get_params(self, params, plugin):
         nif_params = parse_params(params, spec=NIF_PARAMS)
-        extra_params = plug.get('extra_params', {})
+        extra_params = plugin.get('extra_params', {})
         specific_params = parse_params(params, spec=extra_params)
         nif_params.update(specific_params)
+        return nif_params
+
+    def _get_entries(self, params):
+        entry = None
+        if params['informat'] == 'text':
+            entry = Entry(text=params['input'])
+        else:
+            raise NotImplemented('Only text input format implemented')
+        yield entry
+
+    def analyse(self, **api_params):
+        logger.debug("analysing with params: {}".format(api_params))
+        plugin = self._find_plugin(api_params)
+        nif_params = self._get_params(api_params, plugin)
+        resp = Results()
+        if 'with_parameters' in api_params:
+            resp.parameters = nif_params
         try:
-            resp = plug.analyse(**nif_params)
-            resp.analysis.append(plug)
+            entries = []
+            for i in self._get_entries(nif_params):
+                entries += list(plugin.analyse_entry(i, nif_params))
+            resp.entries = entries
+            self.convert_emotions(resp, plugin, nif_params)
+            resp.analysis.append(plugin.id)
             logger.debug("Returning analysis result: {}".format(resp))
+        except Error as ex:
+            logger.exception('Error returning analysis result')
+            resp = ex
         except Exception as ex:
+            logger.exception('Error returning analysis result')
             resp = Error(message=str(ex), status=500)
         return resp
 
+    def _conversion_candidates(self, fromModel, toModel):
+        candidates = self.filter_plugins(**{'@type': 'emotionConversionPlugin'})
+        for name, candidate in candidates.items():
+            for pair in candidate.onyx__doesConversion:
+                logging.debug(pair)
+
+                if pair['onyx:conversionFrom'] == fromModel \
+                   and pair['onyx:conversionTo'] == toModel:
+                    # logging.debug('Found candidate: {}'.format(candidate))
+                    yield candidate
+
+    def convert_emotions(self, resp, plugin, params):
+        """
+        Conversion of all emotions in a response.
+        In addition to converting from one model to another, it has
+        to include the conversion plugin to the analysis list.
+        Needless to say, this is far from an elegant solution, but it works.
+        @todo refactor and clean up
+        """
+        fromModel = plugin.get('onyx:usesEmotionModel', None)
+        toModel = params.get('emotionModel', None)
+        output = params.get('conversion', None)
+        logger.debug('Asked for model: {}'.format(toModel))
+        logger.debug('Analysis plugin uses model: {}'.format(fromModel))
+
+        if not toModel:
+            return
+        try:
+            candidate = next(self._conversion_candidates(fromModel, toModel))
+        except StopIteration:
+            e = Error(('No conversion plugin found for: '
+                       '{} -> {}'.format(fromModel, toModel)))
+            e.original_response = resp
+            e.parameters = params
+            raise e
+        newentries = []
+        for i in resp.entries:
+            if output == "full":
+                newemotions = copy.deepcopy(i.emotions)
+            else:
+                newemotions = []
+            for j in i.emotions:
+                for k in candidate.convert(j, fromModel, toModel, params):
+                    k.prov__wasGeneratedBy = candidate.id
+                    if output == 'nested':
+                        k.prov__wasDerivedFrom = j
+                    newemotions.append(k)
+            i.emotions = newemotions
+            newentries.append(i)
+        resp.entries = newentries
+        resp.analysis.append(candidate.id)
+
     @property
     def default_plugin(self):
-        candidates = self.filter_plugins(is_activated=True)
-        if len(candidates) > 0:
-            candidate = list(candidates.values())[0]
-            logger.debug("Default: {}".format(candidate.name))
-            return candidate
-        else:
-            return None
+        candidate = self._default
+        if not candidate:
+            candidates = self.filter_plugins(is_activated=True)
+            if len(candidates) > 0:
+                candidate = list(candidates.values())[0]
+        logger.debug("Default: {}".format(candidate))
+        return candidate
 
-    def parameters(self, algo):
-        return getattr(
-            self.plugins.get(algo) or self.default_plugin, "extra_params", {})
+    @default_plugin.setter
+    def default_plugin(self, value):
+        if isinstance(value, SenpyPlugin):
+            self._default = value
+        else:
+            self._default = self.plugins[value]
 
     def activate_all(self, sync=False):
         ps = []
@@ -164,7 +252,8 @@ class Senpy(object):
                     plugin.name, ex, traceback.format_exc())
                 logger.error(msg)
                 raise Error(msg)
-        if sync:
+
+        if sync or 'async' in plugin and not plugin.async:
             act()
         else:
             th = Thread(target=act)
@@ -184,34 +273,23 @@ class Senpy(object):
                 plugin.deactivate()
                 logger.info("Plugin deactivated: {}".format(plugin.name))
             except Exception as ex:
-                logger.error("Error deactivating plugin {}: {}".format(
-                    plugin.name, ex))
+                logger.error(
+                    "Error deactivating plugin {}: {}".format(plugin.name, ex))
                 logger.error("Trace: {}".format(traceback.format_exc()))
 
-        if sync:
+        if sync or 'async' in plugin and not plugin.async:
             deact()
         else:
             th = Thread(target=deact)
             th.start()
 
-    def reload_plugin(self, name):
-        logger.debug("Reloading {}".format(name))
-        plugin = self.plugins[name]
-        try:
-            del self.plugins[name]
-            nplug = self._load_plugin(plugin.module, plugin.path)
-            self.plugins[nplug.name] = nplug
-        except Exception as ex:
-            logger.error('Error reloading {}: {}'.format(name, ex))
-            self.plugins[name] = plugin
-
     @classmethod
     def validate_info(cls, info):
-        return all(x in info for x in ('name', 'module', 'version'))
+        return all(x in info for x in ('name', 'module', 'description', 'version'))
 
     def install_deps(self):
         for i in self.plugins.values():
-            self._install_deps(i._info)
+            self._install_deps(i)
 
     @classmethod
     def _install_deps(cls, info=None):
@@ -219,11 +297,19 @@ class Senpy(object):
         if requirements:
             pip_args = []
             pip_args.append('install')
+            pip_args.append('--use-wheel')
             for req in requirements:
                 pip_args.append(req)
             logger.info('Installing requirements: ' + str(requirements))
             pip.main(pip_args)
 
+    @classmethod
+    def _load_module(cls, name, root):
+        sys.path.append(root)
+        tmp = importlib.import_module(name)
+        sys.path.remove(root)
+        return tmp
+
     @classmethod
     def _load_plugin_from_info(cls, info, root):
         if not cls.validate_info(info):
@@ -231,34 +317,21 @@ class Senpy(object):
             return None, None
         module = info["module"]
         name = info["name"]
-        sys.path.append(root)
-        (fp, pathname, desc) = imp.find_module(module, [root, ])
-        try:
-            cls._install_deps(info)
-            tmp = imp.load_module(module, fp, pathname, desc)
-            sys.path.remove(root)
-            candidate = None
-            for _, obj in inspect.getmembers(tmp):
-                if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
-                    logger.debug(("Found plugin class:"
-                                  " {}@{}").format(obj, inspect.getmodule(
-                                      obj)))
-                    candidate = obj
-                    break
-            if not candidate:
-                logger.debug("No valid plugin for: {}".format(module))
-                return
-            module = candidate(info=info)
-            repo_path = root
-            module._repo = Repo(repo_path)
-        except InvalidGitRepositoryError:
-            logger.debug("The plugin {} is not in a Git repository".format(
-                module))
-            module._repo = None
-        except Exception as ex:
-            logger.error("Exception importing {}: {}".format(module, ex))
-            logger.error("Trace: {}".format(traceback.format_exc()))
-            return None, None
+
+        cls._install_deps(info)
+        tmp = cls._load_module(module, root)
+
+        candidate = None
+        for _, obj in inspect.getmembers(tmp):
+            if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
+                logger.debug(("Found plugin class:"
+                              " {}@{}").format(obj, inspect.getmodule(obj)))
+                candidate = obj
+                break
+        if not candidate:
+            logger.debug("No valid plugin for: {}".format(module))
+            return
+        module = candidate(info=info)
         return name, module
 
     @classmethod
@@ -276,7 +349,7 @@ class Senpy(object):
             for root, dirnames, filenames in os.walk(search_folder):
                 for filename in fnmatch.filter(filenames, '*.senpy'):
                     name, plugin = self._load_plugin(root, filename)
-                    if plugin and name not in self._plugin_list:
+                    if plugin and name:
                         plugins[name] = plugin
 
         self._outdated = False
@@ -297,8 +370,8 @@ class Senpy(object):
 
         def matches(plug):
             res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
-            logger.debug("matching {} with {}: {}".format(plug.name, kwargs,
-                                                          res))
+            logger.debug(
+                "matching {} with {}: {}".format(plug.name, kwargs, res))
             return res
 
         if not kwargs:

senpy/models.py

@@ -16,6 +16,9 @@ import jsonref
 import jsonschema
 
 from flask import Response as FlaskResponse
+from pyld import jsonld
+
+from rdflib import Graph
 
 import logging
 
@@ -72,31 +75,60 @@ base_context = Context.load(CONTEXT_PATH)
 
 
 class SenpyMixin(object):
-    context = base_context["@context"]
+    _context = base_context["@context"]
 
-    def flask(self, in_headers=True, headers=None, **kwargs):
+    def flask(self,
+              in_headers=True,
+              headers=None,
+              outformat='json-ld',
+              **kwargs):
         """
         Return the values and error to be used in flask.
         So far, it returns a fixed context. We should store/generate different
         contexts if the plugin adds more aliases.
         """
         headers = headers or {}
-        kwargs["with_context"] = True
-        js = self.jsonld(**kwargs)
-        if in_headers:
-            url = js["@context"]
-            del js["@context"]
+        kwargs["with_context"] = not in_headers
+        content, mimetype = self.serialize(format=outformat,
+                                           with_mime=True,
+                                           **kwargs)
+
+        if outformat == 'json-ld' and in_headers:
             headers.update({
-                "Link": ('<%s>;'
-                         'rel="http://www.w3.org/ns/json-ld#context";'
-                         ' type="application/ld+json"' % url)
+                "Link":
+                ('<%s>;'
+                    'rel="http://www.w3.org/ns/json-ld#context";'
+                    ' type="application/ld+json"' % kwargs.get('context_uri'))
             })
         return FlaskResponse(
-            json.dumps(
-                js, indent=2, sort_keys=True),
+            response=content,
             status=getattr(self, "status", 200),
             headers=headers,
-            mimetype="application/json")
+            mimetype=mimetype)
+
+    def serialize(self, format='json-ld', with_mime=False, **kwargs):
+        js = self.jsonld(**kwargs)
+        if format == 'json-ld':
+            content = json.dumps(js, indent=2, sort_keys=True)
+            mimetype = "application/json"
+        elif format in ['turtle', ]:
+            logger.debug(js)
+            content = json.dumps(js, indent=2, sort_keys=True)
+            g = Graph().parse(
+                data=content,
+                format='json-ld',
+                base=kwargs.get('prefix'),
+                context=self._context)
+            logger.debug(
+                'Parsing with prefix: {}'.format(kwargs.get('prefix')))
+            content = g.serialize(format='turtle').decode('utf-8')
+            mimetype = 'text/{}'.format(format)
+        else:
+            raise Error('Unknown outformat: {}'.format(format))
+        if with_mime:
+            return content, mimetype
+        else:
+            return content
 
     def serializable(self):
         def ser_or_down(item):
@@ -115,28 +147,30 @@ class SenpyMixin(object):
 
         return ser_or_down(self._plain_dict())
 
-    def jsonld(self, with_context=True, context_uri=None):
+    def jsonld(self,
+               with_context=True,
+               context_uri=None,
+               prefix=None,
+               expanded=False):
         ser = self.serializable()
 
-        if with_context:
-            context = []
-            if context_uri:
-                context = context_uri
-            else:
-                context = self.context.copy()
-            if hasattr(self, 'prefix'):
-                # This sets @base for the document, which will be used in
-                # all relative URIs. For example, if a uri is "Example" and
-                # prefix =s "http://example.com", the absolute URI after
-                # expanding with JSON-LD will be "http://example.com/Example"
-
-                prefix_context = {"@base": self.prefix}
-                if isinstance(context, list):
-                    context.append(prefix_context)
-                else:
-                    context = [context, prefix_context]
-            ser["@context"] = context
-        return ser
+        result = jsonld.compact(
+            ser,
+            self._context,
+            options={
+                'base': prefix,
+                'expandContext': self._context,
+                'senpy': prefix
+            })
+        if context_uri:
+            result['@context'] = context_uri
+        if expanded:
+            result = jsonld.expand(
+                result, options={'base': prefix,
+                                 'expandContext': self._context})
+        if not with_context:
+            del result['@context']
+        return result
 
     def to_JSON(self, *args, **kwargs):
         js = json.dumps(self.jsonld(*args, **kwargs), indent=4, sort_keys=True)
@@ -161,13 +195,14 @@ class BaseModel(SenpyMixin, dict):
         if 'id' in kwargs:
             self.id = kwargs.pop('id')
         elif kwargs.pop('_auto_id', True):
-            self.id = '_:{}_{}'.format(
-                type(self).__name__, time.time())
+            self.id = '_:{}_{}'.format(type(self).__name__, time.time())
         temp = dict(*args, **kwargs)
 
-        for obj in [self.schema, ] + self.schema.get('allOf', []):
+        for obj in [
+                self.schema,
+        ] + self.schema.get('allOf', []):
             for k, v in obj.get('properties', {}).items():
-                if 'default' in v:
+                if 'default' in v and k not in temp:
                     temp[k] = copy.deepcopy(v['default'])
 
         for i in temp:
@@ -175,10 +210,6 @@ class BaseModel(SenpyMixin, dict):
             if nk != i:
                 temp[nk] = temp[i]
                 del temp[i]
-        if 'context' in temp:
-            context = temp['context']
-            del temp['context']
-            self.__dict__['context'] = Context.load(context)
         try:
             temp['@type'] = getattr(self, '@type')
         except AttributeError:
@@ -230,14 +261,20 @@ def from_dict(indict):
     return cls(**indict)
 
 
+def from_json(injson):
+    indict = json.loads(injson)
+    return from_dict(indict)
+
+
 def from_schema(name, schema_file=None, base_classes=None):
     base_classes = base_classes or []
     base_classes.append(BaseModel)
     schema_file = schema_file or '{}.json'.format(name)
-    class_name = '{}{}'.format(i[0].upper(), i[1:])
+    class_name = '{}{}'.format(name[0].upper(), name[1:])
     newclass = type(class_name, tuple(base_classes), {})
     setattr(newclass, '@type', name)
     setattr(newclass, 'schema', read_schema(schema_file))
+    setattr(newclass, 'class_name', class_name)
     register(newclass, name)
     return newclass
 
@@ -248,33 +285,44 @@ def _add_from_schema(*args, **kwargs):
     del generatedClass
 
 
-for i in ['response',
-          'results',
-          'entry',
-          'sentiment',
-          'analysis',
-          'emotionSet',
-          'emotion',
-          'emotionModel',
-          'suggestion',
-          'plugin',
-          'emotionPlugin',
-          'sentimentPlugin',
-          'plugins']:
+for i in [
+        'analysis',
+        'emotion',
+        'emotionConversion',
+        'emotionConversionPlugin',
+        'emotionAnalysis',
+        'emotionModel',
+        'emotionPlugin',
+        'emotionSet',
+        'entry',
+        'plugin',
+        'plugins',
+        'response',
+        'results',
+        'sentiment',
+        'sentimentPlugin',
+        'suggestion',
+]:
     _add_from_schema(i)
 
 _ErrorModel = from_schema('error')
 
 
 class Error(SenpyMixin, BaseException):
-    def __init__(self,
-                 message,
-                 *args,
-                 **kwargs):
+    def __init__(self, message, *args, **kwargs):
         super(Error, self).__init__(self, message, message)
         self._error = _ErrorModel(message=message, *args, **kwargs)
         self.message = message
 
+    def __getitem__(self, key):
+        return self._error[key]
+
+    def __setitem__(self, key, value):
+        self._error[key] = value
+
+    def __delitem__(self, key):
+        del self._error[key]
+
     def __getattr__(self, key):
         if key != '_error' and hasattr(self._error, key):
             return getattr(self._error, key)

senpy/plugins/__init__.py

@@ -3,31 +3,50 @@ standard_library.install_aliases()
 
 import inspect
 import os.path
+import os
 import pickle
 import logging
 import tempfile
-from . import models
+import copy
+from .. import models
 
 logger = logging.getLogger(__name__)
 
 
 class SenpyPlugin(models.Plugin):
     def __init__(self, info=None):
+        """
+        Provides a canonical name for plugins and serves as base for other
+        kinds of plugins.
+        """
         if not info:
             raise models.Error(message=("You need to provide configuration"
                                         "information for the plugin."))
         logger.debug("Initialising {}".format(info))
-        super(SenpyPlugin, self).__init__(info)
-        self.id = '{}_{}'.format(self.name, self.version)
-        self._info = info
+        id = 'plugins/{}_{}'.format(info['name'], info['version'])
+        super(SenpyPlugin, self).__init__(id=id, **info)
         self.is_activated = False
 
     def get_folder(self):
         return os.path.dirname(inspect.getfile(self.__class__))
 
     def analyse(self, *args, **kwargs):
-        logger.debug("Analysing with: {} {}".format(self.name, self.version))
-        pass
+        raise NotImplemented(
+            'Your method should implement either analyse or analyse_entry')
+
+    def analyse_entry(self, entry, parameters):
+        """ An implemented plugin should override this method.
+        This base method is here to adapt old style plugins which only
+        implement the *analyse* function.
+        Note that this method may yield an annotated entry or a list of
+        entries (e.g. in a tokenizer)
+        """
+        text = entry['text']
+        params = copy.copy(parameters)
+        params['input'] = text
+        results = self.analyse(**params)
+        for i in results.entries:
+            yield i
 
     def activate(self):
         pass
@@ -35,24 +54,24 @@ class SenpyPlugin(models.Plugin):
     def deactivate(self):
         pass
 
-    def __del__(self):
-        ''' Destructor, to make sure all the resources are freed '''
-        self.deactivate()
 
-
-class SentimentPlugin(SenpyPlugin, models.SentimentPlugin):
+class SentimentPlugin(models.SentimentPlugin, SenpyPlugin):
     def __init__(self, info, *args, **kwargs):
         super(SentimentPlugin, self).__init__(info, *args, **kwargs)
         self.minPolarityValue = float(info.get("minPolarityValue", 0))
         self.maxPolarityValue = float(info.get("maxPolarityValue", 1))
-        self["@type"] = "marl:SentimentAnalysis"
 
 
-class EmotionPlugin(SentimentPlugin, models.EmotionPlugin):
+class EmotionPlugin(models.EmotionPlugin, SenpyPlugin):
     def __init__(self, info, *args, **kwargs):
-        self.minEmotionValue = float(info.get("minEmotionValue", 0))
-        self.maxEmotionValue = float(info.get("maxEmotionValue", 0))
-        self["@type"] = "onyx:EmotionAnalysis"
+        super(EmotionPlugin, self).__init__(info, *args, **kwargs)
+        self.minEmotionValue = float(info.get("minEmotionValue", -1))
+        self.maxEmotionValue = float(info.get("maxEmotionValue", 1))
+
+
+class EmotionConversionPlugin(models.EmotionConversionPlugin, SenpyPlugin):
+    def __init__(self, info, *args, **kwargs):
+        super(EmotionConversionPlugin, self).__init__(info, *args, **kwargs)
 
 
 class ShelfMixin(object):
@@ -73,13 +92,10 @@ class ShelfMixin(object):
 
     @property
     def shelf_file(self):
-        if not hasattr(self, '_shelf_file') or not self._shelf_file:
-            if hasattr(self, '_info') and 'shelf_file' in self._info:
-                self.__dict__['_shelf_file'] = self._info['shelf_file']
-            else:
-                self._shelf_file = os.path.join(tempfile.gettempdir(),
-                                                self.name + '.p')
-        return self._shelf_file
+        if 'shelf_file' not in self or not self['shelf_file']:
+            sd = os.environ.get('SENPY_DATA', tempfile.gettempdir())
+            self.shelf_file = os.path.join(sd, self.name + '.p')
+        return self['shelf_file']
 
     def save(self):
         logger.debug('saving pickle')

senpy/plugins/conversion/centroids.py

@@ -0,0 +1,52 @@
+from senpy.plugins import EmotionConversionPlugin
+from senpy.models import EmotionSet, Emotion, Error
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+class CentroidConversion(EmotionConversionPlugin):
+
+    def _forward_conversion(self, original):
+        """Sum the VAD value of all categories found."""
+        res = Emotion()
+        for e in original.onyx__hasEmotion:
+            category = e.onyx__hasEmotionCategory
+            if category in self.centroids:
+                for dim, value in self.centroids[category].iteritems():
+                    try:
+                        res[dim] += value
+                    except Exception:
+                        res[dim] = value
+        return res
+
+    def _backwards_conversion(self, original):
+        """Find the closest category"""
+        dimensions = list(self.centroids.values())[0]
+
+        def distance(e1, e2):
+            return sum((e1[k] - e2.get(self.aliases[k], 0)) for k in dimensions)
+
+        emotion = ''
+        mindistance = 10000000000000000000000.0
+        for state in self.centroids:
+            d = distance(self.centroids[state], original)
+            if d < mindistance:
+                mindistance = d
+                emotion = state
+        result = Emotion(onyx__hasEmotionCategory=emotion)
+        return result
+
+    def convert(self, emotionSet, fromModel, toModel, params):
+
+        cf, ct = self.centroids_direction
+        logger.debug('{}\n{}\n{}\n{}'.format(emotionSet, fromModel, toModel, params))
+        e = EmotionSet()
+        if fromModel == cf:
+            e.onyx__hasEmotion.append(self._forward_conversion(emotionSet))
+        elif fromModel == ct:
+            for i in emotionSet.onyx__hasEmotion:
+                e.onyx__hasEmotion.append(self._backwards_conversion(i))
+        else:
+            raise Error('EMOTION MODEL NOT KNOWN')
+        yield e

senpy/plugins/conversion/emotion/ekman2vad.senpy

@@ -0,0 +1,38 @@
+---
+name: Ekman2VAD
+module: senpy.plugins.conversion.centroids
+description: Plugin to convert emotion sets from Ekman to VAD
+version: 0.1
+onyx:doesConversion:
+  - onyx:conversionFrom: emoml:big6
+    onyx:conversionTo: emoml:fsre-dimensions
+  - onyx:conversionFrom: emoml:fsre-dimensions
+    onyx:conversionTo: emoml:big6
+centroids:
+  emoml:big6anger:
+    A: 6.95
+    D: 5.1
+    V: 2.7
+  emoml:big6disgust:
+    A: 5.3
+    D: 8.05
+    V: 2.7
+  emoml:big6fear:
+    A: 6.5
+    D: 3.6
+    V: 3.2
+  emoml:big6happiness:
+    A: 7.22
+    D: 6.28
+    V: 8.6
+  emoml:big6sadness:
+    A: 5.21
+    D: 2.82
+    V: 2.21
+centroids_direction:
+  - emoml:big6
+  - emoml:fsre-dimensions
+aliases:
+  A: emoml:arousal
+  V: emoml:valence
+  D: emoml:dominance

senpy/plugins/example/emoRand/emoRand.py

@@ -0,0 +1,18 @@
+import random
+
+from senpy.plugins import EmotionPlugin
+from senpy.models import EmotionSet, Emotion
+
+
+class RmoRandPlugin(EmotionPlugin):
+    def analyse_entry(self, entry, params):
+        category = "emoml:big6happiness"
+        number = max(-1, min(1, random.gauss(0, 0.5)))
+        if number > 0:
+            category = "emoml:big6anger"
+        emotionSet = EmotionSet()
+        emotion = Emotion({"onyx:hasEmotionCategory": category})
+        emotionSet.onyx__hasEmotion.append(emotion)
+        emotionSet.prov__wasGeneratedBy = self.id
+        entry.emotions.append(emotionSet)
+        yield entry

senpy/plugins/example/emoRand/emoRand.senpy

@@ -0,0 +1,9 @@
+---
+name: emoRand
+module: emoRand
+description: A sample plugin that returns a random emotion annotation
+author: "@balkian"
+version: '0.1'
+url: "https://github.com/gsi-upm/senpy-plugins-community"
+requirements: {}
+onyx:usesEmotionModel: "emoml:big6"

senpy/plugins/example/rand/rand.py

@@ -1,29 +1,24 @@
 import random
 
 from senpy.plugins import SentimentPlugin
-from senpy.models import Results, Sentiment, Entry
+from senpy.models import Sentiment
 
 
-class Sentiment140Plugin(SentimentPlugin):
-    def analyse(self, **params):
+class RandPlugin(SentimentPlugin):
+    def analyse_entry(self, entry, params):
         lang = params.get("language", "auto")
 
-        response = Results()
         polarity_value = max(-1, min(1, random.gauss(0.2, 0.2)))
         polarity = "marl:Neutral"
         if polarity_value > 0:
             polarity = "marl:Positive"
         elif polarity_value < 0:
             polarity = "marl:Negative"
-        entry = Entry({"id": ":Entry0", "nif:isString": params["input"]})
         sentiment = Sentiment({
-            "id": ":Sentiment0",
             "marl:hasPolarity": polarity,
             "marl:polarityValue": polarity_value
         })
         sentiment["prov:wasGeneratedBy"] = self.id
-        entry.sentiments = []
         entry.sentiments.append(sentiment)
         entry.language = lang
-        response.entries.append(entry)
-        return response
+        yield entry

senpy/plugins/example/rand/rand.senpy

@@ -0,0 +1,10 @@
+---
+name: rand
+module: rand
+description: A sample plugin that returns a random sentiment annotation
+author: "@balkian"
+version: '0.1'
+url: "https://github.com/gsi-upm/senpy-plugins-community"
+requirements: {}
+marl:maxPolarityValue: '1'
+marl:minPolarityValue: "-1"

senpy/plugins/rand/rand.senpy

@@ -1,18 +0,0 @@
-{
-    "name": "rand",
-    "module": "rand",
-    "description": "What my plugin broadly does",
-    "author": "@balkian",
-    "version": "0.1",
-    "extra_params": {
-        "language": {
-            "@id": "lang_rand",
-            "aliases": ["language", "l"],
-            "required": false,
-            "options": ["es", "en", "auto"]
-        }
-     },
-     "requirements": {},
-     "marl:maxPolarityValue": "1",
-     "marl:minPolarityValue": "-1"
-}

senpy/plugins/sentiment/sentiment140/sentiment140.py

@@ -2,24 +2,22 @@ import requests
 import json
 
 from senpy.plugins import SentimentPlugin
-from senpy.models import Results, Sentiment, Entry
+from senpy.models import Sentiment
 
 
 class Sentiment140Plugin(SentimentPlugin):
-    def analyse(self, **params):
+    def analyse_entry(self, entry, params):
         lang = params.get("language", "auto")
         res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
                             json.dumps({
                                 "language": lang,
                                 "data": [{
-                                    "text": params["input"]
+                                    "text": entry.text
                                 }]
                             }))
-
         p = params.get("prefix", None)
-        response = Results(prefix=p)
-        polarity_value = self.maxPolarityValue * int(res.json()["data"][0][
-            "polarity"]) * 0.25
+        polarity_value = self.maxPolarityValue * int(
+            res.json()["data"][0]["polarity"]) * 0.25
         polarity = "marl:Neutral"
         neutral_value = self.maxPolarityValue / 2.0
         if polarity_value > neutral_value:
@@ -27,9 +25,7 @@ class Sentiment140Plugin(SentimentPlugin):
         elif polarity_value < neutral_value:
             polarity = "marl:Negative"
 
-        entry = Entry(id="Entry0", nif__isString=params["input"])
         sentiment = Sentiment(
-            id="Sentiment0",
             prefix=p,
             marl__hasPolarity=polarity,
             marl__polarityValue=polarity_value)
@@ -37,5 +33,4 @@ class Sentiment140Plugin(SentimentPlugin):
         entry.sentiments = []
         entry.sentiments.append(sentiment)
         entry.language = lang
-        response.entries.append(entry)
-        return response
+        yield entry

senpy/plugins/sentiment/sentiment140/sentiment140.senpy

@@ -0,0 +1,21 @@
+---
+name: sentiment140
+module: sentiment140
+description: "Connects to the sentiment140 free API: http://sentiment140.com"
+author: "@balkian"
+version: '0.2'
+url: "https://github.com/gsi-upm/senpy-plugins-community"
+extra_params:
+  language:
+    "@id": lang_sentiment140
+    aliases:
+    - language
+    - l
+    required: false
+    options:
+    - es
+    - en
+    - auto
+requirements: {}
+maxPolarityValue: 1
+minPolarityValue: 0

senpy/plugins/sentiment140/sentiment140.senpy

@@ -1,18 +0,0 @@
-{
-    "name": "sentiment140",
-    "module": "sentiment140",
-    "description": "What my plugin broadly does",
-    "author": "@balkian",
-    "version": "0.1",
-    "extra_params": {
-        "language": {
-            "@id": "lang_sentiment140",
-            "aliases": ["language", "l"],
-            "required": false,
-            "options": ["es", "en", "auto"]
-        }
-     },
-     "requirements": {},
-     "maxPolarityValue": "1",
-     "minPolarityValue": "0"
-}

senpy/schemas/\

@@ -1,7 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "description": "Senpy analysis",
-  "allOf": [{
-    "$ref": "atom.json"
-  }]
-}

senpy/schemas/context.jsonld

@@ -6,34 +6,51 @@
     "prov": "http://www.w3.org/ns/prov#",
     "nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#",
     "marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#",
-    "onyx": "http://www.gsi.dit.upm.es/ontologies/onyx#",
-    "wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#",
+    "onyx": "http://www.gsi.dit.upm.es/ontologies/onyx/ns#",
+    "wna": "http://www.gsi.dit.upm.es/ontologies/wnaffect/ns#",
+    "emoml": "http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/emotionml/ns#",
     "xsd": "http://www.w3.org/2001/XMLSchema#",
     "topics": {
-        "@id": "dc:subject"
+      "@id": "dc:subject"
     },
     "entities": {
-    "@id": "me:hasEntities"
+      "@id": "me:hasEntities"
     },
     "suggestions": {
-        "@id": "me:hasSuggestions",
-        "@container": "@set"
+      "@id": "me:hasSuggestions",
+      "@container": "@set"
     },
     "emotions": {
-        "@id": "onyx:hasEmotionSet",
-        "@container": "@set"
+      "@id": "onyx:hasEmotionSet",
+      "@container": "@set"
     },
     "sentiments": {
-        "@id": "marl:hasOpinion",
-        "@container": "@set"
+      "@id": "marl:hasOpinion",
+      "@container": "@set"
     },
     "entries": {
-        "@id": "prov:used",
-        "@container": "@set"
+      "@id": "prov:used",
+      "@container": "@set"
     },
     "analysis": {
-        "@id": "prov:wasGeneratedBy"
-
+      "@id": "AnalysisInvolved",
+      "@type": "@id",
+      "@container": "@set"
+    },
+    "prov:wasGeneratedBy": {
+      "@type": "@id"
+    },
+    "onyx:usesEmotionModel": {
+      "@type": "@id"
+    },
+    "onyx:hasEmotionCategory": {
+      "@type": "@id"
+    },
+    "onyx:conversionFrom": {
+      "@type": "@id"
+    },
+    "onyx:conversionTo": {
+      "@type": "@id"
     }
   }
 }

senpy/schemas/emotionAnalysis.json

@@ -6,13 +6,14 @@
     {"$ref": "analysis.json"},
     {"properties":
      {
-       "onyx:hasEmotionModel": {
+       "onyx:usesEmotionModel": {
          "anyOf": [
            {"type": "string"},
            {"$ref": "emotionModel.json"}
          ]
        }
      },
-     "required": ["onyx:hasEmotionModel"]
+     "required": ["onyx:hasEmotionModel",
+                  "@type"]
     }]
 }

senpy/schemas/emotionConversion.json

@@ -0,0 +1,12 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "properties": {
+    "onyx:conversionFrom": {
+      "$ref": "emotionModel.json"
+    },
+    "onyx:conversionTo": {
+      "$ref": "emotionModel.json"
+    }
+  },
+  "required": ["onyx:conversionFrom", "onyx:conversionTo"]
+}

senpy/schemas/emotionConversionPlugin.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "allOf": [
+    {
+      "$ref": "plugin.json"
+    },
+    {
+      "properties": {
+        "onyx:doesConversion": {
+          "type": "array",
+          "items": {
+            "$ref": "emotionConversion.json"
+          }
+        }
+      }
+    }
+  ]
+}

senpy/schemas/emotionPlugin.json

@@ -1,7 +1,7 @@
 {
   "$schema": "http://json-schema.org/draft-04/schema#",
   "type": "object",
-  "$allOf": [
+  "allOf": [
     {
       "$ref": "plugin.json"
     },

senpy/schemas/entry.json

@@ -11,23 +11,28 @@
     },
     "sentiments": {
       "type": "array",
-      "items": {"$ref": "sentiment.json" }
+      "items": {"$ref": "sentiment.json" },
+      "default": []
     },
     "emotions": {
       "type": "array",
-      "items": {"$ref": "emotionSet.json" }
+      "items": {"$ref": "emotionSet.json" },
+      "default": []
     },
     "entities": {
       "type": "array",
-      "items": {"$ref": "entity.json" }
+      "items": {"$ref": "entity.json" },
+      "default": []
     },
     "topics": {
       "type": "array",
-      "items": {"$ref": "topic.json" }
+      "items": {"$ref": "topic.json" },
+      "default": []
     },
     "suggestions": {
       "type": "array",
-      "items": {"$ref": "suggestion.json" }
+      "items": {"$ref": "suggestion.json" },
+      "default": []
     }
   },
   "required": ["@id", "nif:isString"]

senpy/schemas/error.json

@@ -2,7 +2,7 @@
   "$schema": "http://json-schema.org/draft-04/schema#",
   "description": "Base schema for all Senpy objects",
   "type": "object",
-  "$allOf": [
+  "allOf": [
     {"$ref": "atom.json"},
     {
       "properties": {
@@ -10,14 +10,14 @@
           "type": "string"
         },
         "errors": {
-          "type": "list",
+          "type": "array",
           "items": {"type": "object"}
         },
-        "code": {
-          "type": "int"
-        },
-        "required": ["message"]
-      }
+        "status": {
+          "type": "number"
+        }
+      },
+      "required": ["message"]
     }
   ]
 }

senpy/schemas/sentimentPlugin.json

@@ -1,7 +1,7 @@
 {
   "$schema": "http://json-schema.org/draft-04/schema#",
   "type": "object",
-  "$allOf": [
+  "allOf": [
     {
       "$ref": "plugin.json"
     },

senpy/schemas/suggestion.json

@@ -1,4 +1,5 @@
 {
   "$schema": "http://json-schema.org/draft-04/schema#",
-  "type": "object"
+  "type": "object",
+  "required": ["@id", "prov:wasGeneratedBy"]
 }

senpy/static/css/main.css

@@ -8,7 +8,7 @@ body {
 }
 #inputswrapper {
     min-height:100%;
-    background: white;
+    /* background: white; */
     position:relative;
     min-width: 800px;
     height: 100%;
@@ -50,25 +50,16 @@ body {
 #form {
   width: 100%;
 }
-#results {
+.results {
     overflow: auto;
-    padding: 20px;
-    background: lightgray;
-     -moz-border-radius: 20px;
-     -webkit-border-radius: 20px;
-     -khtml-border-radius: 20px;
-     border-radius: 20px;
+    /* padding: 20px; */
+    background: white;
+     /* -moz-border-radius: 20px; */
+     /* -webkit-border-radius: 20px; */
+     /* -khtml-border-radius: 20px; */
+     /* border-radius: 20px; */
 
 }
-#jsonraw {
-    overflow: auto;
-    padding: 20px;
-    background: lightgray;
-     -moz-border-radius: 20px;
-     -webkit-border-radius: 20px;
-     -khtml-border-radius: 20px;
-     border-radius: 20px;
-}
 #input_request { 
     margin-top: 5px;
     display:block;
@@ -156,3 +147,8 @@ textarea{
 #header {
   font-family: 'Architects Daughter', cursive;
 }
+
+#results-div {
+    /* background: white; */
+    display: none;
+}

senpy/static/js/main.js

@@ -32,39 +32,48 @@ $(document).ready(function() {
     var availablePlugins = document.getElementById('availablePlugins');
     plugins = response.plugins;
     for (r in plugins){
-      if (plugins[r]["name"]){
-        if (plugins[r]["name"] == defaultPlugin["name"]){
-          if (plugins[r]["is_activated"]){
-            html+= "<option value=\""+plugins[r]["name"]+"\" selected=\"selected\">"+plugins[r]["name"]+"</option>"
+      plugin = plugins[r]
+      if (plugin["name"]){
+        if (plugin["name"] == defaultPlugin["name"]){
+          if (plugin["is_activated"]){
+            html+= "<option value=\""+plugin["name"]+"\" selected=\"selected\">"+plugin["name"]+"</option>"
           }else{
-            html+= "<option value=\""+plugins[r]["name"]+"\" selected=\"selected\" disabled=\"disabled\">"+plugins[r]["name"]+"</option>"
+            html+= "<option value=\""+plugin["name"]+"\" selected=\"selected\" disabled=\"disabled\">"+plugin["name"]+"</option>"
           }
         }
         else{
-          if (plugins[r]["is_activated"]){
-            html+= "<option value=\""+plugins[r]["name"]+"\">"+plugins[r]["name"]+"</option>"
+          if (plugin["is_activated"]){
+            html+= "<option value=\""+plugin["name"]+"\">"+plugin["name"]+"</option>"
           }
           else{
-            html+= "<option value=\""+plugins[r]["name"]+"\" disabled=\"disabled\">"+plugins[r]["name"]+"</option>"
+            html+= "<option value=\""+plugin["name"]+"\" disabled=\"disabled\">"+plugin["name"]+"</option>"
           }
         }
       }
-      if (plugins[r]["extra_params"]){
-        plugins_params[plugins[r]["name"]]={};
-        for (param in plugins[r]["extra_params"]){
-          if (typeof plugins[r]["extra_params"][param] !="string"){
+      if (plugin["extra_params"]){
+        plugins_params[plugin["name"]]={};
+        for (param in plugin["extra_params"]){
+          if (typeof plugin["extra_params"][param] !="string"){
             var params = new Array();
-            var alias = plugins[r]["extra_params"][param]["aliases"][0];
+            var alias = plugin["extra_params"][param]["aliases"][0];
             params[alias]=new Array();
-            for (option in plugins[r]["extra_params"][param]["options"]){
-              params[alias].push(plugins[r]["extra_params"][param]["options"][option])
+            for (option in plugin["extra_params"][param]["options"]){
+              params[alias].push(plugin["extra_params"][param]["options"][option])
             }
-            plugins_params[plugins[r]["name"]][alias] = (params[alias])
+            plugins_params[plugin["name"]][alias] = (params[alias])
           }
         }
       }
       var pluginList = document.createElement('li');
-      pluginList.innerHTML = "<a href=https://github.com/gsi-upm/senpy-plugins-community>" + plugins[r]["name"] + "</a>" + ": " + plugins[r]["description"]
+      
+      newHtml = ""
+      if(plugin.url) {
+        newHtml= "<a href="+plugin.url+">" + plugin.name + "</a>";
+      }else {
+        newHtml= plugin["name"];
+      }
+      newHtml += ": " + replaceURLWithHTMLLinks(plugin.description);
+      pluginList.innerHTML = newHtml;
       availablePlugins.appendChild(pluginList)
     }
     document.getElementById('plugins').innerHTML = html;
@@ -96,6 +105,10 @@ function change_params(){
 
 function load_JSON(){
       url = "/api";
+      var container = document.getElementById('results');
+      var rawcontainer = document.getElementById("jsonraw");
+      rawcontainer.innerHTML = '';
+      container.innerHTML = '';
       var plugin = document.getElementById("plugins").options[document.getElementById("plugins").selectedIndex].value;
       var input = encodeURIComponent(document.getElementById("input").value);
       url += "?algo="+plugin+"&i="+input
@@ -108,18 +121,14 @@ function load_JSON(){
         }
       }
       var response = JSON.parse($.ajax({type: "GET", url: url , async: false}).responseText);
-      var container = document.getElementById('results');
       var options = {
         mode: 'view'
       };
-      try { 
-        container.removeChild(container.firstChild);
-      }
-      catch(err) {
-      }
       var editor = new JSONEditor(container, options, response);
-      document.getElementById("jsonraw").innerHTML = replaceURLWithHTMLLinks(JSON.stringify(response, undefined, 2))
+      editor.expandAll();
+      rawcontainer.innerHTML = replaceURLWithHTMLLinks(JSON.stringify(response, undefined, 2))
       document.getElementById("input_request").innerHTML = "<a href='"+url+"'>"+url+"</a>"
+      document.getElementById("results-div").style.display = 'block';
       
 
 }

senpy/templates/index.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <meta http-equiv="content-type" content="text/html; charset=utf-8" />
-    <title>Playground</title>
+    <title>Playground {{version}}</title>
 
   </head>
   <script src="static/js/jquery-2.1.1.min.js" ></script>
@@ -25,49 +25,68 @@
         <h3 id="header-title">
           <a href="https://github.com/gsi-upm/senpy" target="_blank">
             <img id="header-logo" class="imsg-responsive" src="static/img/header.png"/></a> Playground
-
         </h3>
+          <h4>v{{ version}}</h4>
       </div>
 
       <ul class="nav nav-tabs" role="tablist">
-        <li role="presentation" class="active"><a class="active" href="#about">About</a></li>
-        <li role="presentation"><a class="active" href="#test">Test it</a></li>
+        <li role="presentation" ><a class="active" href="#about">About</a></li>
+        <li role="presentation"class="active"><a class="active" href="#test">Test it</a></li>
       </ul>
 
       <div class="tab-content">
-        <div class="tab-pane active" id="about">
+        <div class="tab-pane" id="about">
+
           <div class="row">
-            <div class="col-lg-6 ">
-              <div class="well">
-                <h2>Test Senpy</h2>
-                <div>
-                  <p class="text-center">
-                    <a class="btn btn-lg btn-primary" href="#test" role="button">Test it »</a>
+                <div class="col-lg-6">
+                  <h2>About Senpy</h2>
+                  <p>Senpy is a framework to build semantic sentiment and emotion analysis services. It does so by using a mix of web and semantic technologies, such as JSON-LD, RDFlib and Flask.</p>
+                  <p>Senpy makes it easy to develop and publish your own analysis algorithms (plugins in senpy terms).
+                  </p>
+                  <p>
+                    This website is the senpy Playground, which allows you to test the instance of senpy in this server. It provides a user-friendly interface to the functions exposed by the senpy API.
+                    </p>
+                  <p>
+                    Once you get comfortable with the parameters and results, you are encouraged to issue your own requests to the API endpoint, which should be <a href="/api">here</a>.
+                  </p>
+                  <p>
+                    These are some of the things you can do with the API:
+                    <ul>
+                      <li>List all available plugins: <a href="/api/plugins">/api/plugins</a></li>
+                      <li>Get information about the default plugin: <a href="/api/plugins/default">/api/plugins/default</a></li>
+                      <li>Download the JSON-LD context used: <a href="/api/contexts/Results.jsonld">/api/contexts/Results.jsonld</a></li>
+                    </ul>
+
                   </p>
                   
                 </div>
-                
-              </div>
+            <div class="col-lg-6 ">
               <div class="panel panel-default">
-                <div class="panel-heading"><i class="fa fa-sign-in"></i> Follow us on <a href="http://www.github.com/gsi-upm/senpy">GitHub</a></div>
-              </div>
-              <div class="panel panel-default">
-                <div class="panel-heading"><i class="fa fa-child"></i> Enjoy.</div>
+                <div class="panel-heading">
+                Available Plugins
+                </div>
+                <div class="panel-body"><ul id=availablePlugins></ul></div>
               </div>
             </div>
             <div class="col-lg-6 ">
-              <div class="well">
-                <h2>Available Plugins</h2>
-                <div>
-                <span><ul id=availablePlugins></ul></span>
-                </div>
-                
+              <a href="http://senpy.readthedocs.io">
+              <div class="panel panel-default">
+                <div class="panel-heading"><i class="fa fa-book"></i> If you are new to senpy, you might want to read senpy's documentation</div>
+              </div>
+              </a>
+              <a href="http://www.github.com/gsi-upm/senpy">
+              <div class="panel panel-default">
+                <div class="panel-heading"><i class="fa fa-sign-in"></i> Feel free to follow us on GitHub</div>
+              </div>
+              </a>
+              <div class="panel panel-default">
+                <div class="panel-heading"><i class="fa fa-child"></i> Enjoy.</div>
               </div>
             </div>     
           </div>
         </div>
 
-        <div class="tab-pane" id="test">
+        <div class="tab-pane active" id="test">
           <div class="well">
             <form id="form" onsubmit="return getPlugins();" accept-charset="utf-8">
               <div id="inputswrapper">
@@ -81,30 +100,32 @@ I cannot believe it!</textarea></div>
                         <div id ="params">
                         </div>
                 </br>
-                <a id="preview" class="btn btn-lg btn-primary" href="#" onclick="load_JSON()">Analyse!</a>
+                <a id="preview" class="btn btn-lg btn-primary" onclick="load_JSON()">Analyse!</a>
                 <!--<button id="visualise" name="type" type="button">Visualise!</button>-->
+              </div>
             </form>
-              <span id="input_request"></span>
+          </div>
+            <span id="input_request"></span>
+            <div id="results-div">
               <ul class="nav nav-tabs" role="tablist">
                 <li role="presentation" class="active"><a class="active" href="#viewer">Viewer</a></li>
                 <li role="presentation"><a class="active" href="#raw">Raw</a></li>
               </ul>
-              <div class="tab-content">
+              <div class="tab-content" id="results-container">
                 
                 <div class="tab-pane active" id="viewer">
-                    <div id="content">
-                      <pre id="results"></pre>
-                    </div>
+                  <div id="content">
+                    <pre id="results" class="results"></pre>
+                  </div>
                 </div>
                 
                 <div class="tab-pane" id="raw">
-                    <div id="content">
-                       <pre id="jsonraw"></pre>
-                    </div>
+                  <div id="content">
+                    <pre id="jsonraw" class="results"></pre>
+                  </div>
                 </div>
               </div>
             </div>
-          </div>
         </div>
         <a href="http://www.gsi.dit.upm.es" target="_blank"><img class="center-block" src="static/img/gsi.png"/>  </a> 
 

senpy/version.py

@@ -1,4 +1,15 @@
 import os
+import logging
 
-with open(os.path.join(os.path.dirname(__file__), 'VERSION')) as f:
-    __version__ = f.read().strip()
+logger = logging.getLogger(__name__)
+
+ROOT = os.path.dirname(__file__)
+DEFAULT_FILE = os.path.join(ROOT, 'VERSION')
+
+
+def read_version(versionfile=DEFAULT_FILE):
+    with open(versionfile) as f:
+        return f.read().strip()
+
+
+__version__ = read_version()

setup.cfg

@@ -7,3 +7,8 @@ test=pytest
 # finishing the imports. flake8 thinks that we're doing the imports too late,
 # but it's actually ok
 ignore = E402
+max-line-length = 100
+[bdist_wheel]
+universal=1
+[tool:pytest]
+addopts = --cov=senpy --cov-report term-missing

setup.py

@@ -1,7 +1,8 @@
 import pip
 from setuptools import setup
-from pip.req import parse_requirements
 # parse_requirements() returns generator of pip.req.InstallRequirement objects
+from pip.req import parse_requirements
+from senpy import __version__
 
 try:
     install_reqs = parse_requirements(
@@ -12,12 +13,9 @@ except AttributeError:
     install_reqs = parse_requirements("requirements.txt")
     test_reqs = parse_requirements("test-requirements.txt")
 
-# reqs is a list of requirement
-# e.g. ['django==1.5.1', 'mezzanine==1.4.6']
 install_reqs = [str(ir.req) for ir in install_reqs]
 test_reqs = [str(ir.req) for ir in test_reqs]
 
-from senpy import __version__
 
 setup(
     name='senpy',

test-requirements.txt

@@ -1,2 +1,3 @@
-pytest
 mock
+pytest-cov
+pytest

tests/dummy_plugin/dummy.py

@@ -1,7 +0,0 @@
-from senpy.plugins import SentimentPlugin
-from senpy.models import Results
-
-
-class DummyPlugin(SentimentPlugin):
-    def analyse(self, *args, **kwargs):
-        return Results()

tests/dummy_plugin/dummy.senpy

@@ -1,7 +0,0 @@
-{
-    "name": "Dummy",
-    "module": "dummy",
-    "description": "I am dummy",
-    "author": "@balkian",
-    "version": "0.1"
-}

tests/plugins/dummy_plugin/dummy.py

@@ -0,0 +1,7 @@
+from senpy.plugins import SentimentPlugin
+
+
+class DummyPlugin(SentimentPlugin):
+    def analyse_entry(self, entry, params):
+        entry.text = entry.text[::-1]
+        yield entry

tests/plugins/dummy_plugin/dummy.senpy

@@ -0,0 +1,15 @@
+{
+    "name": "Dummy",
+    "module": "dummy",
+    "description": "I am dummy",
+    "author": "@balkian",
+    "version": "0.1",
+    "extra_params": {
+        "example": {
+            "@id": "example_parameter",
+            "aliases": ["example", "ex"],
+            "required": false,
+            "default": 0
+        }
+    }
+}

tests/plugins/dummy_plugin/dummy_required.senpy

@@ -0,0 +1,14 @@
+{
+    "name": "DummyRequired",
+    "module": "dummy",
+    "description": "I am dummy",
+    "author": "@balkian",
+    "version": "0.1",
+    "extra_params": {
+        "example": {
+            "@id": "example_parameter",
+            "aliases": ["example", "ex"],
+            "required": true
+        }
+    }
+}

tests/plugins/sleep_plugin/sleep.py

@@ -1,5 +1,4 @@
 from senpy.plugins import SenpyPlugin
-from senpy.models import Results
 from time import sleep
 
 
@@ -7,6 +6,6 @@ class SleepPlugin(SenpyPlugin):
     def activate(self, *args, **kwargs):
         sleep(self.timeout)
 
-    def analyse(self, *args, **kwargs):
-        sleep(float(kwargs.get("timeout", self.timeout)))
-        return Results()
+    def analyse_entry(self, entry, params):
+        sleep(float(params.get("timeout", self.timeout)))
+        yield entry

tests/test_api.py

@@ -0,0 +1,68 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
+from unittest import TestCase
+from senpy.api import parse_params, API_PARAMS, NIF_PARAMS, WEB_PARAMS
+from senpy.models import Error
+
+
+class APITest(TestCase):
+
+    def test_api_params(self):
+        """The API should not define any required parameters without a default"""
+        parse_params({}, spec=API_PARAMS)
+
+    def test_web_params(self):
+        """The WEB should not define any required parameters without a default"""
+        parse_params({}, spec=WEB_PARAMS)
+
+    def test_basic(self):
+        a = {}
+        try:
+            parse_params(a, spec=NIF_PARAMS)
+            raise AssertionError()
+        except Error:
+            pass
+        a = {'input': 'hello'}
+        p = parse_params(a, spec=NIF_PARAMS)
+        assert 'input' in p
+        b = {'i': 'hello'}
+        p = parse_params(b, spec=NIF_PARAMS)
+        assert 'input' in p
+
+    def test_plugin(self):
+        query = {}
+        plug_params = {
+            'hello': {
+                'aliases': ['hello', 'hiya'],
+                'required': True
+            }
+        }
+        try:
+            parse_params(query, spec=plug_params)
+            raise AssertionError()
+        except Error:
+            pass
+        query['hello'] = 'world'
+        p = parse_params(query, spec=plug_params)
+        assert 'hello' in p
+        assert p['hello'] == 'world'
+        del query['hello']
+
+        query['hiya'] = 'dlrow'
+        p = parse_params(query, spec=plug_params)
+        assert 'hello' in p
+        assert 'hiya' in p
+        assert p['hello'] == 'dlrow'
+
+    def test_default(self):
+        spec = {
+            'hello': {
+                'required': True,
+                'default': 1
+            }
+        }
+        p = parse_params({}, spec=spec)
+        assert 'hello' in p
+        assert p['hello'] == 1

tests/test_blueprints.py

@@ -1,11 +1,10 @@
 import os
 import logging
-import json
 
 from senpy.extensions import Senpy
+from senpy import models
 from flask import Flask
 from unittest import TestCase
-from gevent import sleep
 from itertools import product
 
 
@@ -14,7 +13,7 @@ def check_dict(indic, template):
 
 
 def parse_resp(resp):
-    return json.loads(resp.data.decode('utf-8'))
+    return models.from_json(resp.data.decode('utf-8'))
 
 
 class BlueprintsTest(TestCase):
@@ -26,6 +25,8 @@ class BlueprintsTest(TestCase):
         self.dir = os.path.join(os.path.dirname(__file__), "..")
         self.senpy.add_folder(self.dir)
         self.senpy.activate_plugin("Dummy", sync=True)
+        self.senpy.activate_plugin("DummyRequired", sync=True)
+        self.senpy.default_plugin = 'Dummy'
 
     def assertCode(self, resp, code):
         self.assertEqual(resp.status_code, code)
@@ -35,12 +36,12 @@ class BlueprintsTest(TestCase):
         Calling with no arguments should ask the user for more arguments
         """
         resp = self.client.get("/api/")
-        self.assertCode(resp, 404)
+        self.assertCode(resp, 400)
         js = parse_resp(resp)
         logging.debug(js)
-        assert js["status"] == 404
+        assert js["status"] == 400
         atleast = {
-            "status": 404,
+            "status": 400,
             "message": "Missing or invalid parameters",
         }
         assert check_dict(js, atleast)
@@ -57,6 +58,39 @@ class BlueprintsTest(TestCase):
         assert "@context" in js
         assert "entries" in js
 
+    def test_analysis_extra(self):
+        """
+        Extra params that have a default should
+        """
+        resp = self.client.get("/api/?i=My aloha mohame&algo=Dummy")
+        self.assertCode(resp, 200)
+        js = parse_resp(resp)
+        logging.debug("Got response: %s", js)
+        assert "@context" in js
+        assert "entries" in js
+
+    def test_analysis_extra_required(self):
+        """
+        Extra params that have a required argument that does not
+        have a default should raise an error.
+        """
+        resp = self.client.get("/api/?i=My aloha mohame&algo=DummyRequired")
+        self.assertCode(resp, 400)
+        js = parse_resp(resp)
+        logging.debug("Got response: %s", js)
+        assert isinstance(js, models.Error)
+
+    def test_error(self):
+        """
+        The dummy plugin returns an empty response,\
+        it should contain the context
+        """
+        resp = self.client.get("/api/?i=My aloha mohame&algo=DOESNOTEXIST")
+        self.assertCode(resp, 404)
+        js = parse_resp(resp)
+        logging.debug("Got response: %s", js)
+        assert isinstance(js, models.Error)
+
     def test_list(self):
         """ List the plugins """
         resp = self.client.get("/api/plugins/")
@@ -92,26 +126,7 @@ class BlueprintsTest(TestCase):
         js = parse_resp(resp)
         logging.debug(js)
         assert "@id" in js
-        assert js["@id"] == "Dummy_0.1"
-
-    def test_activate(self):
-        """ Activate and deactivate one plugin """
-        resp = self.client.get("/api/plugins/Dummy/deactivate")
-        self.assertCode(resp, 200)
-        sleep(0.5)
-        resp = self.client.get("/api/plugins/Dummy/")
-        self.assertCode(resp, 200)
-        js = parse_resp(resp)
-        assert "is_activated" in js
-        assert not js["is_activated"]
-        resp = self.client.get("/api/plugins/Dummy/activate")
-        self.assertCode(resp, 200)
-        sleep(0.5)
-        resp = self.client.get("/api/plugins/Dummy/")
-        self.assertCode(resp, 200)
-        js = parse_resp(resp)
-        assert "is_activated" in js
-        assert js["is_activated"]
+        assert js["@id"] == "plugins/Dummy_0.1"
 
     def test_default(self):
         """ Show only one plugin"""
@@ -120,12 +135,7 @@ class BlueprintsTest(TestCase):
         js = parse_resp(resp)
         logging.debug(js)
         assert "@id" in js
-        assert js["@id"] == "Dummy_0.1"
-        resp = self.client.get("/api/plugins/Dummy/deactivate")
-        self.assertCode(resp, 200)
-        sleep(0.5)
-        resp = self.client.get("/api/plugins/default/")
-        self.assertCode(resp, 404)
+        assert js["@id"] == "plugins/Dummy_0.1"
 
     def test_context(self):
         resp = self.client.get("/api/contexts/context.jsonld")

tests/test_client.py

@@ -9,9 +9,10 @@ from senpy.models import Results, Error
 
 
 class Call(dict):
-
     def __init__(self, obj):
         self.obj = obj.jsonld()
+        self.status_code = 200
+        self.content = self.json()
 
     def json(self):
         return self.obj
@@ -29,14 +30,17 @@ class ModelsTest(TestCase):
         with patch('requests.request', return_value=success) as patched:
             resp = client.analyse('hello')
             assert isinstance(resp, Results)
-        patched.assert_called_with(url=endpoint + '/',
-                                   method='GET',
-                                   params={'input': 'hello'})
+        patched.assert_called_with(
+            url=endpoint + '/', method='GET', params={'input': 'hello'})
         error = Call(Error('Nothing'))
         with patch('requests.request', return_value=error) as patched:
-            resp = client.analyse(input='hello', algorithm='NONEXISTENT')
-            assert isinstance(resp, Error)
-        patched.assert_called_with(url=endpoint + '/',
-                                   method='GET',
-                                   params={'input': 'hello',
-                                           'algorithm': 'NONEXISTENT'})
+            try:
+                client.analyse(input='hello', algorithm='NONEXISTENT')
+                raise Exception('Exceptions should be raised. This is not golang')
+            except Error:
+                pass
+        patched.assert_called_with(
+            url=endpoint + '/',
+            method='GET',
+            params={'input': 'hello',
+                    'algorithm': 'NONEXISTENT'})

tests/test_extensions.py

@@ -1,20 +1,27 @@
 from __future__ import print_function
 import os
+from copy import deepcopy
 import logging
 
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
 from functools import partial
 from senpy.extensions import Senpy
-from senpy.models import Error
+from senpy.models import Error, Results, Entry, EmotionSet, Emotion
 from flask import Flask
 from unittest import TestCase
 
 
 class ExtensionsTest(TestCase):
     def setUp(self):
-        self.app = Flask("test_extensions")
+        self.app = Flask('test_extensions')
         self.dir = os.path.join(os.path.dirname(__file__))
-        self.senpy = Senpy(plugin_folder=self.dir, default_plugins=False)
-        self.senpy.init_app(self.app)
+        self.senpy = Senpy(plugin_folder=self.dir,
+                           app=self.app,
+                           default_plugins=False)
         self.senpy.activate_plugin("Dummy", sync=True)
 
     def test_init(self):
@@ -36,20 +43,22 @@ class ExtensionsTest(TestCase):
         info = {
             'name': 'TestPip',
             'module': 'dummy',
+            'description': None,
             'requirements': ['noop'],
             'version': 0
         }
-        root = os.path.join(self.dir, 'dummy_plugin')
+        root = os.path.join(self.dir, 'plugins', 'dummy_plugin')
         name, module = self.senpy._load_plugin_from_info(info, root=root)
         assert name == 'TestPip'
         assert module
         import noop
         dir(noop)
+        self.senpy.install_deps()
 
     def test_installing(self):
         """ Enabling a plugin """
         self.senpy.activate_all(sync=True)
-        assert len(self.senpy.plugins) == 2
+        assert len(self.senpy.plugins) >= 3
         assert self.senpy.plugins["Sleep"].is_activated
 
     def test_disabling(self):
@@ -70,6 +79,11 @@ class ExtensionsTest(TestCase):
         """ Don't analyse if there isn't any plugin installed """
         self.senpy.deactivate_all(sync=True)
         self.assertRaises(Error, partial(self.senpy.analyse, input="tupni"))
+        self.assertRaises(Error,
+                          partial(
+                              self.senpy.analyse,
+                              input="tupni",
+                              algorithm='Dummy'))
 
     def test_analyse(self):
         """ Using a plugin """
@@ -78,8 +92,23 @@ class ExtensionsTest(TestCase):
         r1 = self.senpy.analyse(
             algorithm="Dummy", input="tupni", output="tuptuo")
         r2 = self.senpy.analyse(input="tupni", output="tuptuo")
-        assert r1.analysis[0].id[:5] == "Dummy"
-        assert r2.analysis[0].id[:5] == "Dummy"
+        assert r1.analysis[0] == "plugins/Dummy_0.1"
+        assert r2.analysis[0] == "plugins/Dummy_0.1"
+        assert r1.entries[0].text == 'input'
+
+    def test_analyse_error(self):
+        mm = mock.MagicMock()
+        mm.analyse_entry.side_effect = Error('error on analysis', status=900)
+        self.senpy.plugins['MOCK'] = mm
+        resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
+        assert resp['message'] == 'error on analysis'
+        assert resp['status'] == 900
+        mm.analyse.side_effect = Exception('generic exception on analysis')
+        mm.analyse_entry.side_effect = Exception(
+            'generic exception on analysis')
+        resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
+        assert resp['message'] == 'generic exception on analysis'
+        assert resp['status'] == 500
 
     def test_filtering(self):
         """ Filtering plugins """
@@ -88,5 +117,47 @@ class ExtensionsTest(TestCase):
         assert self.senpy.filter_plugins(name="Dummy", is_activated=True)
         self.senpy.deactivate_plugin("Dummy", sync=True)
         assert not len(
-            self.senpy.filter_plugins(
-                name="Dummy", is_activated=True))
+            self.senpy.filter_plugins(name="Dummy", is_activated=True))
+
+    def test_load_default_plugins(self):
+        senpy = Senpy(plugin_folder=self.dir, default_plugins=True)
+        assert len(senpy.plugins) > 1
+
+    def test_convert_emotions(self):
+        self.senpy.activate_all()
+        plugin = {
+            'id': 'imaginary',
+            'onyx:usesEmotionModel': 'emoml:fsre-dimensions'
+        }
+        eSet1 = EmotionSet()
+        eSet1['onyx:hasEmotion'].append(Emotion({
+            'emoml:arousal': 1,
+            'emoml:potency': 0,
+            'emoml:valence': 0
+        }))
+        response = Results({
+            'entries': [Entry({
+                'text': 'much ado about nothing',
+                'emotions': [eSet1]
+            })]
+        })
+        params = {'emotionModel': 'emoml:big6',
+                  'conversion': 'full'}
+        r1 = deepcopy(response)
+        self.senpy.convert_emotions(r1,
+                                    plugin,
+                                    params)
+        assert len(r1.entries[0].emotions) == 2
+        params['conversion'] = 'nested'
+        r2 = deepcopy(response)
+        self.senpy.convert_emotions(r2,
+                                    plugin,
+                                    params)
+        assert len(r2.entries[0].emotions) == 1
+        assert r2.entries[0].emotions[0]['prov:wasDerivedFrom'] == eSet1
+        params['conversion'] = 'filtered'
+        r3 = deepcopy(response)
+        self.senpy.convert_emotions(r3,
+                                    plugin,
+                                    params)
+        assert len(r3.entries[0].emotions) == 1

tests/test_models.py

@@ -3,20 +3,25 @@ import logging
 import jsonschema
 
 import json
+import rdflib
 from unittest import TestCase
-from senpy.models import Entry, Results, Sentiment, EmotionSet, Error
+from senpy.models import (Emotion,
+                          EmotionAnalysis,
+                          EmotionSet,
+                          Entry,
+                          Error,
+                          Results,
+                          Sentiment)
 from senpy.plugins import SenpyPlugin
 from pprint import pprint
 
 
 class ModelsTest(TestCase):
     def test_jsonld(self):
-        prueba = {"id": "test",
-                  "analysis": [],
-                  "entries": []}
+        prueba = {"id": "test", "analysis": [], "entries": []}
         r = Results(**prueba)
         print("Response's context: ")
-        pprint(r.context)
+        pprint(r._context)
 
         assert r.id == "test"
 
@@ -30,14 +35,11 @@ class ModelsTest(TestCase):
         assert "id" not in j
 
         r6 = Results(**prueba)
-        e = Entry({
-            "@id": "ohno",
-            "nif:isString": "Just testing"
-        })
+        e = Entry({"@id": "ohno", "nif:isString": "Just testing"})
         r6.entries.append(e)
         logging.debug("Reponse 6: %s", r6)
-        assert ("marl" in r6.context)
-        assert ("entries" in r6.context)
+        assert ("marl" in r6._context)
+        assert ("entries" in r6._context)
         j6 = r6.jsonld(with_context=True)
         logging.debug("jsonld: %s", j6)
         assert ("@context" in j6)
@@ -113,5 +115,31 @@ class ModelsTest(TestCase):
         s = str(r)
         assert "_testing" not in s
 
-    def test_frame_response(self):
-        pass
+    def test_turtle(self):
+        """Any model should be serializable as a turtle file"""
+        ana = EmotionAnalysis()
+        res = Results()
+        res.analysis.append(ana)
+        entry = Entry(text='Just testing')
+        eSet = EmotionSet()
+        emotion = Emotion()
+        entry.emotions.append(eSet)
+        res.entries.append(entry)
+        eSet.onyx__hasEmotion.append(emotion)
+        eSet.prov__wasGeneratedBy = ana.id
+        triples = ('ana a :Analysis',
+                   'entry a :entry',
+                   '      nif:isString "Just testing"',
+                   '      onyx:hasEmotionSet eSet',
+                   'eSet a onyx:EmotionSet',
+                   '     prov:wasGeneratedBy ana',
+                   '     onyx:hasEmotion emotion',
+                   'emotion a onyx:Emotion',
+                   'res a :results',
+                   '    me:AnalysisInvoloved ana',
+                   '    prov:used entry')
+
+        t = res.serialize(format='turtle')
+        print(t)
+        g = rdflib.Graph().parse(data=t, format='turtle')
+        assert len(g) == len(triples)

tests/test_plugins.py

@@ -77,6 +77,7 @@ class PluginsTest(TestCase):
         })
         a.activate()
 
+        assert a.shelf_file == self.shelf_file
         res1 = a.analyse(input=1)
         assert res1.entries[0].nif__isString == 1
         res2 = a.analyse(input=1)
@@ -103,3 +104,19 @@ class PluginsTest(TestCase):
         assert b.sh['a'] == 'fromA'
         b.sh['a'] = 'fromB'
         assert b.sh['a'] == 'fromB'
+
+    def test_extra_params(self):
+        ''' Should be able to set extra parameters'''
+        a = ShelfDummyPlugin(info={
+            'name': 'shelve',
+            'version': 'test',
+            'shelf_file': self.shelf_file,
+            'extra_params': {
+                'example': {
+                    'aliases': ['example', 'ex'],
+                    'required': True,
+                    'default': 'nonsense'
+                }
+            }
+        })
+        assert 'example' in a.extra_params