Refactored conversion and postprocessing

Replace algorithm list with a tuple

Closes #56

See merge request senpy/senpy!25

.drone.yml

@@ -0,0 +1,9 @@
+build:
+  image: python:$$PYTHON_VERSION
+  commands:
+    - python setup.py test
+
+matrix:
+  PYTHON_VERSION:
+    - 2.7
+    - 3.4

.gitignore

@@ -2,4 +2,9 @@
 .*
 *egg-info
 dist
+build
 README.html
+__pycache__
+VERSION
+Dockerfile-*
+Dockerfile

.gitlab-ci.yml

@@ -0,0 +1,103 @@
+# Uncomment if you want to use docker-in-docker
+# image: gsiupm/dockermake:latest
+# services:
+# - docker:dind
+# When using dind, it's wise to use the overlayfs driver for
+# improved performance.
+
+stages:
+  - test
+  - push
+  - deploy
+  - clean
+
+before_script:
+  - make -e login
+
+.test: &test_definition
+  stage: test
+  script:
+    - make -e test-$PYTHON_VERSION
+  except:
+    - tags # Avoid unnecessary double testing
+    
+test-3.5:
+  <<: *test_definition
+  variables:
+    PYTHON_VERSION: "3.5"
+
+test-2.7:
+  <<: *test_definition
+  variables:
+    PYTHON_VERSION: "2.7"
+
+push:
+  stage: push
+  script:
+    - make -e push
+  only:
+    - tags
+    - triggers
+    - fix-makefiles
+
+push-latest:
+  stage: push
+  script:
+    - make -e push-latest
+  only:
+    - master
+    - triggers
+    - fix-makefiles
+
+push-github:
+  stage: deploy
+  script:
+    - make -e push-github
+  only:
+    - master
+    - triggers
+    - fix-makefiles
+
+deploy_pypi:
+  stage: deploy
+  script:   # Configure the PyPI credentials, then push the package, and cleanup the creds.
+    - echo "[server-login]" >> ~/.pypirc
+    - echo "repository=https://upload.pypi.org/legacy/" >> ~/.pypirc
+    - echo "username=" ${PYPI_USER} >> ~/.pypirc
+    - echo "password=" ${PYPI_PASSWORD} >> ~/.pypirc
+    - make pip_upload
+    - echo "" > ~/.pypirc && rm ~/.pypirc  # If the above fails, this won't run.
+  only:
+    - /^v?\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
+  except:
+    - branches
+
+deploy:
+  stage: deploy
+  environment: test
+  script:
+    - make -e deploy
+  only:
+    - master
+    - fix-makefiles
+
+push-github:
+  stage: deploy
+  script:
+    - make -e push-github
+  only:
+    - master
+    - triggers
+
+clean :
+  stage: clean
+  script:
+    - make -e clean
+  when: manual
+
+cleanup_py:
+   stage: clean
+   when: always   # this is important; run even if preceding stages failed.
+   script:
+    - rm -vf ~/.pypirc  # we don't want to leave these around, but GitLab may clean up anyway.
+    - docker logout

.makefiles/README.md

@@ -0,0 +1,27 @@
+These makefiles are recipes for several common tasks in different types of projects.
+To add them to your project, simply do:
+
+```
+git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git
+git subtree add --prefix=.makefiles/ makefiles master
+touch Makefile
+echo "include .makefiles/base.mk" >> Makefile
+```
+
+Now you can take advantage of the recipes.
+For instance, to add useful targets for a python project, just add this to your Makefile:
+
+```
+include .makefiles/python.mk
+```
+
+You may need to set special variables like the name of your project or the python versions you're targetting.
+Take a look at each specific `.mk` file for more information, and the `Makefile` in the [senpy](https://lab.cluster.gsi.dit.upm.es/senpy/senpy) project for a real use case.
+
+If you update the makefiles from your repository, make sure to push the changes for review in upstream (this repository):
+
+```
+make makefiles-push
+```
+
+It will automatically commit all unstaged changes in the .makefiles folder.

.makefiles/base.mk

@@ -0,0 +1,36 @@
+export
+NAME ?= $(shell basename $(CURDIR))
+VERSION ?= $(shell git describe --tags --dirty 2>/dev/null)
+
+ifeq ($(VERSION),)
+	VERSION:=unknown
+endif
+
+# Get the location of this makefile.
+MK_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+
+-include .env
+-include ../.env
+
+help:           ## Show this help.
+	@fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/\(.*:\)[^#]*##\s*\(.*\)/\1\t\2/' | column -t -s "	"
+
+config:  ## Load config from the environment. You should run it once in every session before other tasks. Run: eval $(make config)
+	@awk '{ print "export " $$0}' ../.env
+	@awk '{ print "export " $$0}' .env
+	@echo "# Please, run: "
+	@echo "# eval \$$(make config)"
+# If you need to run a command on the key/value pairs, use this:
+# @awk '{ split($$0, a, "="); "echo " a[2] " | base64 -w 0" |& getline b64; print "export " a[1] "=" a[2]; print "export " a[1] "_BASE64=" b64}' .env
+
+ci:  ## Run a task using gitlab-runner. Only use to debug problems in the CI pipeline
+	gitlab-runner exec shell --builds-dir '.builds' --env CI_PROJECT_NAME=$(NAME) ${action}
+
+include $(MK_DIR)/makefiles.mk
+include $(MK_DIR)/docker.mk
+include $(MK_DIR)/git.mk
+
+info:: ## List all variables
+	env
+
+.PHONY:: config help ci

.makefiles/docker.mk

@@ -0,0 +1,51 @@
+ifndef IMAGENAME
+	ifdef CI_REGISTRY_IMAGE
+		IMAGENAME=$(CI_REGISTRY_IMAGE)
+	else
+		IMAGENAME=$(NAME)
+	endif
+endif
+
+IMAGEWTAG?=$(IMAGENAME):$(VERSION)
+DOCKER_FLAGS?=$(-ti)
+DOCKER_CMD?=
+
+docker-login: ## Log in to the registry. It will only be used in the server, or when running a CI task locally (if CI_BUILD_TOKEN is set).
+ifeq ($(CI_BUILD_TOKEN),)
+	@echo "Not logging in to the docker registry" "$(CI_REGISTRY)"
+else
+	@docker login -u gitlab-ci-token -p $(CI_BUILD_TOKEN) $(CI_REGISTRY)
+endif
+ifeq ($(HUB_USER),)
+	@echo "Not logging in to global the docker registry"
+else
+	@docker login -u $(HUB_USER) -p $(HUB_PASSWORD)
+endif
+
+docker-clean: ## Remove docker credentials
+ifeq ($(HUB_USER),)
+else
+	@docker logout
+endif
+
+docker-run: ## Build a generic docker image
+	docker run $(DOCKER_FLAGS) $(IMAGEWTAG) $(DOCKER_CMD)
+
+docker-build: ## Build a generic docker image
+	docker build . -t $(IMAGEWTAG)
+
+docker-push: docker-login ## Push a generic docker image
+	docker push $(IMAGEWTAG)
+
+docker-latest-push: docker-login ## Push the latest image
+	docker tag $(IMAGEWTAG) $(IMAGENAME)
+	docker push $(IMAGENAME)
+
+login:: docker-login
+
+clean:: docker-clean
+
+docker-info:
+	@echo IMAGEWTAG=${IMAGEWTAG}
+
+.PHONY:: docker-login docker-clean login clean

.makefiles/git.mk

@@ -0,0 +1,25 @@
+commit:
+	git commit -a
+
+tag:
+	git tag ${VERSION}
+
+git-push::
+	git push --tags -u origin HEAD
+
+git-pull:
+	git pull --all
+
+push-github: ## Push the code to github. You need to set up GITHUB_DEPLOY_KEY
+ifeq ($(GITHUB_DEPLOY_KEY),)
+else
+	$(eval KEY_FILE := "$(shell mktemp)")
+	@printf '%b' '$(GITHUB_DEPLOY_KEY)' > $(KEY_FILE)
+	@git remote rm github-deploy || true
+	git remote add github-deploy $(GITHUB_REPO)
+	-@GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git fetch github-deploy $(CI_COMMIT_REF_NAME)
+	@GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git push github-deploy HEAD:$(CI_COMMIT_REF_NAME)
+	rm $(KEY_FILE)
+endif
+
+.PHONY:: commit tag git-push git-pull push-github 

.makefiles/k8s.mk

@@ -0,0 +1,51 @@
+# Deployment with Kubernetes
+
+# KUBE_CA_PEM_FILE is the path of a certificate file. It automatically set by GitLab
+# if you enable Kubernetes integration in a project.
+#
+# As of this writing, Kubernetes integration can not be set on a group level, so it has to
+# be manually set in every project.
+# Alternatively, we use a custom KUBE_CA_BUNDLE environment variable, which can be set at
+# the group level. In this case, the variable contains the whole content of the certificate,
+# which we dump to a temporary file
+#
+# Check if the KUBE_CA_PEM_FILE exists. Otherwise, create it from KUBE_CA_BUNDLE
+KUBE_CA_TEMP=false
+ifndef KUBE_CA_PEM_FILE
+KUBE_CA_PEM_FILE:=$$PWD/.ca.crt
+CREATED:=$(shell printf '%b\n' '$(KUBE_CA_BUNDLE)' > $(KUBE_CA_PEM_FILE))
+endif 
+KUBE_TOKEN?=""
+KUBE_NAMESPACE?=$(NAME)
+KUBECTL=docker run --rm -v $(KUBE_CA_PEM_FILE):/tmp/ca.pem -i lachlanevenson/k8s-kubectl --server="$(KUBE_URL)" --token="$(KUBE_TOKEN)" --certificate-authority="/tmp/ca.pem" -n $(KUBE_NAMESPACE)
+CI_COMMIT_REF_NAME?=master
+
+info:: ## Print variables. Useful for debugging.
+	@echo "#KUBERNETES"
+	@echo KUBE_URL=$(KUBE_URL)
+	@echo KUBE_CA_PEM_FILE=$(KUBE_CA_PEM_FILE)
+	@echo KUBE_CA_BUNDLE=$$KUBE_CA_BUNDLE
+	@echo KUBE_TOKEN=$(KUBE_TOKEN)
+	@echo KUBE_NAMESPACE=$(KUBE_NAMESPACE)
+	@echo KUBECTL=$(KUBECTL)
+
+	@echo "#CI"
+	@echo CI_PROJECT_NAME=$(CI_PROJECT_NAME)
+	@echo CI_REGISTRY=$(CI_REGISTRY)
+	@echo CI_REGISTRY_USER=$(CI_REGISTRY_USER)
+	@echo CI_COMMIT_REF_NAME=$(CI_COMMIT_REF_NAME)
+	@echo "CREATED=$(CREATED)"
+
+#
+# Deployment and advanced features
+# 
+
+
+deploy: ## Deploy to kubernetes using the credentials in KUBE_CA_PEM_FILE (or KUBE_CA_BUNDLE ) and TOKEN
+	@ls k8s/*.yaml k8s/*.yml k8s/*.tmpl 2>/dev/null || true
+	@cat k8s/*.yaml k8s/*.yml k8s/*.tmpl 2>/dev/null | envsubst | $(KUBECTL) apply -f -
+
+deploy-check: ## Get the deployed configuration.
+	@$(KUBECTL) get deploy,pods,svc,ingress
+
+.PHONY:: info deploy deploy-check

.makefiles/makefiles.mk

@@ -0,0 +1,15 @@
+makefiles-remote:
+	git ls-remote --exit-code  makefiles 2> /dev/null || git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git
+
+makefiles-commit: makefiles-remote
+	git add -f .makefiles
+	git commit -em "Updated makefiles from ${NAME}"
+
+makefiles-push:
+	git fetch makefiles $(NAME)
+	git subtree push --prefix=.makefiles/ makefiles $(NAME)
+
+makefiles-pull: makefiles-remote
+	git subtree pull --prefix=.makefiles/ makefiles master --squash
+
+.PHONY:: makefiles-remote makefiles-commit makefiles-push makefiles-pull

.makefiles/precommit.mk

@@ -0,0 +1,5 @@
+init: ## Init pre-commit hooks (i.e. enforcing format checking before allowing a commit)
+	pip install --user pre-commit
+	pre-commit install
+
+.PHONY:: init

.makefiles/python.mk

@@ -0,0 +1,101 @@
+PYVERSIONS ?= 3.5
+PYMAIN ?= $(firstword $(PYVERSIONS))
+TARNAME ?= $(NAME)-$(VERSION).tar.gz 
+VERSIONFILE ?= $(NAME)/VERSION
+
+DEVPORT ?= 6000
+
+
+.FORCE:
+
+version: .FORCE
+	@echo $(VERSION) > $(VERSIONFILE)
+	@echo $(VERSION)
+
+yapf: ## Format python code
+	yapf -i -r $(NAME)
+	yapf -i -r tests
+
+dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS)) ## Generate dockerfiles for each python version
+	@unlink Dockerfile >/dev/null
+	ln -s Dockerfile-$(PYMAIN) Dockerfile
+
+Dockerfile-%: Dockerfile.template  ## Generate a specific dockerfile (e.g. Dockerfile-2.7)
+	sed "s/{{PYVERSION}}/$*/" Dockerfile.template > Dockerfile-$*
+
+quick_build: $(addprefix build-, $(PYMAIN))
+
+build: $(addprefix build-, $(PYVERSIONS)) ## Build all images / python versions
+	docker tag $(IMAGEWTAG)-python$(PYMAIN) $(IMAGEWTAG)
+
+build-%: version Dockerfile-%  ## Build a specific version (e.g. build-2.7)
+	docker build -t '$(IMAGEWTAG)-python$*' -f Dockerfile-$* .;
+
+dev-%: ## Launch a specific development environment using docker (e.g. dev-2.7)
+	@docker start $(NAME)-dev$* || (\
+		$(MAKE) build-$*; \
+		docker run -d -w /usr/src/app/ -p $(DEVPORT):5000 -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti --name $(NAME)-dev$* '$(IMAGEWTAG)-python$*'; \
+	)\
+
+	docker exec -ti $(NAME)-dev$* bash
+
+dev: dev-$(PYMAIN) ## Launch a development environment using docker, using the default python version
+
+quick_test: test-$(PYMAIN)
+
+test-%: build-% ## Run setup.py from in an isolated container, built from the base image. (e.g. test-2.7)
+# This speeds tests up because the image has most (if not all) of the dependencies already.
+	docker rm $(NAME)-test-$* || true
+	docker create -ti --name $(NAME)-test-$* --entrypoint="" -w /usr/src/app/ $(IMAGEWTAG)-python$* python setup.py test
+	docker cp . $(NAME)-test-$*:/usr/src/app
+	docker start -a $(NAME)-test-$*
+
+test: $(addprefix test-,$(PYVERSIONS)) ## Run the tests with the main python version
+
+run-%: build-%
+	docker run --rm -p $(DEVPORT):5000 -ti '$(IMAGEWTAG)-python$(PYMAIN)' --default-plugins
+
+run: run-$(PYMAIN)
+
+# Pypy - Upload a package
+
+dist/$(TARNAME): version
+	python setup.py sdist;
+
+sdist: dist/$(TARNAME) ## Generate the distribution file (wheel)
+
+pip_test-%: sdist ## Test the distribution file using pip install and a specific python version (e.g. pip_test-2.7)
+	docker run --rm -v $$PWD/dist:/dist/ python:$* pip install /dist/$(TARNAME);
+
+pip_test: $(addprefix pip_test-,$(PYVERSIONS)) ## Test pip installation with the main python version
+
+pip_upload: pip_test  ## Upload package to pip
+	python setup.py sdist upload ;
+
+# Pushing to docker
+
+push-latest: $(addprefix push-latest-,$(PYVERSIONS)) ## Push the "latest" tag to dockerhub
+	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
+	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGENAME):latest'
+	docker push '$(IMAGENAME):latest'
+	docker push '$(IMAGEWTAG)'
+
+push-latest-%: build-%  ## Push the latest image for a specific python version
+	docker tag $(IMAGENAME):$(VERSION)-python$* $(IMAGENAME):python$*
+	docker push $(IMAGENAME):$(VERSION)-python$*
+	docker push $(IMAGENAME):python$*
+
+push-%: build-%  ## Push the image of the current version (tagged). e.g. push-2.7
+	docker push $(IMAGENAME):$(VERSION)-python$*
+
+push:: $(addprefix push-,$(PYVERSIONS)) ## Push an image with the current version for every python version
+	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
+	docker push  $(IMAGENAME):$(VERSION)
+
+clean:: ## Clean older docker images and containers related to this project and dev environments
+	@docker stop $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true
+	@docker rm $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true
+	@docker ps -a | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true
+	@docker images | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true
+
+.PHONY:: yapf dockerfiles Dockerfile-% quick_build build build-% dev-% quick-dev test quick_test push-latest push-latest-% push-% push version .FORCE

.pre-commit-config.yaml

@@ -0,0 +1,5 @@
+-   repo: git://github.com/pre-commit/pre-commit-hooks
+    sha: e626cd57090d8df0be21e4df0f4e55cc3511d6ab
+    hooks:
+    -   id: flake8
+    -   id: check-json

.travis.yml

@@ -1,6 +1,12 @@
+sudo: required
+
+services:
+  - docker
+
 language: python
-python:
+
-  - "2.7"
+env:
-install: "pip install -r requirements.txt"
+  - PYV=2.7
+  - PYV=3.5
 # run nosetests - Tests
-script: nosetests
+script: make test-$PYV

Dockerfile

@@ -1,3 +0,0 @@
-from python:2.7-onbuild
-
-ENTRYPOINT ["python", "-m", "senpy"]

Dockerfile.template

@@ -0,0 +1,26 @@
+from python:{{PYVERSION}}
+
+MAINTAINER J. Fernando Sánchez <jf.sanchez@upm.es>
+
+RUN apt-get update && apt-get install -y \
+libblas-dev liblapack-dev liblapacke-dev gfortran \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir /cache/ /senpy-plugins /data/
+
+VOLUME /data/
+
+ENV PIP_CACHE_DIR=/cache/ SENPY_DATA=/data
+
+ONBUILD COPY . /senpy-plugins/
+ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
+ONBUILD WORKDIR /senpy-plugins/
+
+
+WORKDIR /usr/src/app
+COPY test-requirements.txt requirements.txt extra-requirements.txt /usr/src/app/
+RUN pip install --no-cache-dir -r test-requirements.txt -r requirements.txt -r extra-requirements.txt
+COPY . /usr/src/app/
+RUN pip install --no-cache-dir --no-index --no-deps --editable .
+
+ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

MANIFEST.in

@@ -1,5 +1,10 @@
 include requirements.txt
 include test-requirements.txt
-include README.md
+include extra-requirements.txt
-include senpy/context.jsonld
+include README.rst
+include senpy/VERSION
 graft senpy/plugins
+graft senpy/schemas
+graft senpy/templates
+graft senpy/static
+graft img

Makefile

@@ -0,0 +1,17 @@
+NAME=senpy
+GITHUB_REPO=git@github.com:gsi-upm/senpy.git
+
+IMAGENAME=gsiupm/senpy
+
+# The first version is the main one (used for quick builds)
+# See .makefiles/python.mk for more info
+PYVERSIONS=3.5 2.7
+
+DEVPORT=5000
+
+action="test-${PYMAIN}"
+GITHUB_REPO=git@github.com:gsi-upm/senpy.git
+
+include .makefiles/base.mk
+include .makefiles/k8s.mk
+include .makefiles/python.mk

Procfile

@@ -1 +1 @@
-web: gunicorn app:app --log-file=-
+web: python -m senpy --host 0.0.0.0 --port $PORT --default-plugins

README.rst

@@ -1,5 +1,5 @@
 .. image:: img/header.png
-   :height: 6em
+   :width: 100%
    :target: http://demos.gsi.dit.upm.es/senpy
 
 .. image:: https://travis-ci.org/gsi-upm/senpy.svg?branch=master
@@ -12,7 +12,7 @@ Have you ever wanted to turn your sentiment analysis algorithms into a service?
 With senpy, now you can.
 It provides all the tools so you just have to worry about improving your algorithms:
 
-`See it in action. <http://demos.gsi.dit.upm.es/senpy>`_
+`See it in action. <http://senpy.cluster.gsi.dit.upm.es/>`_
 
 Installation
 ------------
@@ -23,14 +23,14 @@ Through PIP
 
 .. code:: bash
 
-   pip install --user senpy
+   pip install -U --user senpy
 
    
 Alternatively, you can use the development version:
  
 .. code:: bash
 
-   git clone git@github.com:gsi-upm/senpy
+   git clone http://github.com/gsi-upm/senpy
    cd senpy
    pip install --user .
 
@@ -38,9 +38,56 @@ If you want to install senpy globally, use sudo instead of the ``--user`` flag.
 
 Docker Image
 ************
-Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 balkian/senpy --host 0.0.0.0 --default-plugins``.
+Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy --default-plugins``.
 
-To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins balkian/senpy --host 0.0.0.0 --default-plugins -f /plugins``
+To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --default-plugins -f /plugins``
+
+
+Developing
+----------
+
+Developing/debugging
+********************
+This command will run the senpy container using the latest image available, mounting your current folder so you get your latest code:
+
+.. code:: bash
+
+
+    # Python 3.5
+    make dev
+    # Python 2.7
+    make dev-2.7
+
+Building a docker image
+***********************
+
+.. code:: bash
+
+
+    # Python 3.5
+    make build-3.5
+    # Python 2.7
+    make build-2.7
+
+Testing
+*******
+
+.. code:: bash
+
+
+    make test
+
+Running
+*******
+This command will run the senpy server listening on localhost:5000
+
+.. code:: bash
+
+
+    # Python 3.5
+    make run-3.5
+    # Python 2.7
+    make run-2.7
 
 Usage
 -----
@@ -49,12 +96,14 @@ However, the easiest and recommended way is to just use the command-line tool to
 
 .. code:: bash
 
+
    senpy
 
 or, alternatively:
 
 .. code:: bash
 
+
     python -m senpy
 
 

app.py

@@ -1,43 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-#    Copyright 2014 J. Fernando Sánchez Rada - Grupo de Sistemas Inteligentes
-#                                                       DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-"""
-This is a helper for development. If you want to run Senpy use:
-
-    python -m senpy
-"""
-from gevent.monkey import patch_all; patch_all()
-import gevent
-import config
-from flask import Flask
-from senpy.extensions import Senpy
-import logging
-import os
-from gevent.wsgi import WSGIServer
-
-logging.basicConfig(level=logging.DEBUG)
-
-app = Flask(__name__)
-mypath = os.path.dirname(os.path.realpath(__file__))
-sp = Senpy(app, os.path.join(mypath, "plugins"), default_plugins=True)
-sp.activate_all()
-
-if __name__ == '__main__':
-    import logging
-    logging.basicConfig(level=config.DEBUG)
-    app.debug = config.DEBUG
-    http_server = WSGIServer(('', config.SERVER_PORT), app)
-    http_server.serve_forever()

docker-compose.dev.yml

@@ -0,0 +1,10 @@
+version: '3'
+services:
+    senpy:
+        image: "${IMAGENAME-gsiupm/senpy}:${VERSION-latest}"
+        entrypoint: ["/bin/bash"]
+        working_dir: "/senpy-plugins"
+        ports:
+            - 5000:5000
+        volumes:
+            - ".:/usr/src/app/"

docker-compose.test.yml

@@ -0,0 +1,9 @@
+version: '3'
+services:
+    test:
+        image: "${IMAGENAME-gsiupm/senpy}:${VERSION-dev}"
+        entrypoint: ["py.test"]
+        volumes:
+            - ".:/usr/src/app/"
+        command:
+          []

docker-compose.yml

@@ -0,0 +1,11 @@
+version: '3'
+services:
+    senpy:
+        image: "${IMAGENAME-gsiupm/senpy}:${VERSION-dev}"
+        build:
+          context: .
+          dockerfile: Dockerfile${PYVERSION--2.7}
+        ports:
+            - 5001:5000
+        volumes:
+            - "./data:/data"

docs/SenpyClientUse.ipynb

@@ -0,0 +1,317 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:05:31.465571Z",
+     "start_time": "2017-04-10T19:05:31.458282+02:00"
+    },
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "# Client"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "The built-in senpy client allows you to query any Senpy endpoint. We will illustrate how to use it with the public demo endpoint, and then show you how to spin up your own endpoint using docker."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "Demo Endpoint\n",
+    "-------------"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "To start using senpy, simply create a new Client and point it to your endpoint. In this case, the latest version of Senpy at GSI."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:12.827640Z",
+     "start_time": "2017-04-10T19:29:12.818617+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "from senpy.client import Client\n",
+    "\n",
+    "c = Client('http://latest.senpy.cluster.gsi.dit.upm.es/api')\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "Now, let's use that client analyse some queries:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:14.011657Z",
+     "start_time": "2017-04-10T19:29:13.701808+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "r = c.analyse('I like sugar!!', algorithm='sentiment140')\n",
+    "r"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:08:19.616754Z",
+     "start_time": "2017-04-10T19:08:19.610767+02:00"
+    },
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "As you can see, that gave us the full JSON result. A more concise way to print it would be:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:14.854213Z",
+     "start_time": "2017-04-10T19:29:14.842068+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "for entry in r.entries:\n",
+    "      print('{} -> {}'.format(entry['text'], entry['sentiments'][0]['marl:hasPolarity']))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "We can also obtain a list of available plugins with the client:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:16.245198Z",
+     "start_time": "2017-04-10T19:29:16.056545+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "c.plugins()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "Or, more concisely:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:17.663275Z",
+     "start_time": "2017-04-10T19:29:17.484623+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "c.plugins().keys()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "Local Endpoint\n",
+    "--------------"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "To run your own instance of senpy, just create a docker container with the latest Senpy image. Using `--default-plugins` you will get some extra plugins to start playing with the API."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:20.637539Z",
+     "start_time": "2017-04-10T19:29:19.938322+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "!docker run -ti --name 'SenpyEndpoint' -d -p 6000:5000 gsiupm/senpy:0.8.6 --host 0.0.0.0 --default-plugins"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "To use this endpoint:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:21.263976Z",
+     "start_time": "2017-04-10T19:29:21.260595+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "c_local = Client('http://127.0.0.1:6000/api')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "That's all! After you are done with your analysis, stop the docker container:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:33.226686Z",
+     "start_time": "2017-04-10T19:29:22.392121+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "!docker stop SenpyEndpoint\n",
+    "!docker rm SenpyEndpoint"
+   ]
+  }
+ ],
+ "metadata": {
+  "anaconda-cloud": {},
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.0"
+  },
+  "toc": {
+   "colors": {
+    "hover_highlight": "#DAA520",
+    "running_highlight": "#FF0000",
+    "selected_highlight": "#FFD700"
+   },
+   "moveMenuLeft": true,
+   "nav_menu": {
+    "height": "68px",
+    "width": "252px"
+   },
+   "navigate_menu": true,
+   "number_sections": true,
+   "sideBar": true,
+   "threshold": 4,
+   "toc_cell": false,
+   "toc_section_display": "block",
+   "toc_window_display": false
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}

docs/SenpyClientUse.rst

@@ -0,0 +1,106 @@
+
+Client
+======
+
+Demo Endpoint
+-------------
+
+Import Client and send a request
+
+.. code:: python
+
+    from senpy.client import Client
+    
+    c = Client('http://latest.senpy.cluster.gsi.dit.upm.es/api')
+    r = c.analyse('I like Pizza', algorithm='sentiment140')
+
+Print response
+
+.. code:: python
+
+    for entry in r.entries:
+          print('{} -> {}'.format(entry['text'], entry['sentiments'][0]['marl:hasPolarity']))
+
+
+.. parsed-literal::
+
+    I like Pizza -> marl:Positive
+
+
+Obtain a list of available plugins
+
+.. code:: python
+
+    for plugin in c.request('/plugins')['plugins']:
+        print(plugin['name'])
+
+
+.. parsed-literal::
+
+    emoRand
+    rand
+    sentiment140
+
+
+Local Endpoint
+--------------
+
+Run a docker container with Senpy image and default plugins
+
+.. code::
+
+    docker run -ti --name 'SenpyEndpoint' -d -p 5000:5000 gsiupm/senpy:0.8.6 --host 0.0.0.0 --default-plugins
+
+
+.. parsed-literal::
+
+    a0157cd98057072388bfebeed78a830da7cf0a796f4f1a3fd9188f9f2e5fe562
+
+
+Import client and send a request to localhost
+
+.. code:: python
+
+    c_local = Client('http://127.0.0.1:5000/api')
+    r = c_local.analyse('Hello world', algorithm='sentiment140')
+
+Print response
+
+.. code:: python
+
+    for entry in r.entries:
+          print('{} -> {}'.format(entry['text'], entry['sentiments'][0]['marl:hasPolarity']))
+
+
+.. parsed-literal::
+
+    Hello world -> marl:Neutral
+
+
+Obtain a list of available plugins deployed locally
+
+.. code:: python
+
+    c_local.plugins().keys()
+
+
+.. parsed-literal::
+
+    rand
+    sentiment140
+    emoRand
+
+
+Stop the docker container
+
+.. code:: python
+
+    !docker stop SenpyEndpoint
+    !docker rm SenpyEndpoint
+
+
+.. parsed-literal::
+
+    SenpyEndpoint
+    SenpyEndpoint
+

docs/_static/schemas

@@ -0,0 +1 @@
+../../senpy/schemas/

docs/about.rst

@@ -0,0 +1,11 @@
+About
+--------
+
+If you use Senpy in your research, please cite `Senpy: A Pragmatic Linked Sentiment Analysis Framework <http://gsi.dit.upm.es/index.php/es/investigacion/publicaciones?view=publication&task=show&id=417>`__ (`BibTex <http://gsi.dit.upm.es/index.php/es/investigacion/publicaciones?controller=publications&task=export&format=bibtex&id=417>`__):
+
+.. code-block:: text
+
+	Sánchez-Rada, J. F., Iglesias, C. A., Corcuera, I., & Araque, Ó. (2016, October).
+	Senpy: A Pragmatic Linked Sentiment Analysis Framework.
+	In Data Science and Advanced Analytics (DSAA),
+	2016 IEEE International Conference on (pp. 735-742). IEEE.

docs/api.rst

@@ -1,5 +1,5 @@
 NIF API
-=======
+-------
 .. http:get:: /api
 
    Basic endpoint for sentiment/emotion analysis.
@@ -22,38 +22,32 @@ NIF API
       Content-Type: text/javascript
 
       {
-        "@context": [
+        "@context":"http://127.0.0.1/api/contexts/Results.jsonld",
-          "http://127.0.0.1/static/context.jsonld",
+        "@id":"_:Results_11241245.22",
-      ],
+        "@type":"results"
-      "analysis": [
+        "analysis": [
-        {
+          "plugins/sentiment-140_0.1" 
-          "@id": "SentimentAnalysisExample",
+        ],
-          "@type": "marl:SentimentAnalysis",
+        "entries": [
-          "dc:language": "en", 
+          {  
-          "marl:maxPolarityValue": 10.0,
+            "@id": "_:Entry_11241245.22"
-          "marl:minPolarityValue": 0.0
+            "@type":"entry",
-        }
+            "emotions": [],
-      ],
+            "entities": [],
-      "domain": "wndomains:electronics", 
+            "sentiments": [
-      "entries": [
+              {  
-        {
+                "@id": "Sentiment0",  
-          "opinions": [
+                "@type": "sentiment", 
-            {
+                "marl:hasPolarity": "marl:Negative",
-              "prov:generatedBy": "SentimentAnalysisExample",
+                "marl:polarityValue": 0,
-              "marl:polarityValue": 7.8, 
+                "prefix": ""
-              "marl:hasPolarity": "marl:Positive",
+              }
-              "marl:describesObject": "http://www.gsi.dit.upm.es",
+            ],
-            }
+            "suggestions": [],
-          ],
+            "text": "This text makes me sad.\nwhilst this text makes me happy and surprised at the same time.\nI cannot believe it!",
-          "nif:isString": "I love GSI",
+            "topics": []
-          "strings": [
+          }
-            {
+        ]
-              "nif:anchorOf": "GSI",
-              "nif:taIdentRef": "http://www.gsi.dit.upm.es"
-            }
-          ]
-        }
-       ]
       }
 
    :query i input: No default. Depends on informat and intype
@@ -62,6 +56,7 @@ NIF API
    :query o outformat: one of `turtle` (default), `text`, `json-ld`
    :query p prefix: prefix for the URIs
    :query algo algorithm: algorithm/plugin to use for the analysis. For a list of options, see :http:get:`/api/plugins`. If not provided, the default plugin will be used (:http:get:`/api/plugins/default`).
+   :query algo emotionModel: desired emotion model in the results. If the requested algorithm does not use that emotion model, there are conversion plugins specifically for this. If none of the plugins match, an error will be returned, which includes the results *as is*.
 
    :reqheader Accept: the response content type depends on
                       :mailheader:`Accept` header
@@ -69,6 +64,7 @@ NIF API
                             header of request
    :statuscode 200: no error
    :statuscode 404: service not found
+   :statuscode 400: error while processing the request
 
 .. http:post:: /api
 
@@ -90,56 +86,59 @@ NIF API
 
    .. sourcecode:: http
 
-        {
+      {
-            "@context": {
+        "@id": "plugins/sentiment-140_0.1", 
-                   ...
+        "@type": "sentimentPlugin", 
-            }, 
+        "author": "@balkian", 
-            "sentiment140": {
+        "description": "Sentiment classifier using rule-based classification for English and Spanish. This plugin uses sentiment140 data to perform classification. For more information: http://help.sentiment140.com/for-students/", 
-                "name": "sentiment140", 
+        "extra_params": {
-                "is_activated": true, 
+          "language": {
-                "version": "0.1", 
+            "@id": "lang_sentiment140", 
-                "extra_params": {
+            "aliases": [
-                    "@id": "extra_params_sentiment140_0.1", 
+              "language", 
-                    "language": {
+              "l"
-                        "required": false, 
+            ], 
-                        "@id": "lang_sentiment140", 
+            "options": [
-                        "options": [
+              "es", 
-                            "es", 
+              "en", 
-                            "en", 
+              "auto"
-                            "auto"
+            ], 
-                        ], 
+            "required": false
-                        "aliases": [
+          }
-                            "language", 
+        }, 
-                            "l"
+        "is_activated": true, 
-                        ]
+        "maxPolarityValue": 1.0, 
-                    }
+        "minPolarityValue": 0.0, 
-                }, 
+        "module": "sentiment-140", 
-                "@id": "sentiment140_0.1"
+        "name": "sentiment-140", 
-            }, 
+        "requirements": {}, 
-            "rand": {
+        "version": "0.1"
-                "name": "rand", 
+      }, 
-                "is_activated": true, 
+      {
-                "version": "0.1", 
+        "@id": "plugins/ExamplePlugin_0.1", 
-                "extra_params": {
+        "@type": "sentimentPlugin", 
-                    "@id": "extra_params_rand_0.1", 
+        "author": "@balkian", 
-                    "language": {
+        "custom_attribute": "42", 
-                        "required": false, 
+        "description": "I am just an example", 
-                        "@id": "lang_rand", 
+        "extra_params": {
-                        "options": [
+          "parameter": {
-                            "es", 
+            "@id": "parameter", 
-                            "en", 
+            "aliases": [
-                            "auto"
+              "parameter", 
-                        ], 
+              "param"
-                        "aliases": [
+            ], 
-                            "language", 
+            "default": 42, 
-                            "l"
+            "required": true
-                        ]
+          }
-                    }
+        }, 
-                }, 
+        "is_activated": true, 
-                "@id": "rand_0.1"
+        "maxPolarityValue": 1.0, 
-            }
+        "minPolarityValue": 0.0, 
-        }
+        "module": "example", 
-
+        "name": "ExamplePlugin", 
+        "requirements": "noop", 
+        "version": "0.1"
+      }
 
 .. http:get:: /api/plugins/<pluginname>
 
@@ -148,7 +147,7 @@ NIF API
 
    .. sourcecode:: http
 
-      GET /api/plugins/rand HTTP/1.1
+      GET /api/plugins/rand/ HTTP/1.1
       Host: localhost
       Accept: application/json, text/javascript
 
@@ -158,51 +157,61 @@ NIF API
    .. sourcecode:: http
 
       {
-          "@id": "rand_0.1",
+        "@context": "http://127.0.0.1/api/contexts/ExamplePlugin.jsonld", 
-          "extra_params": {
+        "@id": "plugins/ExamplePlugin_0.1", 
-              "@id": "extra_params_rand_0.1",
+        "@type": "sentimentPlugin", 
-              "language": {
+        "author": "@balkian", 
-                  "@id": "lang_rand",
+        "custom_attribute": "42", 
-                  "aliases": [
+        "description": "I am just an example", 
-                      "language",
+        "extra_params": {
-                      "l"
+          "parameter": {
-                  ],
+            "@id": "parameter", 
-                  "options": [
+            "aliases": [
-                      "es",
+              "parameter", 
-                      "en",
+              "param"
-                      "auto"
+            ], 
-                  ],
+            "default": 42, 
-                  "required": false
+            "required": true
-              }
+          }
-          },
+        }, 
-          "is_activated": true,
+        "is_activated": true, 
-          "name": "rand",
+        "maxPolarityValue": 1.0, 
-          "version": "0.1"
+        "minPolarityValue": 0.0, 
+        "module": "example", 
+        "name": "ExamplePlugin", 
+        "requirements": "noop", 
+        "version": "0.1"
       }
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 .. http:get:: /api/plugins/default
 
    Return the information about the default plugin.
 
-.. http:get:: /api/plugins/<pluginname>/{de}activate
-
-   {De}activate a plugin.
-
-   **Example request**:
-
-   .. sourcecode:: http
-
-      GET /api/plugins/rand/deactivate HTTP/1.1
-      Host: localhost
-      Accept: application/json, text/javascript
-
-
-   **Example response**:
-
-   .. sourcecode:: http
-
-       {
-          "@context": {}, 
-          "message": "Ok"
-       }

docs/apischema.rst

@@ -0,0 +1,7 @@
+API and Examples
+################
+.. toctree::
+
+   vocabularies.rst
+   api.rst
+   examples.rst

docs/bad-examples/plugins/noplugins.json

@@ -0,0 +1,4 @@
+{
+  "@type": "plugins",
+  "plugins": {}
+}

docs/bad-examples/results/example-analysis-as-id-FAIL.json

@@ -0,0 +1,77 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "@type": "results",
+  "analysis": [
+    "me:SAnalysis1",
+    "me:SgAnalysis1",
+    "me:EmotionAnalysis1",
+    "me:NER1",
+    {
+      "description": "missing @id and @type"
+    }
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+        {
+          "@id": "http://micro.blog/status1#char=5,13",
+          "nif:beginIndex": 5,
+          "nif:endIndex": 13,
+          "nif:anchorOf": "Microsoft",
+          "me:references": "http://dbpedia.org/page/Microsoft",
+          "prov:wasGeneratedBy": "me:NER1"
+        },
+        {
+          "@id": "http://micro.blog/status1#char=25,37",
+          "nif:beginIndex": 25,
+          "nif:endIndex": 37,
+          "nif:anchorOf": "Windows Phone",
+          "me:references": "http://dbpedia.org/page/Windows_Phone",
+          "prov:wasGeneratedBy": "me:NER1"
+        }
+      ],
+      "suggestions": [
+        {
+          "@id": "http://micro.blog/status1#char=16,77",
+          "nif:beginIndex": 16,
+          "nif:endIndex": 77,
+          "nif:anchorOf": "put your Windows Phone on your newest #open technology program",
+          "prov:wasGeneratedBy": "me:SgAnalysis1"
+        }
+      ],
+      "sentiments": [
+        {
+          "@id": "http://micro.blog/status1#char=80,97",
+          "nif:beginIndex": 80,
+          "nif:endIndex": 97,
+          "nif:anchorOf": "You'll be awesome.",
+          "marl:hasPolarity": "marl:Positive",
+          "marl:polarityValue": 0.9,
+          "prov:wasGeneratedBy": "me:SAnalysis1"
+        }
+      ],
+      "emotions": [
+        {
+          "@id": "http://micro.blog/status1#char=0,109",
+          "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+          "prov:wasGeneratedBy": "me:EAnalysis1",
+          "onyx:hasEmotion": [
+            {
+              "onyx:hasEmotionCategory": "wna:liking"
+            },
+            {
+              "onyx:hasEmotionCategory": "wna:excitement"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

docs/bad-examples/results/example-basic-FAIL.json

@@ -0,0 +1,18 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "http://example.com#NIFExample",
+  "@type": "results",
+  "analysis": [
+  ],
+  "entries": [
+    {
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:beginIndex": 0,
+      "nif:endIndex": 40,
+      "nif:isString": "My favourite actress is Natalie Portman"
+    }
+  ]
+}

docs/commandline.rst

@@ -0,0 +1,9 @@
+Command line
+============
+
+This video shows how to analyse text directly on the command line using the senpy tool.
+
+.. image:: https://asciinema.org/a/9uwef1ghkjk062cw2t4mhzpyk.png
+   :width: 100%
+   :target: https://asciinema.org/a/9uwef1ghkjk062cw2t4mhzpyk
+   :alt: CLI demo

docs/conf.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# flake8: noqa
 #
 # Senpy documentation build configuration file, created by
 # sphinx-quickstart on Tue Feb 24 08:57:32 2015.
@@ -36,6 +37,7 @@ extensions = [
     'sphinx.ext.todo',
     'sphinxcontrib.httpdomain',
     'sphinx.ext.coverage',
+    'sphinx.ext.autosectionlabel',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -52,20 +54,22 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Senpy'
-copyright = u'2015, J. Fernando Sánchez'
+copyright = u'2016, J. Fernando Sánchez'
+description = u'A framework for sentiment and emotion analysis services'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '0.4'
+# with open('../senpy/VERSION') as f:
+#     version = f.read().strip()
 # The full version, including alpha/beta/rc tags.
-release = '0.4'
+# release = version
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
-#language = None
+language = None
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
@@ -102,14 +106,14 @@ pygments_style = 'sphinx'
 #keep_warnings = False
 
 
+html_theme = 'alabaster'
 # -- Options for HTML output ----------------------------------------------
-if not on_rtd:  # only import and set the theme if we're building docs locally
+# if not on_rtd:  # only import and set the theme if we're building docs locally
-    import sphinx_rtd_theme
+#     import sphinx_rtd_theme
-    html_theme = 'sphinx_rtd_theme'
+#     html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
-    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
-else:
+# else:
-    html_theme = 'default'
+#     html_theme = 'default'
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
@@ -117,7 +121,13 @@ else:
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-#html_theme_options = {}
+html_theme_options = {
+    'logo': 'header.png',
+    'github_user': 'gsi-upm',
+    'github_repo': 'senpy',
+    'github_banner': True,
+}
+
 
 # Add any paths that contain custom themes here, relative to this directory.
 #html_theme_path = []
@@ -157,7 +167,13 @@ html_static_path = ['_static']
 #html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+html_sidebars = {
+    '**': [
+        'about.html',
+        'navigation.html',
+        'searchbox.html',
+        ]
+    }
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.

docs/conversion.rst

@@ -0,0 +1,116 @@
+Conversion
+----------
+
+Senpy includes experimental support for emotion/sentiment conversion plugins.
+
+
+Use
+===
+
+Consider the original query: http://127.0.0.1:5000/api/?i=hello&algo=emoRand
+
+The requested plugin (emoRand) returns emotions using Ekman's model (or big6 in EmotionML):
+
+.. code:: json
+
+
+          ... rest of the document ...
+          {
+            "@type": "emotionSet",
+            "onyx:hasEmotion": {
+                "@type": "emotion",
+                "onyx:hasEmotionCategory": "emoml:big6anger"
+            },
+            "prov:wasGeneratedBy": "plugins/emoRand_0.1"
+          }
+
+          
+
+To get these emotions in VAD space (FSRE dimensions in EmotionML), we'd do this:
+
+http://127.0.0.1:5000/api/?i=hello&algo=emoRand&emotionModel=emoml:fsre-dimensions
+
+This call, provided there is a valid conversion plugin from Ekman's to VAD, would return something like this:
+
+.. code:: json
+
+
+          ... rest of the document ...
+          {
+            "@type": "emotionSet",
+            "onyx:hasEmotion": {
+                "@type": "emotion",
+                "onyx:hasEmotionCategory": "emoml:big6anger"
+                },
+            "prov:wasGeneratedBy": "plugins/emoRand_0.1"
+          }, {
+            "@type": "emotionSet",
+            "onyx:hasEmotion": {
+                "@type": "emotion",
+                "A": 7.22,
+                "D": 6.28,
+                "V": 8.6
+            },
+            "prov:wasGeneratedBy": "plugins/Ekman2VAD_0.1"
+
+          }
+
+
+That is called a *full* response, as it simply adds the converted emotion alongside.
+It is also possible to get the original emotion nested within the new converted emotion, using the `conversion=nested` parameter:
+
+.. code:: json
+
+
+          ... rest of the document ...
+          {
+            "@type": "emotionSet",
+            "onyx:hasEmotion": {
+                "@type": "emotion",
+                "onyx:hasEmotionCategory": "emoml:big6anger"
+                },
+            "prov:wasGeneratedBy": "plugins/emoRand_0.1"
+            "onyx:wasDerivedFrom": {
+                "@type": "emotionSet",
+                "onyx:hasEmotion": {
+                    "@type": "emotion",
+                    "A": 7.22,
+                    "D": 6.28,
+                    "V": 8.6
+                },
+                "prov:wasGeneratedBy": "plugins/Ekman2VAD_0.1"
+             }
+
+          }
+
+
+Lastly, `conversion=filtered` would only return the converted emotions.
+
+Developing a conversion plugin
+================================
+
+Conversion plugins are discovered by the server just like any other plugin.
+The difference is the slightly different API, and the need to specify the `source` and `target` of the conversion.
+For instance, an emotion conversion plugin needs the following:
+
+
+.. code:: yaml
+          
+
+          ---
+          onyx:doesConversion:
+            - onyx:conversionFrom: emoml:big6
+              onyx:conversionTo: emoml:fsre-dimensions
+            - onyx:conversionFrom: emoml:fsre-dimensions
+              onyx:conversionTo: emoml:big6
+
+
+
+
+.. code:: python
+
+
+          class MyConversion(EmotionConversionPlugin):
+
+              def convert(self, emotionSet, fromModel, toModel, params):
+                  pass

docs/demo.rst

@@ -0,0 +1,16 @@
+Demo
+----
+
+There is a demo available on http://senpy.cluster.gsi.dit.upm.es/, where you can test a serie of different plugins.
+You can use the playground (a web interface) or make HTTP requests to the service API.
+
+.. image:: senpy-playground.png
+  :height: 400px
+  :width: 800px
+  :scale: 100 %
+  :align: center
+
+Plugins Demo
+============
+
+The source code and description of the plugins used in the demo is available here: https://lab.cluster.gsi.dit.upm.es/senpy/senpy-plugins-community/.

docs/examples.rst

@@ -0,0 +1,78 @@
+Examples
+------
+All the examples in this page use the :download:`the main schema <_static/schemas/definitions.json>`.
+
+Simple NIF annotation
+.....................
+Description
+,,,,,,,,,,,
+This example covers the basic example in the NIF documentation: `<http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core/nif-core.html>`_.
+
+Representation
+,,,,,,,,,,,,,,
+.. literalinclude:: examples/results/example-basic.json
+   :language: json-ld
+
+Sentiment Analysis
+.....................
+Description
+,,,,,,,,,,,
+This annotation corresponds to the sentiment analysis of an input. The example shows the sentiment represented according to Marl format.
+The sentiments detected are contained in the Sentiments array with their related part of the text.
+
+Representation
+,,,,,,,,,,,,,,
+
+.. literalinclude:: examples/results/example-sentiment.json
+   :emphasize-lines: 5-10,25-33
+   :language: json-ld
+
+Suggestion Mining
+.................
+Description
+,,,,,,,,,,,
+The suggestions schema represented below shows the suggestions detected in the text. Within it, we can find the NIF fields highlighted that corresponds to the text of the detected suggestion. 
+
+Representation
+,,,,,,,,,,,,,,
+
+.. literalinclude:: examples/results/example-suggestion.json
+   :emphasize-lines: 5-8,22-27
+   :language: json-ld
+
+Emotion Analysis
+................
+Description
+,,,,,,,,,,,
+This annotation represents the emotion analysis of an input to Senpy. The emotions are contained in the emotions section with the text that refers to following Onyx format and the emotion model defined beforehand.
+
+Representation
+,,,,,,,,,,,,,,
+
+.. literalinclude:: examples/results/example-emotion.json
+   :language: json-ld
+   :emphasize-lines: 5-8,25-37
+
+Named Entity Recognition
+........................
+Description
+,,,,,,,,,,,
+The Named Entity Recognition is represented as follows. In this particular case, it can be seen within the entities array the entities recognised. For the example input, Microsoft and Windows Phone are the ones detected. 
+Representation
+,,,,,,,,,,,,,,
+
+.. literalinclude:: examples/results/example-ner.json
+   :emphasize-lines: 5-8,19-34
+   :language: json-ld
+
+Complete example
+................
+Description
+,,,,,,,,,,,
+This example covers all of the above cases, integrating all the annotations in the same document.
+
+Representation
+,,,,,,,,,,,,,,
+
+.. literalinclude:: examples/results/example-complete.json
+   :language: json-ld

docs/examples/plugins/noplugins.json

@@ -0,0 +1,5 @@
+{
+  "@type": "plugins",
+  "plugins": [
+    ]
+}

docs/examples/results/example-analysis-as-id.json

@@ -0,0 +1,74 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "@type": "results",
+  "analysis": [
+    "me:SAnalysis1",
+    "me:SgAnalysis1",
+    "me:EmotionAnalysis1",
+    "me:NER1"
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+        {
+          "@id": "http://micro.blog/status1#char=5,13",
+          "nif:beginIndex": 5,
+          "nif:endIndex": 13,
+          "nif:anchorOf": "Microsoft",
+          "me:references": "http://dbpedia.org/page/Microsoft",
+          "prov:wasGeneratedBy": "me:NER1"
+        },
+        {
+          "@id": "http://micro.blog/status1#char=25,37",
+          "nif:beginIndex": 25,
+          "nif:endIndex": 37,
+          "nif:anchorOf": "Windows Phone",
+          "me:references": "http://dbpedia.org/page/Windows_Phone",
+          "prov:wasGeneratedBy": "me:NER1"
+        }
+      ],
+      "suggestions": [
+        {
+          "@id": "http://micro.blog/status1#char=16,77",
+          "nif:beginIndex": 16,
+          "nif:endIndex": 77,
+          "nif:anchorOf": "put your Windows Phone on your newest #open technology program",
+          "prov:wasGeneratedBy": "me:SgAnalysis1"
+        }
+      ],
+      "sentiments": [
+        {
+          "@id": "http://micro.blog/status1#char=80,97",
+          "nif:beginIndex": 80,
+          "nif:endIndex": 97,
+          "nif:anchorOf": "You'll be awesome.",
+          "marl:hasPolarity": "marl:Positive",
+          "marl:polarityValue": 0.9,
+          "prov:wasGeneratedBy": "me:SAnalysis1"
+        }
+      ],
+      "emotions": [
+        {
+          "@id": "http://micro.blog/status1#char=0,109",
+          "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+          "prov:wasGeneratedBy": "me:EAnalysis1",
+          "onyx:hasEmotion": [
+            {
+              "onyx:hasEmotionCategory": "wna:liking"
+            },
+            {
+              "onyx:hasEmotionCategory": "wna:excitement"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

docs/examples/results/example-analysis-mixed-ids.json

@@ -0,0 +1,78 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "@type": "results",
+  "analysis": [
+    "me:SAnalysis1",
+    "me:SgAnalysis1",
+    "me:EmotionAnalysis1",
+    "me:NER1",
+    {
+    "@type": "analysis",
+    "@id": "anonymous"
+    }
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+        {
+          "@id": "http://micro.blog/status1#char=5,13",
+          "nif:beginIndex": 5,
+          "nif:endIndex": 13,
+          "nif:anchorOf": "Microsoft",
+          "me:references": "http://dbpedia.org/page/Microsoft",
+          "prov:wasGeneratedBy": "me:NER1"
+        },
+        {
+          "@id": "http://micro.blog/status1#char=25,37",
+          "nif:beginIndex": 25,
+          "nif:endIndex": 37,
+          "nif:anchorOf": "Windows Phone",
+          "me:references": "http://dbpedia.org/page/Windows_Phone",
+          "prov:wasGeneratedBy": "me:NER1"
+        }
+      ],
+      "suggestions": [
+        {
+          "@id": "http://micro.blog/status1#char=16,77",
+          "nif:beginIndex": 16,
+          "nif:endIndex": 77,
+          "nif:anchorOf": "put your Windows Phone on your newest #open technology program",
+          "prov:wasGeneratedBy": "me:SgAnalysis1"
+        }
+      ],
+      "sentiments": [
+        {
+          "@id": "http://micro.blog/status1#char=80,97",
+          "nif:beginIndex": 80,
+          "nif:endIndex": 97,
+          "nif:anchorOf": "You'll be awesome.",
+          "marl:hasPolarity": "marl:Positive",
+          "marl:polarityValue": 0.9,
+          "prov:wasGeneratedBy": "me:SAnalysis1"
+        }
+      ],
+      "emotions": [
+        {
+          "@id": "http://micro.blog/status1#char=0,109",
+          "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+          "prov:wasGeneratedBy": "me:EAnalysis1",
+          "onyx:hasEmotion": [
+            {
+              "onyx:hasEmotionCategory": "wna:liking"
+            },
+            {
+              "onyx:hasEmotionCategory": "wna:excitement"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

docs/examples/results/example-basic.json

@@ -0,0 +1,19 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "http://example.com#NIFExample",
+  "@type": "results",
+  "analysis": [
+  ],
+  "entries": [
+    {
+      "@id": "http://example.org#char=0,40",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:beginIndex": 0,
+      "nif:endIndex": 40,
+      "nif:isString": "My favourite actress is Natalie Portman"
+    }
+  ]
+}

docs/examples/results/example-complete.json

@@ -0,0 +1,88 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "@type": "results",
+  "analysis": [
+    {
+      "@id": "me:SAnalysis1",
+      "@type": "marl:SentimentAnalysis",
+      "marl:maxPolarityValue": 1,
+      "marl:minPolarityValue": 0
+    },
+    {
+      "@id": "me:SgAnalysis1",
+      "@type": "me:SuggestionAnalysis"
+    },
+    {
+      "@id": "me:EmotionAnalysis1",
+      "@type": "me:EmotionAnalysis"
+    },
+    {
+      "@id": "me:NER1",
+      "@type": "me:NER"
+    }
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+        {
+          "@id": "http://micro.blog/status1#char=5,13",
+          "nif:beginIndex": 5,
+          "nif:endIndex": 13,
+          "nif:anchorOf": "Microsoft",
+          "me:references": "http://dbpedia.org/page/Microsoft",
+          "prov:wasGeneratedBy": "me:NER1"
+        },
+        {
+          "@id": "http://micro.blog/status1#char=25,37",
+          "nif:beginIndex": 25,
+          "nif:endIndex": 37,
+          "nif:anchorOf": "Windows Phone",
+          "me:references": "http://dbpedia.org/page/Windows_Phone",
+          "prov:wasGeneratedBy": "me:NER1"
+        }
+      ],
+      "suggestions": [
+        {
+          "@id": "http://micro.blog/status1#char=16,77",
+          "nif:beginIndex": 16,
+          "nif:endIndex": 77,
+          "nif:anchorOf": "put your Windows Phone on your newest #open technology program",
+          "prov:wasGeneratedBy": "me:SgAnalysis1"
+        }
+      ],
+      "sentiments": [
+        {
+          "@id": "http://micro.blog/status1#char=80,97",
+          "nif:beginIndex": 80,
+          "nif:endIndex": 97,
+          "nif:anchorOf": "You'll be awesome.",
+          "marl:hasPolarity": "marl:Positive",
+          "marl:polarityValue": 0.9,
+          "prov:wasGeneratedBy": "me:SAnalysis1"
+        }
+      ],
+      "emotions": [
+        {
+          "@id": "http://micro.blog/status1#char=0,109",
+          "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+          "prov:wasGeneratedBy": "me:EAnalysis1",
+          "onyx:hasEmotion": [
+            {
+              "onyx:hasEmotionCategory": "wna:liking"
+            },
+            {
+              "onyx:hasEmotionCategory": "wna:excitement"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

docs/examples/results/example-emotion.json

@@ -0,0 +1,42 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "@type": "results",
+  "analysis": [
+    {
+      "@id": "me:EmotionAnalysis1",
+      "@type": "onyx:EmotionAnalysis"
+    }
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+      ],
+      "suggestions": [
+      ],
+      "sentiments": [
+      ],
+      "emotions": [
+        {
+          "@id": "http://micro.blog/status1#char=0,109",
+          "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+          "prov:wasGeneratedBy": "me:EmotionAnalysis1",
+          "onyx:hasEmotion": [
+            {
+              "onyx:hasEmotionCategory": "wna:liking"
+            },
+            {
+              "onyx:hasEmotionCategory": "wna:excitement"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

docs/examples/results/example-ner.json

@@ -0,0 +1,45 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "@type": "results",
+  "analysis": [
+    {
+      "@id": "me:NER1",
+      "@type": "me:NERAnalysis"
+    }
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+        {
+          "@id": "http://micro.blog/status1#char=5,13",
+          "nif:beginIndex": 5,
+          "nif:endIndex": 13,
+          "nif:anchorOf": "Microsoft",
+          "me:references": "http://dbpedia.org/page/Microsoft",
+          "prov:wasGeneratedBy": "me:NER1"
+        },
+        {
+          "@id": "http://micro.blog/status1#char=25,37",
+          "nif:beginIndex": 25,
+          "nif:endIndex": 37,
+          "nif:anchorOf": "Windows Phone",
+          "me:references": "http://dbpedia.org/page/Windows_Phone",
+          "prov:wasGeneratedBy": "me:NER1"
+        }
+      ],
+      "suggestions": [
+      ],
+      "sentiments": [
+      ],
+      "emotionSets": [
+      ]
+    }
+  ]
+}

docs/examples/results/example-pad.json

@@ -0,0 +1,46 @@
+{
+"@context": [
+             "http://mixedemotions-project.eu/ns/context.jsonld",
+             {
+               "emovoc": "http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/emotionml/ns#"
+             }
+  ],
+  "@id": "me:Result1",
+  "@type": "results",
+  "analysis": [
+    {
+      "@id": "me:HesamsAnalysis",
+      "@type": "onyx:EmotionAnalysis",
+      "onyx:usesEmotionModel": "emovoc:pad-dimensions"
+    }
+  ],
+  "entries": [
+    {
+      "@id": "Entry1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:isString": "This is a test string",
+      "entities": [
+      ],
+      "suggestions": [
+      ],
+      "sentiments": [
+      ],
+      "emotions": [
+        {
+          "@id": "Entry1#char=0,21",
+          "nif:anchorOf": "This is a test string",
+          "prov:wasGeneratedBy": "me:HesamAnalysis",
+          "onyx:hasEmotion": [
+            {
+                "emovoc:pleasure": 0.5,
+                "emovoc:arousal": 0.7
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

docs/examples/results/example-sentiment.json

@@ -0,0 +1,40 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "@type": "results",
+  "analysis": [
+    {
+      "@id": "me:SAnalysis1",
+      "@type": "marl:SentimentAnalysis",
+      "marl:maxPolarityValue": 1,
+      "marl:minPolarityValue": 0
+    }
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+      ],
+      "suggestions": [
+      ],
+      "sentiments": [
+        {
+          "@id": "http://micro.blog/status1#char=80,97",
+          "nif:beginIndex": 80,
+          "nif:endIndex": 97,
+          "nif:anchorOf": "You'll be awesome.",
+          "marl:hasPolarity": "marl:Positive",
+          "marl:polarityValue": 0.9,
+          "prov:wasGeneratedBy": "me:SAnalysis1"
+        }
+      ],
+      "emotionSets": [
+      ]
+    }
+  ]
+}

docs/examples/results/example-suggestion.json

@@ -0,0 +1,34 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "@type": "results",
+  "analysis": [
+      "me:SgAnalysis1"
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "prov:wasGeneratedBy": "me:SAnalysis1",
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+      ],
+      "suggestions": [
+        {
+          "@id": "http://micro.blog/status1#char=16,77",
+          "nif:beginIndex": 16,
+          "nif:endIndex": 77,
+          "nif:anchorOf": "put your Windows Phone on your newest #open technology program",
+          "prov:wasGeneratedBy": "me:SgAnalysis1"
+        }
+      ],
+      "sentiments": [
+      ],
+      "emotionSets": [
+      ]
+    }
+  ]
+}

docs/index.rst

@@ -1,16 +1,35 @@
-.. Senpy documentation master file, created by
-   sphinx-quickstart on Tue Feb 24 08:57:32 2015.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
-
 Welcome to Senpy's documentation!
 =================================
+.. image:: https://readthedocs.org/projects/senpy/badge/?version=latest
+   :target: http://senpy.readthedocs.io/en/latest/
+.. image:: https://badge.fury.io/py/senpy.svg
+   :target: https://badge.fury.io/py/senpy
+.. image:: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/badges/master/build.svg
+   :target: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/commits/master
+.. image:: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/badges/master/coverage.svg
+   :target: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/commits/master
+.. image:: https://img.shields.io/pypi/l/requests.svg
+   :target: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/
 
-Contents:
+
+
+Senpy is a framework for sentiment and emotion analysis services.
+Services built with senpy are interchangeable and easy to use because they share a common :doc:`apischema`.
+It also simplifies service development.
+
+.. image:: senpy-architecture.png
+  :width: 100%
+  :align: center
 
 .. toctree::
-   installation
+   :caption: Learn more about senpy:
-   usage
-   api
-   plugins
    :maxdepth: 2
+
+   senpy
+   installation
+   demo
+   usage
+   apischema
+   plugins
+   conversion
+   about

docs/installation.rst

@@ -1,6 +1,16 @@
 Installation
 ------------
-The stable version can be installed in three ways.
+The stable version can be used in two ways: as a system/user library through pip, or as a docker image.
+
+The docker image is the recommended way because it is self-contained and isolated from the system, which means:
+
+  * Downloading and using it is just one command
+  * All dependencies are included
+  * It is OS-independent (MacOS, Windows, GNU/Linux)
+  * Several versions may coexist in the same machine without additional virtual environments
+
+Additionally, you may create your own docker image with your custom plugins, ready to be used by others.
+
 
 Through PIP
 ***********
@@ -22,6 +32,41 @@ If you want to install senpy globally, use sudo instead of the ``--user`` flag.
 
 Docker Image
 ************
-Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 balkian/senpy --host 0.0.0.0 --default-plugins``.
+Build the image or use the pre-built one:   
 
-To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins balkian/senpy --host 0.0.0.0 --default-plugins -f /plugins``
+.. code:: bash
+
+   docker run -ti -p 5000:5000 gsiupm/senpy --host 0.0.0.0 --default-plugins
+
+To add custom plugins, use a docker volume: 
+    
+.. code:: bash
+
+   docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --host 0.0.0.0 --default-plugins -f /plugins
+ 
+
+Python 2
+........
+
+There is a Senpy version for python2 too:
+    
+.. code:: bash
+
+   docker run -ti -p 5000:5000 gsiupm/senpy:python2.7 --host 0.0.0.0 --default-plugins
+
+
+Alias
+.....
+
+If you are using the docker approach regularly, it is advisable to use a script or an alias to simplify your executions:
+
+.. code:: bash
+
+   alias senpy='docker run --rm -ti -p 5000:5000 -v $PWD:/senpy-plugins gsiupm/senpy --default-plugins'
+
+
+Now, you may run senpy from any folder in your computer like so:
+
+.. code:: bash
+
+   senpy --version

docs/plugins-definition.rst

@@ -0,0 +1,113 @@
+Advanced plugin definition
+--------------------------
+In addition to finding plugins defined in source code files, senpy can also load a special type of definition file (`.senpy` files).
+This used to be the only mechanism for loading in earlier versions of senpy.
+
+The definition file contains basic information 
+
+Lastly, it is also possible to add new plugins programmatically.
+
+.. contents:: :local:
+
+What is a plugin?
+=================
+
+A plugin is a program that, given a text, will add annotations to it.
+In practice, a plugin consists of at least two files:
+
+- Definition file: a `.senpy` file that describes the plugin (e.g. what input parameters it accepts, what emotion model it uses).
+- Python module: the actual code that will add annotations to each input.
+
+This separation allows us to deploy plugins that use the same code but employ different parameters.
+For instance, one could use the same classifier and processing in several plugins, but train with different datasets.
+This scenario is particularly useful for evaluation purposes.
+
+The only limitation is that the name of each plugin needs to be unique.
+
+Definition files
+================
+
+The definition file complements and overrides the attributes provided by the plugin.
+It can be written in YAML or JSON.
+The most important attributes are:
+
+* **name**: unique name that senpy will use internally to identify the plugin.
+* **module**: indicates the module that contains the plugin code, which will be automatically loaded by senpy.
+* **version**
+* extra_params: to add parameters to the senpy API when this plugin is requested. Those parameters may be required, and have aliased names. For instance:
+
+  .. code:: yaml
+
+            extra_params:
+                hello_param:
+                    aliases: # required
+                        - hello_param
+                        - hello
+                    required: true
+                    default: Hi you
+                    values:
+                        - Hi you
+                        - Hello y'all
+                        - Howdy
+
+A complete example:
+
+.. code:: yaml
+          
+          name: <Name of the plugin>
+          module: <Python file>
+          version: 0.1
+
+And the json equivalent:
+
+.. code:: json
+
+          {
+            "name": "<Name of the plugin>",
+            "module": "<Python file>",
+            "version": "0.1"
+          }
+
+
+Example plugin with a definition file
+=====================================
+
+In this section, we will implement a basic sentiment analysis plugin.
+To determine the polarity of each entry, the plugin will compare the length of the string to a threshold.
+This threshold will be included in the definition file.
+
+The definition file would look like this:
+
+.. code:: yaml
+
+          name: helloworld
+          module: helloworld
+          version: 0.0
+          threshold: 10
+          description: Hello World
+
+Now, in a file named ``helloworld.py``:
+
+.. code:: python
+
+          #!/bin/env python
+          #helloworld.py
+
+          from senpy import AnalysisPlugin
+          from senpy import Sentiment
+
+
+          class HelloWorld(AnalysisPlugin):
+
+              def analyse_entry(entry, params):
+                  '''Basically do nothing with each entry'''
+
+                  sentiment = Sentiment()
+                  if len(entry.text) < self.threshold:
+                      sentiment['marl:hasPolarity'] = 'marl:Positive'
+                  else:
+                      sentiment['marl:hasPolarity'] = 'marl:Negative'
+                  entry.sentiments.append(sentiment)
+                  yield entry
+
+The complete code of the example plugin is available `here <https://lab.cluster.gsi.dit.upm.es/senpy/plugin-prueba>`__.

docs/plugins.rst

@@ -1,29 +1,236 @@
 Developing new plugins
 ----------------------
+This document contains the minimum to get you started with developing new analysis plugin.
+For an example of conversion plugins, see :doc:`conversion`.
+For a description of definition files, see :doc:`plugins-definition`.
 
-Plugins Interface
+A more step-by-step tutorial with slides is available `here <https://lab.cluster.gsi.dit.upm.es/senpy/senpy-tutorial>`__ 
+
+.. contents:: :local:
+
+What is a plugin?
 =================
 
+A plugin is a python object that can process entries. Given an entry, it will modify it, add annotations to it, or generate new entries.
+
+
+What is an entry?
+=================
+
+Entries are objects that can be annotated.
+In general, they will be a piece of text.
+By default, entries are `NIF contexts <http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core/nif-core.html>`_ represented in JSON-LD format.
+It is a dictionary/JSON object that looks like this:
+
+  .. code:: python
+
+            {
+               "@id": "<unique identifier or blank node name>",
+               "nif:isString": "input text",
+               "sentiments": [ {
+                     ...
+               }
+               ],
+               ...
+            }
+
+Annotations are added to the object like this:
+
+.. code:: python
+
+   entry = Entry()
+   entry.vocabulary__annotationName = 'myvalue'
+   entry['vocabulary:annotationName'] = 'myvalue'
+   entry['annotationNameURI'] = 'myvalue'
+
+Where vocabulary is one of the prefixes defined in the default senpy context, and annotationURI is a full URI.
+The value may be any valid JSON-LD dictionary.
+For simplicity, senpy includes a series of models by default in the ``senpy.models`` module.
+
+
+What are annotations?
+=====================
+They are objects just like entries.
+Senpy ships with several default annotations, including: ``Sentiment``, ``Emotion``, ``EmotionSet``...jk bb
+
+
+What's a plugin made of?
+========================
+
+When receiving a query, senpy selects what plugin or plugins should process each entry, and in what order.
+It also makes sure the every entry and the parameters provided by the user meet the plugin requirements.
+
+Hence, two parts are necessary: 1) the code that will process the entry, and 2) some attributes and metadata that will tell senpy how to interact with the plugin.
+
+In practice, this is what a plugin looks like, tests included:
+
+
+.. literalinclude:: ../senpy/plugins/example/rand_plugin.py
+   :emphasize-lines: 5-11
+   :language: python
+
+
+The lines highlighted contain some information about the plugin.
+In particular, the following information is mandatory:
+
+* A unique name for the class. In our example, Rand.
+* The subclass/type of plugin. This is typically either `SentimentPlugin` or `EmotionPlugin`. However, new types of plugin can be created for different annotations. The only requirement is that these new types inherit from `senpy.Analysis`
+* A description of the plugin. This can be done simply by adding a doc to the class.
+* A version, which should get updated.
+* An author name.
+
+
+Plugins Code
+============
+
 The basic methods in a plugin are:
 
-* __init__
+* analyse_entry: called in every user requests. It takes two parameters: ``Entry``, the entry object, and ``params``, the parameters supplied by the user. It should yield one or more ``Entry`` objects.
-* activate: used to load memory-hungry resources
+* activate: used to load memory-hungry resources. For instance, to train a classifier.
-* deactivate: used to free up resources
+* deactivate: used to free up resources when the plugin is no longer needed.
 
 Plugins are loaded asynchronously, so don't worry if the activate method takes too long. The plugin will be marked as activated once it is finished executing the method.
 
+
+How does senpy find modules?
+============================
+
+Senpy looks for files of two types:
+
+* Python files of the form `senpy_<NAME>.py` or `<NAME>_plugin.py`. In these files, it will look for: 1) Instances that inherit from `senpy.Plugin`, or subclasses of `senpy.Plugin` that can be initialized without a configuration file. i.e. classes that contain all the required attributes for a plugin.
+* Plugin definition files (see :doc:`advanced-plugins`)
+
+Defining additional parameters
+==============================
+
+Your plugin may ask for additional parameters from the users of the service by using the attribute ``extra_params`` in your plugin definition.
+It takes a dictionary, where the keys are the name of the argument/parameter, and the value has the following fields:
+
+* aliases: the different names which can be used in the request to use the parameter.
+* required: if set to true, users need to provide this parameter unless a default is set.
+* options: the different acceptable values of the parameter (i.e. an enum). If set, the value provided must match one of the options.
+* default: the default value of the parameter, if none is provided in the request.
+
+.. code:: python
+
+          "extra_params":{
+             "language": {
+                "aliases": ["language", "lang", "l"],
+                "required": True,
+                "options": ["es", "en"],
+                "default": "es"
+                }
+             }
+
+
+
+Loading data and files
+======================
+
+Most plugins will need access to files (dictionaries, lexicons, etc.).
+These files are usually heavy or under a license that does not allow redistribution.
+For this reason, senpy has a `data_folder` that is separated from the source files.
+The location of this folder is controlled programmatically or by setting the `SENPY_DATA` environment variable.
+
+Plugins have a convenience function `self.open` which will automatically prepend the data folder to relative paths:
+
+
+.. code:: python
+
+          import os
+
+
+          class PluginWithResources(AnalysisPlugin):
+              file_in_data = <FILE PATH>
+              file_in_sources = <FILE PATH>
+
+              def activate(self):
+                  with self.open(self.file_in_data) as f:
+                      self._classifier = train_from_file(f)
+                  file_in_source = os.path.join(self.get_folder(), self.file_in_sources)
+                  with self.open(file_in_source) as f:
+                      pass
+
+         
+It is good practice to specify the paths of these files in the plugin configuration, so the same code can be reused with different resources.
+
+
+Docker image
+============
+
+Add the following dockerfile to your project to generate a docker image with your plugin:
+
+.. code:: dockerfile
+
+   FROM gsiupm/senpy
+
+Once you make sure your plugin works with a specific version of senpy, modify that file to make sure your build will work even if senpy gets updated.
+e.g.:
+
+
+.. code:: dockerfile
+
+   FROM gsiupm/senpy:1.0.1
+
+     
+This will copy your source folder to the image, and install all dependencies.
+Now, to build an image:
+
+.. code:: shell
+
+   docker build . -t gsiupm/exampleplugin
+
+And you can run it with:
+
+.. code:: shell
+
+   docker run -p 5000:5000 gsiupm/exampleplugin
+
+
+If the plugin uses non-source files (:ref:`loading data and files`), the recommended way is to use `SENPY_DATA` folder.
+Data can then be mounted in the container or added to the image.
+The former is recommended for open source plugins with licensed resources, whereas the latter is the most convenient and can be used for private images.
+
+Mounting data:
+
+.. code:: bash
+
+   docker run -v $PWD/data:/data gsiupm/exampleplugin
+
+Adding data to the image:
+
+.. code:: dockerfile
+
+   FROM gsiupm/senpy:1.0.1
+   COPY data /
+
 F.A.Q.
 ======
-If I'm using a classifier, where should I train the classifier?
+What annotations can I use?
-???????????????????????????????????????????????????????????????
+???????????????????????????
+
+You can add almost any annotation to an entry.
+The most common use cases are covered in the :doc:`apischema`.
+
+
+Why does the analyse function yield instead of return?
+??????????????????????????????????????????????????????
+
+This is so that plugins may add new entries to the response or filter some of them.
+For instance, a chunker may split one entry into several.
+On the other hand, a conversion plugin may leave out those entries that do not contain relevant information.
+
+
+If I'm using a classifier, where should I train it?
+???????????????????????????????????????????????????
 
 Training a classifier can be time time consuming. To avoid running the training unnecessarily, you can use ShelfMixin to store the classifier. For instance:
 
 .. code:: python
 
-          from senpy.plugins import ShelfMixin, SenpyPlugin
+          from senpy.plugins import ShelfMixin, AnalysisPlugin
 
-          class MyPlugin(ShelfMixin, SenpyPlugin):
+          class MyPlugin(ShelfMixin, AnalysisPlugin):
               def train(self):
                   ''' Code to train the classifier
                   '''
@@ -40,7 +247,66 @@ Training a classifier can be time time consuming. To avoid running the training
               def deactivate(self):
                   self.close()
 
-You can speficy a 'shelf_file' in your .senpy file. By default the ShelfMixin creates a file based on the plugin name and stores it in that plugin's folder.
+
+By default the ShelfMixin creates a file based on the plugin name and stores it in that plugin's folder.
+However, you can manually specify a 'shelf_file' in your .senpy file.
+
+Shelves may get corrupted if the plugin exists unexpectedly.
+A corrupt shelf prevents the plugin from loading.
+If you do not care about the data in the shelf, you can force your plugin to remove the corrupted file and load anyway, set the  'force_shelf' to True in your plugin and start it again.
+
+How can I turn an external service into a plugin?
+?????????????????????????????????????????????????
+
+This example ilustrate how to implement a plugin that accesses the Sentiment140 service.
+
+.. code:: python
+
+          class Sentiment140Plugin(SentimentPlugin):
+              def analyse_entry(self, entry, params):
+                  text = entry.text
+                  lang = params.get("language", "auto")
+                  res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
+                                      json.dumps({"language": lang,
+                                                  "data": [{"text": text}]
+                                                  }
+                                                 )
+                                      )
+
+                  p = params.get("prefix", None)
+                  polarity_value = self.maxPolarityValue*int(res.json()["data"][0]
+                                                             ["polarity"]) * 0.25
+                  polarity = "marl:Neutral"
+                  neutral_value = self.maxPolarityValue / 2.0
+                  if polarity_value > neutral_value:
+                      polarity = "marl:Positive"
+                  elif polarity_value < neutral_value:
+                      polarity = "marl:Negative"
+
+                  sentiment = Sentiment(id="Sentiment0",
+                                      prefix=p,
+                                      marl__hasPolarity=polarity,
+                                      marl__polarityValue=polarity_value)
+                  sentiment.prov(self)
+                  entry.sentiments.append(sentiment)
+                  yield entry
+
+
+Can I activate a DEBUG mode for my plugin?
+???????????????????????????????????????????
+
+You can activate the DEBUG mode by the command-line tool using the option -d.
+
+.. code:: bash
+
+   senpy -d
+
+
+Additionally, with the ``--pdb`` option you will be dropped into a pdb post mortem shell if an exception is raised.
+
+.. code:: bash
+
+   python -m pdb yourplugin.py
 
 Where can I find more code examples?
 ????????????????????????????????????

docs/requirements.txt

@@ -1 +1,2 @@
 sphinxcontrib-httpdomain>=1.4
+nbsphinx

docs/senpy.rst

@@ -0,0 +1,54 @@
+What is Senpy?
+--------------
+
+Senpy is a framework for text analysis using Linked Data. There are three main applications of Senpy so far: sentiment and emotion analysis, user profiling and entity recoginition. Annotations and Services are compliant with NIF (NLP Interchange Format).
+
+Senpy aims at providing a framework where analysis modules can be integrated easily as plugins, and providing a core functionality for managing tasks such as data validation, user interaction, formatting, logging, translation to linked data, etc. 
+
+The figure below summarizes the typical features in a text analysis service.
+Senpy implements all the common blocks, so developers can focus on what really matters: great analysis algorithms that solve real problems.
+
+.. image:: senpy-framework.png
+  :width: 60%
+  :align: center
+
+
+Senpy for end users
+===================
+
+All services built using senpy share a common interface.
+This allows users to use them (almost) interchangeably.
+Senpy comes with a :ref:`built-in client`.
+
+
+Senpy for service developers
+============================
+
+Senpy is a framework that turns your sentiment or emotion analysis algorithm into a full blown semantic service.
+Senpy takes care of:
+
+  * Interfacing with the user: parameter validation, error handling.
+  * Formatting: JSON-LD, Turtle/n-triples input and output, or simple text input
+  * Linked Data: senpy results are semantically annotated, using a series of well established vocabularies, and sane default URIs.
+  * User interface: a web UI where users can explore your service and test different settings
+  * A client to interact with the service. Currently only available in Python.
+
+Sharing your sentiment analysis with the world has never been easier!
+
+Check out the :doc:`plugins` if you have developed an analysis algorithm (e.g. sentiment analysis) and you want to publish it as a service.
+
+Architecture
+============
+
+The main component of a sentiment analysis service is the algorithm itself. However, for the algorithm to work, it needs to get the appropriate parameters from the user, format the results according to the defined API, interact with the user whn errors occur or more information is needed, etc.
+
+Senpy proposes a modular and dynamic architecture that allows:
+
+* Implementing different algorithms in a extensible way, yet offering a common interface.
+* Offering common services that facilitate development, so developers can focus on implementing new and better algorithms.
+
+The framework consists of two main modules: Senpy core, which is the building block of the service, and Senpy plugins, which consist of the analysis algorithm. The next figure depicts a simplified version of the processes involved in an analysis with the Senpy framework.
+
+.. image:: senpy-architecture.png
+  :width: 100%
+  :align: center

docs/server.rst

@@ -0,0 +1,66 @@
+Server
+======
+
+The senpy server is launched via the `senpy` command:
+
+.. code:: text
+
+    usage: senpy [-h] [--level logging_level] [--debug] [--default-plugins]
+                [--host HOST] [--port PORT] [--plugins-folder PLUGINS_FOLDER]
+                [--only-install] [--only-list] [--data-folder DATA_FOLDER]
+                [--threaded] [--version]
+
+    Run a Senpy server
+
+    optional arguments:
+      -h, --help            show this help message and exit
+      --level logging_level, -l logging_level
+                            Logging level
+      --debug, -d           Run the application in debug mode
+      --default-plugins     Load the default plugins
+      --host HOST           Use 0.0.0.0 to accept requests from any host.
+      --port PORT, -p PORT  Port to listen on.
+      --plugins-folder PLUGINS_FOLDER, -f PLUGINS_FOLDER
+                            Where to look for plugins.
+      --only-install, -i    Do not run a server, only install plugin dependencies
+      --only-list, --list   Do not run a server, only list plugins found
+      --data-folder DATA_FOLDER, --data DATA_FOLDER
+                            Where to look for data. It be set with the SENPY_DATA
+                            environment variable as well.
+      --threaded            Run a threaded server
+      --version, -v         Output the senpy version and exit
+
+
+
+When launched, the server will recursively look for plugins in the specified plugins folder (the current working directory by default).
+For every plugin found, it will download its dependencies, and try to activate it.
+The default server includes a playground and an endpoint with all plugins found.
+
+Let's run senpy with the default plugins:
+
+.. code:: bash
+
+    senpy -f . --default-plugins
+
+Now go to `http://localhost:5000 <http://localhost:5000>`_, you should be greeted by the senpy playground:
+
+.. image:: senpy-playground.png
+   :width: 100%
+   :alt: Playground
+
+The playground is a user-friendly way to test your plugins, but you can always use the service directly:  `http://localhost:5000/api?input=hello <http://localhost:5000/api?input=hello>`_.
+
+
+By default, senpy will listen only on the `127.0.0.1` address.
+That means you can only access the API from your (or localhost).
+You can listen on a different address using the `--host` flag (e.g., 0.0.0.0).
+The default port is 5000.
+You can change it with the `--port` flag. 
+
+For instance, to accept connections on port 6000 on any interface:
+
+.. code:: bash
+
+    senpy --host 0.0.0.0 --port 6000
+
+For more options, see the `--help` page.

docs/usage.rst

@@ -1,20 +1,15 @@
 Usage
 -----
 
-The easiest and recommended way is to just use the command-line tool to load your plugins and launch the server.
+First of all, you need to install the package.
+See :doc:`installation` for instructions.
+Once installed, the `senpy` command should be available. 
 
-.. code:: bash
+.. toctree::
+   :maxdepth: 1
 
-   senpy
+   server
-
+   SenpyClientUse
-Or, alternatively:
+   commandline
-
-.. code:: bash
-
-    python -m senpy
 
 
-This will create a server with any modules found in the current path.
-For more options, see the `--help` page.
-
-Alternatively, you can use the modules included in senpy to build your own application.

docs/vocabularies.rst

@@ -0,0 +1,24 @@
+Vocabularies and model
+======================
+
+The model used in Senpy is based on NIF 2.0 [1], which defines a semantic format and API for improving interoperability among natural language processing services.
+
+Senpy has been applied to sentiment and emotion analysis services using the following vocabularies:
+
+* Marl [2,6], a vocabulary designed to annotate and describe subjetive opinions expressed on the web or in information systems.
+* Onyx [3,5], which is built one the same principles as Marl to annotate and describe emotions, and provides interoperability with Emotion Markup Language.
+
+An overview of the vocabularies and their use can be found in [4].
+
+
+[1] Guidelines for developing NIF-based NLP services, Final Community Group Report 22 December 2015 Available at: https://www.w3.org/2015/09/bpmlod-reports/nif-based-nlp-webservices/
+
+[2] Marl Ontology Specification, available at http://www.gsi.dit.upm.es/ontologies/marl/
+
+[3] Onyx Ontology Specification, available at http://www.gsi.dit.upm.es/ontologies/onyx/
+
+[4] Iglesias, C. A., Sánchez-Rada, J. F., Vulcu, G., & Buitelaar, P. (2017). Linked Data Models for Sentiment and Emotion Analysis in Social Networks. In Sentiment Analysis in Social Networks (pp. 49-69).
+
+[5] Sánchez-Rada, J. F., & Iglesias, C. A. (2016). Onyx: A linked data approach to emotion representation. Information Processing & Management, 52(1), 99-114.
+
+[6] Westerski, A., Iglesias Fernandez, C. A., & Tapia Rico, F. (2011). Linked opinions: Describing sentiments on the structured web of data.

example-plugins/README.md

@@ -0,0 +1,23 @@
+This is a collection of plugins that exemplify certain aspects of plugin development with senpy.
+
+The first series of plugins the `basic` ones.
+Their starting point is a classification function defined in `basic.py`.
+They all include testing and running them as a script will run all tests.
+In ascending order of customization, the plugins are:
+
+* Basic is the simplest plugin of all. It leverages the `SentimentBox` Plugin class to create a plugin out of a classification method, and `MappingMixin` to convert the labels from (`pos`, `neg`) to (`marl:Positive`, `marl:Negative`
+* Basic_box is just like the previous one, but replaces the mixin with a custom function.
+* Basic_configurable is a version of `basic` with a configurable map of emojis for each sentiment.
+* Basic_parameterized like `basic_info`, but users set the map in each query (via `extra_parameters`).
+* Basic_analyse\_entry uses the more general `analyse_entry` method and adds the annotations individually.
+
+
+In rest of the plugins show advanced topics:
+
+* mynoop: shows how to add a definition file with external requirements for a plugin. Doing this with a python-only module would require moving all imports of the requirements to their functions, which is considered bad practice.
+* Async: a barebones example of training a plugin and analyzing data in parallel.
+
+All of the plugins in this folder include a set of test cases and they are periodically tested with the latest version of senpy.
+
+Additioanlly, for an example of stand-alone plugin that can be tested and deployed with docker, take a look at: lab.cluster.gsi.dit.upm.es/senpy/plugin-example
+ bbm

example-plugins/async_plugin.py

@@ -0,0 +1,37 @@
+from senpy import AnalysisPlugin
+
+import multiprocessing
+
+
+def _train(process_number):
+    return process_number
+
+
+class Async(AnalysisPlugin):
+    '''An example of an asynchronous module'''
+    author = '@balkian'
+    version = '0.2'
+    sync = False
+
+    def _do_async(self, num_processes):
+        pool = multiprocessing.Pool(processes=num_processes)
+        values = sorted(pool.map(_train, range(num_processes)))
+
+        return values
+
+    def activate(self):
+        self.value = self._do_async(4)
+
+    def analyse_entry(self, entry, params):
+        values = self._do_async(2)
+        entry.async_values = values
+        yield entry
+
+    test_cases = [
+        {
+            'input': 'any',
+            'expected': {
+                'async_values': [0, 1]
+            }
+        }
+    ]

example-plugins/basic.py

@@ -0,0 +1,23 @@
+#!/usr/local/bin/python
+# coding: utf-8
+
+emoticons = {
+    'pos': [':)', ':]', '=)', ':D'],
+    'neg': [':(', ':[', '=(']
+}
+
+emojis = {
+    'pos': ['😁', '😂', '😃', '😄', '😆', '😅', '😄' '😍'],
+    'neg': ['😢', '😡', '😠', '😞', '😖', '😔', '😓', '😒']
+}
+
+
+def get_polarity(text, dictionaries=[emoticons, emojis]):
+    polarity = 'marl:Neutral'
+    for dictionary in dictionaries:
+        for label, values in dictionary.items():
+            for emoticon in values:
+                if emoticon and emoticon in text:
+                    polarity = label
+                    break
+    return polarity

example-plugins/basic_analyse_entry_plugin.py

@@ -0,0 +1,47 @@
+#!/usr/local/bin/python
+# coding: utf-8
+
+from senpy import easy_test, models, plugins
+
+import basic
+
+
+class BasicAnalyseEntry(plugins.SentimentPlugin):
+    '''Equivalent to Basic, implementing the analyse_entry method'''
+
+    author = '@balkian'
+    version = '0.1'
+
+    mappings = {
+        'pos': 'marl:Positive',
+        'neg': 'marl:Negative',
+        'default': 'marl:Neutral'
+    }
+
+    def analyse_entry(self, entry, params):
+        polarity = basic.get_polarity(entry.text)
+
+        polarity = self.mappings.get(polarity, self.mappings['default'])
+
+        s = models.Sentiment(marl__hasPolarity=polarity)
+        s.prov(self)
+        entry.sentiments.append(s)
+        yield entry
+
+    test_cases = [{
+        'input': 'Hello :)',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'So sad :(',
+        'polarity': 'marl:Negative'
+    }, {
+        'input': 'Yay! Emojis  😁',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'But no emoticons 😢',
+        'polarity': 'marl:Negative'
+    }]
+
+
+if __name__ == '__main__':
+    easy_test()

example-plugins/basic_box_plugin.py

@@ -0,0 +1,41 @@
+#!/usr/local/bin/python
+# coding: utf-8
+
+from senpy import easy_test, SentimentBox
+
+import basic
+
+
+class BasicBox(SentimentBox):
+    ''' A modified version of Basic that also does converts annotations manually'''
+
+    author = '@balkian'
+    version = '0.1'
+
+    mappings = {
+        'pos': 'marl:Positive',
+        'neg': 'marl:Negative',
+        'default': 'marl:Neutral'
+    }
+
+    def predict_one(self, input):
+        output = basic.get_polarity(input)
+        return self.mappings.get(output, self.mappings['default'])
+
+    test_cases = [{
+        'input': 'Hello :)',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'So sad :(',
+        'polarity': 'marl:Negative'
+    }, {
+        'input': 'Yay! Emojis  😁',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'But no emoticons 😢',
+        'polarity': 'marl:Negative'
+    }]
+
+
+if __name__ == '__main__':
+    easy_test()

example-plugins/basic_plugin.py

@@ -0,0 +1,40 @@
+#!/usr/local/bin/python
+# coding: utf-8
+
+from senpy import easy_test, SentimentBox, MappingMixin
+
+import basic
+
+
+class Basic(MappingMixin, SentimentBox):
+    '''Provides sentiment annotation using a lexicon'''
+
+    author = '@balkian'
+    version = '0.1'
+
+    mappings = {
+        'pos': 'marl:Positive',
+        'neg': 'marl:Negative',
+        'default': 'marl:Neutral'
+    }
+
+    def predict_one(self, input):
+        return basic.get_polarity(input)
+
+    test_cases = [{
+        'input': 'Hello :)',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'So sad :(',
+        'polarity': 'marl:Negative'
+    }, {
+        'input': 'Yay! Emojis  😁',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'But no emoticons 😢',
+        'polarity': 'marl:Negative'
+    }]
+
+
+if __name__ == '__main__':
+    easy_test()

example-plugins/configurable_plugin.py

@@ -0,0 +1,105 @@
+#!/usr/local/bin/python
+# coding: utf-8
+
+from senpy import easy_test, models, plugins
+
+import basic
+
+
+class Dictionary(plugins.SentimentPlugin):
+    '''Sentiment annotation using a configurable lexicon'''
+
+    author = '@balkian'
+    version = '0.2'
+
+    dictionaries = [basic.emojis, basic.emoticons]
+
+    mappings = {'pos': 'marl:Positive', 'neg': 'marl:Negative'}
+
+    def analyse_entry(self, entry, params):
+        polarity = basic.get_polarity(entry.text, self.dictionaries)
+        if polarity in self.mappings:
+            polarity = self.mappings[polarity]
+
+        s = models.Sentiment(marl__hasPolarity=polarity)
+        s.prov(self)
+        entry.sentiments.append(s)
+        yield entry
+
+    test_cases = [{
+        'input': 'Hello :)',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'So sad :(',
+        'polarity': 'marl:Negative'
+    }, {
+        'input': 'Yay! Emojis  😁',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'But no emoticons 😢',
+        'polarity': 'marl:Negative'
+    }]
+
+
+class EmojiOnly(Dictionary):
+    '''Sentiment annotation with a basic lexicon of emojis'''
+    dictionaries = [basic.emojis]
+
+    test_cases = [{
+        'input': 'Hello :)',
+        'polarity': 'marl:Neutral'
+    }, {
+        'input': 'So sad :(',
+        'polarity': 'marl:Neutral'
+    }, {
+        'input': 'Yay! Emojis  😁',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'But no emoticons 😢',
+        'polarity': 'marl:Negative'
+    }]
+
+
+class EmoticonsOnly(Dictionary):
+    '''Sentiment annotation with a basic lexicon of emoticons'''
+    dictionaries = [basic.emoticons]
+
+    test_cases = [{
+        'input': 'Hello :)',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'So sad :(',
+        'polarity': 'marl:Negative'
+    }, {
+        'input': 'Yay! Emojis  😁',
+        'polarity': 'marl:Neutral'
+    }, {
+        'input': 'But no emoticons 😢',
+        'polarity': 'marl:Neutral'
+    }]
+
+
+class Salutes(Dictionary):
+    '''Sentiment annotation with a custom lexicon, for illustration purposes'''
+    dictionaries = [{
+        'marl:Positive': ['Hello', '!'],
+        'marl:Negative': ['Good bye', ]
+    }]
+
+    test_cases = [{
+        'input': 'Hello :)',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'Good bye :(',
+        'polarity': 'marl:Negative'
+    }, {
+        'input': 'Yay! Emojis  😁',
+        'polarity': 'marl:Positive'
+    }, {
+        'input': 'But no emoticons 😢',
+        'polarity': 'marl:Neutral'
+    }]
+
+
+if __name__ == '__main__':
+    easy_test()

example-plugins/dummy_plugin.py

@@ -0,0 +1,25 @@
+from senpy import AnalysisPlugin, easy
+
+
+class Dummy(AnalysisPlugin):
+    '''This is a dummy self-contained plugin'''
+    author = '@balkian'
+    version = '0.1'
+
+    def analyse_entry(self, entry, params):
+        entry['nif:isString'] = entry['nif:isString'][::-1]
+        entry.reversed = entry.get('reversed', 0) + 1
+        yield entry
+
+    test_cases = [{
+        'entry': {
+            'nif:isString': 'Hello',
+        },
+        'expected': {
+            'nif:isString': 'olleH'
+        }
+    }]
+
+
+if __name__ == '__main__':
+    easy()

example-plugins/dummy_required_plugin.py

@@ -0,0 +1,40 @@
+from senpy import AnalysisPlugin, easy
+
+
+class DummyRequired(AnalysisPlugin):
+    '''This is a dummy self-contained plugin'''
+    author = '@balkian'
+    version = '0.1'
+    extra_params = {
+        'example': {
+            'description': 'An example parameter',
+            'required': True,
+            'options': ['a', 'b']
+        }
+    }
+
+    def analyse_entry(self, entry, params):
+        entry['nif:isString'] = entry['nif:isString'][::-1]
+        entry.reversed = entry.get('reversed', 0) + 1
+        yield entry
+
+    test_cases = [{
+        'entry': {
+            'nif:isString': 'Hello',
+        },
+        'should_fail': True
+    }, {
+        'entry': {
+            'nif:isString': 'Hello',
+        },
+        'params': {
+            'example': 'a'
+        },
+        'expected': {
+            'nif:isString': 'olleH'
+        }
+    }]
+
+
+if __name__ == '__main__':
+    easy()

example-plugins/mynoop.py

@@ -0,0 +1,24 @@
+import noop
+from senpy.plugins import SentimentPlugin
+
+
+class NoOp(SentimentPlugin):
+    '''This plugin does nothing. Literally nothing.'''
+
+    version = 0
+
+    def analyse_entry(self, entry, *args, **kwargs):
+        yield entry
+
+    def test(self):
+        print(dir(noop))
+        super(NoOp, self).test()
+
+    test_cases = [{
+        'entry': {
+            'nif:isString': 'hello'
+        },
+        'expected': {
+            'nif:isString': 'hello'
+        }
+    }]

example-plugins/mynoop.senpy

@@ -0,0 +1,3 @@
+module: mynoop
+requirements:
+ - noop

example-plugins/parameterized_plugin.py

@@ -0,0 +1,63 @@
+#!/usr/local/bin/python
+# coding: utf-8
+
+from senpy import easy_test, models, plugins
+
+import basic
+
+
+class ParameterizedDictionary(plugins.SentimentPlugin):
+    '''This is a basic self-contained plugin'''
+
+    author = '@balkian'
+    version = '0.2'
+
+    extra_params = {
+        'positive-words': {
+            'description': 'Comma-separated list of words that are considered positive',
+            'aliases': ['positive'],
+            'required': True
+        },
+        'negative-words': {
+            'description': 'Comma-separated list of words that are considered negative',
+            'aliases': ['negative'],
+            'required': False
+        }
+    }
+
+    def analyse_entry(self, entry, params):
+        positive_words = params['positive-words'].split(',')
+        negative_words = params['negative-words'].split(',')
+        dictionary = {
+            'marl:Positive': positive_words,
+            'marl:Negative': negative_words,
+        }
+        polarity = basic.get_polarity(entry.text, [dictionary])
+
+        s = models.Sentiment(marl__hasPolarity=polarity)
+        s.prov(self)
+        entry.sentiments.append(s)
+        yield entry
+
+    test_cases = [
+        {
+            'input': 'Hello :)',
+            'polarity': 'marl:Positive',
+            'parameters': {
+                'positive': "Hello,:)",
+                'negative': "sad,:()"
+            }
+        },
+        {
+            'input': 'Hello :)',
+            'polarity': 'marl:Negative',
+            'parameters': {
+                'positive': "",
+                'negative': "Hello"
+            }
+        }
+    ]
+
+
+if __name__ == '__main__':
+    easy_test()

example-plugins/sklearn/mydata.py

@@ -0,0 +1,33 @@
+'''
+Create a dummy dataset.
+Messages with a happy emoticon are labelled positive
+Messages with a sad emoticon are labelled negative
+'''
+import random
+
+dataset = []
+
+vocabulary = ['hello', 'world', 'senpy', 'cool', 'goodbye', 'random', 'text']
+
+emojimap = {
+    1: [':)', ],
+    -1: [':(', ]
+}
+
+
+for tag, values in emojimap.items():
+    for i in range(1000):
+        msg = ''
+        for j in range(3):
+            msg += random.choice(vocabulary)
+            msg += " "
+        msg += random.choice(values)
+        dataset.append([msg, tag])
+
+
+text = []
+labels = []
+
+for i in dataset:
+    text.append(i[0])
+    labels.append(i[1])

example-plugins/sklearn/mypipeline.py

@@ -0,0 +1,30 @@
+from sklearn.pipeline import Pipeline
+from sklearn.feature_extraction.text import CountVectorizer
+from sklearn.model_selection import train_test_split
+
+from mydata import text, labels
+
+X_train, X_test, y_train, y_test = train_test_split(text, labels, test_size=0.12, random_state=42)
+
+from sklearn.naive_bayes import MultinomialNB
+
+
+count_vec = CountVectorizer(tokenizer=lambda x: x.split())
+clf3 = MultinomialNB()
+pipeline = Pipeline([('cv', count_vec),
+                    ('clf', clf3)])
+
+pipeline.fit(X_train, y_train)
+print('Feature names: {}'.format(count_vec.get_feature_names()))
+print('Class count: {}'.format(clf3.class_count_))
+
+
+if __name__ == '__main__':
+    print('--Results--')
+    tests = [
+        (['The sentiment for senpy should be positive :)', ], 1),
+        (['The sentiment for anything else should be negative :()', ], -1)
+    ]
+    for features, expected in tests:
+        result = pipeline.predict(features)
+        print('Input: {}\nExpected: {}\nGot: {}'.format(features[0], expected, result))

example-plugins/sklearn/pipeline_plugin.py

@@ -0,0 +1,37 @@
+from senpy import SentimentBox, MappingMixin, easy_test
+
+from mypipeline import pipeline
+
+
+class PipelineSentiment(MappingMixin, SentimentBox):
+    '''
+    This is a pipeline plugin that wraps a classifier defined in another module
+    (mypipeline).
+    '''
+    author = '@balkian'
+    version = 0.1
+    maxPolarityValue = 1
+    minPolarityValue = -1
+
+    mappings = {
+        1: 'marl:Positive',
+        -1: 'marl:Negative'
+    }
+
+    def predict_one(self, input):
+        return pipeline.predict([input, ])[0]
+
+    test_cases = [
+        {
+            'input': 'The sentiment for senpy should be positive :)',
+            'polarity': 'marl:Positive'
+        },
+        {
+            'input': 'The sentiment for senpy should be negative :(',
+            'polarity': 'marl:Negative'
+        }
+    ]
+
+
+if __name__ == '__main__':
+    easy_test()

example-plugins/sleep_plugin.py

@@ -0,0 +1,27 @@
+from senpy.plugins import AnalysisPlugin
+from time import sleep
+
+
+class Sleep(AnalysisPlugin):
+    '''Dummy plugin to test async'''
+    author = "@balkian"
+    version = "0.2"
+    timeout = 0.05
+    extra_params = {
+        "timeout": {
+            "@id": "timeout_sleep",
+            "aliases": ["timeout", "to"],
+            "required": False,
+            "default": 0
+        }
+    }
+
+    def activate(self, *args, **kwargs):
+        sleep(self.timeout)
+
+    def analyse_entry(self, entry, params):
+        sleep(float(params.get("timeout", self.timeout)))
+        yield entry
+
+    def test(self):
+        pass

extra-requirements.txt

@@ -0,0 +1 @@
+gsitk

k8s/README.md

@@ -0,0 +1,7 @@
+Deploy senpy to a kubernetes cluster.
+
+Usage:
+
+```
+kubectl apply -f . -n senpy
+```

k8s/senpy-deployment.yaml

@@ -0,0 +1,26 @@
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: senpy-latest
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        role: senpy-latest
+        app: test
+    spec:
+      containers:
+      - name: senpy-latest
+        image: $IMAGEWTAG
+        imagePullPolicy: Always
+        args:
+          - "--default-plugins"
+        resources:
+          limits:
+            memory: "512Mi"
+            cpu: "1000m"
+        ports:
+          - name: web
+            containerPort: 5000

k8s/senpy-ingress.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: senpy-ingress
+spec:
+  rules:
+  - host: latest.senpy.cluster.gsi.dit.upm.es
+    http:
+      paths:
+      - path: /
+        backend:
+          serviceName: senpy-latest
+          servicePort: 5000

k8s/senpy-svc.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: senpy-latest
+spec:
+  type: ClusterIP
+  ports:
+    - port: 5000
+      protocol: TCP
+  selector:
+    role: senpy-latest

requirements.txt

@@ -1,7 +1,15 @@
 Flask>=0.10.1
-gunicorn>=19.0.0
 requests>=2.4.1
-GitPython>=0.3.2.RC1
+tornado>=4.4.3
-gevent>=1.0.1
 PyLD>=0.6.5
-Flask-Testing>=0.4.2
+nltk
+future
+jsonschema
+jsonref
+PyYAML
+rdflib
+rdflib-jsonld
+numpy
+scipy
+scikit-learn
+responses

senpy/__init__.py

@@ -17,3 +17,18 @@
 """
 Sentiment analysis server in Python
 """
+from .version import __version__
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+logger.info('Using senpy version: {}'.format(__version__))
+
+from .utils import easy, easy_load, easy_test  # noqa: F401
+
+from .models import *  # noqa: F401,F403
+from .plugins import *  # noqa: F401,F403
+from .extensions import *  # noqa: F401,F403
+
+__all__ = ['api', 'blueprints', 'cli', 'extensions', 'models', 'plugins']

senpy/__main__.py

@@ -22,60 +22,150 @@ the server.
 
 from flask import Flask
 from senpy.extensions import Senpy
-from gevent.wsgi import WSGIServer
+from senpy.utils import easy_test
-from gevent.monkey import patch_all
+
-import gevent
 import logging
 import os
+import sys
 import argparse
+import senpy
+
+SERVER_PORT = os.environ.get("PORT", 5000)
 
-patch_all(thread=False)
 
 def main():
     parser = argparse.ArgumentParser(description='Run a Senpy server')
-    parser.add_argument('--level',
+    parser.add_argument(
-                        "-l",
+        '--level',
-                        metavar="logging_level",
+        '-l',
-                        type=str,
+        metavar='logging_level',
-                        default="INFO",
+        type=str,
-                        help='Logging level')
+        default="WARN",
-    parser.add_argument('--debug',
+        help='Logging level')
-                        "-d",
+    parser.add_argument(
-                        action='store_true',
+        '--debug',
-                        default=False,
+        '-d',
-                        help='Run the application in debug mode')
+        action='store_true',
-    parser.add_argument('--default-plugins',
+        default=False,
-                        action='store_true',
+        help='Run the application in debug mode')
-                        default=False,
+    parser.add_argument(
-                        help='Load the default plugins')
+        '--default-plugins',
-    parser.add_argument('--host',
+        action='store_true',
-                        type=str,
+        default=False,
-                        default="127.0.0.1",
+        help='Load the default plugins')
-                        help='Use 0.0.0.0 to accept requests from any host.')
+    parser.add_argument(
-    parser.add_argument('--port',
+        '--host',
-                        '-p',
+        type=str,
-                        type=int,
+        default="0.0.0.0",
-                        default=5000,
+        help='Use 0.0.0.0 to accept requests from any host.')
-                        help='Port to listen on.')
+    parser.add_argument(
-    parser.add_argument('--plugins-folder',
+        '--port',
-                        '-f',
+        '-p',
-                        type=str,
+        type=int,
-                        default="plugins",
+        default=SERVER_PORT,
-                        help='Where to look for plugins.')
+        help='Port to listen on.')
+    parser.add_argument(
+        '--plugins-folder',
+        '-f',
+        type=str,
+        default='.',
+        help='Where to look for plugins.')
+    parser.add_argument(
+        '--only-install',
+        '-i',
+        action='store_true',
+        default=False,
+        help='Do not run a server, only install plugin dependencies')
+    parser.add_argument(
+        '--only-test',
+        action='store_true',
+        default=False,
+        help='Do not run a server, just test all plugins')
+    parser.add_argument(
+        '--test',
+        '-t',
+        action='store_true',
+        default=False,
+        help='Test all plugins before launching the server')
+    parser.add_argument(
+        '--only-list',
+        '--list',
+        action='store_true',
+        default=False,
+        help='Do not run a server, only list plugins found')
+    parser.add_argument(
+        '--data-folder',
+        '--data',
+        type=str,
+        default=None,
+        help='Where to look for data. It be set with the SENPY_DATA environment variable as well.')
+    parser.add_argument(
+        '--threaded',
+        action='store_false',
+        default=True,
+        help='Run a threaded server')
+    parser.add_argument(
+        '--no-deps',
+        '-n',
+        action='store_true',
+        default=False,
+        help='Skip installing dependencies')
+    parser.add_argument(
+        '--version',
+        '-v',
+        action='store_true',
+        default=False,
+        help='Output the senpy version and exit')
+    parser.add_argument(
+        '--allow-fail',
+        '--fail',
+        action='store_true',
+        default=False,
+        help='Do not exit if some plugins fail to activate')
     args = parser.parse_args()
-    logging.basicConfig(level=getattr(logging, args.level))
+    if args.version:
+        print('Senpy version {}'.format(senpy.__version__))
+        print(sys.version)
+        exit(1)
+    rl = logging.getLogger()
+    rl.setLevel(getattr(logging, args.level))
     app = Flask(__name__)
     app.debug = args.debug
-    sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins)
+    sp = Senpy(app, args.plugins_folder,
-    sp.activate_all()
+               default_plugins=args.default_plugins,
-    http_server = WSGIServer((args.host, args.port), app)
+               data_folder=args.data_folder)
-    try:
+    if args.only_list:
-        print("Server running on port %s:%d. Ctrl+C to quit" % (args.host,
+        plugins = sp.plugins()
-                                                                args.port))
+        maxname = max(len(x.name) for x in plugins)
-        http_server.serve_forever()
+        maxversion = max(len(x.version) for x in plugins)
-    except KeyboardInterrupt:
+        print('Found {} plugins:'.format(len(plugins)))
-        http_server.stop()
+        for plugin in plugins:
-        print("Bye!")
+            import inspect
+            fpath = inspect.getfile(plugin.__class__)
+            print('\t{: <{maxname}} @ {: <{maxversion}} -> {}'.format(plugin.name,
+                                                                      plugin.version,
+                                                                      fpath,
+                                                                      maxname=maxname,
+                                                                      maxversion=maxversion))
+        return
+    if not args.no_deps:
+        sp.install_deps()
+    if args.only_install:
+        return
+    sp.activate_all(allow_fail=args.allow_fail)
+    if args.test or args.only_test:
+        easy_test(sp.plugins(), debug=args.debug)
+        if args.only_test:
+            return
+    print('Senpy version {}'.format(senpy.__version__))
+    print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,
+                                                            args.port))
+    app.run(args.host,
+            args.port,
+            threaded=args.threaded,
+            debug=app.debug)
+    sp.deactivate_all()
+
 
 if __name__ == '__main__':
     main()

senpy/api.py

@@ -0,0 +1,220 @@
+from future.utils import iteritems
+from .models import Error, Results, Entry, from_string
+import logging
+logger = logging.getLogger(__name__)
+
+boolean = [True, False]
+
+API_PARAMS = {
+    "algorithm": {
+        "aliases": ["algorithms", "a", "algo"],
+        "required": False,
+        "description": ("Algorithms that will be used to process the request."
+                        "It may be a list of comma-separated names."),
+    },
+    "expanded-jsonld": {
+        "@id": "expanded-jsonld",
+        "aliases": ["expanded"],
+        "options": boolean,
+        "required": True,
+        "default": False
+    },
+    "with_parameters": {
+        "aliases": ['withparameters',
+                    'with-parameters'],
+        "options": boolean,
+        "default": False,
+        "required": True
+    },
+    "outformat": {
+        "@id": "outformat",
+        "aliases": ["o"],
+        "default": "json-ld",
+        "required": True,
+        "options": ["json-ld", "turtle", "ntriples"],
+    },
+    "help": {
+        "@id": "help",
+        "description": "Show additional help to know more about the possible parameters",
+        "aliases": ["h"],
+        "required": True,
+        "options": boolean,
+        "default": False
+    },
+    "emotionModel": {
+        "@id": "emotionModel",
+        "aliases": ["emoModel"],
+        "required": False
+    },
+    "conversion": {
+        "@id": "conversion",
+        "description": "How to show the elements that have (not) been converted",
+        "required": True,
+        "options": ["filtered", "nested", "full"],
+        "default": "full"
+    }
+}
+
+EVAL_PARAMS = {
+    "algorithm": {
+        "aliases": ["plug", "p", "plugins", "algorithms", 'algo', 'a', 'plugin'],
+        "description": "Plugins to be evaluated",
+        "required": True,
+        "help": "See activated plugins in /plugins"
+    },
+    "dataset": {
+        "aliases": ["datasets", "data", "d"],
+        "description": "Datasets to be evaluated",
+        "required": True,
+        "help": "See avalaible datasets in /datasets"
+    }
+}
+
+PLUGINS_PARAMS = {
+    "plugin_type": {
+        "@id": "pluginType",
+        "description": 'What kind of plugins to list',
+        "aliases": ["pluginType"],
+        "required": True,
+        "default": 'analysisPlugin'
+    }
+}
+
+WEB_PARAMS = {
+    "inHeaders": {
+        "aliases": ["headers"],
+        "required": True,
+        "default": False,
+        "options": boolean
+    },
+}
+
+CLI_PARAMS = {
+    "plugin_folder": {
+        "aliases": ["folder"],
+        "required": True,
+        "default": "."
+    },
+}
+
+NIF_PARAMS = {
+    "input": {
+        "@id": "input",
+        "aliases": ["i"],
+        "required": True,
+        "help": "Input text"
+    },
+    "intype": {
+        "@id": "intype",
+        "aliases": ["t"],
+        "required": False,
+        "default": "direct",
+        "options": ["direct", "url", "file"],
+    },
+    "informat": {
+        "@id": "informat",
+        "aliases": ["f"],
+        "required": False,
+        "default": "text",
+        "options": ["text", "json-ld"],
+    },
+    "language": {
+        "@id": "language",
+        "aliases": ["l"],
+        "required": False,
+    },
+    "prefix": {
+        "@id": "prefix",
+        "aliases": ["p"],
+        "required": True,
+        "default": "",
+    },
+    "urischeme": {
+        "@id": "urischeme",
+        "aliases": ["u"],
+        "required": False,
+        "default": "RFC5147String",
+        "options": ["RFC5147String", ]
+    }
+}
+
+BUILTIN_PARAMS = {}
+
+for d in [
+        NIF_PARAMS, CLI_PARAMS, WEB_PARAMS, PLUGINS_PARAMS, EVAL_PARAMS,
+        API_PARAMS
+]:
+    for k, v in d.items():
+        BUILTIN_PARAMS[k] = v
+
+
+def parse_params(indict, *specs):
+    if not specs:
+        specs = [NIF_PARAMS]
+    logger.debug("Parsing: {}\n{}".format(indict, specs))
+    outdict = indict.copy()
+    wrong_params = {}
+    for spec in specs:
+        for param, options in iteritems(spec):
+            for alias in options.get("aliases", []):
+                # Replace each alias with the correct name of the parameter
+                if alias in indict and alias != param:
+                    outdict[param] = indict[alias]
+                    del outdict[alias]
+                    continue
+            if param not in outdict:
+                if "default" in options:
+                    # We assume the default is correct
+                    outdict[param] = options["default"]
+                elif options.get("required", False):
+                    wrong_params[param] = spec[param]
+                continue
+            if "options" in options:
+                if options["options"] == boolean:
+                    outdict[param] = str(outdict[param]).lower() in ['true', '1']
+                elif outdict[param] not in options["options"]:
+                    wrong_params[param] = spec[param]
+    if wrong_params:
+        logger.debug("Error parsing: %s", wrong_params)
+        message = Error(
+            status=400,
+            message='Missing or invalid parameters',
+            parameters=outdict,
+            errors=wrong_params)
+        raise message
+    if 'algorithm' in outdict and not isinstance(outdict['algorithm'], tuple):
+        outdict['algorithm'] = tuple(outdict['algorithm'].split(','))
+    return outdict
+
+
+def parse_extra_params(request, plugins=None):
+    plugins = plugins or []
+    params = request.parameters.copy()
+    for plugin in plugins:
+        if plugin:
+            extra_params = parse_params(params, plugin.get('extra_params', {}))
+            for k, v in extra_params.items():
+                if k not in BUILTIN_PARAMS:
+                    if k in params:  # Set by another plugin
+                        del params[k]
+                    else:
+                        params[k] = v
+                params['{}.{}'.format(plugin.name, k)] = v
+    return params
+
+
+def parse_call(params):
+    '''Return a results object based on the parameters used in a call/request.
+    '''
+    params = parse_params(params, NIF_PARAMS)
+    if params['informat'] == 'text':
+        results = Results()
+        entry = Entry(nif__isString=params['input'], id='#')  # Use @base
+        results.entries.append(entry)
+    elif params['informat'] == 'json-ld':
+        results = from_string(params['input'], cls=Results)
+    else:  # pragma: no cover
+        raise NotImplementedError('Informat {} is not implemented'.format(
+            params['informat']))
+    results.parameters = params
+    return results

senpy/blueprints.py

@@ -17,157 +17,226 @@
 """
 Blueprints for Senpy
 """
-from flask import Blueprint, request, current_app
+from flask import (Blueprint, request, current_app, render_template, url_for,
-from .models import Error, Response, Leaf
+                   jsonify, redirect)
+from .models import Error, Response, Help, Plugins, read_schema, dump_schema, Datasets
+from . import api
+from .version import __version__
+from functools import wraps
 
-import json
 import logging
+import json
+import base64
 
 logger = logging.getLogger(__name__)
 
+api_blueprint = Blueprint("api", __name__)
+demo_blueprint = Blueprint("demo", __name__, template_folder='templates')
+ns_blueprint = Blueprint("ns", __name__)
 
-nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__)
+_mimetypes_r = {'json-ld': ['application/ld+json'],
+                'turtle': ['text/turtle'],
+                'ntriples': ['application/n-triples'],
+                'text': ['text/plain']}
 
-BASIC_PARAMS = {
+MIMETYPES = {}
-    "algorithm": {
-        "aliases": ["algorithm", "a", "algo"],
-        "required": False,
-    },
-    "inHeaders": {
-        "aliases": ["inHeaders", "headers"],
-        "required": True,
-        "default": "0"
-    }
-}
 
-LIST_PARAMS = {
+for k, vs in _mimetypes_r.items():
-    "params": {
+    for v in vs:
-        "aliases": ["params", "with_params"],
+        if v in MIMETYPES:
-        "required": False,
+            raise Exception('MIMETYPE {} specified for two formats: {} and {}'.format(v,
-        "default": "0"
+                                                                                      v,
-    },
+                                                                                      MIMETYPES[v]))
-}
+        MIMETYPES[v] = k
+
+DEFAULT_MIMETYPE = 'application/ld+json'
+DEFAULT_FORMAT = 'json-ld'
 
 
-def get_params(req, params=BASIC_PARAMS):
+def get_params(req):
     if req.method == 'POST':
-        indict = req.form
+        indict = req.form.to_dict(flat=True)
     elif req.method == 'GET':
-        indict = req.args
+        indict = req.args.to_dict(flat=True)
     else:
-        raise ValueError("Invalid data")
+        raise Error(message="Invalid data")
-
+    return indict
-    outdict = {}
-    wrong_params = {}
-    for param, options in params.iteritems():
-        if param[0] != "@":  # Exclude json-ld properties
-            logger.debug("Param: %s - Options: %s", param, options)
-            for alias in options["aliases"]:
-                if alias in indict:
-                    outdict[param] = indict[alias]
-            if param not in outdict:
-                if options.get("required", False) and "default" not in options:
-                    wrong_params[param] = params[param]
-                else:
-                    if "default" in options:
-                        outdict[param] = options["default"]
-            else:
-                if "options" in params[param] and \
-                   outdict[param] not in params[param]["options"]:
-                    wrong_params[param] = params[param]
-    if wrong_params:
-        message = Error({"status": 404,
-                         "message": "Missing or invalid parameters",
-                         "parameters": outdict,
-                         "errors": {param: error for param, error in
-                                    wrong_params.iteritems()}
-                         })
-        raise ValueError(message)
-    return outdict
 
 
-def basic_analysis(params):
+def encoded_url(url=None, base=None):
-    response = {"@context":
+    code = ''
-                [("http://demos.gsi.dit.upm.es/"
+    if not url:
-                  "eurosentiment/static/context.jsonld"),
+        if request.method == 'GET':
-                 {
+            url = request.full_path[1:]  # Remove the first slash
-                    "@base": "{}#".format(request.url.encode('utf-8'))
-                  }
-                 ],
-                "analysis": [{"@type": "marl:SentimentAnalysis"}],
-                "entries": []
-                }
-    if "language" in params:
-        response["language"] = params["language"]
-    for idx, sentence in enumerate(params["input"].split(".")):
-        response["entries"].append({
-            "@id": "Sentence{}".format(idx),
-            "nif:isString": sentence
-        })
-    return response
-
-
-@nif_blueprint.route('/', methods=['POST', 'GET'])
-def home():
-    try:
-        params = get_params(request)
-        algo = params.get("algorithm", None)
-        specific_params = current_app.senpy.parameters(algo)
-        logger.debug(
-            "Specific params: %s", json.dumps(specific_params, indent=4))
-        params.update(get_params(request, specific_params))
-        response = current_app.senpy.analyse(**params)
-        in_headers = params["inHeaders"] != "0"
-        return response.flask(in_headers=in_headers)
-    except ValueError as ex:
-        return ex.message.flask()
-
-
-@nif_blueprint.route("/default")
-def default():
-    # return current_app.senpy.default_plugin
-    plug = current_app.senpy.default_plugin
-    if plug:
-        return plugins(action="list", plugin=plug.name)
-    else:
-        error = Error(status=404, message="No plugins found")
-        return error.flask()
-
-
-@nif_blueprint.route('/plugins/', methods=['POST', 'GET'])
-@nif_blueprint.route('/plugins/<plugin>', methods=['POST', 'GET'])
-@nif_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
-def plugins(plugin=None, action="list"):
-    filt = {}
-    sp = current_app.senpy
-    if plugin:
-        filt["name"] = plugin
-    plugs = sp.filter_plugins(**filt)
-    if plugin and not plugs:
-        return "Plugin not found", 400
-    if action == "list":
-        with_params = get_params(request, LIST_PARAMS)["params"] == "1"
-        in_headers = get_params(request, BASIC_PARAMS)["inHeaders"] != "0"
-        if plugin:
-            dic = plugs[plugin]
         else:
-            dic = Response(
+            hash(frozenset(tuple(request.parameters.items())))
-                    {plug: plugs[plug].jsonld(with_params) for plug in plugs},
+            code = 'hash:{}'.format(hash)
-                    frame={})
+
-        return dic.flask(in_headers=in_headers)
+    code = code or base64.urlsafe_b64encode(url.encode()).decode()
-    method = "{}_plugin".format(action)
+
-    if(hasattr(sp, method)):
+    if base:
-        getattr(sp, method)(plugin)
+        return base + code
-        return Leaf(message="Ok").flask()
+    return url_for('api.decode', code=code, _external=True)
+
+
+def decoded_url(code, base=None):
+    if code.startswith('hash:'):
+        raise Exception('Can not decode a URL for a POST request')
+    base = base or request.url_root
+    path = base64.urlsafe_b64decode(code.encode()).decode()
+    return base + path
+
+
+@demo_blueprint.route('/')
+def index():
+    ev = str(get_params(request).get('evaluation', False))
+    evaluation_enabled = ev.lower() not in ['false', 'no', 'none']
+
+    return render_template("index.html",
+                           evaluation=evaluation_enabled,
+                           version=__version__)
+
+
+@api_blueprint.route('/contexts/<entity>.jsonld')
+def context(entity="context"):
+    context = Response._context
+    context['@vocab'] = url_for('ns.index', _external=True)
+    context['endpoint'] = url_for('api.api_root', _external=True)
+    return jsonify({"@context": context})
+
+
+@api_blueprint.route('/d/<code>')
+def decode(code):
+    try:
+        return redirect(decoded_url(code))
+    except Exception:
+        return Error('invalid URL').flask()
+
+
+@ns_blueprint.route('/')  # noqa: F811
+def index():
+    context = Response._context.copy()
+    context['endpoint'] = url_for('api.api_root', _external=True)
+    return jsonify({"@context": context})
+
+
+@api_blueprint.route('/schemas/<schema>')
+def schema(schema="definitions"):
+    try:
+        return dump_schema(read_schema(schema))
+    except Exception as ex:  # Should be FileNotFoundError, but it's missing from py2
+        return Error(message="Schema not found: {}".format(ex), status=404).flask()
+
+
+def basic_api(f):
+    default_params = {
+        'inHeaders': False,
+        'expanded-jsonld': False,
+        'outformat': None,
+        'with_parameters': True,
+    }
+
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        raw_params = get_params(request)
+        logger.info('Getting request: {}'.format(raw_params))
+        headers = {'X-ORIGINAL-PARAMS': json.dumps(raw_params)}
+        params = default_params
+
+        try:
+            params = api.parse_params(raw_params, api.WEB_PARAMS, api.API_PARAMS)
+            if hasattr(request, 'parameters'):
+                request.parameters.update(params)
+            else:
+                request.parameters = params
+            response = f(*args, **kwargs)
+        except (Exception) as ex:
+            if current_app.debug or current_app.config['TESTING']:
+                raise
+            if not isinstance(ex, Error):
+                msg = "{}".format(ex)
+                ex = Error(message=msg, status=500)
+            response = ex
+            response.parameters = raw_params
+            logger.exception(ex)
+
+        if 'parameters' in response and not params['with_parameters']:
+            del response.parameters
+
+        logger.info('Response: {}'.format(response))
+        mime = request.accept_mimetypes\
+                      .best_match(MIMETYPES.keys(),
+                                  DEFAULT_MIMETYPE)
+
+        mimeformat = MIMETYPES.get(mime, DEFAULT_FORMAT)
+        outformat = params['outformat'] or mimeformat
+
+        return response.flask(
+            in_headers=params['inHeaders'],
+            headers=headers,
+            prefix=params.get('prefix', encoded_url()),
+            context_uri=url_for('api.context',
+                                entity=type(response).__name__,
+                                _external=True),
+            outformat=outformat,
+            expanded=params['expanded-jsonld'])
+
+    return decorated_function
+
+
+@api_blueprint.route('/', defaults={'plugin': None}, methods=['POST', 'GET'])
+@api_blueprint.route('/<path:plugin>', methods=['POST', 'GET'])
+@basic_api
+def api_root(plugin):
+    if request.parameters['help']:
+        dic = dict(api.API_PARAMS, **api.NIF_PARAMS)
+        response = Help(valid_parameters=dic)
+        return response
+    req = api.parse_call(request.parameters)
+    if plugin:
+        plugin = plugin.replace('+', '/')
+        plugin = plugin.split('/')
+        req.parameters['algorithm'] = tuple(plugin)
+    results = current_app.senpy.analyse(req)
+    results.analysis = set(i.id for i in results.analysis)
+    return results
+
+
+@api_blueprint.route('/evaluate/', methods=['POST', 'GET'])
+@basic_api
+def evaluate():
+    if request.parameters['help']:
+        dic = dict(api.EVAL_PARAMS)
+        response = Help(parameters=dic)
+        return response
     else:
-        return Error("action '{}' not allowed".format(action)).flask()
+        params = api.parse_params(request.parameters, api.EVAL_PARAMS)
+        response = current_app.senpy.evaluate(params)
+        return response
 
 
-if __name__ == '__main__':
+@api_blueprint.route('/plugins/', methods=['POST', 'GET'])
-    import config
+@basic_api
-    from flask import Flask
+def plugins():
+    sp = current_app.senpy
+    params = api.parse_params(request.parameters, api.PLUGINS_PARAMS)
+    ptype = params.get('plugin_type')
+    plugins = list(sp.plugins(plugin_type=ptype))
+    dic = Plugins(plugins=plugins)
+    return dic
 
-    app = Flask(__name__)
+
-    app.register_blueprint(nif_blueprint)
+@api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
-    app.debug = config.DEBUG
+@basic_api
-    app.run()
+def plugin(plugin):
+    sp = current_app.senpy
+    return sp.get_plugin(plugin)
+
+
+@api_blueprint.route('/datasets/', methods=['POST', 'GET'])
+@basic_api
+def datasets():
+    sp = current_app.senpy
+    datasets = sp.datasets
+    dic = Datasets(datasets=list(datasets.values()))
+    return dic

senpy/cli.py

@@ -0,0 +1,56 @@
+import sys
+from .models import Error
+from .extensions import Senpy
+from . import api
+
+
+def argv_to_dict(argv):
+    '''Turns parameters in the form of '--key value' into a dict {'key': 'value'}
+    '''
+    cli_dict = {}
+
+    for i in range(len(argv)):
+        if argv[i][0] == '-':
+            key = argv[i].strip('-')
+            value = argv[i + 1] if len(argv) > i + 1 else None
+            if not value or value[0] == '-':
+                cli_dict[key] = True
+            else:
+                cli_dict[key] = value
+    return cli_dict
+
+
+def main_function(argv):
+    '''This is the method for unit testing
+    '''
+    params = api.parse_params(argv_to_dict(argv),
+                              api.CLI_PARAMS,
+                              api.API_PARAMS,
+                              api.NIF_PARAMS)
+    plugin_folder = params['plugin_folder']
+    default_plugins = params.get('default-plugins', False)
+    sp = Senpy(default_plugins=default_plugins, plugin_folder=plugin_folder)
+    request = api.parse_call(params)
+    algos = request.parameters.get('algorithm', None)
+    if algos:
+        for algo in algos:
+            sp.activate_plugin(algo)
+    else:
+        sp.activate_all()
+    res = sp.analyse(request)
+    return res
+
+
+def main():
+    '''This method is the entrypoint for the CLI (as configured un setup.py)
+    '''
+    try:
+        res = main_function(sys.argv[1:])
+        print(res.serialize())
+    except Error as err:
+        print(err.serialize())
+        sys.exit(2)
+
+
+if __name__ == '__main__':
+    main()

senpy/client.py

@@ -0,0 +1,51 @@
+import requests
+import logging
+from . import models
+
+logger = logging.getLogger(__name__)
+
+
+class Client(object):
+    def __init__(self, endpoint):
+        self.endpoint = endpoint
+
+    def analyse(self, input, method='GET', **kwargs):
+        return self.request('/', method=method, input=input, **kwargs)
+
+    def evaluate(self, input, method='GET', **kwargs):
+        return self.request('/evaluate', method=method, input=input, **kwargs)
+
+    def plugins(self, *args, **kwargs):
+        resp = self.request(path='/plugins').plugins
+        return {p.name: p for p in resp}
+
+    def datasets(self):
+        resp = self.request(path='/datasets').datasets
+        return {d.name: d for d in resp}
+
+    def request(self, path=None, method='GET', **params):
+        url = '{}{}'.format(self.endpoint.rstrip('/'), path)
+        if method == 'POST':
+            response = requests.post(url=url, data=params)
+        else:
+            response = requests.request(method=method, url=url, params=params)
+
+        try:
+            resp = models.from_dict(response.json())
+        except Exception as ex:
+            logger.error(('There seems to be a problem with the response:\n'
+                          '\tURL: {url}\n'
+                          '\tError: {error}\n'
+                          '\t\n'
+                          '#### Response:\n'
+                          '\tCode: {code}'
+                          '\tContent: {content}'
+                          '\n').format(
+                              error=ex,
+                              url=url,
+                              code=response.status_code,
+                              content=response.content))
+            raise ex
+        if isinstance(resp, models.Error):
+            raise resp
+        return resp

senpy/context.jsonld

@@ -1,42 +0,0 @@
-{
-    "dc": "http://purl.org/dc/terms/",
-    "dc:subject": {
-        "@type": "@id"
-    },
-    "xsd": "http://www.w3.org/2001/XMLSchema#",
-    "marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#",
-    "nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#",
-    "onyx": "http://www.gsi.dit.upm.es/ontologies/onyx/ns#",
-    "emotions": {
-        "@container": "@set",
-        "@id": "onyx:hasEmotionSet"
-    },
-    "opinions": {
-        "@container": "@set",
-        "@id": "marl:hasOpinion"
-    },
-    "prov": "http://www.w3.org/ns/prov#",
-    "rdfs": "http://www.w3.org/2000/01/rdf-schema#",
-    "analysis": {
-        "@container": "@set",
-        "@id": "prov:wasInformedBy"
-    },
-    "entries": {
-        "@container": "@set",
-        "@id": "prov:generated"
-    },
-    "strings": {
-        "@container": "@set",
-        "@reverse": "nif:hasContext"
-    },
-    "date":
-    {
-        "@id": "dc:date",
-        "@type": "xsd:dateTime"
-    },
-    "text": { "@id": "nif:isString" },
-    "wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#",
-    "xsd": "http://www.w3.org/2001/XMLSchema#",
-    "senpy": "http://www.gsi.dit.upm.es/ontologies/senpy/ns#",
-    "@vocab":  "http://www.gsi.dit.upm.es/ontologies/senpy/ns#"
-}

senpy/extensions.py

@@ -1,44 +1,57 @@
 """
+Main class for Senpy.
+It orchestrates plugin (de)activation and analysis.
 """
-import gevent
+from future import standard_library
-from gevent import monkey
+standard_library.install_aliases()
-monkey.patch_all()
 
-from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
+from . import plugins, api
-from .models import Error
+from .models import Error, AggregatedEvaluation
-from .blueprints import nif_blueprint
+from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
 
-from git import Repo, InvalidGitRepositoryError
+from threading import Thread
 from functools import partial
-
 import os
-import fnmatch
+import copy
-import inspect
+import errno
-import sys
-import imp
 import logging
-import traceback
+
-import gevent
+from . import gsitk_compat
-import json
 
 logger = logging.getLogger(__name__)
 
 
 class Senpy(object):
-
     """ Default Senpy extension for Flask """
 
-    def __init__(self, app=None, plugin_folder="plugins", default_plugins=False):
+    def __init__(self,
-        self.app = app
+                 app=None,
+                 plugin_folder=".",
+                 data_folder=None,
+                 default_plugins=False):
 
-        self._search_folders = set()
+        default_data = os.path.join(os.getcwd(), 'senpy_data')
-        self._outdated = True
+        self.data_folder = data_folder or os.environ.get('SENPY_DATA', default_data)
+        try:
+            os.makedirs(self.data_folder)
+        except OSError as e:
+            if e.errno == errno.EEXIST:
+                logger.debug('Data folder exists: {}'.format(self.data_folder))
+            else:  # pragma: no cover
+                raise
+
+        self._default = None
+        self._plugins = {}
+        if plugin_folder:
+            self.add_folder(plugin_folder)
 
-        self.add_folder(plugin_folder)
         if default_plugins:
-            base_folder = os.path.join(os.path.dirname(__file__), "plugins")
+            self.add_folder('plugins', from_root=True)
-            self.add_folder(base_folder)
+        else:
-
+            # Add only conversion plugins
+            self.add_folder(os.path.join('plugins', 'postprocessing'),
+                            from_root=True)
+        self.app = app
         if app is not None:
             self.init_app(app)
 
@@ -53,188 +66,333 @@ class Senpy(object):
         # otherwise fall back to the request context
         if hasattr(app, 'teardown_appcontext'):
             app.teardown_appcontext(self.teardown)
-        else:
+        else:  # pragma: no cover
             app.teardown_request(self.teardown)
-        app.register_blueprint(nif_blueprint)
+        app.register_blueprint(api_blueprint, url_prefix="/api")
+        app.register_blueprint(ns_blueprint, url_prefix="/ns")
+        app.register_blueprint(demo_blueprint, url_prefix="/")
 
-    def add_folder(self, folder):
+    def add_plugin(self, plugin):
+        self._plugins[plugin.name.lower()] = plugin
+
+    def delete_plugin(self, plugin):
+        del self._plugins[plugin.name.lower()]
+
+    def plugins(self, **kwargs):
+        """ Return the plugins registered for a given application. Filtered by criteria  """
+        return list(plugins.pfilter(self._plugins, **kwargs))
+
+    def get_plugin(self, name, default=None):
+        if name == 'default':
+            return self.default_plugin
+        plugin = name.lower()
+        if plugin in self._plugins:
+            return self._plugins[plugin]
+
+        results = self.plugins(id='endpoint:plugins/{}'.format(name))
+
+        if not results:
+            return Error(message="Plugin not found", status=404)
+        return results[0]
+
+    @property
+    def analysis_plugins(self):
+        """ Return only the analysis plugins """
+        return self.plugins(plugin_type='analysisPlugin')
+
+    def add_folder(self, folder, from_root=False):
+        """ Find plugins in this folder and add them to this instance """
+        if from_root:
+            folder = os.path.join(os.path.dirname(__file__), folder)
         logger.debug("Adding folder: %s", folder)
         if os.path.isdir(folder):
-            self._search_folders.add(folder)
+            new_plugins = plugins.from_folder([folder],
-            self._outdated = True
+                                              data_folder=self.data_folder)
-            return True
+            for plugin in new_plugins:
+                self.add_plugin(plugin)
         else:
-            logger.debug("Not a folder: %s", folder)
+            raise AttributeError("Not a folder or does not exist: %s", folder)
-            return False
 
-    def analyse(self, **params):
+    def _get_plugins(self, request):
-        algo = None
+        '''Get a list of plugins that should be run for a specific request'''
-        logger.debug("analysing with params: {}".format(params))
+        if not self.analysis_plugins:
-        if "algorithm" in params:
+            raise Error(
-            algo = params["algorithm"]
+                status=404,
-        elif self.plugins:
+                message=("No plugins found."
-            algo = self.default_plugin and self.default_plugin.name
+                         " Please install one."))
-        if not algo:
+        algos = request.parameters.get('algorithm', None)
-            return Error(status=404,
+        if not algos:
-                         message=("No plugins found."
+            if self.default_plugin:
-                                  " Please install one.").format(algo))
+                algos = [self.default_plugin.name, ]
-        if algo in self.plugins:
-            if self.plugins[algo].is_activated:
-                plug = self.plugins[algo]
-                resp = plug.analyse(**params)
-                resp.analysis.append(plug)
-                logger.debug("Returning analysis result: {}".format(resp))
-                return resp
             else:
-                logger.debug("Plugin not activated: {}".format(algo))
+                raise Error(
-                return Error(status=400,
+                    status=404,
-                             message=("The algorithm '{}'"
+                    message="No default plugin found, and None provided")
-                                      " is not activated yet").format(algo))
+
-        else:
+        plugins = list()
-            logger.debug(("The algorithm '{}' is not valid\n"
+        for algo in algos:
-                          "Valid algorithms: {}").format(algo,
+            algo = algo.lower()
-                                                         self.plugins.keys()))
+            if algo == 'conversion':
-            return Error(status=404,
+                continue  # Allow 'conversion' as a virtual plugin, which does nothing
-                         message="The algorithm '{}' is not valid"
+            if algo not in self._plugins:
-                                 .format(algo))
+                msg = ("The algorithm '{}' is not valid\n"
+                       "Valid algorithms: {}").format(algo,
+                                                      self._plugins.keys())
+                logger.debug(msg)
+                raise Error(status=404, message=msg)
+            plugins.append(self._plugins[algo])
+
+        return plugins
+
+    def _process(self, req, pending, done=None):
+        """
+        Recursively process the entries with the first plugin in the list, and pass the results
+        to the rest of the plugins.
+        """
+        done = done or []
+        if not pending:
+            return req
+
+        plugin = pending[0]
+        results = plugin.process(req, conversions_applied=done)
+        if plugin not in results.analysis:
+            results.analysis.append(plugin)
+        return self._process(results, pending[1:], done)
+
+    def install_deps(self):
+        plugins.install_deps(*self.plugins())
+
+    def analyse(self, request):
+        """
+        Main method that analyses a request, either from CLI or HTTP.
+        It takes a processed request, provided by the user, as returned
+        by api.parse_call().
+        """
+        logger.debug("analysing request: {}".format(request))
+        plugins = self._get_plugins(request)
+        request.parameters = api.parse_extra_params(request, plugins)
+        results = self._process(request, plugins)
+        logger.debug("Got analysis result: {}".format(results))
+        results = self.postprocess(results)
+        logger.debug("Returning post-processed result: {}".format(results))
+        return results
+
+    def convert_emotions(self, resp):
+        """
+        Conversion of all emotions in a response **in place**.
+        In addition to converting from one model to another, it has
+        to include the conversion plugin to the analysis list.
+        Needless to say, this is far from an elegant solution, but it works.
+        @todo refactor and clean up
+        """
+        plugins = resp.analysis
+
+        params = resp.parameters
+        toModel = params.get('emotionModel', None)
+        if not toModel:
+            return resp
+
+        logger.debug('Asked for model: {}'.format(toModel))
+        output = params.get('conversion', None)
+        candidates = {}
+        for plugin in plugins:
+            try:
+                fromModel = plugin.get('onyx:usesEmotionModel', None)
+                candidates[plugin.id] = next(self._conversion_candidates(fromModel, toModel))
+                logger.debug('Analysis plugin {} uses model: {}'.format(
+                    plugin.id, fromModel))
+            except StopIteration:
+                e = Error(('No conversion plugin found for: '
+                           '{} -> {}'.format(fromModel, toModel)),
+                          status=404)
+                e.original_response = resp
+                e.parameters = params
+                raise e
+        newentries = []
+        done = []
+        for i in resp.entries:
+            if output == "full":
+                newemotions = copy.deepcopy(i.emotions)
+            else:
+                newemotions = []
+            for j in i.emotions:
+                plugname = j['prov:wasGeneratedBy']
+                candidate = candidates[plugname]
+                done.append({'plugin': candidate, 'parameters': params})
+                for k in candidate.convert(j, fromModel, toModel, params):
+                    k.prov__wasGeneratedBy = candidate.id
+                    if output == 'nested':
+                        k.prov__wasDerivedFrom = j
+                    newemotions.append(k)
+            i.emotions = newemotions
+            newentries.append(i)
+        resp.entries = newentries
+        return resp
+
+    def _conversion_candidates(self, fromModel, toModel):
+        candidates = self.plugins(plugin_type=plugins.EmotionConversion)
+        for candidate in candidates:
+            for pair in candidate.onyx__doesConversion:
+                logging.debug(pair)
+                if candidate.can_convert(fromModel, toModel):
+                    yield candidate
+
+    def postprocess(self, response):
+        '''
+        Transform the results from the analysis plugins.
+        It has some pre-defined post-processing like emotion conversion,
+        and it also allows plugins to auto-select themselves.
+        '''
+
+        response = self.convert_emotions(response)
+
+        for plug in self.plugins(plugin_type=plugins.PostProcessing):
+            if plug.check(response, response.analysis):
+                response = plug.process(response)
+        return response
+
+    def _get_datasets(self, request):
+        if not self.datasets:
+            raise Error(
+                status=404,
+                message=("No datasets found."
+                         " Please verify DatasetManager"))
+        datasets_name = request.parameters.get('dataset', None).split(',')
+        for dataset in datasets_name:
+            if dataset not in self.datasets:
+                logger.debug(("The dataset '{}' is not valid\n"
+                              "Valid datasets: {}").format(
+                                  dataset, self.datasets.keys()))
+                raise Error(
+                    status=404,
+                    message="The dataset '{}' is not valid".format(dataset))
+        dm = gsitk_compat.DatasetManager()
+        datasets = dm.prepare_datasets(datasets_name)
+        return datasets
+
+    @property
+    def datasets(self):
+        self._dataset_list = {}
+        dm = gsitk_compat.DatasetManager()
+        for item in dm.get_datasets():
+            for key in item:
+                if key in self._dataset_list:
+                    continue
+                properties = item[key]
+                properties['@id'] = key
+                self._dataset_list[key] = properties
+        return self._dataset_list
+
+    def evaluate(self, params):
+        logger.debug("evaluating request: {}".format(params))
+        results = AggregatedEvaluation()
+        results.parameters = params
+        datasets = self._get_datasets(results)
+        plugins = self._get_plugins(results)
+        for eval in plugins.evaluate(plugins, datasets):
+            results.evaluations.append(eval)
+        if 'with_parameters' not in results.parameters:
+            del results.parameters
+        logger.debug("Returning evaluation result: {}".format(results))
+        return results
 
     @property
     def default_plugin(self):
-        candidates = self.filter_plugins(is_activated=True)
+        if not self._default or not self._default.is_activated:
-        if len(candidates) > 0:
+            candidates = self.plugins(
-            candidate = candidates.values()[0]
+                plugin_type='analysisPlugin', is_activated=True)
-            logger.debug("Default: {}".format(candidate))
+            if len(candidates) > 0:
-            return candidate
+                self._default = candidates[0]
+            else:
+                self._default = None
+            logger.debug("Default: {}".format(self._default))
+        return self._default
+
+    @default_plugin.setter
+    def default_plugin(self, value):
+        if isinstance(value, plugins.Plugin):
+            if not value.is_activated:
+                raise AttributeError('The default plugin has to be activated.')
+            self._default = value
+
         else:
-            return None
+            self._default = self._plugins[value.lower()]
 
-    def parameters(self, algo):
+    def activate_all(self, sync=True, allow_fail=False):
-        return getattr(self.plugins.get(algo) or self.default_plugin,
-                       "params",
-                       {})
-
-    def activate_all(self, sync=False):
         ps = []
-        for plug in self.plugins.keys():
+        for plug in self._plugins.keys():
-            ps.append(self.activate_plugin(plug, sync=sync))
+            try:
+                self.activate_plugin(plug, sync=sync)
+            except Exception as ex:
+                if not allow_fail:
+                    raise
+                logger.error('Could not activate {}: {}'.format(plug, ex))
         return ps
 
-    def deactivate_all(self, sync=False):
+    def deactivate_all(self, sync=True):
         ps = []
-        for plug in self.plugins.keys():
+        for plug in self._plugins.keys():
             ps.append(self.deactivate_plugin(plug, sync=sync))
         return ps
 
-    def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
+    def _set_active(self, plugin, active=True, *args, **kwargs):
-        self.plugins[plugin_name].is_activated = active
+        ''' We're using a variable in the plugin itself to activate/deactivate plugins.\
+        Note that plugins may activate themselves by setting this variable.
+        '''
+        plugin.is_activated = active
 
-    def activate_plugin(self, plugin_name, sync=False):
+    def _activate(self, plugin):
-        plugin = self.plugins[plugin_name]
+        success = False
-        def act():
+        with plugin._lock:
-            try:
+            if plugin.is_activated:
-                plugin.activate()
-            except Exception as ex:
-                logger.error("Error activating plugin {}: {}".format(plugin.name,
-                                                                     ex))
-                logger.error("Trace: {}".format(traceback.format_exc()))
-        th = gevent.spawn(act)
-        th.link_value(partial(self._set_active_plugin, plugin_name, True))
-        if sync:
-            th.join()
-        else:
-            return th
-
-    def deactivate_plugin(self, plugin_name, sync=False):
-        plugin = self.plugins[plugin_name]
-        th = gevent.spawn(plugin.deactivate)
-        th.link_value(partial(self._set_active_plugin, plugin_name, False))
-        if sync:
-            th.join()
-        else:
-            return th
-
-    def reload_plugin(self, plugin):
-        logger.debug("Reloading {}".format(plugin))
-        plug = self.plugins[plugin]
-        nplug = self._load_plugin(plug.module, plug.path)
-        del self.plugins[plugin]
-        self.plugins[nplug.name] = nplug
-
-    @staticmethod
-    def _load_plugin(root, filename):
-        logger.debug("Loading plugin: {}".format(filename))
-        fpath = os.path.join(root, filename)
-        with open(fpath, 'r') as f:
-            info = json.load(f)
-        logger.debug("Info: {}".format(info))
-        sys.path.append(root)
-        module = info["module"]
-        name = info["name"]
-        (fp, pathname, desc) = imp.find_module(module, [root, ])
-        try:
-            tmp = imp.load_module(module, fp, pathname, desc)
-            sys.path.remove(root)
-            candidate = None
-            for _, obj in inspect.getmembers(tmp):
-                if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
-                    logger.debug(("Found plugin class:"
-                                  " {}@{}").format(obj, inspect.getmodule(obj))
-                                 )
-                    candidate = obj
-                    break
-            if not candidate:
-                logger.debug("No valid plugin for: {}".format(filename))
                 return
-            module = candidate(info=info)
+            plugin.activate()
-            try:
+            msg = "Plugin activated: {}".format(plugin.name)
-                repo_path = root
+            logger.info(msg)
-                module._repo = Repo(repo_path)
+            success = True
-            except InvalidGitRepositoryError:
+            self._set_active(plugin, success)
-                module._repo = None
+        return success
-        except Exception as ex:
-            logger.error("Exception importing {}: {}".format(filename, ex))
-            logger.error("Trace: {}".format(traceback.format_exc()))
-            return None, None
-        return name, module
 
-    def _load_plugins(self):
+    def activate_plugin(self, plugin_name, sync=True):
-        plugins = {}
+        plugin_name = plugin_name.lower()
-        for search_folder in self._search_folders:
+        if plugin_name not in self._plugins:
-            for root, dirnames, filenames in os.walk(search_folder):
+            raise Error(
-                for filename in fnmatch.filter(filenames, '*.senpy'):
+                message="Plugin not found: {}".format(plugin_name), status=404)
-                    name, plugin = self._load_plugin(root, filename)
+        plugin = self._plugins[plugin_name]
-                    if plugin:
-                        plugins[name] = plugin
 
-        self._outdated = False
+        logger.info("Activating plugin: {}".format(plugin.name))
-        return plugins
+
+        if sync or not getattr(plugin, 'async', True) or getattr(
+                plugin, 'sync', False):
+            return self._activate(plugin)
+        else:
+            th = Thread(target=partial(self._activate, plugin))
+            th.start()
+            return th
+
+    def _deactivate(self, plugin):
+        with plugin._lock:
+            if not plugin.is_activated:
+                return
+            plugin.deactivate()
+            logger.info("Plugin deactivated: {}".format(plugin.name))
+
+    def deactivate_plugin(self, plugin_name, sync=True):
+        plugin_name = plugin_name.lower()
+        if plugin_name not in self._plugins:
+            raise Error(
+                message="Plugin not found: {}".format(plugin_name), status=404)
+        plugin = self._plugins[plugin_name]
+
+        self._set_active(plugin, False)
+
+        if sync or not getattr(plugin, 'async', True) or not getattr(
+                plugin, 'sync', False):
+            self._deactivate(plugin)
+        else:
+            th = Thread(target=partial(self._deactivate, plugin))
+            th.start()
+            return th
 
     def teardown(self, exception):
         pass
-
-    @property
-    def plugins(self):
-        """ Return the plugins registered for a given application.  """
-        if not hasattr(self, 'senpy_plugins') or self._outdated:
-            self.senpy_plugins = self._load_plugins()
-        return self.senpy_plugins
-
-    def filter_plugins(self, **kwargs):
-        """ Filter plugins by different criteria """
-
-        def matches(plug):
-            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
-            logger.debug("matching {} with {}: {}".format(plug.name,
-                                                          kwargs,
-                                                          res))
-            return res
-
-        if not kwargs:
-            return self.plugins
-        else:
-            return {n: p for n, p in self.plugins.items() if matches(p)}
-
-    def sentiment_plugins(self):
-        """ Return only the sentiment plugins """
-        return {p: plugin for p, plugin in self.plugins.items() if
-                isinstance(plugin, SentimentPlugin)}

senpy/gsitk_compat.py

@@ -0,0 +1,31 @@
+import logging
+
+from pkg_resources import parse_version, get_distribution, DistributionNotFound
+
+logger = logging.getLogger(__name__)
+
+MSG = 'GSITK is not (properly) installed.'
+IMPORTMSG = '{} Some functions will be unavailable.'.format(MSG)
+RUNMSG = '{} Install it to use this function.'.format(MSG)
+
+
+def raise_exception(*args, **kwargs):
+    raise Exception(RUNMSG)
+
+
+try:
+    gsitk_distro = get_distribution("gsitk")
+    GSITK_VERSION = parse_version(gsitk_distro.version)
+    GSITK_AVAILABLE = GSITK_VERSION > parse_version("0.1.9.1")  # Earlier versions have a bug
+except DistributionNotFound:
+    GSITK_AVAILABLE = False
+    GSITK_VERSION = ()
+
+if GSITK_AVAILABLE:
+    from gsitk.datasets.datasets import DatasetManager
+    from gsitk.evaluation.evaluation import Evaluation as Eval
+    from sklearn.pipeline import Pipeline
+    modules = locals()
+else:
+    logger.warning(IMPORTMSG)
+    DatasetManager = Eval = Pipeline = raise_exception

senpy/meta.py

@@ -0,0 +1,257 @@
+'''
+Meta-programming for the models.
+'''
+import os
+import json
+import jsonschema
+import inspect
+import copy
+
+from abc import ABCMeta
+from collections import MutableMapping, namedtuple
+
+
+class BaseMeta(ABCMeta):
+    '''
+    Metaclass for models. It extracts the default values for the fields in
+    the model.
+
+    For instance, instances of the following class wouldn't need to mark
+    their version or description on initialization:
+
+    .. code-block:: python
+
+       class MyPlugin(Plugin):
+           version=0.3
+           description='A dull plugin'
+
+
+    Note that these operations could be included in the __init__ of the
+    class, but it would be very inefficient.
+    '''
+    _subtypes = {}
+
+    def __new__(mcs, name, bases, attrs, **kwargs):
+        register_afterwards = False
+        defaults = {}
+
+        attrs = mcs.expand_with_schema(name, attrs)
+        if 'schema' in attrs:
+            register_afterwards = True
+        for base in bases:
+            if hasattr(base, '_defaults'):
+                defaults.update(getattr(base, '_defaults'))
+
+        info, rest = mcs.split_attrs(attrs)
+
+        for i in list(info.keys()):
+            if isinstance(info[i], _Alias):
+                fget, fset, fdel = make_property(info[i].indict)
+                rest[i] = property(fget=fget, fset=fset, fdel=fdel)
+            else:
+                defaults[i] = info[i]
+
+        rest['_defaults'] = defaults
+
+        cls = super(BaseMeta, mcs).__new__(mcs, name, tuple(bases), rest)
+
+        if register_afterwards:
+            mcs.register(cls, defaults['@type'])
+        return cls
+
+    @classmethod
+    def register(mcs, rsubclass, rtype=None):
+        mcs._subtypes[rtype or rsubclass.__name__] = rsubclass
+
+    @staticmethod
+    def expand_with_schema(name, attrs):
+        if 'schema' in attrs:  # Schema specified by name
+            schema_file = '{}.json'.format(attrs['schema'])
+        elif 'schema_file' in attrs:
+            schema_file = attrs['schema_file']
+            del attrs['schema_file']
+        else:
+            return attrs
+
+        if '/' not in 'schema_file':
+            thisdir = os.path.dirname(os.path.realpath(__file__))
+            schema_file = os.path.join(thisdir,
+                                       'schemas',
+                                       schema_file)
+
+        schema_path = 'file://' + schema_file
+
+        with open(schema_file) as f:
+            schema = json.load(f)
+
+        resolver = jsonschema.RefResolver(schema_path, schema)
+        attrs['@type'] = "".join((name[0].lower(), name[1:]))
+        attrs['_schema_file'] = schema_file
+        attrs['schema'] = schema
+        attrs['_validator'] = jsonschema.Draft4Validator(schema, resolver=resolver)
+
+        schema_defaults = BaseMeta.get_defaults(attrs['schema'])
+        attrs.update(schema_defaults)
+
+        return attrs
+
+    @staticmethod
+    def is_func(v):
+        return inspect.isroutine(v) or inspect.ismethod(v) or \
+            inspect.ismodule(v) or isinstance(v, property)
+
+    @staticmethod
+    def is_internal(k):
+        return k[0] == '_' or k == 'schema' or k == 'data'
+
+    @staticmethod
+    def get_key(key):
+        if key[0] != '_':
+            key = key.replace("__", ":", 1)
+        return key
+
+    @staticmethod
+    def split_attrs(attrs):
+        '''
+        Extract the attributes of the class.
+
+        This allows adding default values in the class definition.
+        e.g.:
+        '''
+        isattr = {}
+        rest = {}
+        for key, value in attrs.items():
+            if not (BaseMeta.is_internal(key)) and (not BaseMeta.is_func(value)):
+                isattr[key] = value
+            else:
+                rest[key] = value
+        return isattr, rest
+
+    @staticmethod
+    def get_defaults(schema):
+        temp = {}
+        for obj in [
+                schema,
+        ] + schema.get('allOf', []):
+            for k, v in obj.get('properties', {}).items():
+                if 'default' in v and k not in temp:
+                    temp[k] = v['default']
+        return temp
+
+
+def make_property(key):
+
+    def fget(self):
+        return self[key]
+
+    def fdel(self):
+        del self[key]
+
+    def fset(self, value):
+        self[key] = value
+
+    return fget, fset, fdel
+
+
+class CustomDict(MutableMapping, object):
+    '''
+    A dictionary whose elements can also be accessed as attributes. Since some
+    characters are not valid in the dot-notation, the attribute names also
+    converted. e.g.:
+
+    > d = CustomDict()
+    > d.key = d['ns:name'] = 1
+    > d.key == d['key']
+    True
+    > d.ns__name == d['ns:name']
+    '''
+
+    _defaults = {}
+    _map_attr_key = {'id': '@id'}
+
+    def __init__(self, *args, **kwargs):
+        super(CustomDict, self).__init__()
+        for k, v in self._defaults.items():
+            self[k] = copy.copy(v)
+        for arg in args:
+            self.update(arg)
+        for k, v in kwargs.items():
+            self[self._attr_to_key(k)] = v
+        return self
+
+    def serializable(self):
+        def ser_or_down(item):
+            if hasattr(item, 'serializable'):
+                return item.serializable()
+            elif isinstance(item, dict):
+                temp = dict()
+                for kp in item:
+                    vp = item[kp]
+                    temp[kp] = ser_or_down(vp)
+                return temp
+            elif isinstance(item, list) or isinstance(item, set):
+                return list(ser_or_down(i) for i in item)
+            else:
+                return item
+
+        return ser_or_down(self.as_dict())
+
+    def __getitem__(self, key):
+        key = self._key_to_attr(key)
+        return self.__dict__[key]
+
+    def __setitem__(self, key, value):
+        '''Do not insert data directly, there might be a property in that key. '''
+        key = self._key_to_attr(key)
+        return setattr(self, key, value)
+
+    def as_dict(self):
+        return {self._attr_to_key(k): v for k, v in self.__dict__.items()
+                if not self._internal_key(k)}
+
+    def __iter__(self):
+        return (k for k in self.__dict__ if not self._internal_key(k))
+
+    def __len__(self):
+        return len(self.__dict__)
+
+    def __delitem__(self, key):
+        del self.__dict__[key]
+
+    def update(self, other):
+        for k, v in other.items():
+            self[k] = v
+
+    def _attr_to_key(self, key):
+        key = key.replace("__", ":", 1)
+        key = self._map_attr_key.get(key, key)
+        return key
+
+    def _key_to_attr(self, key):
+        if self._internal_key(key):
+            return key
+        key = key.replace(":", "__", 1)
+        return key
+
+    def __getattr__(self, key):
+        try:
+            return self.__dict__[self._attr_to_key(key)]
+        except KeyError:
+            raise AttributeError
+
+    @staticmethod
+    def _internal_key(key):
+        return key[0] == '_'
+
+    def __str__(self):
+        return str(self.serializable())
+
+    def __repr__(self):
+        return str(self.serializable())
+
+
+_Alias = namedtuple('Alias', 'indict')
+
+
+def alias(key):
+    return _Alias(key)

senpy/models.py

@@ -1,252 +1,351 @@
+'''
+Senpy Models.
+
+This implementation should mirror the JSON schema definition.
+For compatibility with Py3 and for easier debugging, this new version drops
+introspection and adds all arguments to the models.
+'''
+from __future__ import print_function
+from future import standard_library
+standard_library.install_aliases()
+
+from future.utils import with_metaclass
+from past.builtins import basestring
+
+import time
+import copy
 import json
 import os
-from collections import defaultdict
+import jsonref
-from pyld import jsonld
-import logging
 from flask import Response as FlaskResponse
+from pyld import jsonld
+
+import logging
+
+logging.getLogger('rdflib').setLevel(logging.WARN)
+logger = logging.getLogger(__name__)
+
+from rdflib import Graph
 
 
-class Leaf(dict):
+from .meta import BaseMeta, CustomDict, alias
-    _prefix = None
-    _frame = {}
-    _context = {}
 
-    def __init__(self,
+DEFINITIONS_FILE = 'definitions.json'
-                 *args,
+CONTEXT_PATH = os.path.join(
-                 **kwargs):
+    os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld')
 
-        id = kwargs.pop("id", None)
-        context = kwargs.pop("context", self._context)
-        vocab = kwargs.pop("vocab", None)
-        prefix = kwargs.pop("prefix", None)
-        frame = kwargs.pop("frame", None)
-        super(Leaf, self).__init__(*args, **kwargs)
-        if context is not None:
-            self.context = context
-        if frame is not None:
-            self._frame = frame
-        self._prefix = prefix
-        self.id = id
 
-    def __getattr__(self, key):
+def get_schema_path(schema_file, absolute=False):
+    if absolute:
+        return os.path.realpath(schema_file)
+    else:
+        return os.path.join(
+            os.path.dirname(os.path.realpath(__file__)), 'schemas',
+            schema_file)
+
+
+def read_schema(schema_file, absolute=False):
+    schema_path = get_schema_path(schema_file, absolute)
+    schema_uri = 'file://{}'.format(schema_path)
+    with open(schema_path) as f:
+        return jsonref.load(f, base_uri=schema_uri)
+
+
+def dump_schema(schema):
+    return jsonref.dumps(schema)
+
+
+def load_context(context):
+    logging.debug('Loading context: {}'.format(context))
+    if not context:
+        return context
+    elif isinstance(context, list):
+        contexts = []
+        for c in context:
+            contexts.append(load_context(c))
+        return contexts
+    elif isinstance(context, dict):
+        return dict(context)
+    elif isinstance(context, basestring):
         try:
-            return object.__getattr__(self, key)
+            with open(context) as f:
-        except AttributeError:
+                return dict(json.loads(f.read()))
-            try:
+        except IOError:
-                return super(Leaf, self).__getitem__(self._get_key(key))
-            except KeyError:
-                raise AttributeError()
-
-    def __setattr__(self, key, value):
-        try:
-            object.__getattr__(self, key)
-            object.__setattr__(self, key, value)
-        except AttributeError:
-            key = self._get_key(key)
-            if key == "@context":
-                value = self.get_context(value)
-            elif key == "@id":
-                value = self.get_id(value)
-            if key[0] == "_":
-                object.__setattr__(self, key, value)
-            else:
-                if value is None:
-                    try:
-                        super(Leaf, self).__delitem__(key)
-                    except KeyError:
-                        pass
-                else:
-                    super(Leaf, self).__setitem__(key, value)
-
-    def get_id(self, id):
-        """
-        Get id, dealing with prefixes
-        """
-        # This is not the most elegant solution to change the @id attribute,
-        # but it is the quickest way to have it included in the dictionary
-        # without extra boilerplate.
-        if id and self._prefix and ":" not in id:
-            return "{}{}".format(self._prefix, id)
-        else:
-            return id
-
-    def __delattr__(self, key):
-        if key in self.__dict__:
-            del self.__dict__[key]
-        else:
-            super(Leaf, self).__delitem__(self._get_key(key))
-
-    def _get_key(self, key):
-        if key[0] == "_":
-            return key
-        elif key in ["context", "id"]:
-            return "@{}".format(key)
-        else:
-            return key
-
-    @staticmethod
-    def get_context(context):
-        if isinstance(context, list):
-            contexts = []
-            for c in context:
-                contexts.append(Response.get_context(c))
-            return contexts
-        elif isinstance(context, dict):
             return context
-        elif isinstance(context, basestring):
+    else:
-            try:
+        raise AttributeError('Please, provide a valid context')
-                with open(context) as f:
-                    return json.loads(f.read())
-            except IOError:
-                return context
 
-    def compact(self):
-        return jsonld.compact(self, self.get_context(self.context))
 
-    def frame(self, frame=None, options=None):
+base_context = load_context(CONTEXT_PATH)
-        if frame is None:
-            frame = self._frame
-        if options is None:
-            options = {}
-        return jsonld.frame(self, frame, options)
 
-    def jsonld(self, frame=None, options=None,
-               context=None, removeContext=None):
-        if removeContext is None:
-            removeContext = Response._context  # Loop?
-        if frame is None:
-            frame = self._frame
-        if context is None:
-            context = self.context
-        else:
-            context = self.get_context(context)
-        # For some reason, this causes errors with pyld
-        # if options is None:
-            # options = {"expandContext": context.copy() }
-        js = self
-        if frame:
-            logging.debug("Framing: %s", json.dumps(self, indent=4))
-            logging.debug("Framing with %s", json.dumps(frame, indent=4))
-            js = jsonld.frame(js, frame, options)
-            logging.debug("Result: %s", json.dumps(js, indent=4))
-            logging.debug("Compacting with %s", json.dumps(context, indent=4))
-            js = jsonld.compact(js, context, options)
-            logging.debug("Result: %s", json.dumps(js, indent=4))
-        if removeContext == context:
-            del js["@context"]
-        return js
 
-    def to_JSON(self, removeContext=None):
+def register(rsubclass, rtype=None):
-        return json.dumps(self.jsonld(removeContext=removeContext),
+    BaseMeta.register(rsubclass, rtype)
-                          default=lambda o: o.__dict__,
+
-                          sort_keys=True, indent=4)
+
+class BaseModel(with_metaclass(BaseMeta, CustomDict)):
+    '''
+    Entities of the base model are a special kind of dictionary that emulates
+    a JSON-LD object. The structure of the dictionary is checked via JSON-schema.
+    For convenience, the values can also be accessed as attributes
+    (a la Javascript). e.g.:
+
+    >>> myobject.key == myobject['key']
+    True
+    >>> myobject.ns__name == myobject['ns:name']
+    True
+
+    Additionally, subclasses of this class can specify default values for their
+    instances. These defaults are inherited by subclasses. e.g.:
+
+    >>> class NewModel(BaseModel):
+    ...     mydefault = 5
+    >>> n1 = NewModel()
+    >>> n1['mydefault'] == 5
+    True
+    >>> n1.mydefault = 3
+    >>> n1['mydefault'] = 3
+    True
+    >>> n2 = NewModel()
+    >>> n2 == 5
+    True
+    >>> class SubModel(NewModel):
+            pass
+    >>> subn = SubModel()
+    >>> subn.mydefault == 5
+    True
+
+    Lastly, every subclass that also specifies a schema will get registered, so it
+    is possible to deserialize JSON and get the right type.
+    i.e. to recover an instance of the original class from a plain JSON.
+
+    '''
+
+    schema_file = DEFINITIONS_FILE
+    _context = base_context["@context"]
+
+    def __init__(self, *args, **kwargs):
+        auto_id = kwargs.pop('_auto_id', True)
+
+        super(BaseModel, self).__init__(*args, **kwargs)
+
+        if auto_id:
+            self.id
+
+        if '@type' not in self:
+            logger.warn('Created an instance of an unknown model')
+
+    @property
+    def id(self):
+        if '@id' not in self:
+            self['@id'] = '_:{}_{}'.format(type(self).__name__, time.time())
+        return self['@id']
+
+    @id.setter
+    def id(self, value):
+        self['@id'] = value
 
     def flask(self,
               in_headers=False,
-              url="http://demos.gsi.dit.upm.es/senpy/senpy.jsonld"):
+              headers=None,
+              outformat='json-ld',
+              **kwargs):
         """
-        Return the values and error to be used in flask
+        Return the values and error to be used in flask.
+        So far, it returns a fixed context. We should store/generate different
+        contexts if the plugin adds more aliases.
         """
-        js = self.jsonld()
+        headers = headers or {}
-        headers = None
+        kwargs["with_context"] = not in_headers
-        if in_headers:
+        content, mimetype = self.serialize(format=outformat,
-            ctx = js["@context"]
+                                           with_mime=True,
-            headers = {
+                                           **kwargs)
-                "Link": ('<%s>;'
+
-                         'rel="http://www.w3.org/ns/json-ld#context";'
+        if outformat == 'json-ld' and in_headers:
-                         ' type="application/ld+json"' % url)
+            headers.update({
-            }
+                "Link":
-            del js["@context"]
+                ('<%s>;'
-        return FlaskResponse(json.dumps(js, indent=4),
+                    'rel="http://www.w3.org/ns/json-ld#context";'
-                             status=self.get("status", 200),
+                    ' type="application/ld+json"' % kwargs.get('context_uri'))
-                             headers=headers,
+            })
-                             mimetype="application/json")
+        return FlaskResponse(
+            response=content,
+            status=self.get('status', 200),
+            headers=headers,
+            mimetype=mimetype)
+
+    def serialize(self, format='json-ld', with_mime=False, **kwargs):
+        js = self.jsonld(**kwargs)
+        content = json.dumps(js, indent=2, sort_keys=True)
+        if format == 'json-ld':
+            mimetype = "application/json"
+        elif format in ['turtle', 'ntriples']:
+            logger.debug(js)
+            base = kwargs.get('prefix')
+            g = Graph().parse(
+                data=content,
+                format='json-ld',
+                base=base,
+                context=[self._context,
+                         {'@base': base}])
+            logger.debug(
+                'Parsing with prefix: {}'.format(kwargs.get('prefix')))
+            content = g.serialize(format=format,
+                                  base=base).decode('utf-8')
+            mimetype = 'text/{}'.format(format)
+        else:
+            raise Error('Unknown outformat: {}'.format(format))
+        if with_mime:
+            return content, mimetype
+        else:
+            return content
+
+    def jsonld(self,
+               with_context=False,
+               context_uri=None,
+               prefix=None,
+               expanded=False):
+
+        result = self.serializable()
+
+        if expanded:
+            result = jsonld.expand(
+                result, options={'base': prefix,
+                                 'expandContext': self._context})[0]
+        if not with_context:
+            try:
+                del result['@context']
+            except KeyError:
+                pass
+        elif context_uri:
+            result['@context'] = context_uri
+        else:
+            result['@context'] = self._context
+
+        return result
+
+    def validate(self, obj=None):
+        if not obj:
+            obj = self
+        if hasattr(obj, "jsonld"):
+            obj = obj.jsonld()
+        self._validator.validate(obj)
+
+    def prov(self, another):
+        self['prov:wasGeneratedBy'] = another.id
 
 
-class Response(Leaf):
+def subtypes():
-    _context = Leaf.get_context("{}/context.jsonld".format(
+    return BaseMeta._subtypes
-        os.path.dirname(os.path.realpath(__file__))))
-    _frame = {
-        "@context": _context,
-        "analysis": {
-            "@explicit": True,
-            "maxPolarityValue": {},
-            "minPolarityValue": {},
-            "name": {},
-            "version": {},
-        },
-        "entries": {}
-    }
-
-    def __init__(self, *args, **kwargs):
-        context = kwargs.pop("context", None)
-        frame = kwargs.pop("frame", None)
-        if context is None:
-            context = self._context
-        self.context = context
-        super(Response, self).__init__(
-            *args, context=context, frame=frame, **kwargs)
-        if self._frame is not None and "entries" in self._frame:
-            self.analysis = []
-            self.entries = []
-
-    def jsonld(self, frame=None, options=None, context=None, removeContext={}):
-        return super(Response, self).jsonld(frame,
-                                            options,
-                                            context,
-                                            removeContext)
 
 
-class Entry(Leaf):
+def from_dict(indict, cls=None):
-    _context = {
+    if not cls:
-        "@vocab": ("http://persistence.uni-leipzig.org/"
+        target = indict.get('@type', None)
-                   "nlp2rdf/ontologies/nif-core#")
+        cls = BaseModel
-
+        try:
-    }
+            cls = subtypes()[target]
-
+        except KeyError:
-    def __init__(self, text=None, emotion_sets=None, opinions=None, **kwargs):
+            pass
-        super(Entry, self).__init__(**kwargs)
+    outdict = dict()
-        if text:
+    for k, v in indict.items():
-            self.text = text
+        if k == '@context':
-        self.emotionSets = emotion_sets if emotion_sets else []
+            pass
-        self.opinions = opinions if opinions else []
+        elif isinstance(v, dict):
+            v = from_dict(indict[k])
+        elif isinstance(v, list):
+            v = v[:]
+            for ix, v2 in enumerate(v):
+                if isinstance(v2, dict):
+                    v[ix] = from_dict(v2)
+        outdict[k] = copy.copy(v)
+    return cls(**outdict)
 
 
-class Opinion(Leaf):
+def from_string(string, **kwargs):
-    _context = {
+    return from_dict(json.loads(string), **kwargs)
-        "@vocab": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"
-    }
-
-    def __init__(self, polarityValue=None, hasPolarity=None, *args, **kwargs):
-        super(Opinion, self).__init__(*args,
-                                      **kwargs)
-        if polarityValue is not None:
-            self.hasPolarityValue = polarityValue
-        if hasPolarity is not None:
-            self.hasPolarity = hasPolarity
 
 
-class EmotionSet(Leaf):
+def from_json(injson):
-    _context = {}
+    indict = json.loads(injson)
-
+    return from_dict(indict)
-    def __init__(self, emotions=None, *args, **kwargs):
-        if not emotions:
-            emotions = []
-        super(EmotionSet, self).__init__(context=EmotionSet._context,
-                                         *args,
-                                         **kwargs)
-        self.emotions = emotions or []
 
 
-class Emotion(Leaf):
+class Entry(BaseModel):
-    _context = {}
+    schema = 'entry'
+
+    text = alias('nif:isString')
 
 
-class Error(Leaf):
+class Sentiment(BaseModel):
-    # A better pattern would be this:
+    schema = 'sentiment'
-    # http://flask.pocoo.org/docs/0.10/patterns/apierrors/
-    _frame = {}
-    _context = {}
 
-    def __init__(self, *args, **kwargs):
+    polarity = alias('marl:hasPolarity')
+    polarityValue = alias('marl:hasPolarityValue')
+
+
+class Error(BaseModel, Exception):
+    schema = 'error'
+
+    def __init__(self, message='Generic senpy exception', *args, **kwargs):
+        Exception.__init__(self, message)
         super(Error, self).__init__(*args, **kwargs)
+        self.message = message
+
+    def __str__(self):
+        if not hasattr(self, 'errors'):
+            return self.message
+        return '{}:\n\t{}'.format(self.message, self.errors)
+
+    def __hash__(self):
+        return Exception.__hash__(self)
+
+
+# Add the remaining schemas programmatically
+
+def _class_from_schema(name, schema=None, schema_file=None, base_classes=None):
+    base_classes = base_classes or []
+    base_classes.append(BaseModel)
+    attrs = {}
+    if schema:
+        attrs['schema'] = schema
+    elif schema_file:
+        attrs['schema_file'] = schema_file
+    else:
+        attrs['schema'] = name
+    name = "".join((name[0].upper(), name[1:]))
+    return BaseMeta(name, base_classes, attrs)
+
+
+def _add_class_from_schema(*args, **kwargs):
+    generatedClass = _class_from_schema(*args, **kwargs)
+    globals()[generatedClass.__name__] = generatedClass
+    del generatedClass
+
+
+for i in [
+        'aggregatedEvaluation',
+        'analysis',
+        'dataset',
+        'datasets',
+        'emotion',
+        'emotionConversion',
+        'emotionConversionPlugin',
+        'emotionAnalysis',
+        'emotionModel',
+        'emotionPlugin',
+        'emotionSet',
+        'evaluation',
+        'entity',
+        'help',
+        'metric',
+        'plugin',
+        'plugins',
+        'response',
+        'results',
+        'sentimentPlugin',
+        'suggestion',
+        'topic',
+
+]:
+    _add_class_from_schema(i)

senpy/plugins.py

@@ -1,155 +0,0 @@
-
-import inspect
-import os.path
-import shelve
-import logging
-import ConfigParser
-from .models import Response, Leaf
-
-logger = logging.getLogger(__name__)
-
-PARAMS = {
-    "input": {
-        "@id": "input",
-        "aliases": ["i", "input"],
-        "required": True,
-        "help": "Input text"
-    },
-    "informat": {
-        "@id": "informat",
-        "aliases": ["f", "informat"],
-        "required": False,
-        "default": "text",
-        "options": ["turtle", "text"],
-    },
-    "intype": {
-        "@id": "intype",
-        "aliases": ["intype", "t"],
-        "required": False,
-        "default": "direct",
-        "options": ["direct", "url", "file"],
-    },
-    "outformat": {
-        "@id": "outformat",
-        "aliases": ["outformat", "o"],
-        "default": "json-ld",
-        "required": False,
-        "options": ["json-ld"],
-    },
-    "language": {
-        "@id": "language",
-        "aliases": ["language", "l"],
-        "required": False,
-    },
-    "prefix": {
-        "@id": "prefix",
-        "aliases": ["prefix", "p"],
-        "required": True,
-        "default": "",
-    },
-    "urischeme": {
-        "@id": "urischeme",
-        "aliases": ["urischeme", "u"],
-        "required": False,
-        "default": "RFC5147String",
-        "options": "RFC5147String"
-    },
-}
-
-
-class SenpyPlugin(Leaf):
-    _context = Leaf.get_context(Response._context)
-    _frame = {"@context": _context,
-              "name": {},
-              "extra_params": {"@container": "@index"},
-              "@explicit": True,
-              "version": {},
-              "repo": None,
-              "is_activated": {},
-              "params": None,
-              }
-
-    def __init__(self, info=None):
-        if not info:
-            raise ValueError(("You need to provide configuration"
-                              "information for the plugin."))
-        logger.debug("Initialising {}".format(info))
-        super(SenpyPlugin, self).__init__()
-        self.name = info["name"]
-        self.version = info["version"]
-        self.id = "{}_{}".format(self.name, self.version)
-        self.params = info.get("params", PARAMS.copy())
-        if "@id" not in self.params:
-            self.params["@id"] = "params_%s" % self.id
-        self.extra_params = info.get("extra_params", {})
-        self.params.update(self.extra_params.copy())
-        if "@id" not in self.extra_params:
-            self.extra_params["@id"] = "extra_params_%s" % self.id
-        self.is_activated = False
-        self._info = info
-
-    def get_folder(self):
-        return os.path.dirname(inspect.getfile(self.__class__))
-
-    def analyse(self, *args, **kwargs):
-        logger.debug("Analysing with: {} {}".format(self.name, self.version))
-        pass
-
-    def activate(self):
-        pass
-
-    def deactivate(self):
-        pass
-
-    def jsonld(self, parameters=False, *args, **kwargs):
-        nframe = kwargs.pop("frame", self._frame)
-        if parameters:
-            nframe = nframe.copy()
-            nframe["params"] = {}
-        return super(SenpyPlugin, self).jsonld(frame=nframe, *args, **kwargs)
-
-    @property
-    def id(self):
-        return "{}_{}".format(self.name, self.version)
-
-
-class SentimentPlugin(SenpyPlugin):
-
-    def __init__(self, info, *args, **kwargs):
-        super(SentimentPlugin, self).__init__(info, *args, **kwargs)
-        self.minPolarityValue = float(info.get("minPolarityValue", 0))
-        self.maxPolarityValue = float(info.get("maxPolarityValue", 1))
-
-
-class EmotionPlugin(SenpyPlugin):
-
-    def __init__(self, info, *args, **kwargs):
-        resp = super(EmotionPlugin, self).__init__(info, *args, **kwargs)
-        self.minEmotionValue = float(info.get("minEmotionValue", 0))
-        self.maxEmotionValue = float(info.get("maxEmotionValue", 0))
-
-
-class ShelfMixin(object):
-
-    @property
-    def sh(self):
-        if not hasattr(self, '_sh') or not self._sh:
-            self._sh = shelve.open(self.shelf_file, writeback=True)
-        return self._sh
-
-    @sh.deleter
-    def sh(self):
-        if hasattr(self, '_sh'):
-            del(self._sh)
-
-    @property
-    def shelf_file(self):
-        if not hasattr(self, '_shelf_file') or not self._shelf_file:
-            if hasattr(self, '_info') and 'shelf_file' in self._info:
-                self._shelf_file = self._info['shelf_file']
-            else:
-                self._shelf_file = os.path.join(self.get_folder(), self.name + '.db')
-        return self._shelf_file 
-
-    def close(self):
-        del(self._sh)

senpy/plugins/__init__.py

@@ -0,0 +1,733 @@
+from future import standard_library
+standard_library.install_aliases()
+
+from future.utils import with_metaclass
+from functools import partial
+
+import os.path
+import os
+import re
+import pickle
+import logging
+import pprint
+
+import inspect
+import sys
+import subprocess
+import importlib
+import yaml
+import threading
+import nltk
+
+from .. import models, utils
+from .. import api
+from .. import gsitk_compat
+from .. import testing
+
+logger = logging.getLogger(__name__)
+
+
+class PluginMeta(models.BaseMeta):
+    _classes = {}
+
+    def __new__(mcs, name, bases, attrs, **kwargs):
+        plugin_type = []
+        if hasattr(bases[0], 'plugin_type'):
+            plugin_type += bases[0].plugin_type
+        plugin_type.append(name)
+        alias = attrs.get('name', name)
+        attrs['plugin_type'] = plugin_type
+        attrs['name'] = alias
+        if 'description' not in attrs:
+            doc = attrs.get('__doc__', None)
+            if doc:
+                attrs['description'] = doc
+            else:
+                logger.warning(
+                    ('Plugin {} does not have a description. '
+                     'Please, add a short summary to help other developers'
+                     ).format(name))
+        cls = super(PluginMeta, mcs).__new__(mcs, name, bases, attrs)
+
+        if alias in mcs._classes:
+            if os.environ.get('SENPY_TESTING', ""):
+                raise Exception(
+                    ('The type of plugin {} already exists. '
+                     'Please, choose a different name').format(name))
+            else:
+                logger.warning('Overloading plugin class: {}'.format(alias))
+        mcs._classes[alias] = cls
+        return cls
+
+    @classmethod
+    def for_type(cls, ptype):
+        return cls._classes[ptype]
+
+
+class Plugin(with_metaclass(PluginMeta, models.Plugin)):
+    '''
+    Base class for all plugins in senpy.
+    A plugin must provide at least these attributes:
+
+        - version
+        - description (or docstring)
+        - author
+
+    Additionally, they may provide a URL (url) of a repository or website.
+
+    '''
+
+    def __init__(self, info=None, data_folder=None, **kwargs):
+        """
+        Provides a canonical name for plugins and serves as base for other
+        kinds of plugins.
+        """
+        logger.debug("Initialising {}".format(info))
+        super(Plugin, self).__init__(**kwargs)
+        if info:
+            self.update(info)
+        self.validate()
+        self.id = 'endpoint:plugins/{}_{}'.format(self['name'],
+                                                  self['version'])
+        self.is_activated = False
+        self._lock = threading.Lock()
+        self._directory = os.path.abspath(
+            os.path.dirname(inspect.getfile(self.__class__)))
+
+        data_folder = data_folder or os.getcwd()
+        subdir = os.path.join(data_folder, self.name)
+
+        self._data_paths = [
+            data_folder,
+            subdir,
+            self._directory,
+            os.path.join(self._directory, 'data'),
+        ]
+
+        if os.path.exists(subdir):
+            data_folder = subdir
+        self.data_folder = data_folder
+
+        self._log = logging.getLogger('{}.{}'.format(__name__, self.name))
+
+    @property
+    def log(self):
+        return self._log
+
+    def validate(self):
+        missing = []
+        for x in ['name', 'description', 'version']:
+            if x not in self:
+                missing.append(x)
+        if missing:
+            raise models.Error(
+                'Missing configuration parameters: {}'.format(missing))
+
+    def get_folder(self):
+        return os.path.dirname(inspect.getfile(self.__class__))
+
+    def activate(self):
+        pass
+
+    def deactivate(self):
+        pass
+
+    def process(self, request, **kwargs):
+        """
+        An implemented plugin should override this method.
+        Here, we assume that a process_entries method exists."""
+        newentries = list(
+            self.process_entries(request.entries, request.parameters))
+        request.entries = newentries
+        return request
+
+    def process_entries(self, entries, parameters):
+        for entry in entries:
+            self.log.debug('Processing entry with plugin {}: {}'.format(
+                self, entry))
+            results = self.process_entry(entry, parameters)
+            if inspect.isgenerator(results):
+                for result in results:
+                    yield result
+            else:
+                yield results
+
+    def process_entry(self, entry, parameters):
+        """
+        This base method is here to adapt plugins which only
+        implement the *process* function.
+        Note that this method may yield an annotated entry or a list of
+        entries (e.g. in a tokenizer)
+        """
+        raise NotImplementedError(
+            'You need to implement process, process_entries or process_entry in your plugin'
+        )
+
+    def test(self, test_cases=None):
+        if not test_cases:
+            if not hasattr(self, 'test_cases'):
+                raise AttributeError(
+                    ('Plugin {} [{}] does not have any defined '
+                     'test cases').format(self.id,
+                                          inspect.getfile(self.__class__)))
+            test_cases = self.test_cases
+        for case in test_cases:
+            try:
+                self.test_case(case)
+                self.log.debug('Test case passed:\n{}'.format(
+                    pprint.pformat(case)))
+            except Exception as ex:
+                self.log.warning('Test case failed:\n{}'.format(
+                    pprint.pformat(case)))
+                raise
+
+    def test_case(self, case, mock=testing.MOCK_REQUESTS):
+        if 'entry' not in case and 'input' in case:
+            entry = models.Entry(_auto_id=False)
+            entry.nif__isString = case['input']
+            case['entry'] = entry
+        entry = models.Entry(case['entry'])
+        given_parameters = case.get('params', case.get('parameters', {}))
+        expected = case.get('expected', None)
+        should_fail = case.get('should_fail', False)
+        responses = case.get('responses', [])
+
+        try:
+            request = models.Response()
+            request.parameters = api.parse_params(given_parameters,
+                                                  self.extra_params)
+            request.entries = [
+                entry,
+            ]
+
+            method = partial(self.process, request)
+
+            if mock:
+                res = method()
+            else:
+                with testing.patch_all_requests(responses):
+                    res = method()
+
+            if not isinstance(expected, list):
+                expected = [expected]
+            utils.check_template(res.entries, expected)
+            res.validate()
+        except models.Error:
+            if should_fail:
+                return
+            raise
+        assert not should_fail
+
+    def find_file(self, fname):
+        for p in self._data_paths:
+            alternative = os.path.join(p, fname)
+            if os.path.exists(alternative):
+                return alternative
+        raise IOError('File does not exist: {}'.format(fname))
+
+    def open(self, fpath, mode='r'):
+        if 'w' in mode:
+            # When writing, only use absolute paths or data_folder
+            if not os.path.isabs(fpath):
+                fpath = os.path.join(self.data_folder, fpath)
+        else:
+            fpath = self.find_file(fpath)
+
+        return open(fpath, mode=mode)
+
+    def serve(self, debug=True, **kwargs):
+        utils.easy(plugin_list=[self, ], plugin_folder=None, debug=debug, **kwargs)
+
+
+# For backwards compatibility
+SenpyPlugin = Plugin
+
+
+class Analysis(Plugin):
+    '''
+    A subclass of Plugin that analyses text and provides an annotation.
+    '''
+
+    def analyse(self, request, parameters):
+        return super(Analysis, self).process(request)
+
+    def analyse_entries(self, entries, parameters):
+        for i in super(Analysis, self).process_entries(entries, parameters):
+            yield i
+
+    def process(self, request, **kwargs):
+        return self.analyse(request, request.parameters)
+
+    def process_entries(self, entries, parameters):
+        for i in self.analyse_entries(entries, parameters):
+            yield i
+
+    def process_entry(self, entry, parameters, **kwargs):
+        if hasattr(self, 'analyse_entry'):
+            for i in self.analyse_entry(entry, parameters):
+                yield i
+        else:
+            super(Analysis, self).process_entry(entry, parameters, **kwargs)
+
+
+AnalysisPlugin = Analysis
+
+
+class Conversion(Plugin):
+    '''
+    A subclass of Plugins that convert between different annotation models.
+    e.g. a conversion of emotion models, or normalization of sentiment values.
+    '''
+
+    def process(self, response, plugins=None, **kwargs):
+        plugins = plugins or []
+        newentries = []
+        for entry in response.entries:
+            newentries.append(
+                self.convert_entry(entry, response.parameters, plugins))
+        response.entries = newentries
+        return response
+
+    def convert_entry(self, entry, parameters, conversions_applied):
+        raise NotImplementedError(
+            'You should implement a way to convert each entry, or a custom process method'
+        )
+
+
+ConversionPlugin = Conversion
+
+
+class SentimentPlugin(Analysis, models.SentimentPlugin):
+    '''
+    Sentiment plugins provide sentiment annotation (using Marl)
+    '''
+    minPolarityValue = 0
+    maxPolarityValue = 1
+
+    def test_case(self, case):
+        if 'polarity' in case:
+            expected = case.get('expected', {})
+            s = models.Sentiment(_auto_id=False)
+            s.marl__hasPolarity = case['polarity']
+            if 'sentiments' not in expected:
+                expected['sentiments'] = []
+            expected['sentiments'].append(s)
+            case['expected'] = expected
+        super(SentimentPlugin, self).test_case(case)
+
+
+class EmotionPlugin(Analysis, models.EmotionPlugin):
+    '''
+    Emotion plugins provide emotion annotation (using Onyx)
+    '''
+    minEmotionValue = 0
+    maxEmotionValue = 1
+
+
+class EmotionConversion(Conversion):
+    '''
+    A subclass of Conversion that converts emotion annotations using different models
+    '''
+
+    def can_convert(self, fromModel, toModel):
+        '''
+        Whether this plugin can convert from fromModel to toModel.
+        If fromModel is None, it is interpreted as "any Model"
+        '''
+        for pair in self.onyx__doesConversion:
+            if (pair['onyx:conversionTo'] == toModel) and \
+               ((fromModel is None) or (pair['onyx:conversionFrom'] == fromModel)):
+                return True
+        return False
+
+
+EmotionConversionPlugin = EmotionConversion
+
+
+class PostProcessing(Plugin):
+    def check(self, request, plugins):
+        '''Should this plugin be run for this request?'''
+        return False
+
+
+class Box(AnalysisPlugin):
+    '''
+    Black box plugins delegate analysis to a function.
+    The flow is like so:
+
+    .. code-block::
+
+                   entry --> input() --> predict_one() --> output() --> entry'
+
+
+    In other words: their ``input`` method convers a query (entry and a set of parameters) into
+    the input to the box method. The ``output`` method convers the results given by the box into
+    an entry that senpy can handle.
+    '''
+
+    def input(self, entry, params=None):
+        '''Transforms a query (entry+param) into an input for the black box'''
+        return entry
+
+    def output(self, output, entry=None, params=None):
+        '''Transforms the results of the black box into an entry'''
+        return output
+
+    def predict_one(self, input):
+        raise NotImplementedError(
+            'You should define the behavior of this plugin')
+
+    def process_entries(self, entries, params):
+        for entry in entries:
+            input = self.input(entry=entry, params=params)
+            results = self.predict_one(input=input)
+            yield self.output(output=results, entry=entry, params=params)
+
+    def fit(self, X=None, y=None):
+        return self
+
+    def transform(self, X):
+        return [self.predict_one(x) for x in X]
+
+    def predict(self, X):
+        return self.transform(X)
+
+    def fit_transform(self, X, y):
+        self.fit(X, y)
+        return self.transform(X)
+
+    def as_pipe(self):
+        pipe = gsitk_compat.Pipeline([('plugin', self)])
+        pipe.name = self.name
+        return pipe
+
+
+class TextBox(Box):
+    '''A black box plugin that takes only text as input'''
+
+    def input(self, entry, params):
+        entry = super(TextBox, self).input(entry, params)
+        return entry['nif:isString']
+
+
+class SentimentBox(TextBox, SentimentPlugin):
+    '''
+    A box plugin where the output is only a polarity label or a tuple (polarity, polarityValue)
+    '''
+
+    def output(self, output, entry, **kwargs):
+        s = models.Sentiment()
+        try:
+            label, value = output
+        except ValueError:
+            label, value = output, None
+        s.prov(self)
+        s.polarity = label
+        if value is not None:
+            s.polarityValue = value
+        entry.sentiments.append(s)
+        return entry
+
+
+class EmotionBox(TextBox, EmotionPlugin):
+    '''
+    A box plugin where the output is only an a tuple of emotion labels
+    '''
+
+    def output(self, output, entry, **kwargs):
+        if not isinstance(output, list):
+            output = [output]
+        s = models.EmotionSet()
+        entry.emotions.append(s)
+        for label in output:
+            e = models.Emotion(onyx__hasEmotionCategory=label)
+            s.append(e)
+        return entry
+
+
+class MappingMixin(object):
+    @property
+    def mappings(self):
+        return self._mappings
+
+    @mappings.setter
+    def mappings(self, value):
+        self._mappings = value
+
+    def output(self, output, entry, params):
+        output = self.mappings.get(output, self.mappings.get(
+            'default', output))
+        return super(MappingMixin, self).output(
+            output=output, entry=entry, params=params)
+
+
+class ShelfMixin(object):
+    @property
+    def sh(self):
+        if not hasattr(self, '_sh') or self._sh is None:
+            self._sh = {}
+            if os.path.isfile(self.shelf_file):
+                try:
+                    with self.open(self.shelf_file, 'rb') as p:
+                        self._sh = pickle.load(p)
+                except (IndexError, EOFError, pickle.UnpicklingError):
+                    self.log.warning('Corrupted shelf file: {}'.format(
+                        self.shelf_file))
+                    if not self.get('force_shelf', False):
+                        raise
+        return self._sh
+
+    @sh.deleter
+    def sh(self):
+        if os.path.isfile(self.shelf_file):
+            os.remove(self.shelf_file)
+            del self._sh
+        self.save()
+
+    @sh.setter
+    def sh(self, value):
+        self._sh = value
+
+    @property
+    def shelf_file(self):
+        if not hasattr(self, '_shelf_file') or not self._shelf_file:
+            self._shelf_file = os.path.join(self.data_folder, self.name + '.p')
+        return self._shelf_file
+
+    @shelf_file.setter
+    def shelf_file(self, value):
+        self._shelf_file = value
+
+    def save(self):
+        self.log.debug('Saving pickle')
+        if hasattr(self, '_sh') and self._sh is not None:
+            with self.open(self.shelf_file, 'wb') as f:
+                pickle.dump(self._sh, f)
+
+
+def pfilter(plugins, plugin_type=Analysis, **kwargs):
+    """ Filter plugins by different criteria """
+    if isinstance(plugins, models.Plugins):
+        plugins = plugins.plugins
+    elif isinstance(plugins, dict):
+        plugins = plugins.values()
+    logger.debug('#' * 100)
+    logger.debug('plugin_type {}'.format(plugin_type))
+    if plugin_type:
+        if isinstance(plugin_type, PluginMeta):
+            plugin_type = plugin_type.__name__
+        try:
+            plugin_type = plugin_type[0].upper() + plugin_type[1:]
+            pclass = globals()[plugin_type]
+            logger.debug('Class: {}'.format(pclass))
+            candidates = filter(lambda x: isinstance(x, pclass), plugins)
+        except KeyError:
+            raise models.Error('{} is not a valid type'.format(plugin_type))
+    else:
+        candidates = plugins
+
+    logger.debug(candidates)
+
+    def matches(plug):
+        res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
+        logger.debug("matching {} with {}: {}".format(plug.name, kwargs, res))
+        return res
+
+    if kwargs:
+        candidates = filter(matches, candidates)
+    return candidates
+
+
+def load_module(name, root=None):
+    if root:
+        sys.path.append(root)
+    tmp = importlib.import_module(name)
+    if root:
+        sys.path.remove(root)
+    return tmp
+
+
+def _log_subprocess_output(process):
+    for line in iter(process.stdout.readline, b''):
+        logger.info('%r', line)
+    for line in iter(process.stderr.readline, b''):
+        logger.error('%r', line)
+
+
+def install_deps(*plugins):
+    installed = False
+    nltk_resources = set()
+    for info in plugins:
+        requirements = info.get('requirements', [])
+        if requirements:
+            pip_args = [sys.executable, '-m', 'pip', 'install']
+            for req in requirements:
+                pip_args.append(req)
+            logger.info('Installing requirements: ' + str(requirements))
+            process = subprocess.Popen(
+                pip_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            _log_subprocess_output(process)
+            exitcode = process.wait()
+            installed = True
+            if exitcode != 0:
+                raise models.Error(
+                    "Dependencies not properly installed: {}".format(pip_args))
+        nltk_resources |= set(info.get('nltk_resources', []))
+
+    installed |= nltk.download(list(nltk_resources))
+    return installed
+
+
+is_plugin_file = re.compile(r'.*\.senpy$|senpy_[a-zA-Z0-9_]+\.py$|'
+                            '^(?!test_)[a-zA-Z0-9_]+_plugin.py$')
+
+
+def find_plugins(folders):
+    for search_folder in folders:
+        for root, dirnames, filenames in os.walk(search_folder):
+            # Do not look for plugins in hidden or special folders
+            dirnames[:] = [d for d in dirnames if d[0] not in ['.', '_']]
+            for filename in filter(is_plugin_file.match, filenames):
+                fpath = os.path.join(root, filename)
+                yield fpath
+
+
+def from_path(fpath, install_on_fail=False, **kwargs):
+    logger.debug("Loading plugin from {}".format(fpath))
+    if fpath.endswith('.py'):
+        # We asume root is the dir of the file, and module is the name of the file
+        root = os.path.dirname(fpath)
+        module = os.path.basename(fpath)[:-3]
+        for instance in _from_module_name(module=module, root=root, **kwargs):
+            yield instance
+    else:
+        info = parse_plugin_info(fpath)
+        yield from_info(info, install_on_fail=install_on_fail, **kwargs)
+
+
+def from_folder(folders, loader=from_path, **kwargs):
+    plugins = []
+    for fpath in find_plugins(folders):
+        for plugin in loader(fpath, **kwargs):
+            plugins.append(plugin)
+    return plugins
+
+
+def from_info(info, root=None, install_on_fail=True, **kwargs):
+    if any(x not in info for x in ('module', )):
+        raise ValueError('Plugin info is not valid: {}'.format(info))
+    module = info["module"]
+
+    if not root and '_path' in info:
+        root = os.path.dirname(info['_path'])
+
+    fun = partial(one_from_module, module, root=root, info=info, **kwargs)
+    try:
+        return fun()
+    except (ImportError, LookupError):
+        install_deps(info)
+        return fun()
+
+
+def parse_plugin_info(fpath):
+    logger.debug("Parsing plugin info: {}".format(fpath))
+    with open(fpath, 'r') as f:
+        info = yaml.load(f)
+    info['_path'] = fpath
+    return info
+
+
+def from_module(module, **kwargs):
+
+    if inspect.ismodule(module):
+        res = _from_loaded_module(module, **kwargs)
+    else:
+        res = _from_module_name(module, **kwargs)
+    for p in res:
+        yield p
+
+
+def one_from_module(module, root, info, **kwargs):
+    if '@type' in info:
+        cls = PluginMeta.from_type(info['@type'])
+        return cls(info=info, **kwargs)
+    instance = next(
+        from_module(module=module, root=root, info=info, **kwargs), None)
+    if not instance:
+        raise Exception("No valid plugin for: {}".format(module))
+    return instance
+
+
+def _classes_in_module(module):
+    for _, obj in inspect.getmembers(module):
+        if inspect.isclass(obj) and inspect.getmodule(obj) == module:
+            logger.debug(("Found plugin class:"
+                          " {}@{}").format(obj, inspect.getmodule(obj)))
+            yield obj
+
+
+def _instances_in_module(module):
+    for _, obj in inspect.getmembers(module):
+        if isinstance(obj, Plugin) and inspect.getmodule(obj) == module:
+            logger.debug(("Found plugin instance:"
+                          " {}@{}").format(obj, inspect.getmodule(obj)))
+            yield obj
+
+
+def _from_module_name(module, root, info=None, **kwargs):
+    module = load_module(module, root)
+    for plugin in _from_loaded_module(
+            module=module, root=root, info=info, **kwargs):
+        yield plugin
+
+
+def _from_loaded_module(module, info=None, **kwargs):
+    for cls in _classes_in_module(module):
+        yield cls(info=info, **kwargs)
+    for instance in _instances_in_module(module):
+        yield instance
+
+
+def evaluate(plugins, datasets, **kwargs):
+    ev = gsitk_compat.Eval(
+        tuples=None,
+        datasets=datasets,
+        pipelines=[plugin.as_pipe() for plugin in plugins])
+    ev.evaluate()
+    results = ev.results
+    evaluations = evaluations_to_JSONLD(results, **kwargs)
+    return evaluations
+
+
+def evaluations_to_JSONLD(results, flatten=False):
+    '''
+    Map the evaluation results to a JSONLD scheme
+    '''
+
+    evaluations = list()
+    metric_names = ['accuracy', 'precision_macro', 'recall_macro',
+                    'f1_macro', 'f1_weighted', 'f1_micro', 'f1_macro']
+
+    for index, row in results.iterrows():
+        evaluation = models.Evaluation()
+        if row.get('CV', True):
+            evaluation['@type'] = ['StaticCV', 'Evaluation']
+        evaluation.evaluatesOn = row['Dataset']
+        evaluation.evaluates = row['Model']
+        i = 0
+        if flatten:
+            metric = models.Metric()
+            for name in metric_names:
+                metric[name] = row[name]
+            evaluation.metrics.append(metric)
+        else:
+            # We should probably discontinue this representation
+            for name in metric_names:
+                metric = models.Metric()
+                metric['@id'] = 'Metric' + str(i)
+                metric['@type'] = name.capitalize()
+                metric.value = row[name]
+                evaluation.metrics.append(metric)
+                i += 1
+        evaluations.append(evaluation)
+    return evaluations

senpy/plugins/example/emoRand/emoRand.py

@@ -0,0 +1,34 @@
+import random
+
+from senpy.plugins import EmotionPlugin
+from senpy.models import EmotionSet, Emotion, Entry
+
+
+class EmoRand(EmotionPlugin):
+    name = "emoRand"
+    description = 'A sample plugin that returns a random emotion annotation'
+    author = '@balkian'
+    version = '0.1'
+    url = "https://github.com/gsi-upm/senpy-plugins-community"
+    requirements = {}
+    onyx__usesEmotionModel = "emoml:big6"
+
+    def analyse_entry(self, entry, params):
+        category = "emoml:big6happiness"
+        number = max(-1, min(1, random.gauss(0, 0.5)))
+        if number > 0:
+            category = "emoml:big6anger"
+        emotionSet = EmotionSet()
+        emotion = Emotion({"onyx:hasEmotionCategory": category})
+        emotionSet.onyx__hasEmotion.append(emotion)
+        emotionSet.prov__wasGeneratedBy = self.id
+        entry.emotions.append(emotionSet)
+        yield entry
+
+    def test(self):
+        params = dict()
+        results = list()
+        for i in range(100):
+            res = next(self.analyse_entry(Entry(nif__isString="Hello"), params))
+            res.validate()
+            results.append(res.emotions[0]['onyx:hasEmotion'][0]['onyx:hasEmotionCategory'])

senpy/plugins/example/emorand_plugin.py

@@ -0,0 +1,32 @@
+import random
+
+from senpy.plugins import EmotionPlugin
+from senpy.models import EmotionSet, Emotion, Entry
+
+
+class EmoRand(EmotionPlugin):
+    '''A sample plugin that returns a random emotion annotation'''
+    author = '@balkian'
+    version = '0.1'
+    url = "https://github.com/gsi-upm/senpy-plugins-community"
+    onyx__usesEmotionModel = "emoml:big6"
+
+    def analyse_entry(self, entry, params):
+        category = "emoml:big6happiness"
+        number = max(-1, min(1, random.gauss(0, 0.5)))
+        if number > 0:
+            category = "emoml:big6anger"
+        emotionSet = EmotionSet()
+        emotion = Emotion({"onyx:hasEmotionCategory": category})
+        emotionSet.onyx__hasEmotion.append(emotion)
+        emotionSet.prov__wasGeneratedBy = self.id
+        entry.emotions.append(emotionSet)
+        yield entry
+
+    def test(self):
+        params = dict()
+        results = list()
+        for i in range(100):
+            res = next(self.analyse_entry(Entry(nif__isString="Hello"), params))
+            res.validate()
+            results.append(res.emotions[0]['onyx:hasEmotion'][0]['onyx:hasEmotionCategory'])

senpy/plugins/example/rand_plugin.py

@@ -0,0 +1,36 @@
+import random
+from senpy import SentimentPlugin, Sentiment, Entry
+
+
+class Rand(SentimentPlugin):
+    '''A sample plugin that returns a random sentiment annotation'''
+    author = "@balkian"
+    version = '0.1'
+    url = "https://github.com/gsi-upm/senpy-plugins-community"
+    marl__maxPolarityValue = '1'
+    marl__minPolarityValue = "-1"
+
+    def analyse_entry(self, entry, params):
+        polarity_value = max(-1, min(1, random.gauss(0.2, 0.2)))
+        polarity = "marl:Neutral"
+        if polarity_value > 0:
+            polarity = "marl:Positive"
+        elif polarity_value < 0:
+            polarity = "marl:Negative"
+        sentiment = Sentiment(marl__hasPolarity=polarity,
+                              marl__polarityValue=polarity_value)
+        sentiment.prov(self)
+        entry.sentiments.append(sentiment)
+        yield entry
+
+    def test(self):
+        '''Run several random analyses.'''
+        params = dict()
+        results = list()
+        for i in range(50):
+            res = next(self.analyse_entry(Entry(nif__isString="Hello"),
+                                          params))
+            res.validate()
+            results.append(res.sentiments[0]['marl:hasPolarity'])
+        assert 'marl:Positive' in results
+        assert 'marl:Negative' in results