Change name to split, according to issue #37

* Add git fetch to (try to) fix github push from gitlab

.drone.yml

@@ -0,0 +1,9 @@
+build:
+  image: python:$$PYTHON_VERSION
+  commands:
+    - python setup.py test
+
+matrix:
+  PYTHON_VERSION:
+    - 2.7
+    - 3.4

.gitignore

@@ -2,4 +2,9 @@
 .*
 *egg-info
 dist
+build
 README.html
+__pycache__
+VERSION
+Dockerfile-*
+Dockerfile

.gitlab-ci.yml

@@ -0,0 +1,106 @@
+# Uncomment if you want to use docker-in-docker
+# image: gsiupm/dockermake:latest
+# services:
+# - docker:dind
+# When using dind, it's wise to use the overlayfs driver for
+# improved performance.
+
+stages:
+  - test
+  - push
+  - deploy
+  - clean
+
+before_script:
+  - docker login -u $HUB_USER -p $HUB_PASSWORD
+
+.test: &test_definition
+  stage: test
+  script:
+    - make -e test-$PYTHON_VERSION
+    
+test-3.5:
+  <<: *test_definition
+  variables:
+    PYTHON_VERSION: "3.5"
+
+test-2.7:
+  <<: *test_definition
+  variables:
+    PYTHON_VERSION: "2.7"
+
+.image: &image_definition
+  stage: push
+  script:
+    - make -e push-$PYTHON_VERSION
+  only:
+    - tags
+    - triggers
+
+push-3.5:
+  <<: *image_definition
+  variables:
+    PYTHON_VERSION: "3.5"
+
+push-2.7:
+  <<: *image_definition
+  variables:
+    PYTHON_VERSION: "2.7"
+
+push-latest:
+  <<: *image_definition
+  variables:
+    PYTHON_VERSION: latest
+  only:
+    - master
+    - triggers
+
+push-github:
+  stage: deploy
+  script:
+    - make -e push-github
+  only:
+    - master
+    - triggers
+
+deploy_pypi:
+  stage: deploy
+  script:   # Configure the PyPI credentials, then push the package, and cleanup the creds.
+    - echo "[server-login]" >> ~/.pypirc
+    - echo "username=" ${PYPI_USER} >> ~/.pypirc
+    - echo "password=" ${PYPI_PASSWORD} >> ~/.pypirc
+    - make pip_upload
+    - echo "" > ~/.pypirc && rm ~/.pypirc  # If the above fails, this won't run.
+  only:
+    - /^v?\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
+  except:
+    - branches
+
+deploy:
+  stage: deploy
+  environment: test
+  script:
+    - make -e deploy
+  only:
+    - master
+
+push-github:
+  stage: deploy
+  script:
+    - make -e push-github
+  only:
+    - master
+    - triggers
+
+clean :
+  stage: clean
+  script:
+    - make -e clean
+  when: manual
+
+cleanup_py:
+   stage: clean
+   when: always   # this is important; run even if preceding stages failed.
+   script:
+    - rm -vf ~/.pypirc  # we don't want to leave these around, but GitLab may clean up anyway.
+    - docker logout

.pre-commit-config.yaml

@@ -0,0 +1,5 @@
+-   repo: git://github.com/pre-commit/pre-commit-hooks
+    sha: e626cd57090d8df0be21e4df0f4e55cc3511d6ab
+    hooks:
+    -   id: flake8
+    -   id: check-json

.travis.yml

@@ -1,6 +1,12 @@
+sudo: required
+
+services:
+  - docker
+
 language: python
-python:
-  - "2.7"
-install: "pip install -r requirements.txt"
+
+env:
+  - PYV=2.7
+  - PYV=3.5
 # run nosetests - Tests
-script: nosetests
+script: make test-$PYV

Dockerfile

@@ -1,3 +0,0 @@
-from python:2.7-onbuild
-
-ENTRYPOINT ["python", "-m", "senpy"]

Dockerfile.template

@@ -0,0 +1,22 @@
+from python:{{PYVERSION}}
+
+MAINTAINER J. Fernando Sánchez <jf.sanchez@upm.es>
+
+RUN mkdir /cache/ /senpy-plugins /data/
+
+VOLUME /data/
+
+ENV PIP_CACHE_DIR=/cache/ SENPY_DATA=/data
+
+ONBUILD COPY . /senpy-plugins/
+ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
+ONBUILD WORKDIR /senpy-plugins/
+
+
+WORKDIR /usr/src/app
+COPY test-requirements.txt requirements.txt /usr/src/app/
+RUN pip install --use-wheel -r test-requirements.txt -r requirements.txt
+COPY . /usr/src/app/
+RUN pip install --no-index --no-deps --editable .
+
+ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

MANIFEST.in

@@ -1,5 +1,9 @@
 include requirements.txt
 include test-requirements.txt
-include README.md
-include senpy/context.jsonld
+include README.rst
+include senpy/VERSION
 graft senpy/plugins
+graft senpy/schemas
+graft senpy/templates
+graft senpy/static
+graft img

Makefile

@@ -0,0 +1,147 @@
+NAME=senpy
+VERSION=$(shell git describe --tags --dirty 2>/dev/null)
+GITHUB_REPO=git@github.com:gsi-upm/senpy.git
+
+IMAGENAME=gsiupm/senpy
+IMAGEWTAG=$(IMAGENAME):$(VERSION)
+
+PYVERSIONS=3.5 2.7
+PYMAIN=$(firstword $(PYVERSIONS))
+
+DEVPORT=5000
+
+TARNAME=$(NAME)-$(VERSION).tar.gz 
+action="test-${PYMAIN}"
+GITHUB_REPO=git@github.com:gsi-upm/senpy.git
+
+KUBE_CA_PEM_FILE=""
+KUBE_URL=""
+KUBE_TOKEN=""
+KUBE_NAMESPACE=$(NAME)
+KUBECTL=docker run --rm -v $(KUBE_CA_PEM_FILE):/tmp/ca.pem -v $$PWD:/tmp/cwd/ -i lachlanevenson/k8s-kubectl --server="$(KUBE_URL)" --token="$(KUBE_TOKEN)" --certificate-authority="/tmp/ca.pem" -n $(KUBE_NAMESPACE)
+CI_REGISTRY=docker.io
+CI_REGISTRY_USER=gitlab
+CI_BUILD_TOKEN=""
+CI_COMMIT_REF_NAME=master
+
+
+
+all: build run
+
+.FORCE:
+
+version: .FORCE
+	@echo $(VERSION) > $(NAME)/VERSION
+	@echo $(VERSION)
+
+yapf:
+	yapf -i -r $(NAME)
+	yapf -i -r tests
+
+init:
+	pip install --user pre-commit
+	pre-commit install
+
+dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS))
+	@unlink Dockerfile >/dev/null
+	ln -s Dockerfile-$(PYMAIN) Dockerfile
+
+Dockerfile-%: Dockerfile.template
+	sed "s/{{PYVERSION}}/$*/" Dockerfile.template > Dockerfile-$*
+
+quick_build: $(addprefix build-, $(PYMAIN))
+
+build: $(addprefix build-, $(PYVERSIONS))
+
+build-%: version Dockerfile-%
+	docker build -t '$(IMAGEWTAG)-python$*' --cache-from $(IMAGENAME):python$* -f Dockerfile-$* .;
+
+quick_test: $(addprefix test-,$(PYMAIN))
+
+dev-%:
+	@docker start $(NAME)-dev$* || (\
+		$(MAKE) build-$*; \
+		docker run -d -w /usr/src/app/ -p $(DEVPORT):5000 -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti --name $(NAME)-dev$* '$(IMAGEWTAG)-python$*'; \
+	)\
+
+	docker exec -ti $(NAME)-dev$* bash
+
+dev: dev-$(PYMAIN)
+
+test-all: $(addprefix test-,$(PYVERSIONS))
+
+test-%:
+	docker run --rm --entrypoint /usr/local/bin/python -w /usr/src/app $(IMAGENAME):python$*  setup.py test
+
+test: test-$(PYMAIN)
+
+dist/$(TARNAME): version
+	python setup.py sdist;
+
+sdist: dist/$(TARNAME)
+
+pip_test-%: sdist
+	docker run --rm -v $$PWD/dist:/dist/ python:$* pip install /dist/$(TARNAME);
+
+pip_test: $(addprefix pip_test-,$(PYVERSIONS))
+
+pip_upload: pip_test 
+	python setup.py sdist upload ;
+
+clean:
+	@docker ps -a | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true
+	@docker images | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true
+	@docker stop $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true
+	@docker rm $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true
+
+git_commit:
+	git commit -a
+
+git_tag:
+	git tag ${VERSION}
+
+git_push:
+	git push --tags origin master
+
+run-%: build-%
+	docker run --rm -p $(DEVPORT):5000 -ti '$(IMAGEWTAG)-python$(PYMAIN)' --default-plugins
+
+run: run-$(PYMAIN)
+
+push-latest: $(addprefix push-latest-,$(PYVERSIONS))
+	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
+	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGENAME)'
+	docker push '$(IMAGENAME):latest'
+	docker push '$(IMAGEWTAG)'
+
+push-latest-%: build-%
+	docker tag $(IMAGENAME):$(VERSION)-python$* $(IMAGENAME):python$*
+	docker push $(IMAGENAME):$(VERSION)-python$*
+	docker push $(IMAGENAME):python$*
+
+push-%: build-%
+	docker push $(IMAGENAME):$(VERSION)-python$*
+
+push: $(addprefix push-,$(PYVERSIONS))
+	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
+	docker push  $(IMAGENAME):$(VERSION)
+
+push-github:
+	$(eval KEY_FILE := $(shell mktemp))
+	@echo "$$GITHUB_DEPLOY_KEY" > $(KEY_FILE)
+	@git remote rm github-deploy || true
+	git remote add github-deploy $(GITHUB_REPO)
+	@GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git fetch github-deploy $(CI_COMMIT_REF_NAME) || true
+	@GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git push github-deploy $(CI_COMMIT_REF_NAME)
+	rm $(KEY_FILE)
+
+ci:
+	gitlab-runner exec docker --docker-volumes /var/run/docker.sock:/var/run/docker.sock --env CI_PROJECT_NAME=$(NAME) ${action}
+
+deploy:
+	@$(KUBECTL) delete secret $(CI_REGISTRY) || true
+	@$(KUBECTL) create secret docker-registry $(CI_REGISTRY) --docker-server=$(CI_REGISTRY) --docker-username=$(CI_REGISTRY_USER) --docker-email=$(CI_REGISTRY_USER) --docker-password=$(CI_BUILD_TOKEN)
+	@$(KUBECTL) apply -f /tmp/cwd/k8s/
+
+
+.PHONY: test test-% test-all build-% build test pip_test run yapf push-main push-% dev ci version .FORCE deploy

Procfile

@@ -1 +1 @@
-web: gunicorn app:app --log-file=-
+web: python -m senpy --host 0.0.0.0 --port $PORT --default-plugins

README.rst

@@ -12,7 +12,7 @@ Have you ever wanted to turn your sentiment analysis algorithms into a service?
 With senpy, now you can.
 It provides all the tools so you just have to worry about improving your algorithms:
 
-`See it in action. <http://demos.gsi.dit.upm.es/senpy>`_
+`See it in action. <http://senpy.cluster.gsi.dit.upm.es/>`_
 
 Installation
 ------------
@@ -23,14 +23,14 @@ Through PIP
 
 .. code:: bash
 
-   pip install --user senpy
+   pip install -U --user senpy
 
    
 Alternatively, you can use the development version:
  
 .. code:: bash
 
-   git clone git@github.com:gsi-upm/senpy
+   git clone http://github.com/gsi-upm/senpy
    cd senpy
    pip install --user .
 
@@ -38,9 +38,56 @@ If you want to install senpy globally, use sudo instead of the ``--user`` flag.
 
 Docker Image
 ************
-Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 balkian/senpy --host 0.0.0.0 --default-plugins``.
+Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy --default-plugins``.
 
-To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins balkian/senpy --host 0.0.0.0 --default-plugins -f /plugins``
+To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --default-plugins -f /plugins``
+
+
+Developing
+----------
+
+Developing/debugging
+********************
+This command will run the senpy container using the latest image available, mounting your current folder so you get your latest code:
+
+.. code:: bash
+
+
+    # Python 3.5
+    make dev
+    # Python 2.7
+    make dev-2.7
+
+Building a docker image
+***********************
+
+.. code:: bash
+
+
+    # Python 3.5
+    make build-3.5
+    # Python 2.7
+    make build-2.7
+
+Testing
+*******
+
+.. code:: bash
+
+
+    make test
+
+Running
+*******
+This command will run the senpy server listening on localhost:5000
+
+.. code:: bash
+
+
+    # Python 3.5
+    make run-3.5
+    # Python 2.7
+    make run-2.7
 
 Usage
 -----
@@ -49,12 +96,14 @@ However, the easiest and recommended way is to just use the command-line tool to
 
 .. code:: bash
 
+
    senpy
 
 or, alternatively:
 
 .. code:: bash
 
+
     python -m senpy
 
 

app.py

@@ -1,43 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-#    Copyright 2014 J. Fernando Sánchez Rada - Grupo de Sistemas Inteligentes
-#                                                       DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-"""
-This is a helper for development. If you want to run Senpy use:
-
-    python -m senpy
-"""
-from gevent.monkey import patch_all; patch_all()
-import gevent
-import config
-from flask import Flask
-from senpy.extensions import Senpy
-import logging
-import os
-from gevent.wsgi import WSGIServer
-
-logging.basicConfig(level=logging.DEBUG)
-
-app = Flask(__name__)
-mypath = os.path.dirname(os.path.realpath(__file__))
-sp = Senpy(app, os.path.join(mypath, "plugins"), default_plugins=True)
-sp.activate_all()
-
-if __name__ == '__main__':
-    import logging
-    logging.basicConfig(level=config.DEBUG)
-    app.debug = config.DEBUG
-    http_server = WSGIServer(('', config.SERVER_PORT), app)
-    http_server.serve_forever()

docs/SenpyClientUse.ipynb

@@ -0,0 +1,317 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:05:31.465571Z",
+     "start_time": "2017-04-10T19:05:31.458282+02:00"
+    },
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "# Client"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "The built-in senpy client allows you to query any Senpy endpoint. We will illustrate how to use it with the public demo endpoint, and then show you how to spin up your own endpoint using docker."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "Demo Endpoint\n",
+    "-------------"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "To start using senpy, simply create a new Client and point it to your endpoint. In this case, the latest version of Senpy at GSI."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:12.827640Z",
+     "start_time": "2017-04-10T19:29:12.818617+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "from senpy.client import Client\n",
+    "\n",
+    "c = Client('http://latest.senpy.cluster.gsi.dit.upm.es/api')\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "Now, let's use that client analyse some queries:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:14.011657Z",
+     "start_time": "2017-04-10T19:29:13.701808+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "r = c.analyse('I like sugar!!', algorithm='sentiment140')\n",
+    "r"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:08:19.616754Z",
+     "start_time": "2017-04-10T19:08:19.610767+02:00"
+    },
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "As you can see, that gave us the full JSON result. A more concise way to print it would be:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:14.854213Z",
+     "start_time": "2017-04-10T19:29:14.842068+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "for entry in r.entries:\n",
+    "      print('{} -> {}'.format(entry['text'], entry['sentiments'][0]['marl:hasPolarity']))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "We can also obtain a list of available plugins with the client:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:16.245198Z",
+     "start_time": "2017-04-10T19:29:16.056545+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "c.plugins()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "Or, more concisely:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:17.663275Z",
+     "start_time": "2017-04-10T19:29:17.484623+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "c.plugins().keys()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "Local Endpoint\n",
+    "--------------"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "To run your own instance of senpy, just create a docker container with the latest Senpy image. Using `--default-plugins` you will get some extra plugins to start playing with the API."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:20.637539Z",
+     "start_time": "2017-04-10T19:29:19.938322+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "!docker run -ti --name 'SenpyEndpoint' -d -p 6000:5000 gsiupm/senpy:0.8.6 --host 0.0.0.0 --default-plugins"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "To use this endpoint:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:21.263976Z",
+     "start_time": "2017-04-10T19:29:21.260595+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "c_local = Client('http://127.0.0.1:6000/api')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "That's all! After you are done with your analysis, stop the docker container:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:33.226686Z",
+     "start_time": "2017-04-10T19:29:22.392121+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "!docker stop SenpyEndpoint\n",
+    "!docker rm SenpyEndpoint"
+   ]
+  }
+ ],
+ "metadata": {
+  "anaconda-cloud": {},
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.0"
+  },
+  "toc": {
+   "colors": {
+    "hover_highlight": "#DAA520",
+    "running_highlight": "#FF0000",
+    "selected_highlight": "#FFD700"
+   },
+   "moveMenuLeft": true,
+   "nav_menu": {
+    "height": "68px",
+    "width": "252px"
+   },
+   "navigate_menu": true,
+   "number_sections": true,
+   "sideBar": true,
+   "threshold": 4,
+   "toc_cell": false,
+   "toc_section_display": "block",
+   "toc_window_display": false
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}

docs/_static/schemas

@@ -0,0 +1 @@
+../../senpy/schemas/

docs/about.rst

@@ -0,0 +1,11 @@
+About
+--------
+
+If you use Senpy in your research, please cite `Senpy: A Pragmatic Linked Sentiment Analysis Framework <http://gsi.dit.upm.es/index.php/es/investigacion/publicaciones?view=publication&task=show&id=417>`__ (`BibTex <http://gsi.dit.upm.es/index.php/es/investigacion/publicaciones?controller=publications&task=export&format=bibtex&id=417>`__):
+
+.. code-block:: text
+
+	Sánchez-Rada, J. F., Iglesias, C. A., Corcuera, I., & Araque, Ó. (2016, October).
+	Senpy: A Pragmatic Linked Sentiment Analysis Framework.
+	In Data Science and Advanced Analytics (DSAA),
+	2016 IEEE International Conference on (pp. 735-742). IEEE.

docs/api.rst

@@ -1,5 +1,5 @@
 NIF API
-=======
+-------
 .. http:get:: /api
 
    Basic endpoint for sentiment/emotion analysis.
@@ -22,38 +22,32 @@ NIF API
       Content-Type: text/javascript
 
       {
-        "@context": [
-          "http://127.0.0.1/static/context.jsonld",
-      ],
-      "analysis": [
-        {
-          "@id": "SentimentAnalysisExample",
-          "@type": "marl:SentimentAnalysis",
-          "dc:language": "en", 
-          "marl:maxPolarityValue": 10.0,
-          "marl:minPolarityValue": 0.0
-        }
-      ],
-      "domain": "wndomains:electronics", 
-      "entries": [
-        {
-          "opinions": [
-            {
-              "prov:generatedBy": "SentimentAnalysisExample",
-              "marl:polarityValue": 7.8, 
-              "marl:hasPolarity": "marl:Positive",
-              "marl:describesObject": "http://www.gsi.dit.upm.es",
-            }
-          ],
-          "nif:isString": "I love GSI",
-          "strings": [
-            {
-              "nif:anchorOf": "GSI",
-              "nif:taIdentRef": "http://www.gsi.dit.upm.es"
-            }
-          ]
-        }
-       ]
+        "@context":"http://127.0.0.1/api/contexts/Results.jsonld",
+        "@id":"_:Results_11241245.22",
+        "@type":"results"
+        "analysis": [
+          "plugins/sentiment-140_0.1" 
+        ],
+        "entries": [
+          {  
+            "@id": "_:Entry_11241245.22"
+            "@type":"entry",
+            "emotions": [],
+            "entities": [],
+            "sentiments": [
+              {  
+                "@id": "Sentiment0",  
+                "@type": "sentiment", 
+                "marl:hasPolarity": "marl:Negative",
+                "marl:polarityValue": 0,
+                "prefix": ""
+              }
+            ],
+            "suggestions": [],
+            "text": "This text makes me sad.\nwhilst this text makes me happy and surprised at the same time.\nI cannot believe it!",
+            "topics": []
+          }
+        ]
       }
 
    :query i input: No default. Depends on informat and intype
@@ -62,6 +56,7 @@ NIF API
    :query o outformat: one of `turtle` (default), `text`, `json-ld`
    :query p prefix: prefix for the URIs
    :query algo algorithm: algorithm/plugin to use for the analysis. For a list of options, see :http:get:`/api/plugins`. If not provided, the default plugin will be used (:http:get:`/api/plugins/default`).
+   :query algo emotionModel: desired emotion model in the results. If the requested algorithm does not use that emotion model, there are conversion plugins specifically for this. If none of the plugins match, an error will be returned, which includes the results *as is*.
 
    :reqheader Accept: the response content type depends on
                       :mailheader:`Accept` header
@@ -69,6 +64,7 @@ NIF API
                             header of request
    :statuscode 200: no error
    :statuscode 404: service not found
+   :statuscode 400: error while processing the request
 
 .. http:post:: /api
 
@@ -90,56 +86,59 @@ NIF API
 
    .. sourcecode:: http
 
-        {
-            "@context": {
-                   ...
-            }, 
-            "sentiment140": {
-                "name": "sentiment140", 
-                "is_activated": true, 
-                "version": "0.1", 
-                "extra_params": {
-                    "@id": "extra_params_sentiment140_0.1", 
-                    "language": {
-                        "required": false, 
-                        "@id": "lang_sentiment140", 
-                        "options": [
-                            "es", 
-                            "en", 
-                            "auto"
-                        ], 
-                        "aliases": [
-                            "language", 
-                            "l"
-                        ]
-                    }
-                }, 
-                "@id": "sentiment140_0.1"
-            }, 
-            "rand": {
-                "name": "rand", 
-                "is_activated": true, 
-                "version": "0.1", 
-                "extra_params": {
-                    "@id": "extra_params_rand_0.1", 
-                    "language": {
-                        "required": false, 
-                        "@id": "lang_rand", 
-                        "options": [
-                            "es", 
-                            "en", 
-                            "auto"
-                        ], 
-                        "aliases": [
-                            "language", 
-                            "l"
-                        ]
-                    }
-                }, 
-                "@id": "rand_0.1"
-            }
-        }
-
+      {
+        "@id": "plugins/sentiment-140_0.1", 
+        "@type": "sentimentPlugin", 
+        "author": "@balkian", 
+        "description": "Sentiment classifier using rule-based classification for English and Spanish. This plugin uses sentiment140 data to perform classification. For more information: http://help.sentiment140.com/for-students/", 
+        "extra_params": {
+          "language": {
+            "@id": "lang_sentiment140", 
+            "aliases": [
+              "language", 
+              "l"
+            ], 
+            "options": [
+              "es", 
+              "en", 
+              "auto"
+            ], 
+            "required": false
+          }
+        }, 
+        "is_activated": true, 
+        "maxPolarityValue": 1.0, 
+        "minPolarityValue": 0.0, 
+        "module": "sentiment-140", 
+        "name": "sentiment-140", 
+        "requirements": {}, 
+        "version": "0.1"
+      }, 
+      {
+        "@id": "plugins/ExamplePlugin_0.1", 
+        "@type": "sentimentPlugin", 
+        "author": "@balkian", 
+        "custom_attribute": "42", 
+        "description": "I am just an example", 
+        "extra_params": {
+          "parameter": {
+            "@id": "parameter", 
+            "aliases": [
+              "parameter", 
+              "param"
+            ], 
+            "default": 42, 
+            "required": true
+          }
+        }, 
+        "is_activated": true, 
+        "maxPolarityValue": 1.0, 
+        "minPolarityValue": 0.0, 
+        "module": "example", 
+        "name": "ExamplePlugin", 
+        "requirements": "noop", 
+        "version": "0.1"
+      }
 
 .. http:get:: /api/plugins/<pluginname>
 
@@ -148,7 +147,7 @@ NIF API
 
    .. sourcecode:: http
 
-      GET /api/plugins/rand HTTP/1.1
+      GET /api/plugins/rand/ HTTP/1.1
       Host: localhost
       Accept: application/json, text/javascript
 
@@ -158,51 +157,61 @@ NIF API
    .. sourcecode:: http
 
       {
-          "@id": "rand_0.1",
-          "extra_params": {
-              "@id": "extra_params_rand_0.1",
-              "language": {
-                  "@id": "lang_rand",
-                  "aliases": [
-                      "language",
-                      "l"
-                  ],
-                  "options": [
-                      "es",
-                      "en",
-                      "auto"
-                  ],
-                  "required": false
-              }
-          },
-          "is_activated": true,
-          "name": "rand",
-          "version": "0.1"
+        "@context": "http://127.0.0.1/api/contexts/ExamplePlugin.jsonld", 
+        "@id": "plugins/ExamplePlugin_0.1", 
+        "@type": "sentimentPlugin", 
+        "author": "@balkian", 
+        "custom_attribute": "42", 
+        "description": "I am just an example", 
+        "extra_params": {
+          "parameter": {
+            "@id": "parameter", 
+            "aliases": [
+              "parameter", 
+              "param"
+            ], 
+            "default": 42, 
+            "required": true
+          }
+        }, 
+        "is_activated": true, 
+        "maxPolarityValue": 1.0, 
+        "minPolarityValue": 0.0, 
+        "module": "example", 
+        "name": "ExamplePlugin", 
+        "requirements": "noop", 
+        "version": "0.1"
       }
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 .. http:get:: /api/plugins/default
 
    Return the information about the default plugin.
 
-.. http:get:: /api/plugins/<pluginname>/{de}activate
-
-   {De}activate a plugin.
-
-   **Example request**:
-
-   .. sourcecode:: http
-
-      GET /api/plugins/rand/deactivate HTTP/1.1
-      Host: localhost
-      Accept: application/json, text/javascript
-
-
-   **Example response**:
-
-   .. sourcecode:: http
-
-       {
-          "@context": {}, 
-          "message": "Ok"
-       }

docs/apischema.rst

@@ -0,0 +1,7 @@
+API and Examples
+################
+.. toctree::
+
+   vocabularies.rst
+   api.rst
+   examples.rst

docs/bad-examples/plugins/noplugins.json

@@ -0,0 +1,4 @@
+{
+  "plugins": [
+    ]
+}

docs/bad-examples/results/example-analysis-as-id-FAIL.json

@@ -0,0 +1,78 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "@type": "results",
+  "analysis": [
+    "me:SAnalysis1",
+    "me:SgAnalysis1",
+    "me:EmotionAnalysis1",
+    "me:NER1",
+    {
+      "@type": "analysis",
+      "@id": "wrong"
+    }
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+        {
+          "@id": "http://micro.blog/status1#char=5,13",
+          "nif:beginIndex": 5,
+          "nif:endIndex": 13,
+          "nif:anchorOf": "Microsoft",
+          "me:references": "http://dbpedia.org/page/Microsoft",
+          "prov:wasGeneratedBy": "me:NER1"
+        },
+        {
+          "@id": "http://micro.blog/status1#char=25,37",
+          "nif:beginIndex": 25,
+          "nif:endIndex": 37,
+          "nif:anchorOf": "Windows Phone",
+          "me:references": "http://dbpedia.org/page/Windows_Phone",
+          "prov:wasGeneratedBy": "me:NER1"
+        }
+      ],
+      "suggestions": [
+        {
+          "@id": "http://micro.blog/status1#char=16,77",
+          "nif:beginIndex": 16,
+          "nif:endIndex": 77,
+          "nif:anchorOf": "put your Windows Phone on your newest #open technology program",
+          "prov:wasGeneratedBy": "me:SgAnalysis1"
+        }
+      ],
+      "sentiments": [
+        {
+          "@id": "http://micro.blog/status1#char=80,97",
+          "nif:beginIndex": 80,
+          "nif:endIndex": 97,
+          "nif:anchorOf": "You'll be awesome.",
+          "marl:hasPolarity": "marl:Positive",
+          "marl:polarityValue": 0.9,
+          "prov:wasGeneratedBy": "me:SAnalysis1"
+        }
+      ],
+      "emotions": [
+        {
+          "@id": "http://micro.blog/status1#char=0,109",
+          "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+          "prov:wasGeneratedBy": "me:EAnalysis1",
+          "onyx:hasEmotion": [
+            {
+              "onyx:hasEmotionCategory": "wna:liking"
+            },
+            {
+              "onyx:hasEmotionCategory": "wna:excitement"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

docs/bad-examples/results/example-basic-FAIL.json

@@ -0,0 +1,18 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "http://example.com#NIFExample",
+  "@type": "results",
+  "analysis": [
+  ],
+  "entries": [
+    {
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:beginIndex": 0,
+      "nif:endIndex": 40,
+      "nif:isString": "My favourite actress is Natalie Portman"
+    }
+  ]
+}

docs/commandline.rst

@@ -0,0 +1,9 @@
+Command line
+============
+
+This video shows how to analyse text directly on the command line using the senpy tool.
+
+.. image:: https://asciinema.org/a/9uwef1ghkjk062cw2t4mhzpyk.png
+   :width: 100%
+   :target: https://asciinema.org/a/9uwef1ghkjk062cw2t4mhzpyk
+   :alt: CLI demo

docs/conf.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# flake8: noqa
 #
 # Senpy documentation build configuration file, created by
 # sphinx-quickstart on Tue Feb 24 08:57:32 2015.
@@ -36,6 +37,7 @@ extensions = [
     'sphinx.ext.todo',
     'sphinxcontrib.httpdomain',
     'sphinx.ext.coverage',
+    'sphinx.ext.autosectionlabel',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -52,20 +54,22 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Senpy'
-copyright = u'2015, J. Fernando Sánchez'
+copyright = u'2016, J. Fernando Sánchez'
+description = u'A framework for sentiment and emotion analysis services'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '0.4'
+# with open('../senpy/VERSION') as f:
+#     version = f.read().strip()
 # The full version, including alpha/beta/rc tags.
-release = '0.4'
+# release = version
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
-#language = None
+language = None
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
@@ -102,14 +106,14 @@ pygments_style = 'sphinx'
 #keep_warnings = False
 
 
+html_theme = 'alabaster'
 # -- Options for HTML output ----------------------------------------------
-if not on_rtd:  # only import and set the theme if we're building docs locally
-    import sphinx_rtd_theme
-    html_theme = 'sphinx_rtd_theme'
-    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+# if not on_rtd:  # only import and set the theme if we're building docs locally
+#     import sphinx_rtd_theme
+#     html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
-else:
-    html_theme = 'default'
+# else:
+#     html_theme = 'default'
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
@@ -117,7 +121,13 @@ else:
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-#html_theme_options = {}
+html_theme_options = {
+    'logo': 'header.png',
+    'github_user': 'gsi-upm',
+    'github_repo': 'senpy',
+    'github_banner': True,
+}
+
 
 # Add any paths that contain custom themes here, relative to this directory.
 #html_theme_path = []
@@ -157,7 +167,13 @@ html_static_path = ['_static']
 #html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+html_sidebars = {
+    '**': [
+        'about.html',
+        'navigation.html',
+        'searchbox.html',
+        ]
+    }
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.

docs/conversion.rst

@@ -0,0 +1,116 @@
+Conversion
+----------
+
+Senpy includes experimental support for emotion/sentiment conversion plugins.
+
+
+Use
+===
+
+Consider the original query: http://127.0.0.1:5000/api/?i=hello&algo=emoRand
+
+The requested plugin (emoRand) returns emotions using Ekman's model (or big6 in EmotionML):
+
+.. code:: json
+
+
+          ... rest of the document ...
+          {
+            "@type": "emotionSet",
+            "onyx:hasEmotion": {
+                "@type": "emotion",
+                "onyx:hasEmotionCategory": "emoml:big6anger"
+            },
+            "prov:wasGeneratedBy": "plugins/emoRand_0.1"
+          }
+
+          
+
+To get these emotions in VAD space (FSRE dimensions in EmotionML), we'd do this:
+
+http://127.0.0.1:5000/api/?i=hello&algo=emoRand&emotionModel=emoml:fsre-dimensions
+
+This call, provided there is a valid conversion plugin from Ekman's to VAD, would return something like this:
+
+.. code:: json
+
+
+          ... rest of the document ...
+          {
+            "@type": "emotionSet",
+            "onyx:hasEmotion": {
+                "@type": "emotion",
+                "onyx:hasEmotionCategory": "emoml:big6anger"
+                },
+            "prov:wasGeneratedBy": "plugins/emoRand_0.1"
+          }, {
+            "@type": "emotionSet",
+            "onyx:hasEmotion": {
+                "@type": "emotion",
+                "A": 7.22,
+                "D": 6.28,
+                "V": 8.6
+            },
+            "prov:wasGeneratedBy": "plugins/Ekman2VAD_0.1"
+
+          }
+
+
+That is called a *full* response, as it simply adds the converted emotion alongside.
+It is also possible to get the original emotion nested within the new converted emotion, using the `conversion=nested` parameter:
+
+.. code:: json
+
+
+          ... rest of the document ...
+          {
+            "@type": "emotionSet",
+            "onyx:hasEmotion": {
+                "@type": "emotion",
+                "onyx:hasEmotionCategory": "emoml:big6anger"
+                },
+            "prov:wasGeneratedBy": "plugins/emoRand_0.1"
+            "onyx:wasDerivedFrom": {
+                "@type": "emotionSet",
+                "onyx:hasEmotion": {
+                    "@type": "emotion",
+                    "A": 7.22,
+                    "D": 6.28,
+                    "V": 8.6
+                },
+                "prov:wasGeneratedBy": "plugins/Ekman2VAD_0.1"
+             }
+
+          }
+
+
+Lastly, `conversion=filtered` would only return the converted emotions.
+
+Developing a conversion plugin
+================================
+
+Conversion plugins are discovered by the server just like any other plugin.
+The difference is the slightly different API, and the need to specify the `source` and `target` of the conversion.
+For instance, an emotion conversion plugin needs the following:
+
+
+.. code:: yaml
+          
+
+          ---
+          onyx:doesConversion:
+            - onyx:conversionFrom: emoml:big6
+              onyx:conversionTo: emoml:fsre-dimensions
+            - onyx:conversionFrom: emoml:fsre-dimensions
+              onyx:conversionTo: emoml:big6
+
+
+
+
+.. code:: python
+
+
+          class MyConversion(EmotionConversionPlugin):
+
+              def convert(self, emotionSet, fromModel, toModel, params):
+                  pass

docs/demo.rst

@@ -0,0 +1,16 @@
+Demo
+----
+
+There is a demo available on http://senpy.cluster.gsi.dit.upm.es/, where you can test a serie of different plugins.
+You can use the playground (a web interface) or make HTTP requests to the service API.
+
+.. image:: senpy-playground.png
+  :height: 400px
+  :width: 800px
+  :scale: 100 %
+  :align: center
+
+Plugins Demo
+============
+
+The source code and description of the plugins used in the demo is available here: https://lab.cluster.gsi.dit.upm.es/senpy/senpy-plugins-community/.

docs/examples.rst

@@ -0,0 +1,78 @@
+Examples
+------
+All the examples in this page use the :download:`the main schema <_static/schemas/definitions.json>`.
+
+Simple NIF annotation
+.....................
+Description
+,,,,,,,,,,,
+This example covers the basic example in the NIF documentation: `<http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core/nif-core.html>`_.
+
+Representation
+,,,,,,,,,,,,,,
+.. literalinclude:: examples/results/example-basic.json
+   :language: json-ld
+
+Sentiment Analysis
+.....................
+Description
+,,,,,,,,,,,
+This annotation corresponds to the sentiment analysis of an input. The example shows the sentiment represented according to Marl format.
+The sentiments detected are contained in the Sentiments array with their related part of the text.
+
+Representation
+,,,,,,,,,,,,,,
+
+.. literalinclude:: examples/results/example-sentiment.json
+   :emphasize-lines: 5-10,25-33
+   :language: json-ld
+
+Suggestion Mining
+.................
+Description
+,,,,,,,,,,,
+The suggestions schema represented below shows the suggestions detected in the text. Within it, we can find the NIF fields highlighted that corresponds to the text of the detected suggestion. 
+
+Representation
+,,,,,,,,,,,,,,
+
+.. literalinclude:: examples/results/example-suggestion.json
+   :emphasize-lines: 5-8,22-27
+   :language: json-ld
+
+Emotion Analysis
+................
+Description
+,,,,,,,,,,,
+This annotation represents the emotion analysis of an input to Senpy. The emotions are contained in the emotions section with the text that refers to following Onyx format and the emotion model defined beforehand.
+
+Representation
+,,,,,,,,,,,,,,
+
+.. literalinclude:: examples/results/example-emotion.json
+   :language: json-ld
+   :emphasize-lines: 5-8,25-37
+
+Named Entity Recognition
+........................
+Description
+,,,,,,,,,,,
+The Named Entity Recognition is represented as follows. In this particular case, it can be seen within the entities array the entities recognised. For the example input, Microsoft and Windows Phone are the ones detected. 
+Representation
+,,,,,,,,,,,,,,
+
+.. literalinclude:: examples/results/example-ner.json
+   :emphasize-lines: 5-8,19-34
+   :language: json-ld
+
+Complete example
+................
+Description
+,,,,,,,,,,,
+This example covers all of the above cases, integrating all the annotations in the same document.
+
+Representation
+,,,,,,,,,,,,,,
+
+.. literalinclude:: examples/results/example-complete.json
+   :language: json-ld

docs/examples/plugins/noplugins.json

@@ -0,0 +1,5 @@
+{
+  "@type": "plugins",
+  "plugins": [
+    ]
+}

docs/examples/results/example-analysis-as-id.json

@@ -0,0 +1,74 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "@type": "results",
+  "analysis": [
+    "me:SAnalysis1",
+    "me:SgAnalysis1",
+    "me:EmotionAnalysis1",
+    "me:NER1"
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+        {
+          "@id": "http://micro.blog/status1#char=5,13",
+          "nif:beginIndex": 5,
+          "nif:endIndex": 13,
+          "nif:anchorOf": "Microsoft",
+          "me:references": "http://dbpedia.org/page/Microsoft",
+          "prov:wasGeneratedBy": "me:NER1"
+        },
+        {
+          "@id": "http://micro.blog/status1#char=25,37",
+          "nif:beginIndex": 25,
+          "nif:endIndex": 37,
+          "nif:anchorOf": "Windows Phone",
+          "me:references": "http://dbpedia.org/page/Windows_Phone",
+          "prov:wasGeneratedBy": "me:NER1"
+        }
+      ],
+      "suggestions": [
+        {
+          "@id": "http://micro.blog/status1#char=16,77",
+          "nif:beginIndex": 16,
+          "nif:endIndex": 77,
+          "nif:anchorOf": "put your Windows Phone on your newest #open technology program",
+          "prov:wasGeneratedBy": "me:SgAnalysis1"
+        }
+      ],
+      "sentiments": [
+        {
+          "@id": "http://micro.blog/status1#char=80,97",
+          "nif:beginIndex": 80,
+          "nif:endIndex": 97,
+          "nif:anchorOf": "You'll be awesome.",
+          "marl:hasPolarity": "marl:Positive",
+          "marl:polarityValue": 0.9,
+          "prov:wasGeneratedBy": "me:SAnalysis1"
+        }
+      ],
+      "emotions": [
+        {
+          "@id": "http://micro.blog/status1#char=0,109",
+          "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+          "prov:wasGeneratedBy": "me:EAnalysis1",
+          "onyx:hasEmotion": [
+            {
+              "onyx:hasEmotionCategory": "wna:liking"
+            },
+            {
+              "onyx:hasEmotionCategory": "wna:excitement"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

docs/examples/results/example-basic.json

@@ -0,0 +1,19 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "http://example.com#NIFExample",
+  "@type": "results",
+  "analysis": [
+  ],
+  "entries": [
+    {
+      "@id": "http://example.org#char=0,40",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:beginIndex": 0,
+      "nif:endIndex": 40,
+      "nif:isString": "My favourite actress is Natalie Portman"
+    }
+  ]
+}

docs/examples/results/example-complete.json

@@ -0,0 +1,88 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "@type": "results",
+  "analysis": [
+    {
+      "@id": "me:SAnalysis1",
+      "@type": "marl:SentimentAnalysis",
+      "marl:maxPolarityValue": 1,
+      "marl:minPolarityValue": 0
+    },
+    {
+      "@id": "me:SgAnalysis1",
+      "@type": "me:SuggestionAnalysis"
+    },
+    {
+      "@id": "me:EmotionAnalysis1",
+      "@type": "me:EmotionAnalysis"
+    },
+    {
+      "@id": "me:NER1",
+      "@type": "me:NER"
+    }
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+        {
+          "@id": "http://micro.blog/status1#char=5,13",
+          "nif:beginIndex": 5,
+          "nif:endIndex": 13,
+          "nif:anchorOf": "Microsoft",
+          "me:references": "http://dbpedia.org/page/Microsoft",
+          "prov:wasGeneratedBy": "me:NER1"
+        },
+        {
+          "@id": "http://micro.blog/status1#char=25,37",
+          "nif:beginIndex": 25,
+          "nif:endIndex": 37,
+          "nif:anchorOf": "Windows Phone",
+          "me:references": "http://dbpedia.org/page/Windows_Phone",
+          "prov:wasGeneratedBy": "me:NER1"
+        }
+      ],
+      "suggestions": [
+        {
+          "@id": "http://micro.blog/status1#char=16,77",
+          "nif:beginIndex": 16,
+          "nif:endIndex": 77,
+          "nif:anchorOf": "put your Windows Phone on your newest #open technology program",
+          "prov:wasGeneratedBy": "me:SgAnalysis1"
+        }
+      ],
+      "sentiments": [
+        {
+          "@id": "http://micro.blog/status1#char=80,97",
+          "nif:beginIndex": 80,
+          "nif:endIndex": 97,
+          "nif:anchorOf": "You'll be awesome.",
+          "marl:hasPolarity": "marl:Positive",
+          "marl:polarityValue": 0.9,
+          "prov:wasGeneratedBy": "me:SAnalysis1"
+        }
+      ],
+      "emotions": [
+        {
+          "@id": "http://micro.blog/status1#char=0,109",
+          "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+          "prov:wasGeneratedBy": "me:EAnalysis1",
+          "onyx:hasEmotion": [
+            {
+              "onyx:hasEmotionCategory": "wna:liking"
+            },
+            {
+              "onyx:hasEmotionCategory": "wna:excitement"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

docs/examples/results/example-emotion.json

@@ -0,0 +1,42 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "@type": "results",
+  "analysis": [
+    {
+      "@id": "me:EmotionAnalysis1",
+      "@type": "onyx:EmotionAnalysis"
+    }
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+      ],
+      "suggestions": [
+      ],
+      "sentiments": [
+      ],
+      "emotions": [
+        {
+          "@id": "http://micro.blog/status1#char=0,109",
+          "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+          "prov:wasGeneratedBy": "me:EmotionAnalysis1",
+          "onyx:hasEmotion": [
+            {
+              "onyx:hasEmotionCategory": "wna:liking"
+            },
+            {
+              "onyx:hasEmotionCategory": "wna:excitement"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

docs/examples/results/example-ner.json

@@ -0,0 +1,45 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "@type": "results",
+  "analysis": [
+    {
+      "@id": "me:NER1",
+      "@type": "me:NERAnalysis"
+    }
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+        {
+          "@id": "http://micro.blog/status1#char=5,13",
+          "nif:beginIndex": 5,
+          "nif:endIndex": 13,
+          "nif:anchorOf": "Microsoft",
+          "me:references": "http://dbpedia.org/page/Microsoft",
+          "prov:wasGeneratedBy": "me:NER1"
+        },
+        {
+          "@id": "http://micro.blog/status1#char=25,37",
+          "nif:beginIndex": 25,
+          "nif:endIndex": 37,
+          "nif:anchorOf": "Windows Phone",
+          "me:references": "http://dbpedia.org/page/Windows_Phone",
+          "prov:wasGeneratedBy": "me:NER1"
+        }
+      ],
+      "suggestions": [
+      ],
+      "sentiments": [
+      ],
+      "emotionSets": [
+      ]
+    }
+  ]
+}

docs/examples/results/example-pad.json

@@ -0,0 +1,46 @@
+{
+"@context": [
+             "http://mixedemotions-project.eu/ns/context.jsonld",
+             {
+               "emovoc": "http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/emotionml/ns#"
+             }
+  ],
+  "@id": "me:Result1",
+  "@type": "results",
+  "analysis": [
+    {
+      "@id": "me:HesamsAnalysis",
+      "@type": "onyx:EmotionAnalysis",
+      "onyx:usesEmotionModel": "emovoc:pad-dimensions"
+    }
+  ],
+  "entries": [
+    {
+      "@id": "Entry1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:isString": "This is a test string",
+      "entities": [
+      ],
+      "suggestions": [
+      ],
+      "sentiments": [
+      ],
+      "emotions": [
+        {
+          "@id": "Entry1#char=0,21",
+          "nif:anchorOf": "This is a test string",
+          "prov:wasGeneratedBy": "me:HesamAnalysis",
+          "onyx:hasEmotion": [
+            {
+                "emovoc:pleasure": 0.5,
+                "emovoc:arousal": 0.7
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

docs/examples/results/example-sentiment.json

@@ -0,0 +1,40 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "@type": "results",
+  "analysis": [
+    {
+      "@id": "me:SAnalysis1",
+      "@type": "marl:SentimentAnalysis",
+      "marl:maxPolarityValue": 1,
+      "marl:minPolarityValue": 0
+    }
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+      ],
+      "suggestions": [
+      ],
+      "sentiments": [
+        {
+          "@id": "http://micro.blog/status1#char=80,97",
+          "nif:beginIndex": 80,
+          "nif:endIndex": 97,
+          "nif:anchorOf": "You'll be awesome.",
+          "marl:hasPolarity": "marl:Positive",
+          "marl:polarityValue": 0.9,
+          "prov:wasGeneratedBy": "me:SAnalysis1"
+        }
+      ],
+      "emotionSets": [
+      ]
+    }
+  ]
+}

docs/examples/results/example-suggestion.json

@@ -0,0 +1,37 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "@type": "results",
+  "analysis": [
+    {
+      "@id": "me:SgAnalysis1",
+      "@type": "me:SuggestionAnalysis"
+    }
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "prov:wasGeneratedBy": "me:SAnalysis1",
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+      ],
+      "suggestions": [
+        {
+          "@id": "http://micro.blog/status1#char=16,77",
+          "nif:beginIndex": 16,
+          "nif:endIndex": 77,
+          "nif:anchorOf": "put your Windows Phone on your newest #open technology program",
+          "prov:wasGeneratedBy": "me:SgAnalysis1"
+        }
+      ],
+      "sentiments": [
+      ],
+      "emotionSets": [
+      ]
+    }
+  ]
+}

docs/index.rst

@@ -1,16 +1,35 @@
-.. Senpy documentation master file, created by
-   sphinx-quickstart on Tue Feb 24 08:57:32 2015.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
-
 Welcome to Senpy's documentation!
 =================================
+.. image:: https://readthedocs.org/projects/senpy/badge/?version=latest
+   :target: http://senpy.readthedocs.io/en/latest/
+.. image:: https://badge.fury.io/py/senpy.svg
+   :target: https://badge.fury.io/py/senpy
+.. image:: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/badges/master/build.svg
+   :target: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/commits/master
+.. image:: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/badges/master/coverage.svg
+   :target: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/commits/master
+.. image:: https://img.shields.io/pypi/l/requests.svg
+   :target: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/
 
-Contents:
+
+
+Senpy is a framework for sentiment and emotion analysis services.
+Services built with senpy are interchangeable and easy to use because they share a common :doc:`apischema`.
+It also simplifies service development.
+
+.. image:: senpy-architecture.png
+  :width: 100%
+  :align: center
 
 .. toctree::
-   installation
-   usage
-   api
-   plugins
+   :caption: Learn more about senpy:
    :maxdepth: 2
+
+   senpy
+   installation
+   demo
+   usage
+   apischema
+   plugins
+   conversion
+   about

docs/installation.rst

@@ -1,6 +1,16 @@
 Installation
 ------------
-The stable version can be installed in three ways.
+The stable version can be used in two ways: as a system/user library through pip, or as a docker image.
+
+The docker image is the recommended way because it is self-contained and isolated from the system, which means:
+
+  * Downloading and using it is just one command
+  * All dependencies are included
+  * It is OS-independent (MacOS, Windows, GNU/Linux)
+  * Several versions may coexist in the same machine without additional virtual environments
+
+Additionally, you may create your own docker image with your custom plugins, ready to be used by others.
+
 
 Through PIP
 ***********
@@ -22,6 +32,41 @@ If you want to install senpy globally, use sudo instead of the ``--user`` flag.
 
 Docker Image
 ************
-Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 balkian/senpy --host 0.0.0.0 --default-plugins``.
+Build the image or use the pre-built one:   
 
-To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins balkian/senpy --host 0.0.0.0 --default-plugins -f /plugins``
+.. code:: bash
+
+   docker run -ti -p 5000:5000 gsiupm/senpy --host 0.0.0.0 --default-plugins
+
+To add custom plugins, use a docker volume: 
+    
+.. code:: bash
+
+   docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --host 0.0.0.0 --default-plugins -f /plugins
+ 
+
+Python 2
+........
+
+There is a Senpy version for python2 too:
+    
+.. code:: bash
+
+   docker run -ti -p 5000:5000 gsiupm/senpy:python2.7 --host 0.0.0.0 --default-plugins
+
+
+Alias
+.....
+
+If you are using the docker approach regularly, it is advisable to use a script or an alias to simplify your executions:
+
+.. code:: bash
+
+   alias senpy='docker run --rm -ti -p 5000:5000 -v $PWD:/senpy-plugins gsiupm/senpy --default-plugins'
+
+
+Now, you may run senpy from any folder in your computer like so:
+
+.. code:: bash
+
+   senpy --version

docs/plugins.rst

@@ -1,29 +1,264 @@
 Developing new plugins
 ----------------------
+This document describes how to develop a new analysis plugin. For an example of conversion plugins, see :doc:`conversion`.
 
-Plugins Interface
+A more step-by-step tutorial with slides is available `here <https://lab.cluster.gsi.dit.upm.es/senpy/senpy-tutorial>`__ 
+
+.. contents:: :local:
+
+What is a plugin?
 =================
 
+A plugin is a program that, given a text, will add annotations to it.
+In practice, a plugin consists of at least two files:
+
+- Definition file: a `.senpy` file that describes the plugin (e.g. what input parameters it accepts, what emotion model it uses).
+- Python module: the actual code that will add annotations to each input.
+
+This separation allows us to deploy plugins that use the same code but employ different parameters.
+For instance, one could use the same classifier and processing in several plugins, but train with different datasets.
+This scenario is particularly useful for evaluation purposes.
+
+The only limitation is that the name of each plugin needs to be unique.
+
+Plugin Definition files
+=======================
+
+The definition file contains all the attributes of the plugin, and can be written in YAML or JSON.
+When the server is launched, it will recursively search for definition files in the plugin folder (the current folder, by default).
+The most important attributes are:
+
+* **name**: unique name that senpy will use internally to identify the plugin.
+* **module**: indicates the module that contains the plugin code, which will be automatically loaded by senpy.
+* **version**
+* extra_params: to add parameters to the senpy API when this plugin is requested. Those parameters may be required, and have aliased names. For instance:
+
+  .. code:: yaml
+
+            extra_params:
+                hello_param:
+                    aliases: # required
+                        - hello_param
+                        - hello
+                    required: true
+                    default: Hi you
+                    values:
+                        - Hi you
+                        - Hello y'all
+                        - Howdy
+
+  Parameter validation will fail if a required parameter without a default has not been provided, or if the definition includes a set of values and the provided one does not match one of them.
+
+
+A complete example:
+
+.. code:: yaml
+          
+          name: <Name of the plugin>
+          module: <Python file>
+          version: 0.1
+
+And the json equivalent:
+
+.. code:: json
+
+          {
+            "name": "<Name of the plugin>",
+            "module": "<Python file>",
+            "version": "0.1"
+          }
+
+
+Plugins Code
+============
+
 The basic methods in a plugin are:
 
 * __init__
 * activate: used to load memory-hungry resources
 * deactivate: used to free up resources
+* analyse_entry: called in every user requests. It takes two parameters: ``Entry``, the entry object, and ``params``, the parameters supplied by the user. It should yield one or more ``Entry`` objects.
 
 Plugins are loaded asynchronously, so don't worry if the activate method takes too long. The plugin will be marked as activated once it is finished executing the method.
 
+Entries
+=======
+
+Entries are objects that can be annotated.
+By default, entries are `NIF contexts <http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core/nif-core.html>`_ represented in JSON-LD format.
+Annotations are added to the object like this:
+
+.. code:: python
+
+   entry = Entry()
+   entry.vocabulary__annotationName = 'myvalue'
+   entry['vocabulary:annotationName'] = 'myvalue'
+   entry['annotationNameURI'] = 'myvalue'
+
+Where vocabulary is one of the prefixes defined in the default senpy context, and annotationURI is a full URI.
+The value may be any valid JSON-LD dictionary.
+For simplicity, senpy includes a series of models by default in the ``senpy.models`` module.
+
+
+Example plugin
+==============
+
+In this section, we will implement a basic sentiment analysis plugin.
+To determine the polarity of each entry, the plugin will compare the length of the string to a threshold.
+This threshold will be included in the definition file.
+
+The definition file would look like this:
+
+.. code:: yaml
+
+          name: helloworld
+          module: helloworld
+          version: 0.0
+          threshold: 10
+          description: Hello World
+
+Now, in a file named ``helloworld.py``:
+
+.. code:: python
+
+          #!/bin/env python
+          #helloworld.py
+
+          from senpy.plugins import AnalysisPlugin
+          from senpy.models import Sentiment
+
+
+          class HelloWorld(AnalysisPlugin):
+
+              def analyse_entry(entry, params):
+                  '''Basically do nothing with each entry'''
+
+                  sentiment = Sentiment()
+                  if len(entry.text) < self.threshold:
+                      sentiment['marl:hasPolarity'] = 'marl:Positive'
+                  else:
+                      sentiment['marl:hasPolarity'] = 'marl:Negative'
+                  entry.sentiments.append(sentiment)
+                  yield entry
+
+The complete code of the example plugin is available `here <https://lab.cluster.gsi.dit.upm.es/senpy/plugin-prueba>`__.
+
+Loading data and files
+======================
+
+Most plugins will need access to files (dictionaries, lexicons, etc.).
+It is good practice to specify the paths of these files in the plugin configuration, so the same code can be reused with different resources.
+
+
+.. code:: yaml
+
+          name: dictworld
+          module: dictworld
+          dictionary_path: <PATH OF THE FILE>
+         
+The path can be either absolute, or relative.
+
+From absolute paths
+???????????????????
+
+Absolute paths (such as ``/data/dictionary.csv`` are straightfoward:
+
+.. code:: python
+
+   with open(os.path.join(self.dictionary_path) as f:
+      ...
+
+From relative paths
+???????????????????
+Since plugins are loading dynamically, relative paths will refer to the current working directory.
+Instead, what you usually want is to load files *relative to the plugin source folder*, like so:
+
+
+:: 
+
+   .
+   ..
+   plugin.senpy
+   plugin.py
+   dictionary.csv
+
+For this, we need to first get the path of your source folder first, like so:
+
+.. code:: python
+
+   import os
+   root = os.path.realpath(__file__)
+   with open(os.path.join(root, self.dictionary_path) as f:
+       ...
+
+
+Docker image
+============
+
+Add the following dockerfile to your project to generate a docker image with your plugin:
+
+.. code:: dockerfile
+
+   FROM gsiupm/senpy:0.8.8
+
+This will copy your source folder to the image, and install all dependencies.
+Now, to build an image:
+
+.. code:: shell
+
+   docker build . -t gsiupm/exampleplugin
+
+And you can run it with:
+
+.. code:: shell
+
+   docker run -p 5000:5000 gsiupm/exampleplugin
+
+
+If the plugin non-source files (:ref:`loading data and files`), the recommended way is to use absolute paths.
+Data can then be mounted in the container or added to the image.
+The former is recommended for open source plugins with licensed resources, whereas the latter is the most convenient and can be used for private images.
+
+Mounting data:
+
+.. code:: bash
+
+   docker run -v $PWD/data:/data gsiupm/exampleplugin
+
+Adding data to the image:
+
+.. code:: dockerfile
+
+   FROM gsiupm/senpy:0.8.8
+   COPY data /
+
 F.A.Q.
 ======
-If I'm using a classifier, where should I train the classifier?
-???????????????????????????????????????????????????????????????
+What annotations can I use?
+???????????????????????????
+
+You can add almost any annotation to an entry.
+The most common use cases are covered in the :doc:`apischema`.
+
+
+Why does the analyse function yield instead of return?
+??????????????????????????????????????????????????????
+
+This is so that plugins may add new entries to the response or filter some of them.
+For instance, a `context detection` plugin may add a new entry for each context in the original entry.
+On the other hand, a conversion plugin may leave out those entries that do not contain relevant information.
+
+
+If I'm using a classifier, where should I train it?
+???????????????????????????????????????????????????
 
 Training a classifier can be time time consuming. To avoid running the training unnecessarily, you can use ShelfMixin to store the classifier. For instance:
 
 .. code:: python
 
-          from senpy.plugins import ShelfMixin, SenpyPlugin
+          from senpy.plugins import ShelfMixin, AnalysisPlugin
 
-          class MyPlugin(ShelfMixin, SenpyPlugin):
+          class MyPlugin(ShelfMixin, AnalysisPlugin):
               def train(self):
                   ''' Code to train the classifier
                   '''
@@ -40,7 +275,103 @@ Training a classifier can be time time consuming. To avoid running the training
               def deactivate(self):
                   self.close()
 
-You can speficy a 'shelf_file' in your .senpy file. By default the ShelfMixin creates a file based on the plugin name and stores it in that plugin's folder.
+You can specify a 'shelf_file' in your .senpy file. By default the ShelfMixin creates a file based on the plugin name and stores it in that plugin's folder.
+
+Shelves may get corrupted if the plugin exists unexpectedly.
+A corrupt shelf prevents the plugin from loading.
+If you do not care about the pickle, you can force your plugin to remove the corrupted file and load anyway, set the  'force_shelf' to True in your .senpy file.
+
+How can I turn an external service into a plugin?
+?????????????????????????????????????????????????
+
+This example ilustrate how to implement a plugin that accesses the Sentiment140 service.
+
+.. code:: python
+
+          class Sentiment140Plugin(SentimentPlugin):
+              def analyse_entry(self, entry, params):
+                  text = entry.text
+                  lang = params.get("language", "auto")
+                  res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
+                                      json.dumps({"language": lang,
+                                                  "data": [{"text": text}]
+                                                  }
+                                                 )
+                                      )
+
+                  p = params.get("prefix", None)
+                  polarity_value = self.maxPolarityValue*int(res.json()["data"][0]
+                                                             ["polarity"]) * 0.25
+                  polarity = "marl:Neutral"
+                  neutral_value = self.maxPolarityValue / 2.0
+                  if polarity_value > neutral_value:
+                      polarity = "marl:Positive"
+                  elif polarity_value < neutral_value:
+                      polarity = "marl:Negative"
+
+                  sentiment = Sentiment(id="Sentiment0",
+                                      prefix=p,
+                                      marl__hasPolarity=polarity,
+                                      marl__polarityValue=polarity_value)
+                  sentiment.prov__wasGeneratedBy = self.id
+                  entry.sentiments.append(sentiment)
+                  yield entry
+
+
+Can my plugin require additional parameters from the user?
+??????????????????????????????????????????????????????????
+
+You can add extra parameters in the definition file under the attribute ``extra_params``.
+It takes a dictionary, where the keys are the name of the argument/parameter, and the value has the following fields:
+
+* aliases: the different names which can be used in the request to use the parameter.
+* required: if set to true, users need to provide this parameter unless a default is set.
+* options: the different acceptable values of the parameter (i.e. an enum). If set, the value provided must match one of the options.
+* default: the default value of the parameter, if none is provided in the request.
+
+.. code:: python
+
+          extra_params
+             language:
+                aliases:
+                    - language
+                    - lang
+                    - l
+                required: true,
+                options:
+                    - es
+                    - en
+                default: es
+
+This example shows how to introduce a parameter associated with language.
+The extraction of this paremeter is used in the analyse method of the Plugin interface.
+
+.. code:: python
+
+          lang = params.get("language")
+
+Where can I set up variables for using them in my plugin?
+?????????????????????????????????????????????????????????
+
+You can add these variables in the definition file with the structure of attribute-value pairs.
+
+Every field added to the definition file is available to the plugin instance.
+
+Can I activate a DEBUG mode for my plugin?
+???????????????????????????????????????????
+
+You can activate the DEBUG mode by the command-line tool using the option -d.
+
+.. code:: bash
+
+   senpy -d
+
+
+Additionally, with the ``--pdb`` option you will be dropped into a pdb post mortem shell if an exception is raised.
+
+.. code:: bash
+
+   senpy --pdb
 
 Where can I find more code examples?
 ????????????????????????????????????

docs/requirements.txt

@@ -1 +1,2 @@
 sphinxcontrib-httpdomain>=1.4
+nbsphinx

docs/senpy.rst

@@ -0,0 +1,51 @@
+What is Senpy?
+--------------
+
+Web services can get really complex: data validation, user interaction, formatting, logging., etc. 
+The figure below summarizes the typical features in an analysis service.
+Senpy implements all the common blocks, so developers can focus on what really matters: great analysis algorithms that solve real problems.
+
+.. image:: senpy-framework.png
+  :width: 60%
+  :align: center
+
+
+Senpy for end users
+===================
+
+All services built using senpy share a common interface.
+This allows users to use them (almost) interchangeably.
+Senpy comes with a :ref:`built-in client`.
+
+
+Senpy for service developers
+============================
+
+Senpy is a framework that turns your sentiment or emotion analysis algorithm into a full blown semantic service.
+Senpy takes care of:
+
+  * Interfacing with the user: parameter validation, error handling.
+  * Formatting: JSON-LD, Turtle/n-triples input and output, or simple text input
+  * Linked Data: senpy results are semantically annotated, using a series of well established vocabularies, and sane default URIs.
+  * User interface: a web UI where users can explore your service and test different settings
+  * A client to interact with the service. Currently only available in Python.
+
+Sharing your sentiment analysis with the world has never been easier!
+
+Check out the :doc:`plugins` if you have developed an analysis algorithm (e.g. sentiment analysis) and you want to publish it as a service.
+
+Architecture
+============
+
+The main component of a sentiment analysis service is the algorithm itself. However, for the algorithm to work, it needs to get the appropriate parameters from the user, format the results according to the defined API, interact with the user whn errors occur or more information is needed, etc.
+
+Senpy proposes a modular and dynamic architecture that allows:
+
+* Implementing different algorithms in a extensible way, yet offering a common interface.
+* Offering common services that facilitate development, so developers can focus on implementing new and better algorithms.
+
+The framework consists of two main modules: Senpy core, which is the building block of the service, and Senpy plugins, which consist of the analysis algorithm. The next figure depicts a simplified version of the processes involved in an analysis with the Senpy framework.
+
+.. image:: senpy-architecture.png
+  :width: 100%
+  :align: center

docs/server.rst

@@ -0,0 +1,58 @@
+Server
+======
+
+The senpy server is launched via the `senpy` command:
+
+.. code:: text
+
+    usage: senpy [-h] [--level logging_level] [--debug] [--default-plugins]
+                [--host HOST] [--port PORT] [--plugins-folder PLUGINS_FOLDER]
+                [--only-install]
+
+    Run a Senpy server
+
+    optional arguments:
+    -h, --help            show this help message and exit
+    --level logging_level, -l logging_level
+                            Logging level
+    --debug, -d           Run the application in debug mode
+    --default-plugins     Load the default plugins
+    --host HOST           Use 0.0.0.0 to accept requests from any host.
+    --port PORT, -p PORT  Port to listen on.
+    --plugins-folder PLUGINS_FOLDER, -f PLUGINS_FOLDER
+                            Where to look for plugins.
+    --only-install, -i    Do not run a server, only install plugin dependencies
+
+
+When launched, the server will recursively look for plugins in the specified plugins folder (the current working directory by default).
+For every plugin found, it will download its dependencies, and try to activate it.
+The default server includes a playground and an endpoint with all plugins found.
+
+Let's run senpy with the default plugins:
+
+.. code:: bash
+
+    senpy -f . --default-plugins
+
+Now go to `http://localhost:5000 <http://localhost:5000>`_, you should be greeted by the senpy playground:
+
+.. image:: senpy-playground.png
+   :width: 100%
+   :alt: Playground
+
+The playground is a user-friendly way to test your plugins, but you can always use the service directly:  `http://localhost:5000/api?input=hello <http://localhost:5000/api?input=hello>`_.
+
+
+By default, senpy will listen only on the `127.0.0.1` address.
+That means you can only access the API from your (or localhost).
+You can listen on a different address using the `--host` flag (e.g., 0.0.0.0).
+The default port is 5000.
+You can change it with the `--port` flag. 
+
+For instance, to accept connections on port 6000 on any interface:
+
+.. code:: bash
+
+    senpy --host 0.0.0.0 --port 6000
+
+For more options, see the `--help` page.

docs/usage.rst

@@ -1,20 +1,15 @@
 Usage
 -----
 
-The easiest and recommended way is to just use the command-line tool to load your plugins and launch the server.
+First of all, you need to install the package.
+See :doc:`installation` for instructions.
+Once installed, the `senpy` command should be available. 
 
-.. code:: bash
+.. toctree::
+   :maxdepth: 1
 
-   senpy
-
-Or, alternatively:
-
-.. code:: bash
-
-    python -m senpy
+   server
+   SenpyClientUse
+   commandline
 
 
-This will create a server with any modules found in the current path.
-For more options, see the `--help` page.
-
-Alternatively, you can use the modules included in senpy to build your own application.

docs/vocabularies.rst

@@ -0,0 +1,8 @@
+Vocabularies and model
+======================
+
+The model used in Senpy is based on the following vocabularies:
+
+* Marl, a vocabulary designed to annotate and describe subjetive opinions expressed on the web or in information systems.
+* Onyx, which is built one the same principles as Marl to annotate and describe emotions, and provides interoperability with Emotion Markup Language.
+* NIF 2.0, which defines a semantic format and APO for improving interoperability among natural language processing services

k8s/README.md

@@ -0,0 +1,7 @@
+Deploy senpy to a kubernetes cluster.
+
+Usage:
+
+```
+kubectl apply -f . -n senpy
+```

k8s/senpy-deployment.yaml

@@ -0,0 +1,26 @@
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: senpy-latest
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        role: senpy-latest
+        app: test
+    spec:
+      containers:
+      - name: senpy-latest
+        image: gsiupm/senpy:latest
+        imagePullPolicy: Always
+        args:
+          - "--default-plugins"
+        resources:
+          limits:
+            memory: "512Mi"
+            cpu: "1000m"
+        ports:
+          - name: web
+            containerPort: 5000

k8s/senpy-ingress.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: senpy-ingress
+spec:
+  rules:
+  - host: latest.senpy.cluster.gsi.dit.upm.es
+    http:
+      paths:
+      - path: /
+        backend:
+          serviceName: senpy-latest
+          servicePort: 5000

k8s/senpy-svc.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: senpy-latest
+spec:
+  type: ClusterIP
+  ports:
+    - port: 5000
+      protocol: TCP
+  selector:
+    role: senpy-latest

requirements.txt

@@ -1,7 +1,11 @@
 Flask>=0.10.1
-gunicorn>=19.0.0
 requests>=2.4.1
-GitPython>=0.3.2.RC1
-gevent>=1.0.1
+tornado>=4.4.3
 PyLD>=0.6.5
-Flask-Testing>=0.4.2
+six
+future
+jsonschema
+jsonref
+PyYAML
+rdflib
+rdflib-jsonld

senpy/__init__.py

@@ -17,3 +17,12 @@
 """
 Sentiment analysis server in Python
 """
+from .version import __version__
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+logger.info('Using senpy version: {}'.format(__version__))
+
+__all__ = ['api', 'blueprints', 'cli', 'extensions', 'models', 'plugins']

senpy/__main__.py

@@ -22,60 +22,92 @@ the server.
 
 from flask import Flask
 from senpy.extensions import Senpy
-from gevent.wsgi import WSGIServer
-from gevent.monkey import patch_all
-import gevent
+
 import logging
 import os
 import argparse
+import senpy
+
+SERVER_PORT = os.environ.get("PORT", 5000)
 
-patch_all(thread=False)
 
 def main():
     parser = argparse.ArgumentParser(description='Run a Senpy server')
-    parser.add_argument('--level',
-                        "-l",
-                        metavar="logging_level",
-                        type=str,
-                        default="INFO",
-                        help='Logging level')
-    parser.add_argument('--debug',
-                        "-d",
-                        action='store_true',
-                        default=False,
-                        help='Run the application in debug mode')
-    parser.add_argument('--default-plugins',
-                        action='store_true',
-                        default=False,
-                        help='Load the default plugins')
-    parser.add_argument('--host',
-                        type=str,
-                        default="127.0.0.1",
-                        help='Use 0.0.0.0 to accept requests from any host.')
-    parser.add_argument('--port',
-                        '-p',
-                        type=int,
-                        default=5000,
-                        help='Port to listen on.')
-    parser.add_argument('--plugins-folder',
-                        '-f',
-                        type=str,
-                        default="plugins",
-                        help='Where to look for plugins.')
+    parser.add_argument(
+        '--level',
+        '-l',
+        metavar='logging_level',
+        type=str,
+        default="INFO",
+        help='Logging level')
+    parser.add_argument(
+        '--debug',
+        '-d',
+        action='store_true',
+        default=False,
+        help='Run the application in debug mode')
+    parser.add_argument(
+        '--default-plugins',
+        action='store_true',
+        default=False,
+        help='Load the default plugins')
+    parser.add_argument(
+        '--host',
+        type=str,
+        default="0.0.0.0",
+        help='Use 0.0.0.0 to accept requests from any host.')
+    parser.add_argument(
+        '--port',
+        '-p',
+        type=int,
+        default=SERVER_PORT,
+        help='Port to listen on.')
+    parser.add_argument(
+        '--plugins-folder',
+        '-f',
+        type=str,
+        default='plugins',
+        help='Where to look for plugins.')
+    parser.add_argument(
+        '--only-install',
+        '-i',
+        action='store_true',
+        default=False,
+        help='Do not run a server, only install plugin dependencies')
+    parser.add_argument(
+        '--threaded',
+        action='store_false',
+        default=True,
+        help='Run a threaded server')
+    parser.add_argument(
+        '--version',
+        '-v',
+        action='store_true',
+        default=False,
+        help='Output the senpy version and exit')
     args = parser.parse_args()
-    logging.basicConfig(level=getattr(logging, args.level))
+    if args.version:
+        print('Senpy version {}'.format(senpy.__version__))
+        exit(1)
+    logging.basicConfig()
+    rl = logging.getLogger()
+    rl.setLevel(getattr(logging, args.level))
     app = Flask(__name__)
     app.debug = args.debug
     sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins)
+    if args.only_install:
+        sp.install_deps()
+        return
     sp.activate_all()
-    http_server = WSGIServer((args.host, args.port), app)
-    try:
-        print("Server running on port %s:%d. Ctrl+C to quit" % (args.host,
-                                                                args.port))
-        http_server.serve_forever()
-    except KeyboardInterrupt:
-        http_server.stop()
-        print("Bye!")
+    print('Senpy version {}'.format(senpy.__version__))
+    print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,
+                                                            args.port))
+    app.run(args.host,
+            args.port,
+            threaded=args.threaded,
+            debug=app.debug)
+    sp.deactivate_all()
+
 
 if __name__ == '__main__':
     main()

senpy/api.py

@@ -0,0 +1,131 @@
+from future.utils import iteritems
+from .models import Error
+import logging
+logger = logging.getLogger(__name__)
+
+API_PARAMS = {
+    "algorithm": {
+        "aliases": ["algorithm", "a", "algo"],
+        "required": False,
+    },
+    "outformat": {
+        "@id": "outformat",
+        "aliases": ["outformat", "o"],
+        "default": "json-ld",
+        "required": True,
+        "options": ["json-ld", "turtle"],
+    },
+    "expanded-jsonld": {
+        "@id": "expanded-jsonld",
+        "aliases": ["expanded", "expanded-jsonld"],
+        "required": True,
+        "default": 0
+    },
+    "emotionModel": {
+        "@id": "emotionModel",
+        "aliases": ["emotionModel", "emoModel"],
+        "required": False
+    },
+    "plugin_type": {
+        "@id": "pluginType",
+        "description": 'What kind of plugins to list',
+        "aliases": ["pluginType", "plugin_type"],
+        "required": True,
+        "default": "analysisPlugin"
+    },
+    "conversion": {
+        "@id": "conversion",
+        "description": "How to show the elements that have (not) been converted",
+        "required": True,
+        "options": ["filtered", "nested", "full"],
+        "default": "full"
+    }
+}
+
+WEB_PARAMS = {
+    "inHeaders": {
+        "aliases": ["inHeaders", "headers"],
+        "required": True,
+        "default": "0"
+    },
+}
+
+CLI_PARAMS = {
+    "plugin_folder": {
+        "aliases": ["plugin_folder", "folder"],
+        "required": True,
+        "default": "."
+    },
+}
+
+NIF_PARAMS = {
+    "input": {
+        "@id": "input",
+        "aliases": ["i", "input"],
+        "required": True,
+        "help": "Input text"
+    },
+    "informat": {
+        "@id": "informat",
+        "aliases": ["f", "informat"],
+        "required": False,
+        "default": "text",
+        "options": ["turtle", "text", "json-ld"],
+    },
+    "intype": {
+        "@id": "intype",
+        "aliases": ["intype", "t"],
+        "required": False,
+        "default": "direct",
+        "options": ["direct", "url", "file"],
+    },
+    "language": {
+        "@id": "language",
+        "aliases": ["language", "l"],
+        "required": False,
+    },
+    "prefix": {
+        "@id": "prefix",
+        "aliases": ["prefix", "p"],
+        "required": True,
+        "default": "",
+    },
+    "urischeme": {
+        "@id": "urischeme",
+        "aliases": ["urischeme", "u"],
+        "required": False,
+        "default": "RFC5147String",
+        "options": "RFC5147String"
+    },
+}
+
+
+def parse_params(indict, spec=NIF_PARAMS):
+    logger.debug("Parsing: {}\n{}".format(indict, spec))
+    outdict = indict.copy()
+    wrong_params = {}
+    for param, options in iteritems(spec):
+        if param[0] != "@":  # Exclude json-ld properties
+            for alias in options.get("aliases", []):
+                if alias in indict:
+                    outdict[param] = indict[alias]
+            if param not in outdict:
+                if options.get("required", False) and "default" not in options:
+                    wrong_params[param] = spec[param]
+                else:
+                    if "default" in options:
+                        outdict[param] = options["default"]
+            else:
+                if "options" in spec[param] and \
+                   outdict[param] not in spec[param]["options"]:
+                    wrong_params[param] = spec[param]
+    if wrong_params:
+        logger.debug("Error parsing: %s", wrong_params)
+        message = Error(
+            status=400,
+            message="Missing or invalid parameters",
+            parameters=outdict,
+            errors={param: error
+                    for param, error in iteritems(wrong_params)})
+        raise message
+    return outdict

senpy/blueprints.py

@@ -17,157 +17,130 @@
 """
 Blueprints for Senpy
 """
-from flask import Blueprint, request, current_app
-from .models import Error, Response, Leaf
+from flask import (Blueprint, request, current_app, render_template, url_for,
+                   jsonify)
+from .models import Error, Response, Plugins, read_schema
+from .api import WEB_PARAMS, API_PARAMS, parse_params
+from .version import __version__
+from functools import wraps
 
-import json
 import logging
+import json
 
 logger = logging.getLogger(__name__)
 
-
-nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__)
-
-BASIC_PARAMS = {
-    "algorithm": {
-        "aliases": ["algorithm", "a", "algo"],
-        "required": False,
-    },
-    "inHeaders": {
-        "aliases": ["inHeaders", "headers"],
-        "required": True,
-        "default": "0"
-    }
-}
-
-LIST_PARAMS = {
-    "params": {
-        "aliases": ["params", "with_params"],
-        "required": False,
-        "default": "0"
-    },
-}
+api_blueprint = Blueprint("api", __name__)
+demo_blueprint = Blueprint("demo", __name__)
+ns_blueprint = Blueprint("ns", __name__)
 
 
-def get_params(req, params=BASIC_PARAMS):
+def get_params(req):
     if req.method == 'POST':
-        indict = req.form
+        indict = req.form.to_dict(flat=True)
     elif req.method == 'GET':
-        indict = req.args
+        indict = req.args.to_dict(flat=True)
     else:
-        raise ValueError("Invalid data")
+        raise Error(message="Invalid data")
+    return indict
 
-    outdict = {}
-    wrong_params = {}
-    for param, options in params.iteritems():
-        if param[0] != "@":  # Exclude json-ld properties
-            logger.debug("Param: %s - Options: %s", param, options)
-            for alias in options["aliases"]:
-                if alias in indict:
-                    outdict[param] = indict[alias]
-            if param not in outdict:
-                if options.get("required", False) and "default" not in options:
-                    wrong_params[param] = params[param]
-                else:
-                    if "default" in options:
-                        outdict[param] = options["default"]
+
+@demo_blueprint.route('/')
+def index():
+    return render_template("index.html", version=__version__)
+
+
+@api_blueprint.route('/contexts/<entity>.jsonld')
+def context(entity="context"):
+    context = Response._context
+    context['@vocab'] = url_for('ns.index', _external=True)
+    return jsonify({"@context": context})
+
+
+@ns_blueprint.route('/')  # noqa: F811
+def index():
+    context = Response._context
+    context['@vocab'] = url_for('.ns', _external=True)
+    return jsonify({"@context": context})
+
+
+@api_blueprint.route('/schemas/<schema>')
+def schema(schema="definitions"):
+    try:
+        return jsonify(read_schema(schema))
+    except Exception:  # Should be FileNotFoundError, but it's missing from py2
+        return Error(message="Schema not found", status=404).flask()
+
+
+def basic_api(f):
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        raw_params = get_params(request)
+        headers = {'X-ORIGINAL-PARAMS': json.dumps(raw_params)}
+        # Get defaults
+        web_params = parse_params({}, spec=WEB_PARAMS)
+        api_params = parse_params({}, spec=API_PARAMS)
+
+        outformat = 'json-ld'
+        try:
+            print('Getting request:')
+            print(request)
+            web_params = parse_params(raw_params, spec=WEB_PARAMS)
+            api_params = parse_params(raw_params, spec=API_PARAMS)
+            if hasattr(request, 'params'):
+                request.params.update(api_params)
             else:
-                if "options" in params[param] and \
-                   outdict[param] not in params[param]["options"]:
-                    wrong_params[param] = params[param]
-    if wrong_params:
-        message = Error({"status": 404,
-                         "message": "Missing or invalid parameters",
-                         "parameters": outdict,
-                         "errors": {param: error for param, error in
-                                    wrong_params.iteritems()}
-                         })
-        raise ValueError(message)
-    return outdict
+                request.params = api_params
+            response = f(*args, **kwargs)
+        except Error as ex:
+            response = ex
+            logger.error(ex)
+            if current_app.debug:
+                raise
+
+        in_headers = web_params['inHeaders'] != "0"
+        expanded = api_params['expanded-jsonld']
+        outformat = api_params['outformat']
+
+        return response.flask(
+            in_headers=in_headers,
+            headers=headers,
+            prefix=url_for('.api', _external=True),
+            context_uri=url_for('api.context',
+                                entity=type(response).__name__,
+                                _external=True),
+            outformat=outformat,
+            expanded=expanded)
+
+    return decorated_function
 
 
-def basic_analysis(params):
-    response = {"@context":
-                [("http://demos.gsi.dit.upm.es/"
-                  "eurosentiment/static/context.jsonld"),
-                 {
-                    "@base": "{}#".format(request.url.encode('utf-8'))
-                  }
-                 ],
-                "analysis": [{"@type": "marl:SentimentAnalysis"}],
-                "entries": []
-                }
-    if "language" in params:
-        response["language"] = params["language"]
-    for idx, sentence in enumerate(params["input"].split(".")):
-        response["entries"].append({
-            "@id": "Sentence{}".format(idx),
-            "nif:isString": sentence
-        })
+@api_blueprint.route('/', methods=['POST', 'GET'])
+@basic_api
+def api():
+    response = current_app.senpy.analyse(**request.params)
     return response
 
 
-@nif_blueprint.route('/', methods=['POST', 'GET'])
-def home():
-    try:
-        params = get_params(request)
-        algo = params.get("algorithm", None)
-        specific_params = current_app.senpy.parameters(algo)
-        logger.debug(
-            "Specific params: %s", json.dumps(specific_params, indent=4))
-        params.update(get_params(request, specific_params))
-        response = current_app.senpy.analyse(**params)
-        in_headers = params["inHeaders"] != "0"
-        return response.flask(in_headers=in_headers)
-    except ValueError as ex:
-        return ex.message.flask()
-
-
-@nif_blueprint.route("/default")
-def default():
-    # return current_app.senpy.default_plugin
-    plug = current_app.senpy.default_plugin
-    if plug:
-        return plugins(action="list", plugin=plug.name)
-    else:
-        error = Error(status=404, message="No plugins found")
-        return error.flask()
-
-
-@nif_blueprint.route('/plugins/', methods=['POST', 'GET'])
-@nif_blueprint.route('/plugins/<plugin>', methods=['POST', 'GET'])
-@nif_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
-def plugins(plugin=None, action="list"):
-    filt = {}
+@api_blueprint.route('/plugins/', methods=['POST', 'GET'])
+@basic_api
+def plugins():
     sp = current_app.senpy
-    if plugin:
-        filt["name"] = plugin
-    plugs = sp.filter_plugins(**filt)
-    if plugin and not plugs:
-        return "Plugin not found", 400
-    if action == "list":
-        with_params = get_params(request, LIST_PARAMS)["params"] == "1"
-        in_headers = get_params(request, BASIC_PARAMS)["inHeaders"] != "0"
-        if plugin:
-            dic = plugs[plugin]
-        else:
-            dic = Response(
-                    {plug: plugs[plug].jsonld(with_params) for plug in plugs},
-                    frame={})
-        return dic.flask(in_headers=in_headers)
-    method = "{}_plugin".format(action)
-    if(hasattr(sp, method)):
-        getattr(sp, method)(plugin)
-        return Leaf(message="Ok").flask()
+    ptype = request.params.get('plugin_type')
+    plugins = sp.filter_plugins(plugin_type=ptype)
+    dic = Plugins(plugins=list(plugins.values()))
+    return dic
+
+
+@api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
+@basic_api
+def plugin(plugin=None):
+    sp = current_app.senpy
+    if plugin == 'default' and sp.default_plugin:
+        return sp.default_plugin
+    plugins = sp.filter_plugins(
+        id='plugins/{}'.format(plugin)) or sp.filter_plugins(name=plugin)
+    if plugins:
+        response = list(plugins.values())[0]
     else:
-        return Error("action '{}' not allowed".format(action)).flask()
-
-
-if __name__ == '__main__':
-    import config
-    from flask import Flask
-
-    app = Flask(__name__)
-    app.register_blueprint(nif_blueprint)
-    app.debug = config.DEBUG
-    app.run()
+        return Error(message="Plugin not found", status=404)
+    return response

senpy/cli.py

@@ -0,0 +1,52 @@
+import sys
+from .models import Error
+from .api import parse_params, CLI_PARAMS
+from .extensions import Senpy
+
+
+def argv_to_dict(argv):
+    '''Turns parameters in the form of '--key value' into a dict {'key': 'value'}
+    '''
+    cli_dict = {}
+
+    for i in range(len(argv)):
+        if argv[i][0] == '-':
+            key = argv[i].strip('-')
+            value = argv[i + 1] if len(argv) > i + 1 else None
+            if value and value[0] == '-':
+                cli_dict[key] = ""
+            else:
+                cli_dict[key] = value
+    return cli_dict
+
+
+def parse_cli(argv):
+    cli_dict = argv_to_dict(argv)
+    cli_params = parse_params(cli_dict, spec=CLI_PARAMS)
+    return cli_params, cli_dict
+
+
+def main_function(argv):
+    '''This is the method for unit testing
+    '''
+    cli_params, cli_dict = parse_cli(argv)
+    plugin_folder = cli_params['plugin_folder']
+    sp = Senpy(default_plugins=False, plugin_folder=plugin_folder)
+    sp.activate_all(sync=True)
+    res = sp.analyse(**cli_dict)
+    return res
+
+
+def main():
+    '''This method is the entrypoint for the CLI (as configured un setup.py)
+    '''
+    try:
+        res = main_function(sys.argv[1:])
+        print(res.to_JSON())
+    except Error as err:
+        print(err.to_JSON())
+        sys.exit(2)
+
+
+if __name__ == '__main__':
+    main()

senpy/client.py

@@ -0,0 +1,41 @@
+import requests
+import logging
+from . import models
+from .plugins import default_plugin_type
+
+logger = logging.getLogger(__name__)
+
+
+class Client(object):
+    def __init__(self, endpoint):
+        self.endpoint = endpoint
+
+    def analyse(self, input, method='GET', **kwargs):
+        return self.request('/', method=method, input=input, **kwargs)
+
+    def plugins(self, ptype=default_plugin_type):
+        resp = self.request(path='/plugins', plugin_type=ptype).plugins
+        return {p.name: p for p in resp}
+
+    def request(self, path=None, method='GET', **params):
+        url = '{}{}'.format(self.endpoint, path)
+        response = requests.request(method=method, url=url, params=params)
+        try:
+            resp = models.from_dict(response.json())
+        except Exception as ex:
+            logger.error(('There seems to be a problem with the response:\n'
+                          '\tURL: {url}\n'
+                          '\tError: {error}\n'
+                          '\t\n'
+                          '#### Response:\n'
+                          '\tCode: {code}'
+                          '\tContent: {content}'
+                          '\n').format(
+                              error=ex,
+                              url=url,
+                              code=response.status_code,
+                              content=response.content))
+            raise ex
+        if isinstance(resp, models.Error):
+            raise resp
+        return resp

senpy/context.jsonld

@@ -1,42 +0,0 @@
-{
-    "dc": "http://purl.org/dc/terms/",
-    "dc:subject": {
-        "@type": "@id"
-    },
-    "xsd": "http://www.w3.org/2001/XMLSchema#",
-    "marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#",
-    "nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#",
-    "onyx": "http://www.gsi.dit.upm.es/ontologies/onyx/ns#",
-    "emotions": {
-        "@container": "@set",
-        "@id": "onyx:hasEmotionSet"
-    },
-    "opinions": {
-        "@container": "@set",
-        "@id": "marl:hasOpinion"
-    },
-    "prov": "http://www.w3.org/ns/prov#",
-    "rdfs": "http://www.w3.org/2000/01/rdf-schema#",
-    "analysis": {
-        "@container": "@set",
-        "@id": "prov:wasInformedBy"
-    },
-    "entries": {
-        "@container": "@set",
-        "@id": "prov:generated"
-    },
-    "strings": {
-        "@container": "@set",
-        "@reverse": "nif:hasContext"
-    },
-    "date":
-    {
-        "@id": "dc:date",
-        "@type": "xsd:dateTime"
-    },
-    "text": { "@id": "nif:isString" },
-    "wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#",
-    "xsd": "http://www.w3.org/2001/XMLSchema#",
-    "senpy": "http://www.gsi.dit.upm.es/ontologies/senpy/ns#",
-    "@vocab":  "http://www.gsi.dit.upm.es/ontologies/senpy/ns#"
-}

senpy/extensions.py

@@ -1,43 +1,59 @@
 """
+Main class for Senpy.
+It orchestrates plugin (de)activation and analysis.
 """
-import gevent
-from gevent import monkey
-monkey.patch_all()
+from future import standard_library
+standard_library.install_aliases()
 
-from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
-from .models import Error
-from .blueprints import nif_blueprint
+from . import plugins
+from .plugins import SenpyPlugin
+from .models import Error, Entry, Results, from_string
+from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
+from .api import API_PARAMS, NIF_PARAMS, parse_params
 
-from git import Repo, InvalidGitRepositoryError
-from functools import partial
+from threading import Thread
 
 import os
+import copy
 import fnmatch
 import inspect
 import sys
-import imp
+import importlib
 import logging
 import traceback
-import gevent
-import json
+import yaml
+import subprocess
 
 logger = logging.getLogger(__name__)
 
 
-class Senpy(object):
+def log_subprocess_output(process):
+    for line in iter(process.stdout.readline, b''):
+        logger.info('%r', line)
+    for line in iter(process.stderr.readline, b''):
+        logger.error('%r', line)
 
+
+class Senpy(object):
     """ Default Senpy extension for Flask """
 
-    def __init__(self, app=None, plugin_folder="plugins", default_plugins=False):
+    def __init__(self,
+                 app=None,
+                 plugin_folder=".",
+                 default_plugins=False):
         self.app = app
-
         self._search_folders = set()
+        self._plugin_list = []
         self._outdated = True
+        self._default = None
 
         self.add_folder(plugin_folder)
         if default_plugins:
-            base_folder = os.path.join(os.path.dirname(__file__), "plugins")
-            self.add_folder(base_folder)
+            self.add_folder('plugins', from_root=True)
+        else:
+            # Add only conversion plugins
+            self.add_folder(os.path.join('plugins', 'conversion'),
+                            from_root=True)
 
         if app is not None:
             self.init_app(app)
@@ -55,63 +71,191 @@ class Senpy(object):
             app.teardown_appcontext(self.teardown)
         else:
             app.teardown_request(self.teardown)
-        app.register_blueprint(nif_blueprint)
+        app.register_blueprint(api_blueprint, url_prefix="/api")
+        app.register_blueprint(ns_blueprint, url_prefix="/ns")
+        app.register_blueprint(demo_blueprint, url_prefix="/")
 
-    def add_folder(self, folder):
+    def add_folder(self, folder, from_root=False):
+        if from_root:
+            folder = os.path.join(os.path.dirname(__file__), folder)
         logger.debug("Adding folder: %s", folder)
         if os.path.isdir(folder):
             self._search_folders.add(folder)
             self._outdated = True
-            return True
         else:
             logger.debug("Not a folder: %s", folder)
-            return False
 
-    def analyse(self, **params):
-        algo = None
-        logger.debug("analysing with params: {}".format(params))
-        if "algorithm" in params:
-            algo = params["algorithm"]
-        elif self.plugins:
-            algo = self.default_plugin and self.default_plugin.name
-        if not algo:
-            return Error(status=404,
-                         message=("No plugins found."
-                                  " Please install one.").format(algo))
-        if algo in self.plugins:
-            if self.plugins[algo].is_activated:
-                plug = self.plugins[algo]
-                resp = plug.analyse(**params)
-                resp.analysis.append(plug)
-                logger.debug("Returning analysis result: {}".format(resp))
-                return resp
-            else:
-                logger.debug("Plugin not activated: {}".format(algo))
-                return Error(status=400,
-                             message=("The algorithm '{}'"
-                                      " is not activated yet").format(algo))
+    def _find_plugins(self, params):
+        if not self.analysis_plugins:
+            raise Error(
+                status=404,
+                message=("No plugins found."
+                         " Please install one."))
+        api_params = parse_params(params, spec=API_PARAMS)
+        algos = None
+        if "algorithm" in api_params and api_params["algorithm"]:
+            algos = api_params["algorithm"].split(',')
+        elif self.default_plugin:
+            algos = [self.default_plugin.name, ]
         else:
-            logger.debug(("The algorithm '{}' is not valid\n"
-                          "Valid algorithms: {}").format(algo,
-                                                         self.plugins.keys()))
-            return Error(status=404,
-                         message="The algorithm '{}' is not valid"
-                                 .format(algo))
+            raise Error(
+                status=404,
+                message="No default plugin found, and None provided")
+
+        plugins = list()
+        for algo in algos:
+            if algo not in self.plugins:
+                logger.debug(("The algorithm '{}' is not valid\n"
+                              "Valid algorithms: {}").format(algo,
+                                                             self.plugins.keys()))
+                raise Error(
+                    status=404,
+                    message="The algorithm '{}' is not valid".format(algo))
+
+            if not self.plugins[algo].is_activated:
+                logger.debug("Plugin not activated: {}".format(algo))
+                raise Error(
+                    status=400,
+                    message=("The algorithm '{}'"
+                             " is not activated yet").format(algo))
+            plugins.append(self.plugins[algo])
+        return plugins
+
+    def _get_params(self, params, plugin=None):
+        nif_params = parse_params(params, spec=NIF_PARAMS)
+        if plugin:
+            extra_params = plugin.get('extra_params', {})
+            specific_params = parse_params(params, spec=extra_params)
+            nif_params.update(specific_params)
+        return nif_params
+
+    def _get_entries(self, params):
+        if params['informat'] == 'text':
+            results = Results()
+            entry = Entry(text=params['input'])
+            results.entries.append(entry)
+        elif params['informat'] == 'json-ld':
+            results = from_string(params['input'], cls=Results)
+        else:
+            raise NotImplemented('Informat {} is not implemented'.format(params['informat']))
+        return results
+
+    def _process_entries(self, entries, plugins, nif_params):
+        if not plugins:
+            for i in entries:
+                yield i
+            return
+        plugin = plugins[0]
+        specific_params = self._get_params(nif_params, plugin)
+        results = plugin.analyse_entries(entries, specific_params)
+        for i in self._process_entries(results, plugins[1:], nif_params):
+            yield i
+
+    def _process_response(self, resp, plugins, nif_params):
+        entries = resp.entries
+        resp.entries = []
+        for plug in plugins:
+            resp.analysis.append(plug.id)
+        for i in self._process_entries(entries, plugins, nif_params):
+            resp.entries.append(i)
+        return resp
+
+    def analyse(self, **api_params):
+        """
+        Main method that analyses a request, either from CLI or HTTP.
+        It uses a dictionary of parameters, provided by the user.
+        """
+        logger.debug("analysing with params: {}".format(api_params))
+        plugins = self._find_plugins(api_params)
+        nif_params = self._get_params(api_params)
+        resp = self._get_entries(nif_params)
+        if 'with_parameters' in api_params:
+            resp.parameters = nif_params
+        try:
+            resp = self._process_response(resp, plugins, nif_params)
+            self.convert_emotions(resp, plugins, nif_params)
+            logger.debug("Returning analysis result: {}".format(resp))
+        except (Error, Exception) as ex:
+            if not isinstance(ex, Error):
+                ex = Error(message=str(ex), status=500)
+            logger.exception('Error returning analysis result')
+            raise ex
+        return resp
+
+    def _conversion_candidates(self, fromModel, toModel):
+        candidates = self.filter_plugins(plugin_type='emotionConversionPlugin')
+        for name, candidate in candidates.items():
+            for pair in candidate.onyx__doesConversion:
+                logging.debug(pair)
+
+                if pair['onyx:conversionFrom'] == fromModel \
+                   and pair['onyx:conversionTo'] == toModel:
+                    # logging.debug('Found candidate: {}'.format(candidate))
+                    yield candidate
+
+    def convert_emotions(self, resp, plugins, params):
+        """
+        Conversion of all emotions in a response **in place**.
+        In addition to converting from one model to another, it has
+        to include the conversion plugin to the analysis list.
+        Needless to say, this is far from an elegant solution, but it works.
+        @todo refactor and clean up
+        """
+        toModel = params.get('emotionModel', None)
+        if not toModel:
+            return
+
+        logger.debug('Asked for model: {}'.format(toModel))
+        output = params.get('conversion', None)
+        candidates = {}
+        for plugin in plugins:
+            try:
+                fromModel = plugin.get('onyx:usesEmotionModel', None)
+                candidates[plugin.id] = next(self._conversion_candidates(fromModel, toModel))
+                logger.debug('Analysis plugin {} uses model: {}'.format(plugin.id, fromModel))
+            except StopIteration:
+                e = Error(('No conversion plugin found for: '
+                           '{} -> {}'.format(fromModel, toModel)))
+                e.original_response = resp
+                e.parameters = params
+                raise e
+        newentries = []
+        for i in resp.entries:
+            if output == "full":
+                newemotions = copy.deepcopy(i.emotions)
+            else:
+                newemotions = []
+            for j in i.emotions:
+                plugname = j['prov:wasGeneratedBy']
+                candidate = candidates[plugname]
+                resp.analysis.append(candidate.id)
+                for k in candidate.convert(j, fromModel, toModel, params):
+                    k.prov__wasGeneratedBy = candidate.id
+                    if output == 'nested':
+                        k.prov__wasDerivedFrom = j
+                    newemotions.append(k)
+            i.emotions = newemotions
+            newentries.append(i)
+        resp.entries = newentries
+        resp.analysis = list(set(resp.analysis))
 
     @property
     def default_plugin(self):
-        candidates = self.filter_plugins(is_activated=True)
-        if len(candidates) > 0:
-            candidate = candidates.values()[0]
-            logger.debug("Default: {}".format(candidate))
-            return candidate
-        else:
-            return None
+        candidate = self._default
+        if not candidate:
+            candidates = self.filter_plugins(plugin_type='analysisPlugin',
+                                             is_activated=True)
+            if len(candidates) > 0:
+                candidate = list(candidates.values())[0]
+        logger.debug("Default: {}".format(candidate))
+        return candidate
 
-    def parameters(self, algo):
-        return getattr(self.plugins.get(algo) or self.default_plugin,
-                       "params",
-                       {})
+    @default_plugin.setter
+    def default_plugin(self, value):
+        if isinstance(value, SenpyPlugin):
+            self._default = value
+        else:
+            self._default = self.plugins[value]
 
     def activate_all(self, sync=False):
         ps = []
@@ -126,84 +270,139 @@ class Senpy(object):
         return ps
 
     def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
+        ''' We're using a variable in the plugin itself to activate/deactive plugins.\
+        Note that plugins may activate themselves by setting this variable.
+        '''
         self.plugins[plugin_name].is_activated = active
 
     def activate_plugin(self, plugin_name, sync=False):
-        plugin = self.plugins[plugin_name]
+        try:
+            plugin = self.plugins[plugin_name]
+        except KeyError:
+            raise Error(
+                message="Plugin not found: {}".format(plugin_name), status=404)
+
+        logger.info("Activating plugin: {}".format(plugin.name))
+
         def act():
+            success = False
             try:
                 plugin.activate()
+                msg = "Plugin activated: {}".format(plugin.name)
+                logger.info(msg)
+                success = True
+                self._set_active_plugin(plugin_name, success)
             except Exception as ex:
-                logger.error("Error activating plugin {}: {}".format(plugin.name,
-                                                                     ex))
-                logger.error("Trace: {}".format(traceback.format_exc()))
-        th = gevent.spawn(act)
-        th.link_value(partial(self._set_active_plugin, plugin_name, True))
-        if sync:
-            th.join()
+                msg = "Error activating plugin {} - {} : \n\t{}".format(
+                    plugin.name, ex, traceback.format_exc())
+                logger.error(msg)
+                raise Error(msg)
+
+        if sync or 'async' in plugin and not plugin.async:
+            act()
         else:
+            th = Thread(target=act)
+            th.start()
             return th
 
     def deactivate_plugin(self, plugin_name, sync=False):
-        plugin = self.plugins[plugin_name]
-        th = gevent.spawn(plugin.deactivate)
-        th.link_value(partial(self._set_active_plugin, plugin_name, False))
-        if sync:
-            th.join()
+        try:
+            plugin = self.plugins[plugin_name]
+        except KeyError:
+            raise Error(
+                message="Plugin not found: {}".format(plugin_name), status=404)
+
+        self._set_active_plugin(plugin_name, False)
+
+        def deact():
+            try:
+                plugin.deactivate()
+                logger.info("Plugin deactivated: {}".format(plugin.name))
+            except Exception as ex:
+                logger.error(
+                    "Error deactivating plugin {}: {}".format(plugin.name, ex))
+                logger.error("Trace: {}".format(traceback.format_exc()))
+
+        if sync or 'async' in plugin and not plugin.async:
+            deact()
         else:
+            th = Thread(target=deact)
+            th.start()
             return th
 
-    def reload_plugin(self, plugin):
-        logger.debug("Reloading {}".format(plugin))
-        plug = self.plugins[plugin]
-        nplug = self._load_plugin(plug.module, plug.path)
-        del self.plugins[plugin]
-        self.plugins[nplug.name] = nplug
+    @classmethod
+    def validate_info(cls, info):
+        return all(x in info for x in ('name', 'module', 'description', 'version'))
 
-    @staticmethod
-    def _load_plugin(root, filename):
-        logger.debug("Loading plugin: {}".format(filename))
-        fpath = os.path.join(root, filename)
-        with open(fpath, 'r') as f:
-            info = json.load(f)
-        logger.debug("Info: {}".format(info))
+    def install_deps(self):
+        for i in self.plugins.values():
+            self._install_deps(i)
+
+    @classmethod
+    def _install_deps(cls, info=None):
+        requirements = info.get('requirements', [])
+        if requirements:
+            pip_args = ['pip']
+            pip_args.append('install')
+            pip_args.append('--use-wheel')
+            for req in requirements:
+                pip_args.append(req)
+            logger.info('Installing requirements: ' + str(requirements))
+            process = subprocess.Popen(pip_args,
+                                       stdout=subprocess.PIPE,
+                                       stderr=subprocess.PIPE)
+            log_subprocess_output(process)
+            exitcode = process.wait()
+            if exitcode != 0:
+                raise Error("Dependencies not properly installed")
+
+    @classmethod
+    def _load_module(cls, name, root):
         sys.path.append(root)
+        tmp = importlib.import_module(name)
+        sys.path.remove(root)
+        return tmp
+
+    @classmethod
+    def _load_plugin_from_info(cls, info, root):
+        if not cls.validate_info(info):
+            logger.warn('The module info is not valid.\n\t{}'.format(info))
+            return None, None
         module = info["module"]
         name = info["name"]
-        (fp, pathname, desc) = imp.find_module(module, [root, ])
-        try:
-            tmp = imp.load_module(module, fp, pathname, desc)
-            sys.path.remove(root)
-            candidate = None
-            for _, obj in inspect.getmembers(tmp):
-                if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
-                    logger.debug(("Found plugin class:"
-                                  " {}@{}").format(obj, inspect.getmodule(obj))
-                                 )
-                    candidate = obj
-                    break
-            if not candidate:
-                logger.debug("No valid plugin for: {}".format(filename))
-                return
-            module = candidate(info=info)
-            try:
-                repo_path = root
-                module._repo = Repo(repo_path)
-            except InvalidGitRepositoryError:
-                module._repo = None
-        except Exception as ex:
-            logger.error("Exception importing {}: {}".format(filename, ex))
-            logger.error("Trace: {}".format(traceback.format_exc()))
-            return None, None
+
+        cls._install_deps(info)
+        tmp = cls._load_module(module, root)
+
+        candidate = None
+        for _, obj in inspect.getmembers(tmp):
+            if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
+                logger.debug(("Found plugin class:"
+                              " {}@{}").format(obj, inspect.getmodule(obj)))
+                candidate = obj
+                break
+        if not candidate:
+            logger.debug("No valid plugin for: {}".format(module))
+            return
+        module = candidate(info=info)
         return name, module
 
+    @classmethod
+    def _load_plugin(cls, root, filename):
+        fpath = os.path.join(root, filename)
+        logger.debug("Loading plugin: {}".format(fpath))
+        with open(fpath, 'r') as f:
+            info = yaml.load(f)
+        logger.debug("Info: {}".format(info))
+        return cls._load_plugin_from_info(info, root)
+
     def _load_plugins(self):
         plugins = {}
         for search_folder in self._search_folders:
             for root, dirnames, filenames in os.walk(search_folder):
                 for filename in fnmatch.filter(filenames, '*.senpy'):
                     name, plugin = self._load_plugin(root, filename)
-                    if plugin:
+                    if plugin and name:
                         plugins[name] = plugin
 
         self._outdated = False
@@ -215,26 +414,14 @@ class Senpy(object):
     @property
     def plugins(self):
         """ Return the plugins registered for a given application.  """
-        if not hasattr(self, 'senpy_plugins') or self._outdated:
-            self.senpy_plugins = self._load_plugins()
-        return self.senpy_plugins
+        if self._outdated:
+            self._plugin_list = self._load_plugins()
+        return self._plugin_list
 
     def filter_plugins(self, **kwargs):
-        """ Filter plugins by different criteria """
+        return plugins.pfilter(self.plugins, **kwargs)
 
-        def matches(plug):
-            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
-            logger.debug("matching {} with {}: {}".format(plug.name,
-                                                          kwargs,
-                                                          res))
-            return res
-
-        if not kwargs:
-            return self.plugins
-        else:
-            return {n: p for n, p in self.plugins.items() if matches(p)}
-
-    def sentiment_plugins(self):
-        """ Return only the sentiment plugins """
-        return {p: plugin for p, plugin in self.plugins.items() if
-                isinstance(plugin, SentimentPlugin)}
+    @property
+    def analysis_plugins(self):
+        """ Return only the analysis plugins """
+        return self.filter_plugins(plugin_type='analysisPlugin')

senpy/models.py

@@ -1,252 +1,367 @@
+'''
+Senpy Models.
+
+This implementation should mirror the JSON schema definition.
+For compatibility with Py3 and for easier debugging, this new version drops
+introspection and adds all arguments to the models.
+'''
+from __future__ import print_function
+from six import string_types
+
+import time
+import copy
 import json
 import os
-from collections import defaultdict
-from pyld import jsonld
-import logging
+import jsonref
+import jsonschema
+
 from flask import Response as FlaskResponse
+from pyld import jsonld
+
+from rdflib import Graph
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+DEFINITIONS_FILE = 'definitions.json'
+CONTEXT_PATH = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld')
 
 
-class Leaf(dict):
-    _prefix = None
-    _frame = {}
-    _context = {}
+def get_schema_path(schema_file, absolute=False):
+    if absolute:
+        return os.path.realpath(schema_file)
+    else:
+        return os.path.join(
+            os.path.dirname(os.path.realpath(__file__)), 'schemas',
+            schema_file)
 
-    def __init__(self,
-                 *args,
-                 **kwargs):
 
-        id = kwargs.pop("id", None)
-        context = kwargs.pop("context", self._context)
-        vocab = kwargs.pop("vocab", None)
-        prefix = kwargs.pop("prefix", None)
-        frame = kwargs.pop("frame", None)
-        super(Leaf, self).__init__(*args, **kwargs)
-        if context is not None:
-            self.context = context
-        if frame is not None:
-            self._frame = frame
-        self._prefix = prefix
-        self.id = id
+def read_schema(schema_file, absolute=False):
+    schema_path = get_schema_path(schema_file, absolute)
+    schema_uri = 'file://{}'.format(schema_path)
+    with open(schema_path) as f:
+        return jsonref.load(f, base_uri=schema_uri)
+
+
+base_schema = read_schema(DEFINITIONS_FILE)
+
+
+class Context(dict):
+    @staticmethod
+    def load(context):
+        logging.debug('Loading context: {}'.format(context))
+        if not context:
+            return context
+        elif isinstance(context, list):
+            contexts = []
+            for c in context:
+                contexts.append(Context.load(c))
+            return contexts
+        elif isinstance(context, dict):
+            return Context(context)
+        elif isinstance(context, string_types):
+            try:
+                with open(context) as f:
+                    return Context(json.loads(f.read()))
+            except IOError:
+                return context
+        else:
+            raise AttributeError('Please, provide a valid context')
+
+
+base_context = Context.load(CONTEXT_PATH)
+
+
+class SenpyMixin(object):
+    _context = base_context["@context"]
+
+    def flask(self,
+              in_headers=True,
+              headers=None,
+              outformat='json-ld',
+              **kwargs):
+        """
+        Return the values and error to be used in flask.
+        So far, it returns a fixed context. We should store/generate different
+        contexts if the plugin adds more aliases.
+        """
+        headers = headers or {}
+        kwargs["with_context"] = not in_headers
+        content, mimetype = self.serialize(format=outformat,
+                                           with_mime=True,
+                                           **kwargs)
+
+        if outformat == 'json-ld' and in_headers:
+            headers.update({
+                "Link":
+                ('<%s>;'
+                    'rel="http://www.w3.org/ns/json-ld#context";'
+                    ' type="application/ld+json"' % kwargs.get('context_uri'))
+            })
+        return FlaskResponse(
+            response=content,
+            status=getattr(self, "status", 200),
+            headers=headers,
+            mimetype=mimetype)
+
+    def serialize(self, format='json-ld', with_mime=False, **kwargs):
+        js = self.jsonld(**kwargs)
+        if format == 'json-ld':
+            content = json.dumps(js, indent=2, sort_keys=True)
+            mimetype = "application/json"
+        elif format in ['turtle', ]:
+            logger.debug(js)
+            content = json.dumps(js, indent=2, sort_keys=True)
+            g = Graph().parse(
+                data=content,
+                format='json-ld',
+                base=kwargs.get('prefix'),
+                context=self._context)
+            logger.debug(
+                'Parsing with prefix: {}'.format(kwargs.get('prefix')))
+            content = g.serialize(format='turtle').decode('utf-8')
+            mimetype = 'text/{}'.format(format)
+        else:
+            raise Error('Unknown outformat: {}'.format(format))
+        if with_mime:
+            return content, mimetype
+        else:
+            return content
+
+    def serializable(self):
+        def ser_or_down(item):
+            if hasattr(item, 'serializable'):
+                return item.serializable()
+            elif isinstance(item, dict):
+                temp = dict()
+                for kp in item:
+                    vp = item[kp]
+                    temp[kp] = ser_or_down(vp)
+                return temp
+            elif isinstance(item, list):
+                return list(ser_or_down(i) for i in item)
+            else:
+                return item
+
+        return ser_or_down(self._plain_dict())
+
+    def jsonld(self,
+               with_context=True,
+               context_uri=None,
+               prefix=None,
+               expanded=False):
+        ser = self.serializable()
+
+        result = jsonld.compact(
+            ser,
+            self._context,
+            options={
+                'base': prefix,
+                'expandContext': self._context,
+                'senpy': prefix
+            })
+        if context_uri:
+            result['@context'] = context_uri
+        if expanded:
+            result = jsonld.expand(
+                result, options={'base': prefix,
+                                 'expandContext': self._context})
+        if not with_context:
+            del result['@context']
+        return result
+
+    def to_JSON(self, *args, **kwargs):
+        js = json.dumps(self.jsonld(*args, **kwargs), indent=4, sort_keys=True)
+        return js
+
+    def validate(self, obj=None):
+        if not obj:
+            obj = self
+        if hasattr(obj, "jsonld"):
+            obj = obj.jsonld()
+        jsonschema.validate(obj, self.schema)
+
+    def __str__(self):
+        return str(self.to_JSON())
+
+
+class BaseModel(SenpyMixin, dict):
+
+    schema = base_schema
+
+    def __init__(self, *args, **kwargs):
+        if 'id' in kwargs:
+            self.id = kwargs.pop('id')
+        elif kwargs.pop('_auto_id', True):
+            self.id = '_:{}_{}'.format(type(self).__name__, time.time())
+        temp = dict(*args, **kwargs)
+
+        for obj in [
+                self.schema,
+        ] + self.schema.get('allOf', []):
+            for k, v in obj.get('properties', {}).items():
+                if 'default' in v and k not in temp:
+                    temp[k] = copy.deepcopy(v['default'])
+
+        for i in temp:
+            nk = self._get_key(i)
+            if nk != i:
+                temp[nk] = temp[i]
+                del temp[i]
+        try:
+            temp['@type'] = getattr(self, '@type')
+        except AttributeError:
+            logger.warn('Creating an instance of an unknown model')
+
+        super(BaseModel, self).__init__(temp)
+
+    def _get_key(self, key):
+        key = key.replace("__", ":", 1)
+        return key
+
+    def __setitem__(self, key, value):
+        dict.__setitem__(self, key, value)
+
+    def __delitem__(self, key):
+        dict.__delitem__(self, key)
 
     def __getattr__(self, key):
         try:
-            return object.__getattr__(self, key)
-        except AttributeError:
-            try:
-                return super(Leaf, self).__getitem__(self._get_key(key))
-            except KeyError:
-                raise AttributeError()
+            return self.__getitem__(self._get_key(key))
+        except KeyError:
+            raise AttributeError(key)
 
     def __setattr__(self, key, value):
-        try:
-            object.__getattr__(self, key)
-            object.__setattr__(self, key, value)
-        except AttributeError:
-            key = self._get_key(key)
-            if key == "@context":
-                value = self.get_context(value)
-            elif key == "@id":
-                value = self.get_id(value)
-            if key[0] == "_":
-                object.__setattr__(self, key, value)
-            else:
-                if value is None:
-                    try:
-                        super(Leaf, self).__delitem__(key)
-                    except KeyError:
-                        pass
-                else:
-                    super(Leaf, self).__setitem__(key, value)
-
-    def get_id(self, id):
-        """
-        Get id, dealing with prefixes
-        """
-        # This is not the most elegant solution to change the @id attribute,
-        # but it is the quickest way to have it included in the dictionary
-        # without extra boilerplate.
-        if id and self._prefix and ":" not in id:
-            return "{}{}".format(self._prefix, id)
-        else:
-            return id
+        self.__setitem__(self._get_key(key), value)
 
     def __delattr__(self, key):
-        if key in self.__dict__:
-            del self.__dict__[key]
+        try:
+            object.__delattr__(self, key)
+        except AttributeError:
+            self.__delitem__(self._get_key(key))
+
+    def _plain_dict(self):
+        d = {k: v for (k, v) in self.items() if k[0] != "_"}
+        if 'id' in d:
+            d["@id"] = d.pop('id')
+        return d
+
+
+_subtypes = {}
+
+
+def register(rsubclass, rtype=None):
+    _subtypes[rtype or rsubclass.__name__] = rsubclass
+
+
+def from_dict(indict, cls=None):
+    if not cls:
+        target = indict.get('@type', None)
+        try:
+            if target and target in _subtypes:
+                cls = _subtypes[target]
+            else:
+                cls = BaseModel
+        except Exception:
+            cls = BaseModel
+    outdict = dict()
+    for k, v in indict.items():
+        if k == '@context':
+            pass
+        elif isinstance(v, dict):
+            v = from_dict(indict[k])
+        elif isinstance(v, list):
+            for ix, v2 in enumerate(v):
+                if isinstance(v2, dict):
+                    v[ix] = from_dict(v2)
+        outdict[k] = v
+    return cls(**outdict)
+
+
+def from_string(string, **kwargs):
+    return from_dict(json.loads(string), **kwargs)
+
+
+def from_json(injson):
+    indict = json.loads(injson)
+    return from_dict(indict)
+
+
+def from_schema(name, schema_file=None, base_classes=None):
+    base_classes = base_classes or []
+    base_classes.append(BaseModel)
+    schema_file = schema_file or '{}.json'.format(name)
+    class_name = '{}{}'.format(name[0].upper(), name[1:])
+    newclass = type(class_name, tuple(base_classes), {})
+    setattr(newclass, '@type', name)
+    setattr(newclass, 'schema', read_schema(schema_file))
+    setattr(newclass, 'class_name', class_name)
+    register(newclass, name)
+    return newclass
+
+
+def _add_from_schema(*args, **kwargs):
+    generatedClass = from_schema(*args, **kwargs)
+    globals()[generatedClass.__name__] = generatedClass
+    del generatedClass
+
+
+for i in [
+        'analysis',
+        'emotion',
+        'emotionConversion',
+        'emotionConversionPlugin',
+        'emotionAnalysis',
+        'emotionModel',
+        'emotionPlugin',
+        'emotionSet',
+        'entry',
+        'plugin',
+        'plugins',
+        'response',
+        'results',
+        'sentiment',
+        'sentimentPlugin',
+        'suggestion',
+]:
+    _add_from_schema(i)
+
+_ErrorModel = from_schema('error')
+
+
+class Error(SenpyMixin, Exception):
+    def __init__(self, message, *args, **kwargs):
+        super(Error, self).__init__(self, message, message)
+        self._error = _ErrorModel(message=message, *args, **kwargs)
+        self.message = message
+
+    def __getitem__(self, key):
+        return self._error[key]
+
+    def __setitem__(self, key, value):
+        self._error[key] = value
+
+    def __delitem__(self, key):
+        del self._error[key]
+
+    def __getattr__(self, key):
+        if key != '_error' and hasattr(self._error, key):
+            return getattr(self._error, key)
+        raise AttributeError(key)
+
+    def __setattr__(self, key, value):
+        if key != '_error':
+            return setattr(self._error, key, value)
         else:
-            super(Leaf, self).__delitem__(self._get_key(key))
+            super(Error, self).__setattr__(key, value)
 
-    def _get_key(self, key):
-        if key[0] == "_":
-            return key
-        elif key in ["context", "id"]:
-            return "@{}".format(key)
-        else:
-            return key
+    def __delattr__(self, key):
+        delattr(self._error, key)
 
-    @staticmethod
-    def get_context(context):
-        if isinstance(context, list):
-            contexts = []
-            for c in context:
-                contexts.append(Response.get_context(c))
-            return contexts
-        elif isinstance(context, dict):
-            return context
-        elif isinstance(context, basestring):
-            try:
-                with open(context) as f:
-                    return json.loads(f.read())
-            except IOError:
-                return context
-
-    def compact(self):
-        return jsonld.compact(self, self.get_context(self.context))
-
-    def frame(self, frame=None, options=None):
-        if frame is None:
-            frame = self._frame
-        if options is None:
-            options = {}
-        return jsonld.frame(self, frame, options)
-
-    def jsonld(self, frame=None, options=None,
-               context=None, removeContext=None):
-        if removeContext is None:
-            removeContext = Response._context  # Loop?
-        if frame is None:
-            frame = self._frame
-        if context is None:
-            context = self.context
-        else:
-            context = self.get_context(context)
-        # For some reason, this causes errors with pyld
-        # if options is None:
-            # options = {"expandContext": context.copy() }
-        js = self
-        if frame:
-            logging.debug("Framing: %s", json.dumps(self, indent=4))
-            logging.debug("Framing with %s", json.dumps(frame, indent=4))
-            js = jsonld.frame(js, frame, options)
-            logging.debug("Result: %s", json.dumps(js, indent=4))
-            logging.debug("Compacting with %s", json.dumps(context, indent=4))
-            js = jsonld.compact(js, context, options)
-            logging.debug("Result: %s", json.dumps(js, indent=4))
-        if removeContext == context:
-            del js["@context"]
-        return js
-
-    def to_JSON(self, removeContext=None):
-        return json.dumps(self.jsonld(removeContext=removeContext),
-                          default=lambda o: o.__dict__,
-                          sort_keys=True, indent=4)
-
-    def flask(self,
-              in_headers=False,
-              url="http://demos.gsi.dit.upm.es/senpy/senpy.jsonld"):
-        """
-        Return the values and error to be used in flask
-        """
-        js = self.jsonld()
-        headers = None
-        if in_headers:
-            ctx = js["@context"]
-            headers = {
-                "Link": ('<%s>;'
-                         'rel="http://www.w3.org/ns/json-ld#context";'
-                         ' type="application/ld+json"' % url)
-            }
-            del js["@context"]
-        return FlaskResponse(json.dumps(js, indent=4),
-                             status=self.get("status", 200),
-                             headers=headers,
-                             mimetype="application/json")
+    def __str__(self):
+        return str(self.to_JSON(with_context=False))
 
 
-class Response(Leaf):
-    _context = Leaf.get_context("{}/context.jsonld".format(
-        os.path.dirname(os.path.realpath(__file__))))
-    _frame = {
-        "@context": _context,
-        "analysis": {
-            "@explicit": True,
-            "maxPolarityValue": {},
-            "minPolarityValue": {},
-            "name": {},
-            "version": {},
-        },
-        "entries": {}
-    }
-
-    def __init__(self, *args, **kwargs):
-        context = kwargs.pop("context", None)
-        frame = kwargs.pop("frame", None)
-        if context is None:
-            context = self._context
-        self.context = context
-        super(Response, self).__init__(
-            *args, context=context, frame=frame, **kwargs)
-        if self._frame is not None and "entries" in self._frame:
-            self.analysis = []
-            self.entries = []
-
-    def jsonld(self, frame=None, options=None, context=None, removeContext={}):
-        return super(Response, self).jsonld(frame,
-                                            options,
-                                            context,
-                                            removeContext)
-
-
-class Entry(Leaf):
-    _context = {
-        "@vocab": ("http://persistence.uni-leipzig.org/"
-                   "nlp2rdf/ontologies/nif-core#")
-
-    }
-
-    def __init__(self, text=None, emotion_sets=None, opinions=None, **kwargs):
-        super(Entry, self).__init__(**kwargs)
-        if text:
-            self.text = text
-        self.emotionSets = emotion_sets if emotion_sets else []
-        self.opinions = opinions if opinions else []
-
-
-class Opinion(Leaf):
-    _context = {
-        "@vocab": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"
-    }
-
-    def __init__(self, polarityValue=None, hasPolarity=None, *args, **kwargs):
-        super(Opinion, self).__init__(*args,
-                                      **kwargs)
-        if polarityValue is not None:
-            self.hasPolarityValue = polarityValue
-        if hasPolarity is not None:
-            self.hasPolarity = hasPolarity
-
-
-class EmotionSet(Leaf):
-    _context = {}
-
-    def __init__(self, emotions=None, *args, **kwargs):
-        if not emotions:
-            emotions = []
-        super(EmotionSet, self).__init__(context=EmotionSet._context,
-                                         *args,
-                                         **kwargs)
-        self.emotions = emotions or []
-
-
-class Emotion(Leaf):
-    _context = {}
-
-
-class Error(Leaf):
-    # A better pattern would be this:
-    # http://flask.pocoo.org/docs/0.10/patterns/apierrors/
-    _frame = {}
-    _context = {}
-
-    def __init__(self, *args, **kwargs):
-        super(Error, self).__init__(*args, **kwargs)
+register(Error, 'error')

senpy/plugins.py

@@ -1,155 +0,0 @@
-
-import inspect
-import os.path
-import shelve
-import logging
-import ConfigParser
-from .models import Response, Leaf
-
-logger = logging.getLogger(__name__)
-
-PARAMS = {
-    "input": {
-        "@id": "input",
-        "aliases": ["i", "input"],
-        "required": True,
-        "help": "Input text"
-    },
-    "informat": {
-        "@id": "informat",
-        "aliases": ["f", "informat"],
-        "required": False,
-        "default": "text",
-        "options": ["turtle", "text"],
-    },
-    "intype": {
-        "@id": "intype",
-        "aliases": ["intype", "t"],
-        "required": False,
-        "default": "direct",
-        "options": ["direct", "url", "file"],
-    },
-    "outformat": {
-        "@id": "outformat",
-        "aliases": ["outformat", "o"],
-        "default": "json-ld",
-        "required": False,
-        "options": ["json-ld"],
-    },
-    "language": {
-        "@id": "language",
-        "aliases": ["language", "l"],
-        "required": False,
-    },
-    "prefix": {
-        "@id": "prefix",
-        "aliases": ["prefix", "p"],
-        "required": True,
-        "default": "",
-    },
-    "urischeme": {
-        "@id": "urischeme",
-        "aliases": ["urischeme", "u"],
-        "required": False,
-        "default": "RFC5147String",
-        "options": "RFC5147String"
-    },
-}
-
-
-class SenpyPlugin(Leaf):
-    _context = Leaf.get_context(Response._context)
-    _frame = {"@context": _context,
-              "name": {},
-              "extra_params": {"@container": "@index"},
-              "@explicit": True,
-              "version": {},
-              "repo": None,
-              "is_activated": {},
-              "params": None,
-              }
-
-    def __init__(self, info=None):
-        if not info:
-            raise ValueError(("You need to provide configuration"
-                              "information for the plugin."))
-        logger.debug("Initialising {}".format(info))
-        super(SenpyPlugin, self).__init__()
-        self.name = info["name"]
-        self.version = info["version"]
-        self.id = "{}_{}".format(self.name, self.version)
-        self.params = info.get("params", PARAMS.copy())
-        if "@id" not in self.params:
-            self.params["@id"] = "params_%s" % self.id
-        self.extra_params = info.get("extra_params", {})
-        self.params.update(self.extra_params.copy())
-        if "@id" not in self.extra_params:
-            self.extra_params["@id"] = "extra_params_%s" % self.id
-        self.is_activated = False
-        self._info = info
-
-    def get_folder(self):
-        return os.path.dirname(inspect.getfile(self.__class__))
-
-    def analyse(self, *args, **kwargs):
-        logger.debug("Analysing with: {} {}".format(self.name, self.version))
-        pass
-
-    def activate(self):
-        pass
-
-    def deactivate(self):
-        pass
-
-    def jsonld(self, parameters=False, *args, **kwargs):
-        nframe = kwargs.pop("frame", self._frame)
-        if parameters:
-            nframe = nframe.copy()
-            nframe["params"] = {}
-        return super(SenpyPlugin, self).jsonld(frame=nframe, *args, **kwargs)
-
-    @property
-    def id(self):
-        return "{}_{}".format(self.name, self.version)
-
-
-class SentimentPlugin(SenpyPlugin):
-
-    def __init__(self, info, *args, **kwargs):
-        super(SentimentPlugin, self).__init__(info, *args, **kwargs)
-        self.minPolarityValue = float(info.get("minPolarityValue", 0))
-        self.maxPolarityValue = float(info.get("maxPolarityValue", 1))
-
-
-class EmotionPlugin(SenpyPlugin):
-
-    def __init__(self, info, *args, **kwargs):
-        resp = super(EmotionPlugin, self).__init__(info, *args, **kwargs)
-        self.minEmotionValue = float(info.get("minEmotionValue", 0))
-        self.maxEmotionValue = float(info.get("maxEmotionValue", 0))
-
-
-class ShelfMixin(object):
-
-    @property
-    def sh(self):
-        if not hasattr(self, '_sh') or not self._sh:
-            self._sh = shelve.open(self.shelf_file, writeback=True)
-        return self._sh
-
-    @sh.deleter
-    def sh(self):
-        if hasattr(self, '_sh'):
-            del(self._sh)
-
-    @property
-    def shelf_file(self):
-        if not hasattr(self, '_shelf_file') or not self._shelf_file:
-            if hasattr(self, '_info') and 'shelf_file' in self._info:
-                self._shelf_file = self._info['shelf_file']
-            else:
-                self._shelf_file = os.path.join(self.get_folder(), self.name + '.db')
-        return self._shelf_file 
-
-    def close(self):
-        del(self._sh)

senpy/plugins/__init__.py

@@ -0,0 +1,162 @@
+from future import standard_library
+standard_library.install_aliases()
+
+import inspect
+import os.path
+import os
+import pickle
+import logging
+import tempfile
+import copy
+from .. import models
+from ..api import API_PARAMS
+
+logger = logging.getLogger(__name__)
+
+
+class Plugin(models.Plugin):
+    def __init__(self, info=None):
+        """
+        Provides a canonical name for plugins and serves as base for other
+        kinds of plugins.
+        """
+        if not info:
+            raise models.Error(message=("You need to provide configuration"
+                                        "information for the plugin."))
+        logger.debug("Initialising {}".format(info))
+        id = 'plugins/{}_{}'.format(info['name'], info['version'])
+        super(Plugin, self).__init__(id=id, **info)
+        self.is_activated = False
+
+    def get_folder(self):
+        return os.path.dirname(inspect.getfile(self.__class__))
+
+    def activate(self):
+        pass
+
+    def deactivate(self):
+        pass
+
+
+SenpyPlugin = Plugin
+
+
+class AnalysisPlugin(Plugin):
+
+    def analyse(self, *args, **kwargs):
+        raise NotImplemented(
+            'Your method should implement either analyse or analyse_entry')
+
+    def analyse_entry(self, entry, parameters):
+        """ An implemented plugin should override this method.
+        This base method is here to adapt old style plugins which only
+        implement the *analyse* function.
+        Note that this method may yield an annotated entry or a list of
+        entries (e.g. in a tokenizer)
+        """
+        text = entry['text']
+        params = copy.copy(parameters)
+        params['input'] = text
+        results = self.analyse(**params)
+        for i in results.entries:
+            yield i
+
+    def analyse_entries(self, entries, parameters):
+        for entry in entries:
+            logger.debug('Analysing entry with plugin {}: {}'.format(self, entry))
+            for result in self.analyse_entry(entry, parameters):
+                yield result
+
+
+class ConversionPlugin(Plugin):
+    pass
+
+
+class SentimentPlugin(models.SentimentPlugin, AnalysisPlugin):
+    def __init__(self, info, *args, **kwargs):
+        super(SentimentPlugin, self).__init__(info, *args, **kwargs)
+        self.minPolarityValue = float(info.get("minPolarityValue", 0))
+        self.maxPolarityValue = float(info.get("maxPolarityValue", 1))
+
+
+class EmotionPlugin(models.EmotionPlugin, AnalysisPlugin):
+    def __init__(self, info, *args, **kwargs):
+        super(EmotionPlugin, self).__init__(info, *args, **kwargs)
+        self.minEmotionValue = float(info.get("minEmotionValue", -1))
+        self.maxEmotionValue = float(info.get("maxEmotionValue", 1))
+
+
+class EmotionConversionPlugin(models.EmotionConversionPlugin, ConversionPlugin):
+    pass
+
+
+class ShelfMixin(object):
+    @property
+    def sh(self):
+        if not hasattr(self, '_sh') or self._sh is None:
+            self.__dict__['_sh'] = {}
+            if os.path.isfile(self.shelf_file):
+                try:
+                    self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb'))
+                except (IndexError, EOFError, pickle.UnpicklingError):
+                    logger.warning('{} has a corrupted shelf file!'.format(self.id))
+                    if not self.get('force_shelf', False):
+                        raise
+        return self._sh
+
+    @sh.deleter
+    def sh(self):
+        if os.path.isfile(self.shelf_file):
+            os.remove(self.shelf_file)
+            del self.__dict__['_sh']
+        self.save()
+
+    @property
+    def shelf_file(self):
+        if 'shelf_file' not in self or not self['shelf_file']:
+            sd = os.environ.get('SENPY_DATA', tempfile.gettempdir())
+            self.shelf_file = os.path.join(sd, self.name + '.p')
+        return self['shelf_file']
+
+    def save(self):
+        logger.debug('saving pickle')
+        if hasattr(self, '_sh') and self._sh is not None:
+            with open(self.shelf_file, 'wb') as f:
+                pickle.dump(self._sh, f)
+
+
+default_plugin_type = API_PARAMS['plugin_type']['default']
+
+
+def pfilter(plugins, **kwargs):
+    """ Filter plugins by different criteria """
+    if isinstance(plugins, models.Plugins):
+        plugins = plugins.plugins
+    elif isinstance(plugins, dict):
+        plugins = plugins.values()
+    ptype = kwargs.pop('plugin_type', default_plugin_type)
+    logger.debug('#' * 100)
+    logger.debug('ptype {}'.format(ptype))
+    if ptype:
+        try:
+            ptype = ptype[0].upper() + ptype[1:]
+            pclass = globals()[ptype]
+            logger.debug('Class: {}'.format(pclass))
+            candidates = filter(lambda x: isinstance(x, pclass),
+                                plugins)
+        except KeyError:
+            raise models.Error('{} is not a valid type'.format(ptype))
+    else:
+        candidates = plugins
+
+    logger.debug(candidates)
+
+    def matches(plug):
+        res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
+        logger.debug(
+            "matching {} with {}: {}".format(plug.name, kwargs, res))
+        return res
+
+    if kwargs:
+        candidates = filter(matches, candidates)
+    return {p.name: p for p in candidates}

senpy/plugins/conversion/emotion/centroids.py

@@ -0,0 +1,102 @@
+from senpy.plugins import EmotionConversionPlugin
+from senpy.models import EmotionSet, Emotion, Error
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+class CentroidConversion(EmotionConversionPlugin):
+    def __init__(self, info):
+        if 'centroids' not in info:
+            raise Error('Centroid conversion plugins should provide '
+                        'the centroids in their senpy file')
+        if 'onyx:doesConversion' not in info:
+            if 'centroids_direction' not in info:
+                raise Error('Please, provide centroids direction')
+
+            cf, ct = info['centroids_direction']
+            info['onyx:doesConversion'] = [{
+                'onyx:conversionFrom': cf,
+                'onyx:conversionTo': ct
+            }, {
+                'onyx:conversionFrom': ct,
+                'onyx:conversionTo': cf
+            }]
+
+        if 'aliases' in info:
+            aliases = info['aliases']
+            ncentroids = {}
+            for k1, v1 in info['centroids'].items():
+                nv1 = {}
+                for k2, v2 in v1.items():
+                    nv1[aliases.get(k2, k2)] = v2
+                ncentroids[aliases.get(k1, k1)] = nv1
+            info['centroids'] = ncentroids
+
+        super(CentroidConversion, self).__init__(info)
+
+        self.dimensions = set()
+        for c in self.centroids.values():
+            self.dimensions.update(c.keys())
+        self.neutralPoints = self.get("neutralPoints", dict())
+        if not self.neutralPoints:
+            for i in self.dimensions:
+                self.neutralPoints[i] = self.get("neutralValue", 0)
+
+    def _forward_conversion(self, original):
+        """Sum the VAD value of all categories found weighted by intensity.
+        Intensities are scaled by onyx:maxIntensityValue if it is present, else maxIntensityValue
+        is assumed to be one. Emotion entries that do not have onxy:hasEmotionIntensity specified
+        are assumed to have maxIntensityValue. Emotion entries that do not have
+        onyx:hasEmotionCategory specified are ignored."""
+        res = Emotion()
+        maxIntensity = float(original.get("onyx:maxIntensityValue", 1))
+        for e in original.onyx__hasEmotion:
+            category = e.get("onyx:hasEmotionCategory", None)
+            if not category:
+                continue
+            intensity = e.get("onyx:hasEmotionIntensity", maxIntensity) / maxIntensity
+            if not intensity:
+                continue
+            centroid = self.centroids.get(category, None)
+            if centroid:
+                for dim, value in centroid.items():
+                    neutral = self.neutralPoints[dim]
+                    if dim not in res:
+                        res[dim] = 0
+                    res[dim] += (value - neutral) * intensity + neutral
+        return res
+
+    def _backwards_conversion(self, original):
+        """Find the closest category"""
+        centroids = self.centroids
+        neutralPoints = self.neutralPoints
+        dimensions = self.dimensions
+
+        def distance_k(centroid, original, k):
+            # k component of the distance between the value and a given centroid
+            return (centroid.get(k, neutralPoints[k]) - original.get(k, neutralPoints[k]))**2
+
+        def distance(centroid):
+            return sum(distance_k(centroid, original, k) for k in dimensions)
+
+        emotion = min(centroids, key=lambda x: distance(centroids[x]))
+
+        result = Emotion(onyx__hasEmotionCategory=emotion)
+        result.onyx__algorithmConfidence = distance(centroids[emotion])
+        return result
+
+    def convert(self, emotionSet, fromModel, toModel, params):
+
+        cf, ct = self.centroids_direction
+        logger.debug(
+            '{}\n{}\n{}\n{}'.format(emotionSet, fromModel, toModel, params))
+        e = EmotionSet()
+        if fromModel == cf and toModel == ct:
+            e.onyx__hasEmotion.append(self._forward_conversion(emotionSet))
+        elif fromModel == ct and toModel == cf:
+            for i in emotionSet.onyx__hasEmotion:
+                e.onyx__hasEmotion.append(self._backwards_conversion(i))
+        else:
+            raise Error('EMOTION MODEL NOT KNOWN')
+        yield e

senpy/plugins/conversion/emotion/ekman2fsre.senpy

@@ -0,0 +1,39 @@
+---
+name: Ekman2FSRE
+module: senpy.plugins.conversion.emotion.centroids
+description: Plugin to convert emotion sets from Ekman to VAD
+version: 0.1
+# No need to specify onyx:doesConversion because centroids.py adds it automatically from centroids_direction
+centroids:
+  anger:
+    A: 6.95
+    D: 5.1
+    V: 2.7
+  disgust:
+    A: 5.3
+    D: 8.05
+    V: 2.7
+  fear:
+    A: 6.5
+    D: 3.6
+    V: 3.2
+  happiness:
+    A: 7.22
+    D: 6.28
+    V: 8.6
+  sadness:
+    A: 5.21
+    D: 2.82
+    V: 2.21
+centroids_direction:
+  - emoml:big6
+  - emoml:fsre-dimensions
+aliases: # These are aliases for any key in the centroid, to avoid repeating a long name several times
+  A: emoml:arousal
+  V: emoml:valence
+  D: emoml:dominance
+  anger: emoml:big6anger
+  disgust: emoml:big6disgust
+  fear: emoml:big6fear
+  happiness: emoml:big6happiness
+  sadness: emoml:big6sadness

senpy/plugins/conversion/emotion/ekman2vad.senpy

@@ -0,0 +1,44 @@
+---
+name: Ekman2PAD
+module: senpy.plugins.conversion.emotion.centroids
+description: Plugin to convert emotion sets from Ekman to VAD
+version: 0.1
+# No need to specify onyx:doesConversion because centroids.py adds it automatically from centroids_direction
+origin:
+  # Point in VAD space with no emotion (aka Neutral)
+  A: 5.0
+  D: 5.0
+  V: 5.0
+centroids:
+  anger:
+    A: 6.95
+    D: 5.1
+    V: 2.7
+  disgust:
+    A: 5.3
+    D: 8.05
+    V: 2.7
+  fear:
+    A: 6.5
+    D: 3.6
+    V: 3.2
+  happiness:
+    A: 7.22
+    D: 6.28
+    V: 8.6
+  sadness:
+    A: 5.21
+    D: 2.82
+    V: 2.21
+centroids_direction:
+  - emoml:big6
+  - emoml:pad
+aliases: # These are aliases for any key in the centroid, to avoid repeating a long name several times
+  A: emoml:arousal
+  V: emoml:valence
+  D: emoml:dominance
+  anger: emoml:big6anger
+  disgust: emoml:big6disgust
+  fear: emoml:big6fear
+  happiness: emoml:big6happiness
+  sadness: emoml:big6sadness

senpy/plugins/example/emoRand/emoRand.py

@@ -0,0 +1,18 @@
+import random
+
+from senpy.plugins import EmotionPlugin
+from senpy.models import EmotionSet, Emotion
+
+
+class RmoRandPlugin(EmotionPlugin):
+    def analyse_entry(self, entry, params):
+        category = "emoml:big6happiness"
+        number = max(-1, min(1, random.gauss(0, 0.5)))
+        if number > 0:
+            category = "emoml:big6anger"
+        emotionSet = EmotionSet()
+        emotion = Emotion({"onyx:hasEmotionCategory": category})
+        emotionSet.onyx__hasEmotion.append(emotion)
+        emotionSet.prov__wasGeneratedBy = self.id
+        entry.emotions.append(emotionSet)
+        yield entry

senpy/plugins/example/emoRand/emoRand.senpy

@@ -0,0 +1,9 @@
+---
+name: emoRand
+module: emoRand
+description: A sample plugin that returns a random emotion annotation
+author: "@balkian"
+version: '0.1'
+url: "https://github.com/gsi-upm/senpy-plugins-community"
+requirements: {}
+onyx:usesEmotionModel: "emoml:big6"

senpy/plugins/example/rand/rand.py

@@ -0,0 +1,24 @@
+import random
+
+from senpy.plugins import SentimentPlugin
+from senpy.models import Sentiment
+
+
+class RandPlugin(SentimentPlugin):
+    def analyse_entry(self, entry, params):
+        lang = params.get("language", "auto")
+
+        polarity_value = max(-1, min(1, random.gauss(0.2, 0.2)))
+        polarity = "marl:Neutral"
+        if polarity_value > 0:
+            polarity = "marl:Positive"
+        elif polarity_value < 0:
+            polarity = "marl:Negative"
+        sentiment = Sentiment({
+            "marl:hasPolarity": polarity,
+            "marl:polarityValue": polarity_value
+        })
+        sentiment["prov:wasGeneratedBy"] = self.id
+        entry.sentiments.append(sentiment)
+        entry.language = lang
+        yield entry

senpy/plugins/example/rand/rand.senpy

@@ -0,0 +1,10 @@
+---
+name: rand
+module: rand
+description: A sample plugin that returns a random sentiment annotation
+author: "@balkian"
+version: '0.1'
+url: "https://github.com/gsi-upm/senpy-plugins-community"
+requirements: {}
+marl:maxPolarityValue: '1'
+marl:minPolarityValue: "-1"

senpy/plugins/rand/rand.py

@@ -1,31 +0,0 @@
-import json
-import random
-
-from senpy.plugins import SentimentPlugin
-from senpy.models import Response, Opinion, Entry
-
-
-class Sentiment140Plugin(SentimentPlugin):
-    def analyse(self, **params):
-        lang = params.get("language", "auto")
-
-        p = params.get("prefix", None)
-        response = Response(prefix=p)
-        polarity_value = max(-1, min(1, random.gauss(0.2, 0.2)))
-        polarity = "marl:Neutral"
-        if polarity_value > 0:
-            polarity = "marl:Positive"
-        elif polarity_value < 0:
-            polarity = "marl:Negative"
-        entry = Entry(id="Entry0",
-                      text=params["input"],
-                      prefix=p)
-        opinion = Opinion(id="Opinion0",
-                          prefix=p,
-                          hasPolarity=polarity,
-                          polarityValue=polarity_value)
-        opinion["prov:wasGeneratedBy"] = self.id
-        entry.opinions.append(opinion)
-        entry.language = lang
-        response.entries.append(entry)
-        return response

senpy/plugins/rand/rand.senpy

@@ -1,18 +0,0 @@
-{
-    "name": "rand",
-    "module": "rand",
-    "description": "What my plugin broadly does",
-    "author": "@balkian",
-    "version": "0.1",
-    "extra_params": {
-        "language": {
-            "@id": "lang_rand",
-            "aliases": ["language", "l"],
-            "required": false,
-            "options": ["es", "en", "auto"]
-        }
-     },
-     "requirements": {},
-     "marl:maxPolarityValue": "1",
-     "marl:minPolarityValue": "-1"
-}

senpy/plugins/sentiment/sentiment140/sentiment140.py

@@ -0,0 +1,36 @@
+import requests
+import json
+
+from senpy.plugins import SentimentPlugin
+from senpy.models import Sentiment
+
+
+class Sentiment140Plugin(SentimentPlugin):
+    def analyse_entry(self, entry, params):
+        lang = params.get("language", "auto")
+        res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
+                            json.dumps({
+                                "language": lang,
+                                "data": [{
+                                    "text": entry.text
+                                }]
+                            }))
+        p = params.get("prefix", None)
+        polarity_value = self.maxPolarityValue * int(
+            res.json()["data"][0]["polarity"]) * 0.25
+        polarity = "marl:Neutral"
+        neutral_value = self.maxPolarityValue / 2.0
+        if polarity_value > neutral_value:
+            polarity = "marl:Positive"
+        elif polarity_value < neutral_value:
+            polarity = "marl:Negative"
+
+        sentiment = Sentiment(
+            prefix=p,
+            marl__hasPolarity=polarity,
+            marl__polarityValue=polarity_value)
+        sentiment.prov__wasGeneratedBy = self.id
+        entry.sentiments = []
+        entry.sentiments.append(sentiment)
+        entry.language = lang
+        yield entry

senpy/plugins/sentiment/sentiment140/sentiment140.senpy

@@ -0,0 +1,21 @@
+---
+name: sentiment140
+module: sentiment140
+description: "Connects to the sentiment140 free API: http://sentiment140.com"
+author: "@balkian"
+version: '0.2'
+url: "https://github.com/gsi-upm/senpy-plugins-community"
+extra_params:
+  language:
+    "@id": lang_sentiment140
+    aliases:
+    - language
+    - l
+    required: false
+    options:
+    - es
+    - en
+    - auto
+requirements: {}
+maxPolarityValue: 1
+minPolarityValue: 0

senpy/plugins/sentiment140/sentiment140.py

@@ -1,39 +0,0 @@
-import requests
-import json
-
-from senpy.plugins import SentimentPlugin
-from senpy.models import Response, Opinion, Entry
-
-
-class Sentiment140Plugin(SentimentPlugin):
-    def analyse(self, **params):
-        lang = params.get("language", "auto")
-        res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
-                            json.dumps({"language": lang,
-                                        "data": [{"text": params["input"]}]
-                                        }
-                                       )
-                            )
-
-        p = params.get("prefix", None)
-        response = Response(prefix=p)
-        polarity_value = self.maxPolarityValue*int(res.json()["data"][0]
-                                                   ["polarity"]) * 0.25
-        polarity = "marl:Neutral"
-        neutral_value = self.maxPolarityValue / 2.0
-        if polarity_value > neutral_value:
-            polarity = "marl:Positive"
-        elif polarity_value < neutral_value:
-            polarity = "marl:Negative"
-        entry = Entry(id="Entry0",
-                      text=params["input"],
-                      prefix=p)
-        opinion = Opinion(id="Opinion0",
-                          prefix=p,
-                          hasPolarity=polarity,
-                          polarityValue=polarity_value)
-        opinion["prov:wasGeneratedBy"] = self.id
-        entry.opinions.append(opinion)
-        entry.language = lang
-        response.entries.append(entry)
-        return response

senpy/plugins/sentiment140/sentiment140.senpy

@@ -1,18 +0,0 @@
-{
-    "name": "sentiment140",
-    "module": "sentiment140",
-    "description": "What my plugin broadly does",
-    "author": "@balkian",
-    "version": "0.1",
-    "extra_params": {
-        "language": {
-            "@id": "lang_sentiment140",
-            "aliases": ["language", "l"],
-            "required": false,
-            "options": ["es", "en", "auto"]
-        }
-     },
-     "requirements": {},
-     "maxPolarityValue": "1",
-     "minPolarityValue": "0"
-}

senpy/plugins/split/split.py

@@ -0,0 +1,30 @@
+from senpy.plugins import AnalysisPlugin
+from senpy.models import Entry
+from nltk.tokenize.punkt import PunktSentenceTokenizer
+from nltk.tokenize.simple import LineTokenizer
+import nltk
+class SplitPlugin(AnalysisPlugin):
+    def activate(self):
+        nltk.download('punkt')
+    
+    def analyse_entry(self, entry, params):
+        chunker_type = params.get("delimiter", "sentence")
+        original_id = entry.id
+        original_text = entry.get("text", None)
+        if chunker_type == "sentence":
+            tokenizer = PunktSentenceTokenizer()
+            chars = tokenizer.span_tokenize(original_text)
+            for i, sentence in enumerate(tokenizer.tokenize(original_text)):
+                e = Entry()
+                e.text = sentence
+                e.id = original_id + "#char={},{}".format(chars[i][0], chars[i][1])
+                yield e
+        if chunker_type == "paragraph":
+            tokenizer = LineTokenizer()
+            chars = tokenizer.span_tokenize(original_text)
+            for i, paragraph in enumerate(tokenizer.tokenize(original_text)):
+                e = Entry() 
+                e.text = paragraph
+                chars = [char for char in chars]
+                e.id = original_id + "#char={},{}".format(chars[i][0], chars[i][1])
+                yield e

senpy/plugins/split/split.senpy

@@ -0,0 +1,18 @@
+---
+name: split
+module: split
+description: A sample plugin that chunks input text
+author: "@militarpancho"
+version: '0.1'
+url: "https://github.com/gsi-upm/senpy"
+requirements: {nltk}
+extra_params:
+  delimiter:
+    aliases:
+    - type
+    - t
+    required: false
+    default: sentence
+    options:
+    - sentence
+    - paragraph

senpy/schemas/analysis.json

@@ -0,0 +1,15 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Senpy analysis",
+  "type": "object",
+  "properties": {
+    "@id": {
+      "type": "string"
+    },
+    "@type": {
+      "type": "string",
+      "description": "Type of the analysis. e.g. marl:SentimentAnalysis"
+    }
+  },
+  "required": ["@id", "@type"]
+}

senpy/schemas/atom.json

@@ -0,0 +1,15 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Base schema for all Senpy objects",
+  "type": "object",
+  "properties": {
+    "@id": {
+      "type": "string"
+    },
+    "@type": {
+      "type": "string",
+      "description": "Type of the atom. e.g., 'onyx:EmotionAnalysis', 'nif:Entry'"
+    }
+  },
+  "required": ["@id", "@type"]
+}

senpy/schemas/context.json

@@ -0,0 +1,5 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "JSON-LD Context",
+  "type": ["array", "string", "object"]
+}

senpy/schemas/context.jsonld

@@ -0,0 +1,62 @@
+{
+  "@context": {
+    "@vocab": "http://www.gsi.dit.upm.es/ontologies/senpy#",
+    "dc": "http://dublincore.org/2012/06/14/dcelements#",
+    "me": "http://www.mixedemotions-project.eu/ns/model#",
+    "prov": "http://www.w3.org/ns/prov#",
+    "nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#",
+    "marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#",
+    "onyx": "http://www.gsi.dit.upm.es/ontologies/onyx/ns#",
+    "wna": "http://www.gsi.dit.upm.es/ontologies/wnaffect/ns#",
+    "emoml": "http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/emotionml/ns#",
+    "xsd": "http://www.w3.org/2001/XMLSchema#",
+    "topics": {
+      "@id": "dc:subject"
+    },
+    "entities": {
+      "@id": "me:hasEntities"
+    },
+    "suggestions": {
+      "@id": "me:hasSuggestions",
+      "@container": "@set"
+    },
+    "emotions": {
+      "@id": "onyx:hasEmotionSet",
+      "@container": "@set"
+    },
+    "sentiments": {
+      "@id": "marl:hasOpinion",
+      "@container": "@set"
+    },
+    "entries": {
+      "@id": "prov:used",
+      "@container": "@set"
+    },
+    "analysis": {
+      "@id": "AnalysisInvolved",
+      "@type": "@id",
+      "@container": "@set"
+    },
+    "options": {
+      "@container": "@set"
+    },
+    "plugins": {
+      "@container": "@set"
+    },
+    "prov:wasGeneratedBy": {
+      "@type": "@id"
+    },
+    "onyx:usesEmotionModel": {
+      "@type": "@id"
+    },
+    "onyx:hasEmotionCategory": {
+      "@type": "@id"
+    },
+    "onyx:conversionFrom": {
+      "@type": "@id"
+    },
+    "onyx:conversionTo": {
+      "@type": "@id"
+    }
+  }
+}

senpy/schemas/definitions.json

@@ -0,0 +1,45 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "Results": {
+    "$ref": "results.json"
+  },
+  "Context": {
+    "$ref": "context.json"
+  },
+  "Analysis": {
+    "$ref": "analysis.json"
+  },
+  "Entry": {
+    "$ref": "entry.json"
+  },
+  "Sentiment": {
+    "$ref": "sentiment.json"
+  },
+  "EmotionSet": {
+    "$ref": "emotionSet.json"
+  },
+  "Emotion": {
+    "$ref": "emotion.json"
+  },
+  "EmotionModel": {
+    "$ref": "emotionModel.json"
+  },
+  "Entity": {
+    "$ref": "entity.json"
+  },
+  "Topic": {
+    "$ref": "topic.json"
+  },
+  "Suggestion": {
+    "$ref": "suggestion.json"
+  },
+  "Plugins": {
+    "$ref": "plugin.json"
+  },
+  "Plugin": {
+    "$ref": "plugin.json"
+  },
+  "Response": {
+    "$ref": "response.json"
+  }
+}

senpy/schemas/dimensions.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "properties": {
+    "name": {"type": "string"},
+    "maxValue": {"type": "number"},
+    "minValue": {"type": "number"}
+  },
+  "required": ["name", "maxValue", "minValue"]
+}

senpy/schemas/emotion.json

@@ -0,0 +1,4 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object"
+}

senpy/schemas/emotionAnalysis.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Senpy Emotion analysis",
+  "type": "object",
+  "allOf": [
+    {"$ref": "analysis.json"},
+    {"properties":
+     {
+       "onyx:usesEmotionModel": {
+         "anyOf": [
+           {"type": "string"},
+           {"$ref": "emotionModel.json"}
+         ]
+       }
+     },
+     "required": ["onyx:hasEmotionModel",
+                  "@type"]
+    }]
+}

senpy/schemas/emotionConversion.json

@@ -0,0 +1,12 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "properties": {
+    "onyx:conversionFrom": {
+      "$ref": "emotionModel.json"
+    },
+    "onyx:conversionTo": {
+      "$ref": "emotionModel.json"
+    }
+  },
+  "required": ["onyx:conversionFrom", "onyx:conversionTo"]
+}

senpy/schemas/emotionConversionPlugin.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "allOf": [
+    {
+      "$ref": "plugin.json"
+    },
+    {
+      "properties": {
+        "onyx:doesConversion": {
+          "type": "array",
+          "items": {
+            "$ref": "emotionConversion.json"
+          }
+        }
+      }
+    }
+  ]
+}

senpy/schemas/emotionModel.json

@@ -0,0 +1,27 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "properties": {
+    "@id": {"type": "string"},
+    "nif:beginIndex": {"type": "integer"},
+    "nif:endIndex": {"type": "integer"},
+    "nif:anchorOf": {
+      "description": "Piece of context that contains the Sentiment",
+      "type": "string"
+    },
+    "onyx:hasDimension": {
+      "type": "array",
+      "items": {
+        "$ref": "dimensions.json"
+      },
+      "uniqueItems": true
+    },
+    "onyx:hasEmotionCategory": {
+      "type": "array",
+      "items": {
+        "$ref": "emotion.json"
+      },
+      "default": []
+    }
+  },
+  "required": ["@id", "onyx:hasEmotion"]
+}

senpy/schemas/emotionPlugin.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "allOf": [
+    {
+      "$ref": "plugin.json"
+    },
+    {
+      "properties": {
+        "onyx:usesEmotionModel": {
+          "type": "array",
+          "items": {
+            "$ref": "emotionModel.json"
+          }
+        }
+      }
+    }
+  ]
+}

senpy/schemas/emotionSet.json

@@ -0,0 +1,24 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "properties": {
+    "@id": {"type": "string"},
+    "nif:beginIndex": {"type": "integer"},
+    "nif:endIndex": {"type": "integer"},
+    "nif:anchorOf": {
+      "description": "Piece of context that contains the Sentiment",
+      "type": "string"
+    },
+    "onyx:hasEmotion": {
+      "type": "array",
+      "items": {
+        "$ref": "emotion.json"
+      },
+      "default": []
+    },
+    "prov:wasGeneratedBy": {
+      "type": "string",
+      "description": "The ID of the analysis that generated this Emotion. The full object should be included in the \"analysis\" property of the root object"
+    }
+  },
+  "required": ["@id", "prov:wasGeneratedBy", "onyx:hasEmotion"]
+}

senpy/schemas/entity.json

@@ -0,0 +1,4 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object"
+}

senpy/schemas/entry.json

@@ -0,0 +1,39 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "name": "Entry",
+  "properties": {
+    "@id": {
+      "type": "string"
+    },
+    "nif:isString": {
+      "description": "String contained in this Context",
+      "type": "string"
+    },
+    "sentiments": {
+      "type": "array",
+      "items": {"$ref": "sentiment.json" },
+      "default": []
+    },
+    "emotions": {
+      "type": "array",
+      "items": {"$ref": "emotionSet.json" },
+      "default": []
+    },
+    "entities": {
+      "type": "array",
+      "items": {"$ref": "entity.json" },
+      "default": []
+    },
+    "topics": {
+      "type": "array",
+      "items": {"$ref": "topic.json" },
+      "default": []
+    },
+    "suggestions": {
+      "type": "array",
+      "items": {"$ref": "suggestion.json" },
+      "default": []
+    }
+  },
+  "required": ["@id", "nif:isString"]
+}

senpy/schemas/error.json

@@ -0,0 +1,23 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Base schema for all Senpy objects",
+  "type": "object",
+  "allOf": [
+    {"$ref": "atom.json"},
+    {
+      "properties": {
+        "message": {
+          "type": "string"
+        },
+        "errors": {
+          "type": "array",
+          "items": {"type": "object"}
+        },
+        "status": {
+          "type": "number"
+        }
+      },
+      "required": ["message"]
+    }
+  ]
+}

senpy/schemas/plugin.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "required": ["@id", "extra_params"],
+  "properties": {
+    "@id": {
+      "type": "string",
+      "description": "Unique identifier for the plugin, usually comprised of the name of the plugin and the version."
+    },
+    "name": {
+      "type": "string",
+      "description": "The name of the plugin, which will be used in the algorithm detection phase"
+    },
+    "extra_params": {
+      "type": "object",
+      "default": {}
+    }
+  }
+}

senpy/schemas/plugins.json

@@ -0,0 +1,17 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "allOf": [
+    {"$ref": "response.json"},
+    {
+      "properties": {
+        "plugins": {
+          "type": "array",
+          "default": [],
+          "items": {
+            "$ref": "plugin.json"
+          }
+        }
+      }
+    }
+  ]
+}

senpy/schemas/response.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "properties": {
+    "@type": {"type": "string"}
+  },
+  "required": ["@type"]
+
+}

senpy/schemas/results.json

@@ -0,0 +1,45 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "allOf": [
+    {"$ref": "response.json"},
+    {
+      "title": "Results",
+      "description": "The results of an analysis",
+      "type": "object",
+      "properties": {
+        "@context": {
+          "$ref": "context.json"
+        },
+        "@type": {
+          "default": "results"
+        },
+        "@id": {
+          "description": "ID of the analysis",
+          "type": "string"
+        },
+        "analysis": {
+          "default": [],
+          "type": "array",
+          "items": {
+            "anyOf": [
+              {
+                "$ref": "analysis.json"
+              },{
+                "type": "string"
+              }
+            ]
+          }
+        },
+        "entries": {
+          "type": "array",
+          "default": [],
+          "items": {
+            "$ref": "entry.json"
+          }
+        }
+
+      },
+      "required": ["@id", "analysis", "entries"]
+    }
+  ]
+}