remove pytest warning errors

In the previous commit I naively configured pytest to
treat any warning as an error, excep for numpy
pendingdeprecationwarning errors,
This causes the build in py2.7 to fail, due to some
warnings from nltk.

.gitignore

@@ -7,5 +7,4 @@ README.html
 __pycache__
 VERSION
 Dockerfile-*
-Dockerfile
+Dockerfile
-senpy_data

.gitlab-ci.yml

@@ -4,130 +4,100 @@
 # - docker:dind
 # When using dind, it's wise to use the overlayfs driver for
 # improved performance.
+
 stages:
   - test
-  - publish
+  - push
-  - test_image
   - deploy
+  - clean
 
-variables:
+before_script:
-  KUBENS: senpy
+  - make -e login
-  LATEST_IMAGE: "${HUB_REPO}:${CI_COMMIT_SHORT_SHA}"
-  SENPY_DATA: "/senpy-data/" # This is configured in the CI job
-  NLTK_DATA: "/senpy-data/nltk_data" # Store NLTK downloaded data
 
-docker:
+.test: &test_definition
-  stage: publish
-  image:
-    name: gcr.io/kaniko-project/executor:debug
-    entrypoint: [""]
-  variables:
-    PYTHON_VERSION: "3.10"
-  tags:
-    - docker
-  script:
-    - echo $CI_COMMIT_TAG > senpy/VERSION
-    - sed "s/{{PYVERSION}}/$PYTHON_VERSION/" Dockerfile.template > Dockerfile
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"},\"https://index.docker.io/v1/\":{\"auth\":\"$HUB_AUTH\"}}}" > /kaniko/.docker/config.json
-    # The skip-tls-verify flag is there because our registry certificate is self signed
-    - /kaniko/executor --context $CI_PROJECT_DIR --skip-tls-verify --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG --destination $HUB_REPO:$CI_COMMIT_TAG
-  only:
-    - tags
-
-docker-latest:
-  stage: publish
-  image:
-    name: gcr.io/kaniko-project/executor:debug
-    entrypoint: [""]
-  variables:
-    PYTHON_VERSION: "3.10"
-  tags:
-    - docker
-  script:
-    - echo git.${CI_COMMIT_SHORT_SHA} > senpy/VERSION
-    - sed "s/{{PYVERSION}}/$PYTHON_VERSION/" Dockerfile.template > Dockerfile
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"},\"https://index.docker.io/v1/\":{\"auth\":\"$HUB_AUTH\"}}}" > /kaniko/.docker/config.json
-    # The skip-tls-verify flag is there because our registry certificate is self signed
-    - /kaniko/executor --context $CI_PROJECT_DIR --skip-tls-verify --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $LATEST_IMAGE --destination "${HUB_REPO}:latest"
-  only:
-    refs:
-      - master
-
-testimage:
-  only:
-    - tags
-  tags:
-    - docker
-  stage: test_image
-  image: "$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG"
-  script:
-    - python -m senpy --no-run --test
-
-testpy37:
-  tags:
-    - docker
-  variables:
-    SENPY_STRICT: "false"
-  image: python:3.7
   stage: test
   script:
-    - pip install -r requirements.txt -r test-requirements.txt
+    - make -e test-$PYTHON_VERSION
-    - python setup.py test
+  except:
-
+    - tags # Avoid unnecessary double testing
-testpy310:
+    
-  tags:
+test-3.5:
-    - docker
+  <<: *test_definition
   variables:
-    SENPY_STRICT: "true"
+    PYTHON_VERSION: "3.5"
-  image: python:3.10
-  stage: test
-  script:
-    - pip install -r requirements.txt -r test-requirements.txt -r extra-requirements.txt
-    - python setup.py test
 
-push_pypi:
+test-2.7:
+  <<: *test_definition
+  variables:
+    PYTHON_VERSION: "2.7"
+
+push:
+  stage: push
+  script:
+    - make -e push
   only:
     - tags
-  tags:
+    - triggers
-    - docker
+    - fix-makefiles
-  image: python:3.10
-  stage: publish
-  script:
-    - echo $CI_COMMIT_TAG > senpy/VERSION
-    - pip install twine
-    - python setup.py sdist bdist_wheel
-    - TWINE_PASSWORD=$PYPI_PASSWORD TWINE_USERNAME=$PYPI_USERNAME python -m twine upload dist/*
 
-check_pypi:
+push-latest:
+  stage: push
+  script:
+    - make -e push-latest
   only:
-    - tags
+    - master
-  tags:
+    - triggers
-    - docker
+    - fix-makefiles
-  image: python:3.10
+
+push-github:
   stage: deploy
   script:
-    - pip install senpy==$CI_COMMIT_TAG
+    - make -e push-github
-  # Allow PYPI to update its index before we try to install 
-  when: delayed
-  start_in: 10 minutes
-
-latest-demo:
   only:
-    refs:
+    - master
-      - master
+    - triggers
-  tags:
+    - fix-makefiles
-    - docker
+
-  image: alpine/k8s:1.22.6
+deploy_pypi:
   stage: deploy
-  environment: production
+  script:   # Configure the PyPI credentials, then push the package, and cleanup the creds.
-  variables:
+    - echo "[server-login]" >> ~/.pypirc
-    KUBECONFIG: "/kubeconfig"
+    - echo "repository=https://upload.pypi.org/legacy/" >> ~/.pypirc
-    # Same image as docker-latest
+    - echo "username=" ${PYPI_USER} >> ~/.pypirc
-    IMAGEWTAG: "${LATEST_IMAGE}"
+    - echo "password=" ${PYPI_PASSWORD} >> ~/.pypirc
-    KUBEAPP: "senpy"
+    - make pip_upload
+    - echo "" > ~/.pypirc && rm ~/.pypirc  # If the above fails, this won't run.
+  only:
+    - /^v?\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
+  except:
+    - branches
+
+deploy:
+  stage: deploy
+  environment: test
   script:
-  - echo "${KUBECONFIG_RAW}" > $KUBECONFIG
+    - make -e deploy
-  - kubectl --kubeconfig $KUBECONFIG version
+  only:
-  - cd k8s/
+    - master
-  - cat *.yaml *.tmpl 2>/dev/null | envsubst | kubectl --kubeconfig $KUBECONFIG apply --namespace ${KUBENS:-default} -f -
+    - fix-makefiles
-  - kubectl --kubeconfig $KUBECONFIG get all,ing -l app=${KUBEAPP} --namespace=${KUBENS:-default}
+
+push-github:
+  stage: deploy
+  script:
+    - make -e push-github
+  only:
+    - master
+    - triggers
+
+clean :
+  stage: clean
+  script:
+    - make -e clean
+  when: manual
+
+cleanup_py:
+   stage: clean
+   when: always   # this is important; run even if preceding stages failed.
+   script:
+    - rm -vf ~/.pypirc  # we don't want to leave these around, but GitLab may clean up anyway.
+    - docker logout

.makefiles/README.md

@@ -2,7 +2,7 @@ These makefiles are recipes for several common tasks in different types of proje
 To add them to your project, simply do:
 
 ```
-git remote add makefiles ssh://git@lab.gsi.upm.es:2200/docs/templates/makefiles.git
+git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git
 git subtree add --prefix=.makefiles/ makefiles master
 touch Makefile
 echo "include .makefiles/base.mk" >> Makefile
@@ -16,7 +16,7 @@ include .makefiles/python.mk
 ```
 
 You may need to set special variables like the name of your project or the python versions you're targetting.
-Take a look at each specific `.mk` file for more information, and the `Makefile` in the [senpy](https://lab.gsi.upm.es/senpy/senpy) project for a real use case.
+Take a look at each specific `.mk` file for more information, and the `Makefile` in the [senpy](https://lab.cluster.gsi.dit.upm.es/senpy/senpy) project for a real use case.
 
 If you update the makefiles from your repository, make sure to push the changes for review in upstream (this repository):
 

.makefiles/makefiles.mk

@@ -1,5 +1,5 @@
 makefiles-remote:
-	git ls-remote --exit-code  makefiles 2> /dev/null || git remote add makefiles ssh://git@lab.gsi.upm.es:2200/docs/templates/makefiles.git
+	git ls-remote --exit-code  makefiles 2> /dev/null || git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git
 
 makefiles-commit: makefiles-remote
 	git add -f .makefiles

.makefiles/python.mk

@@ -29,7 +29,7 @@ build: $(addprefix build-, $(PYVERSIONS)) ## Build all images / python versions
 	docker tag $(IMAGEWTAG)-python$(PYMAIN) $(IMAGEWTAG)
 
 build-%: version Dockerfile-%  ## Build a specific version (e.g. build-2.7)
-	docker build --pull -t '$(IMAGEWTAG)-python$*' -f Dockerfile-$* .;
+	docker build -t '$(IMAGEWTAG)-python$*' -f Dockerfile-$* .;
 
 dev-%: ## Launch a specific development environment using docker (e.g. dev-2.7)
 	@docker start $(NAME)-dev$* || (\

.readthedocs.yaml

@@ -1,22 +0,0 @@
-# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
-
-version: 2
-
-# Set the OS, Python version and other tools you might need
-
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.10"
-
-# Build documentation in the "docs/" directory with Sphinx
-sphinx:
-  configuration: docs/conf.py
-
-# formats:
-#     - pdf
-#     - epub
-
-python:
-   install:
-   - requirements: docs/requirements.txt

.travis.yml

@@ -1,43 +1,12 @@
 sudo: required
 
-matrix:
+services:
-  allow_failures:
+  - docker
-    # Windows is experimental in Travis.
+
-    # As of this writing, senpy installs but hangs on tests that use the flask test client (e.g. blueprints)
+language: python
-    - os: windows
+
-  include:
+env:
-    - os: linux
+  - PYV=2.7
-      language: python
+  - PYV=3.5
-      python: 3.4
+# run nosetests - Tests
-      before_install:
+script: make test-$PYV
-        - pip install --upgrade --force-reinstall pandas
-    - os: linux
-      language: python
-      python: 3.5
-    - os: linux
-      language: python
-      python: 3.6
-    - os: linux
-      language: python
-      python: 3.7
-    - os: osx
-      language: generic
-      addons:
-        homebrew:
-          # update: true
-          packages: python3
-      before_install:
-        - python3 -m pip install --upgrade virtualenv
-        - virtualenv -p python3 --system-site-packages "$HOME/venv"
-        - source "$HOME/venv/bin/activate"
-    - os: windows
-      language: bash
-      before_install:
-        - choco install -y python3
-        - python -m pip install --upgrade pip
-      env: PATH=/c/Python37:/c/Python37/Scripts:$PATH
-# command to run tests
-# 'python' points to Python 2.7 on macOS but points to Python 3.7 on Linux and Windows
-# 'python3' is a 'command not found' error on Windows but 'py' works on Windows only
-script:
-  - python3 setup.py test || python setup.py test

CHANGELOG.md

@@ -1,86 +0,0 @@
-# Changelog
-All notable changes to this project will be documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-## [Unreleased]
-### Changed
-* Changed the underlying model to use `pydantic` models
-* Plugin interface slightly changed. Activation should be performed in the `activate` method
-* data directory selection logic is slightly modified, and will choose one of the following (in this order): `data_folder` (argument), `$SENPY_DATA` or `$CWD`
-* Plugins cannot be deactivated
-### Fixed
-* Tests now detect errors on optional plugins
-### Added
-* The code of many senpy community plugins have been included by default. However, additional files (e.g., licensed data) and/or installing additional dependencies may be necessary for some plugins. Read each plugin's documentation for more information.
-* `optional` attribute in plugins. Optional plugins may fail to load or activate but the server will be started regardless, unless running in strict mode
-* Option in shelf plugins to ignore pickling errors
-### Removed
-* `--only-install`, `--only-test` and `--only-list` flags were removed in favor of `--no-run` + `--install`/`--test`/`--dependencies`
-
-
-## [1.0.6]
-### Fixed
-* Plugins now get activated for testing
-## [1.0.1]
-### Added
-* License headers
-* Description for PyPI (setup.py)
-
-### Changed
-* The evaluation tab shows datasets inline, and a tooltip shows the number of instances
-* The docs should be clearer now
-
-## [1.0.0]
-### Fixed
-* Restored hash changing function in `main.js`
-
-## 0.20
-
-### Added
-* Objects can control the keys that will be used in `serialize`/`jsonld`/`as_dict` by specifying a list of keys in `terse_keys`.
-e.g.
-```python
->>> class MyModel(senpy.models.BaseModel):
-...   _terse_keys = ['visible']
-...   invisible = 5
-...   visible = 1
-...
->>> m = MyModel(id='testing')
->>> m.jsonld()
-{'invisible': 5, 'visible': 1, '@id': 'testing'}
->>> m.jsonld(verbose=False)
-{'visible': 1}
-```
-* Configurable logging format.
-* Added default terse keys for the most common classes (entry, sentiment, emotion...).
-* Flag parameters (boolean) are set to true even when no value is added (e.g. `&verbose` is the same as `&verbose=true`).
-* Plugin and parameter descriptions are now formatted with (showdown)[https://github.com/showdownjs/showdown].
-* The web UI requests extra_parameters from the server. This is useful for pipelines. See #52
-* First batch of semantic tests (using SPARQL)
-* `Plugin.path()` method to get a file path from a relative path (using the senpy data folder)
-
-### Changed
-* `install_deps` now checks what requirements are already met before installing with pip.
-* Help is now provided verbosely by default
-* Other outputs are terse by default. This means some properties are now hidden unless verbose is set.
-* `sentiments` and `emotions` are now `marl:hasOpinion` and `onyx:hasEmotionSet`, respectively.
-* Nicer logging format
-* Context aliases (e.g. `sentiments` and `emotions` properties) have been replaced with the original properties (e.g. `marl:hasOpinion` and `onyx:hasEmotionSet**), to use aliases, pass the `aliases** parameter.
-* Several UI improvements
-  * Dedicated tab to show the list of plugins
-  * URLs in plugin descriptions are shown as links
-  * The format of the response is selected by clicking on a tab instead of selecting from a drop-down
-  * list of examples
-  * Bootstrap v4
-* RandEmotion and RandSentiment are no longer included in the base set of plugins
-* The `--plugin-folder` option can be used more than once, and every folder will be added to the app.
-
-### Deprecated
-### Removed
-* Python 2.7 is no longer test or officially supported
-### Fixed
-* Plugin descriptions are now dedented when they are extracted from the docstring.
-### Security
-

Dockerfile.template

@@ -6,20 +6,21 @@ RUN apt-get update && apt-get install -y \
 libblas-dev liblapack-dev liblapacke-dev gfortran \
  && rm -rf /var/lib/apt/lists/*
 
-RUN mkdir -p /cache/ /senpy-plugins /data/
+RUN mkdir /cache/ /senpy-plugins /data/
 
 VOLUME /data/
 
 ENV PIP_CACHE_DIR=/cache/ SENPY_DATA=/data
 
+ONBUILD COPY . /senpy-plugins/
+ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
+ONBUILD WORKDIR /senpy-plugins/
+
+
 WORKDIR /usr/src/app
 COPY test-requirements.txt requirements.txt extra-requirements.txt /usr/src/app/
 RUN pip install --no-cache-dir -r test-requirements.txt -r requirements.txt -r extra-requirements.txt
 COPY . /usr/src/app/
 RUN pip install --no-cache-dir --no-index --no-deps --editable .
 
-ONBUILD COPY . /senpy-plugins/
-ONBUILD RUN python -m senpy -i --no-run -f /senpy-plugins
-ONBUILD WORKDIR /senpy-plugins/
-
 ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

MANIFEST.in

@@ -2,10 +2,9 @@ include requirements.txt
 include test-requirements.txt
 include extra-requirements.txt
 include README.rst
-include LICENSE.txt
 include senpy/VERSION
 graft senpy/plugins
 graft senpy/schemas
 graft senpy/templates
 graft senpy/static
-graft img
+graft img

Makefile

@@ -5,7 +5,7 @@ IMAGENAME=gsiupm/senpy
 
 # The first version is the main one (used for quick builds)
 # See .makefiles/python.mk for more info
-PYVERSIONS ?= 3.10 3.7
+PYVERSIONS=3.5 2.7
 
 DEVPORT=5000
 

Procfile

@@ -1 +1 @@
-web: python -m senpy --host 0.0.0.0 --port $PORT
+web: python -m senpy --host 0.0.0.0 --port $PORT --default-plugins

README.rst

@@ -1,25 +1,18 @@
 .. image:: img/header.png
    :width: 100%
-   :target: http://senpy.gsi.upm.es
+   :target: http://demos.gsi.dit.upm.es/senpy
 
-.. image:: https://readthedocs.org/projects/senpy/badge/?version=latest
+.. image:: https://travis-ci.org/gsi-upm/senpy.svg?branch=master
-  :target: http://senpy.readthedocs.io/en/latest/
+   :target: https://travis-ci.org/gsi-upm/senpy
-.. image:: https://badge.fury.io/py/senpy.svg
-  :target: https://badge.fury.io/py/senpy
-.. image:: https://travis-ci.org/gsi-upm/senpy.svg
-  :target: https://github.com/gsi-upm/senpy/senpy/tree/master
-.. image:: https://img.shields.io/pypi/l/requests.svg
-  :target: https://lab.gsi.upm.es/senpy/senpy/
 
-     
 Senpy lets you create sentiment analysis web services easily, fast and using a well known API.
-As a bonus, Senpy services use semantic vocabularies (e.g. `NIF <http://persistence.uni-leipzig.org/nlp2rdf/>`_, `Marl <http://www.gsi.upm.es/ontologies/marl>`_, `Onyx <http://www.gsi.upm.es/ontologies/onyx>`_) and formats (turtle, JSON-LD, xml-rdf).
+As a bonus, senpy services use semantic vocabularies (e.g. `NIF <http://persistence.uni-leipzig.org/nlp2rdf/>`_, `Marl <http://www.gsi.dit.upm.es/ontologies/marl>`_, `Onyx <http://www.gsi.dit.upm.es/ontologies/onyx>`_) and formats (turtle, JSON-LD, xml-rdf).
 
 Have you ever wanted to turn your sentiment analysis algorithms into a service?
-With Senpy, now you can.
+With senpy, now you can.
 It provides all the tools so you just have to worry about improving your algorithms:
 
-`See it in action. <http://senpy.gsi.upm.es/>`_
+`See it in action. <http://senpy.cluster.gsi.dit.upm.es/>`_
 
 Installation
 ------------
@@ -41,36 +34,20 @@ Alternatively, you can use the development version:
    cd senpy
    pip install --user .
 
-If you want to install Senpy globally, use sudo instead of the ``--user`` flag.
+If you want to install senpy globally, use sudo instead of the ``--user`` flag.
 
 Docker Image
 ************
-Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy``.
+Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy --default-plugins``.
 
-To add custom plugins, add a volume and tell Senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy -f /plugins``
+To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --default-plugins -f /plugins``
 
 
-Compatibility
--------------
-
-Senpy should run on any major operating system.
-Its code is pure Python, and the only limitations are imposed by its dependencies (e.g., nltk, pandas).
-
-Currently, the CI/CD pipeline tests the code on:
-
-* GNU/Linux with Python versions 3.7+ (3.10+ recommended for all plugins)
-* MacOS and homebrew's python3
-* Windows 10 and chocolatey's python3
-
-The latest PyPI package is verified to install on Ubuntu, Debian and Arch Linux.
-
-If you have trouble installing Senpy on your platform, see `Having problems?`_.
-
 Developing
 ----------
 
-Running/debugging
+Developing/debugging
-*****************
+********************
 This command will run the senpy container using the latest image available, mounting your current folder so you get your latest code:
 
 .. code:: bash
@@ -133,7 +110,7 @@ or, alternatively:
 This will create a server with any modules found in the current path.
 For more options, see the `--help` page.
 
-Alternatively, you can use the modules included in Senpy to build your own application.
+Alternatively, you can use the modules included in senpy to build your own application.
 
 Deploying on Heroku
 -------------------
@@ -141,31 +118,13 @@ Use a free heroku instance to share your service with the world.
 Just use the example Procfile in this repository, or build your own.
 
 
+`DEMO on heroku <http://senpy.herokuapp.com>`_
+
+
 For more information, check out the `documentation <http://senpy.readthedocs.org>`_.
 ------------------------------------------------------------------------------------
 
 
-Python 2.x compatibility
-------------------------
-
-Keeping compatibility between python 2.7 and 3.x is not always easy, especially for a framework that deals both with text and web requests.
-Hence, starting February 2019, this project will no longer make efforts to support python 2.7, which will reach its end of life in 2020.
-Most of the functionality should still work, and the compatibility shims will remain for now, but we cannot make any guarantees at this point.
-Instead, the maintainers will focus their efforts on keeping the codebase compatible across different Python 3.3+ versions, including upcoming ones.
-We apologize for the inconvenience.
-
-
-Having problems?
-----------------
-
-Please, file a new issue `on GitHub <https://github.com/gsi-upm/senpy/issues>`_ including enough details to reproduce the bug, including:
-
-* Operating system
-* Version of Senpy (or docker tag)
-* Installed libraries
-* Relevant logs
-* A simple code example
-
 Acknowledgement
 ---------------
 This development has been partially funded by the European Union through the MixedEmotions Project (project number H2020 655632), as part of the `RIA ICT 15 Big data and Open Data Innovation and take-up` programme.

TODO.md

@@ -1,9 +0,0 @@
-* Upload context to www.gsi.upm.es/ontologies/senpy
-* Use context from gsi.upm.es
-* Add example of morality analysis
-
-* Simplify plugin loading
-* Simplify models
-    * Remove json schemas and generate pydantic objects
-    * Remove some of the meta-programming magic
-* Migrate to openapi/quart

config.py

@@ -0,0 +1,4 @@
+import os
+
+SERVER_PORT = os.environ.get("SERVER_PORT", 5000)
+DEBUG = os.environ.get("DEBUG", True)

docs/Evaluation.ipynb

@@ -1,592 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Evaluating Services"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Sentiment analysis plugins can also be evaluated on a series of pre-defined datasets.\n",
-    "This can be done in three ways: through the Web UI (playground), through the web API and programmatically.\n",
-    "\n",
-    "Regardless of the way you perform the evaluation, you will need to specify a plugin (service) that you want to evaluate, and a series of datasets on which it should be evaluated.\n",
-    "\n",
-    "to evaluate a plugin on a dataset, senpy use the plugin to predict the sentiment in each entry in the dataset.\n",
-    "These predictions are compared with the expected values to produce several metrics, such as: accuracy, precision and f1-score.\n",
-    "\n",
-    "**note**: the evaluation process might take long for plugins that use external services, such as `sentiment140`.\n",
-    "\n",
-    "**note**: plugins are assumed to be pre-trained and invariant. i.e., the prediction for an entry should "
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Web UI (Playground)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "The playground should contain a tab for Evaluation, where you can select any plugin that can be evaluated, and the set of datasets that you want to test the plugin on.\n",
-    "\n",
-    "For example, the image below shows the results of the `sentiment-vader` plugin on the `vader` and `sts` datasets:\n",
-    "\n",
-    "\n",
-    "![](eval_table.png)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Web API"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "The api exposes an endpoint (`/evaluate`), which accents the plugin and the set of datasets on which it should be evaluated."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "The following code is not necessary, but it will display the results better:"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Here is a simple call using the requests library:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<style>.output_html .hll { background-color: #ffffcc }\n",
-       ".output_html  { background: #f8f8f8; }\n",
-       ".output_html .c { color: #408080; font-style: italic } /* Comment */\n",
-       ".output_html .err { border: 1px solid #FF0000 } /* Error */\n",
-       ".output_html .k { color: #008000; font-weight: bold } /* Keyword */\n",
-       ".output_html .o { color: #666666 } /* Operator */\n",
-       ".output_html .ch { color: #408080; font-style: italic } /* Comment.Hashbang */\n",
-       ".output_html .cm { color: #408080; font-style: italic } /* Comment.Multiline */\n",
-       ".output_html .cp { color: #BC7A00 } /* Comment.Preproc */\n",
-       ".output_html .cpf { color: #408080; font-style: italic } /* Comment.PreprocFile */\n",
-       ".output_html .c1 { color: #408080; font-style: italic } /* Comment.Single */\n",
-       ".output_html .cs { color: #408080; font-style: italic } /* Comment.Special */\n",
-       ".output_html .gd { color: #A00000 } /* Generic.Deleted */\n",
-       ".output_html .ge { font-style: italic } /* Generic.Emph */\n",
-       ".output_html .gr { color: #FF0000 } /* Generic.Error */\n",
-       ".output_html .gh { color: #000080; font-weight: bold } /* Generic.Heading */\n",
-       ".output_html .gi { color: #00A000 } /* Generic.Inserted */\n",
-       ".output_html .go { color: #888888 } /* Generic.Output */\n",
-       ".output_html .gp { color: #000080; font-weight: bold } /* Generic.Prompt */\n",
-       ".output_html .gs { font-weight: bold } /* Generic.Strong */\n",
-       ".output_html .gu { color: #800080; font-weight: bold } /* Generic.Subheading */\n",
-       ".output_html .gt { color: #0044DD } /* Generic.Traceback */\n",
-       ".output_html .kc { color: #008000; font-weight: bold } /* Keyword.Constant */\n",
-       ".output_html .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */\n",
-       ".output_html .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */\n",
-       ".output_html .kp { color: #008000 } /* Keyword.Pseudo */\n",
-       ".output_html .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */\n",
-       ".output_html .kt { color: #B00040 } /* Keyword.Type */\n",
-       ".output_html .m { color: #666666 } /* Literal.Number */\n",
-       ".output_html .s { color: #BA2121 } /* Literal.String */\n",
-       ".output_html .na { color: #7D9029 } /* Name.Attribute */\n",
-       ".output_html .nb { color: #008000 } /* Name.Builtin */\n",
-       ".output_html .nc { color: #0000FF; font-weight: bold } /* Name.Class */\n",
-       ".output_html .no { color: #880000 } /* Name.Constant */\n",
-       ".output_html .nd { color: #AA22FF } /* Name.Decorator */\n",
-       ".output_html .ni { color: #999999; font-weight: bold } /* Name.Entity */\n",
-       ".output_html .ne { color: #D2413A; font-weight: bold } /* Name.Exception */\n",
-       ".output_html .nf { color: #0000FF } /* Name.Function */\n",
-       ".output_html .nl { color: #A0A000 } /* Name.Label */\n",
-       ".output_html .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */\n",
-       ".output_html .nt { color: #008000; font-weight: bold } /* Name.Tag */\n",
-       ".output_html .nv { color: #19177C } /* Name.Variable */\n",
-       ".output_html .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */\n",
-       ".output_html .w { color: #bbbbbb } /* Text.Whitespace */\n",
-       ".output_html .mb { color: #666666 } /* Literal.Number.Bin */\n",
-       ".output_html .mf { color: #666666 } /* Literal.Number.Float */\n",
-       ".output_html .mh { color: #666666 } /* Literal.Number.Hex */\n",
-       ".output_html .mi { color: #666666 } /* Literal.Number.Integer */\n",
-       ".output_html .mo { color: #666666 } /* Literal.Number.Oct */\n",
-       ".output_html .sa { color: #BA2121 } /* Literal.String.Affix */\n",
-       ".output_html .sb { color: #BA2121 } /* Literal.String.Backtick */\n",
-       ".output_html .sc { color: #BA2121 } /* Literal.String.Char */\n",
-       ".output_html .dl { color: #BA2121 } /* Literal.String.Delimiter */\n",
-       ".output_html .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */\n",
-       ".output_html .s2 { color: #BA2121 } /* Literal.String.Double */\n",
-       ".output_html .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */\n",
-       ".output_html .sh { color: #BA2121 } /* Literal.String.Heredoc */\n",
-       ".output_html .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */\n",
-       ".output_html .sx { color: #008000 } /* Literal.String.Other */\n",
-       ".output_html .sr { color: #BB6688 } /* Literal.String.Regex */\n",
-       ".output_html .s1 { color: #BA2121 } /* Literal.String.Single */\n",
-       ".output_html .ss { color: #19177C } /* Literal.String.Symbol */\n",
-       ".output_html .bp { color: #008000 } /* Name.Builtin.Pseudo */\n",
-       ".output_html .fm { color: #0000FF } /* Name.Function.Magic */\n",
-       ".output_html .vc { color: #19177C } /* Name.Variable.Class */\n",
-       ".output_html .vg { color: #19177C } /* Name.Variable.Global */\n",
-       ".output_html .vi { color: #19177C } /* Name.Variable.Instance */\n",
-       ".output_html .vm { color: #19177C } /* Name.Variable.Magic */\n",
-       ".output_html .il { color: #666666 } /* Literal.Number.Integer.Long */</style><div class=\"highlight\"><pre><span></span><span class=\"p\">{</span>\n",
-       "  <span class=\"nt\">&quot;@context&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;http://senpy.gsi.upm.es/api/contexts/YXBpL2V2YWx1YXRlLz9hbGdvPXNlbnRpbWVudC12YWRlciZkYXRhc2V0PXZhZGVyJTJDc3RzJm91dGZvcm1hdD1qc29uLWxkIw%3D%3D&quot;</span><span class=\"p\">,</span>\n",
-       "  <span class=\"nt\">&quot;@type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;AggregatedEvaluation&quot;</span><span class=\"p\">,</span>\n",
-       "  <span class=\"nt\">&quot;senpy:evaluations&quot;</span><span class=\"p\">:</span> <span class=\"p\">[</span>\n",
-       "    <span class=\"p\">{</span>\n",
-       "      <span class=\"nt\">&quot;@type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;Evaluation&quot;</span><span class=\"p\">,</span>\n",
-       "      <span class=\"nt\">&quot;evaluates&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;endpoint:plugins/sentiment-vader_0.1.1__vader&quot;</span><span class=\"p\">,</span>\n",
-       "      <span class=\"nt\">&quot;evaluatesOn&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;vader&quot;</span><span class=\"p\">,</span>\n",
-       "      <span class=\"nt\">&quot;metrics&quot;</span><span class=\"p\">:</span> <span class=\"p\">[</span>\n",
-       "        <span class=\"p\">{</span>\n",
-       "          <span class=\"nt\">&quot;@type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;Accuracy&quot;</span><span class=\"p\">,</span>\n",
-       "          <span class=\"nt\">&quot;value&quot;</span><span class=\"p\">:</span> <span class=\"mf\">0.6907142857142857</span>\n",
-       "        <span class=\"p\">},</span>\n",
-       "        <span class=\"p\">{</span>\n",
-       "          <span class=\"nt\">&quot;@type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;Precision_macro&quot;</span><span class=\"p\">,</span>\n",
-       "          <span class=\"nt\">&quot;value&quot;</span><span class=\"p\">:</span> <span class=\"mf\">0.34535714285714286</span>\n",
-       "        <span class=\"p\">},</span>\n",
-       "        <span class=\"p\">{</span>\n",
-       "          <span class=\"nt\">&quot;@type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;Recall_macro&quot;</span><span class=\"p\">,</span>\n",
-       "          <span class=\"nt\">&quot;value&quot;</span><span class=\"p\">:</span> <span class=\"mf\">0.5</span>\n",
-       "        <span class=\"p\">},</span>\n",
-       "        <span class=\"p\">{</span>\n",
-       "          <span class=\"nt\">&quot;@type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;F1_macro&quot;</span><span class=\"p\">,</span>\n",
-       "          <span class=\"nt\">&quot;value&quot;</span><span class=\"p\">:</span> <span class=\"mf\">0.40853400929446554</span>\n",
-       "        <span class=\"p\">},</span>\n",
-       "        <span class=\"p\">{</span>\n",
-       "          <span class=\"nt\">&quot;@type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;F1_weighted&quot;</span><span class=\"p\">,</span>\n",
-       "          <span class=\"nt\">&quot;value&quot;</span><span class=\"p\">:</span> <span class=\"mf\">0.5643605528396403</span>\n",
-       "        <span class=\"p\">},</span>\n",
-       "        <span class=\"p\">{</span>\n",
-       "          <span class=\"nt\">&quot;@type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;F1_micro&quot;</span><span class=\"p\">,</span>\n",
-       "          <span class=\"nt\">&quot;value&quot;</span><span class=\"p\">:</span> <span class=\"mf\">0.6907142857142857</span>\n",
-       "        <span class=\"p\">},</span>\n",
-       "        <span class=\"p\">{</span>\n",
-       "          <span class=\"nt\">&quot;@type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;F1_macro&quot;</span><span class=\"p\">,</span>\n",
-       "          <span class=\"nt\">&quot;value&quot;</span><span class=\"p\">:</span> <span class=\"mf\">0.40853400929446554</span>\n",
-       "        <span class=\"p\">}</span>\n",
-       "      <span class=\"p\">]</span>\n",
-       "    <span class=\"p\">},</span>\n",
-       "    <span class=\"p\">{</span>\n",
-       "      <span class=\"nt\">&quot;@type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;Evaluation&quot;</span><span class=\"p\">,</span>\n",
-       "      <span class=\"nt\">&quot;evaluates&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;endpoint:plugins/sentiment-vader_0.1.1__sts&quot;</span><span class=\"p\">,</span>\n",
-       "      <span class=\"nt\">&quot;evaluatesOn&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;sts&quot;</span><span class=\"p\">,</span>\n",
-       "      <span class=\"nt\">&quot;metrics&quot;</span><span class=\"p\">:</span> <span class=\"p\">[</span>\n",
-       "        <span class=\"p\">{</span>\n",
-       "          <span class=\"nt\">&quot;@type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;Accuracy&quot;</span><span class=\"p\">,</span>\n",
-       "          <span class=\"nt\">&quot;value&quot;</span><span class=\"p\">:</span> <span class=\"mf\">0.3107177974434612</span>\n",
-       "        <span class=\"p\">},</span>\n",
-       "        <span class=\"p\">{</span>\n",
-       "          <span class=\"nt\">&quot;@type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;Precision_macro&quot;</span><span class=\"p\">,</span>\n",
-       "          <span class=\"nt\">&quot;value&quot;</span><span class=\"p\">:</span> <span class=\"mf\">0.1553588987217306</span>\n",
-       "        <span class=\"p\">},</span>\n",
-       "        <span class=\"p\">{</span>\n",
-       "          <span class=\"nt\">&quot;@type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;Recall_macro&quot;</span><span class=\"p\">,</span>\n",
-       "          <span class=\"nt\">&quot;value&quot;</span><span class=\"p\">:</span> <span class=\"mf\">0.5</span>\n",
-       "        <span class=\"p\">},</span>\n",
-       "        <span class=\"p\">{</span>\n",
-       "          <span class=\"nt\">&quot;@type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;F1_macro&quot;</span><span class=\"p\">,</span>\n",
-       "          <span class=\"nt\">&quot;value&quot;</span><span class=\"p\">:</span> <span class=\"mf\">0.23705926481620407</span>\n",
-       "        <span class=\"p\">},</span>\n",
-       "        <span class=\"p\">{</span>\n",
-       "          <span class=\"nt\">&quot;@type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;F1_weighted&quot;</span><span class=\"p\">,</span>\n",
-       "          <span class=\"nt\">&quot;value&quot;</span><span class=\"p\">:</span> <span class=\"mf\">0.14731706525451424</span>\n",
-       "        <span class=\"p\">},</span>\n",
-       "        <span class=\"p\">{</span>\n",
-       "          <span class=\"nt\">&quot;@type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;F1_micro&quot;</span><span class=\"p\">,</span>\n",
-       "          <span class=\"nt\">&quot;value&quot;</span><span class=\"p\">:</span> <span class=\"mf\">0.3107177974434612</span>\n",
-       "        <span class=\"p\">},</span>\n",
-       "        <span class=\"p\">{</span>\n",
-       "          <span class=\"nt\">&quot;@type&quot;</span><span class=\"p\">:</span> <span class=\"s2\">&quot;F1_macro&quot;</span><span class=\"p\">,</span>\n",
-       "          <span class=\"nt\">&quot;value&quot;</span><span class=\"p\">:</span> <span class=\"mf\">0.23705926481620407</span>\n",
-       "        <span class=\"p\">}</span>\n",
-       "      <span class=\"p\">]</span>\n",
-       "    <span class=\"p\">}</span>\n",
-       "  <span class=\"p\">]</span>\n",
-       "<span class=\"p\">}</span>\n",
-       "</pre></div>\n"
-      ],
-      "text/latex": [
-       "\\begin{Verbatim}[commandchars=\\\\\\{\\}]\n",
-       "\\PY{p}{\\PYZob{}}\n",
-       "  \\PY{n+nt}{\\PYZdq{}@context\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}http://senpy.gsi.upm.es/api/contexts/YXBpL2V2YWx1YXRlLz9hbGdvPXNlbnRpbWVudC12YWRlciZkYXRhc2V0PXZhZGVyJTJDc3RzJm91dGZvcm1hdD1qc29uLWxkIw\\PYZpc{}3D\\PYZpc{}3D\\PYZdq{}}\\PY{p}{,}\n",
-       "  \\PY{n+nt}{\\PYZdq{}@type\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}AggregatedEvaluation\\PYZdq{}}\\PY{p}{,}\n",
-       "  \\PY{n+nt}{\\PYZdq{}senpy:evaluations\\PYZdq{}}\\PY{p}{:} \\PY{p}{[}\n",
-       "    \\PY{p}{\\PYZob{}}\n",
-       "      \\PY{n+nt}{\\PYZdq{}@type\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}Evaluation\\PYZdq{}}\\PY{p}{,}\n",
-       "      \\PY{n+nt}{\\PYZdq{}evaluates\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}endpoint:plugins/sentiment\\PYZhy{}vader\\PYZus{}0.1.1\\PYZus{}\\PYZus{}vader\\PYZdq{}}\\PY{p}{,}\n",
-       "      \\PY{n+nt}{\\PYZdq{}evaluatesOn\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}vader\\PYZdq{}}\\PY{p}{,}\n",
-       "      \\PY{n+nt}{\\PYZdq{}metrics\\PYZdq{}}\\PY{p}{:} \\PY{p}{[}\n",
-       "        \\PY{p}{\\PYZob{}}\n",
-       "          \\PY{n+nt}{\\PYZdq{}@type\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}Accuracy\\PYZdq{}}\\PY{p}{,}\n",
-       "          \\PY{n+nt}{\\PYZdq{}value\\PYZdq{}}\\PY{p}{:} \\PY{l+m+mf}{0.6907142857142857}\n",
-       "        \\PY{p}{\\PYZcb{}}\\PY{p}{,}\n",
-       "        \\PY{p}{\\PYZob{}}\n",
-       "          \\PY{n+nt}{\\PYZdq{}@type\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}Precision\\PYZus{}macro\\PYZdq{}}\\PY{p}{,}\n",
-       "          \\PY{n+nt}{\\PYZdq{}value\\PYZdq{}}\\PY{p}{:} \\PY{l+m+mf}{0.34535714285714286}\n",
-       "        \\PY{p}{\\PYZcb{}}\\PY{p}{,}\n",
-       "        \\PY{p}{\\PYZob{}}\n",
-       "          \\PY{n+nt}{\\PYZdq{}@type\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}Recall\\PYZus{}macro\\PYZdq{}}\\PY{p}{,}\n",
-       "          \\PY{n+nt}{\\PYZdq{}value\\PYZdq{}}\\PY{p}{:} \\PY{l+m+mf}{0.5}\n",
-       "        \\PY{p}{\\PYZcb{}}\\PY{p}{,}\n",
-       "        \\PY{p}{\\PYZob{}}\n",
-       "          \\PY{n+nt}{\\PYZdq{}@type\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}F1\\PYZus{}macro\\PYZdq{}}\\PY{p}{,}\n",
-       "          \\PY{n+nt}{\\PYZdq{}value\\PYZdq{}}\\PY{p}{:} \\PY{l+m+mf}{0.40853400929446554}\n",
-       "        \\PY{p}{\\PYZcb{}}\\PY{p}{,}\n",
-       "        \\PY{p}{\\PYZob{}}\n",
-       "          \\PY{n+nt}{\\PYZdq{}@type\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}F1\\PYZus{}weighted\\PYZdq{}}\\PY{p}{,}\n",
-       "          \\PY{n+nt}{\\PYZdq{}value\\PYZdq{}}\\PY{p}{:} \\PY{l+m+mf}{0.5643605528396403}\n",
-       "        \\PY{p}{\\PYZcb{}}\\PY{p}{,}\n",
-       "        \\PY{p}{\\PYZob{}}\n",
-       "          \\PY{n+nt}{\\PYZdq{}@type\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}F1\\PYZus{}micro\\PYZdq{}}\\PY{p}{,}\n",
-       "          \\PY{n+nt}{\\PYZdq{}value\\PYZdq{}}\\PY{p}{:} \\PY{l+m+mf}{0.6907142857142857}\n",
-       "        \\PY{p}{\\PYZcb{}}\\PY{p}{,}\n",
-       "        \\PY{p}{\\PYZob{}}\n",
-       "          \\PY{n+nt}{\\PYZdq{}@type\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}F1\\PYZus{}macro\\PYZdq{}}\\PY{p}{,}\n",
-       "          \\PY{n+nt}{\\PYZdq{}value\\PYZdq{}}\\PY{p}{:} \\PY{l+m+mf}{0.40853400929446554}\n",
-       "        \\PY{p}{\\PYZcb{}}\n",
-       "      \\PY{p}{]}\n",
-       "    \\PY{p}{\\PYZcb{}}\\PY{p}{,}\n",
-       "    \\PY{p}{\\PYZob{}}\n",
-       "      \\PY{n+nt}{\\PYZdq{}@type\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}Evaluation\\PYZdq{}}\\PY{p}{,}\n",
-       "      \\PY{n+nt}{\\PYZdq{}evaluates\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}endpoint:plugins/sentiment\\PYZhy{}vader\\PYZus{}0.1.1\\PYZus{}\\PYZus{}sts\\PYZdq{}}\\PY{p}{,}\n",
-       "      \\PY{n+nt}{\\PYZdq{}evaluatesOn\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}sts\\PYZdq{}}\\PY{p}{,}\n",
-       "      \\PY{n+nt}{\\PYZdq{}metrics\\PYZdq{}}\\PY{p}{:} \\PY{p}{[}\n",
-       "        \\PY{p}{\\PYZob{}}\n",
-       "          \\PY{n+nt}{\\PYZdq{}@type\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}Accuracy\\PYZdq{}}\\PY{p}{,}\n",
-       "          \\PY{n+nt}{\\PYZdq{}value\\PYZdq{}}\\PY{p}{:} \\PY{l+m+mf}{0.3107177974434612}\n",
-       "        \\PY{p}{\\PYZcb{}}\\PY{p}{,}\n",
-       "        \\PY{p}{\\PYZob{}}\n",
-       "          \\PY{n+nt}{\\PYZdq{}@type\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}Precision\\PYZus{}macro\\PYZdq{}}\\PY{p}{,}\n",
-       "          \\PY{n+nt}{\\PYZdq{}value\\PYZdq{}}\\PY{p}{:} \\PY{l+m+mf}{0.1553588987217306}\n",
-       "        \\PY{p}{\\PYZcb{}}\\PY{p}{,}\n",
-       "        \\PY{p}{\\PYZob{}}\n",
-       "          \\PY{n+nt}{\\PYZdq{}@type\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}Recall\\PYZus{}macro\\PYZdq{}}\\PY{p}{,}\n",
-       "          \\PY{n+nt}{\\PYZdq{}value\\PYZdq{}}\\PY{p}{:} \\PY{l+m+mf}{0.5}\n",
-       "        \\PY{p}{\\PYZcb{}}\\PY{p}{,}\n",
-       "        \\PY{p}{\\PYZob{}}\n",
-       "          \\PY{n+nt}{\\PYZdq{}@type\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}F1\\PYZus{}macro\\PYZdq{}}\\PY{p}{,}\n",
-       "          \\PY{n+nt}{\\PYZdq{}value\\PYZdq{}}\\PY{p}{:} \\PY{l+m+mf}{0.23705926481620407}\n",
-       "        \\PY{p}{\\PYZcb{}}\\PY{p}{,}\n",
-       "        \\PY{p}{\\PYZob{}}\n",
-       "          \\PY{n+nt}{\\PYZdq{}@type\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}F1\\PYZus{}weighted\\PYZdq{}}\\PY{p}{,}\n",
-       "          \\PY{n+nt}{\\PYZdq{}value\\PYZdq{}}\\PY{p}{:} \\PY{l+m+mf}{0.14731706525451424}\n",
-       "        \\PY{p}{\\PYZcb{}}\\PY{p}{,}\n",
-       "        \\PY{p}{\\PYZob{}}\n",
-       "          \\PY{n+nt}{\\PYZdq{}@type\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}F1\\PYZus{}micro\\PYZdq{}}\\PY{p}{,}\n",
-       "          \\PY{n+nt}{\\PYZdq{}value\\PYZdq{}}\\PY{p}{:} \\PY{l+m+mf}{0.3107177974434612}\n",
-       "        \\PY{p}{\\PYZcb{}}\\PY{p}{,}\n",
-       "        \\PY{p}{\\PYZob{}}\n",
-       "          \\PY{n+nt}{\\PYZdq{}@type\\PYZdq{}}\\PY{p}{:} \\PY{l+s+s2}{\\PYZdq{}F1\\PYZus{}macro\\PYZdq{}}\\PY{p}{,}\n",
-       "          \\PY{n+nt}{\\PYZdq{}value\\PYZdq{}}\\PY{p}{:} \\PY{l+m+mf}{0.23705926481620407}\n",
-       "        \\PY{p}{\\PYZcb{}}\n",
-       "      \\PY{p}{]}\n",
-       "    \\PY{p}{\\PYZcb{}}\n",
-       "  \\PY{p}{]}\n",
-       "\\PY{p}{\\PYZcb{}}\n",
-       "\\end{Verbatim}\n"
-      ],
-      "text/plain": [
-       "{\n",
-       "  \"@context\": \"http://senpy.gsi.upm.es/api/contexts/YXBpL2V2YWx1YXRlLz9hbGdvPXNlbnRpbWVudC12YWRlciZkYXRhc2V0PXZhZGVyJTJDc3RzJm91dGZvcm1hdD1qc29uLWxkIw%3D%3D\",\n",
-       "  \"@type\": \"AggregatedEvaluation\",\n",
-       "  \"senpy:evaluations\": [\n",
-       "    {\n",
-       "      \"@type\": \"Evaluation\",\n",
-       "      \"evaluates\": \"endpoint:plugins/sentiment-vader_0.1.1__vader\",\n",
-       "      \"evaluatesOn\": \"vader\",\n",
-       "      \"metrics\": [\n",
-       "        {\n",
-       "          \"@type\": \"Accuracy\",\n",
-       "          \"value\": 0.6907142857142857\n",
-       "        },\n",
-       "        {\n",
-       "          \"@type\": \"Precision_macro\",\n",
-       "          \"value\": 0.34535714285714286\n",
-       "        },\n",
-       "        {\n",
-       "          \"@type\": \"Recall_macro\",\n",
-       "          \"value\": 0.5\n",
-       "        },\n",
-       "        {\n",
-       "          \"@type\": \"F1_macro\",\n",
-       "          \"value\": 0.40853400929446554\n",
-       "        },\n",
-       "        {\n",
-       "          \"@type\": \"F1_weighted\",\n",
-       "          \"value\": 0.5643605528396403\n",
-       "        },\n",
-       "        {\n",
-       "          \"@type\": \"F1_micro\",\n",
-       "          \"value\": 0.6907142857142857\n",
-       "        },\n",
-       "        {\n",
-       "          \"@type\": \"F1_macro\",\n",
-       "          \"value\": 0.40853400929446554\n",
-       "        }\n",
-       "      ]\n",
-       "    },\n",
-       "    {\n",
-       "      \"@type\": \"Evaluation\",\n",
-       "      \"evaluates\": \"endpoint:plugins/sentiment-vader_0.1.1__sts\",\n",
-       "      \"evaluatesOn\": \"sts\",\n",
-       "      \"metrics\": [\n",
-       "        {\n",
-       "          \"@type\": \"Accuracy\",\n",
-       "          \"value\": 0.3107177974434612\n",
-       "        },\n",
-       "        {\n",
-       "          \"@type\": \"Precision_macro\",\n",
-       "          \"value\": 0.1553588987217306\n",
-       "        },\n",
-       "        {\n",
-       "          \"@type\": \"Recall_macro\",\n",
-       "          \"value\": 0.5\n",
-       "        },\n",
-       "        {\n",
-       "          \"@type\": \"F1_macro\",\n",
-       "          \"value\": 0.23705926481620407\n",
-       "        },\n",
-       "        {\n",
-       "          \"@type\": \"F1_weighted\",\n",
-       "          \"value\": 0.14731706525451424\n",
-       "        },\n",
-       "        {\n",
-       "          \"@type\": \"F1_micro\",\n",
-       "          \"value\": 0.3107177974434612\n",
-       "        },\n",
-       "        {\n",
-       "          \"@type\": \"F1_macro\",\n",
-       "          \"value\": 0.23705926481620407\n",
-       "        }\n",
-       "      ]\n",
-       "    }\n",
-       "  ]\n",
-       "}"
-      ]
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import requests\n",
-    "from IPython.display import Code\n",
-    "\n",
-    "endpoint = 'http://senpy.gsi.upm.es/api'\n",
-    "res = requests.get(f'{endpoint}/evaluate',\n",
-    "                   params={\"algo\": \"sentiment-vader\",\n",
-    "                           \"dataset\": \"vader,sts\",\n",
-    "                           'outformat': 'json-ld'\n",
-    "                          })\n",
-    "Code(res.text, language='json')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Programmatically (expert)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "A third option is to evaluate plugins manually without launching the server.\n",
-    "\n",
-    "This option is particularly interesting for advanced users that want faster iterations and evaluation results, and for automation.\n",
-    "\n",
-    "We would first need an instance of a plugin.\n",
-    "In this example we will use the Sentiment140 plugin that is included in every senpy installation:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 22,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from senpy.plugins.sentiment import sentiment140_plugin"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 23,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "s140 = sentiment140_plugin.Sentiment140()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Then, we need to know what datasets are available.\n",
-    "We can list all datasets and basic stats (e.g., number of instances and labels used) like this:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 32,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "vader {'instances': 4200, 'labels': [1, -1]}\n",
-      "sts {'instances': 4200, 'labels': [1, -1]}\n",
-      "imdb_unsup {'instances': 50000, 'labels': [1, -1]}\n",
-      "imdb {'instances': 50000, 'labels': [1, -1]}\n",
-      "sst {'instances': 11855, 'labels': [1, -1]}\n",
-      "multidomain {'instances': 38548, 'labels': [1, -1]}\n",
-      "sentiment140 {'instances': 1600000, 'labels': [1, -1]}\n",
-      "semeval07 {'instances': 'None', 'labels': [1, -1]}\n",
-      "semeval14 {'instances': 7838, 'labels': [1, -1]}\n",
-      "pl04 {'instances': 4000, 'labels': [1, -1]}\n",
-      "pl05 {'instances': 10662, 'labels': [1, -1]}\n",
-      "semeval13 {'instances': 6259, 'labels': [1, -1]}\n"
-     ]
-    }
-   ],
-   "source": [
-    "from senpy.gsitk_compat import datasets\n",
-    "for k, d in datasets.items():\n",
-    "    print(k, d['stats'])"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Now, we will evaluate our plugin in one of the smallest datasets, `sts`:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 37,
-   "metadata": {
-    "scrolled": false
-   },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "[{\n",
-       "     \"@type\": \"Evaluation\",\n",
-       "     \"evaluates\": \"endpoint:plugins/sentiment140_0.2\",\n",
-       "     \"evaluatesOn\": \"sts\",\n",
-       "     \"metrics\": [\n",
-       "         {\n",
-       "             \"@type\": \"Accuracy\",\n",
-       "             \"value\": 0.872173058013766\n",
-       "         },\n",
-       "         {\n",
-       "             \"@type\": \"Precision_macro\",\n",
-       "             \"value\": 0.9035254323131467\n",
-       "         },\n",
-       "         {\n",
-       "             \"@type\": \"Recall_macro\",\n",
-       "             \"value\": 0.8021249029415483\n",
-       "         },\n",
-       "         {\n",
-       "             \"@type\": \"F1_macro\",\n",
-       "             \"value\": 0.8320673712021136\n",
-       "         },\n",
-       "         {\n",
-       "             \"@type\": \"F1_weighted\",\n",
-       "             \"value\": 0.8631351567604358\n",
-       "         },\n",
-       "         {\n",
-       "             \"@type\": \"F1_micro\",\n",
-       "             \"value\": 0.872173058013766\n",
-       "         },\n",
-       "         {\n",
-       "             \"@type\": \"F1_macro\",\n",
-       "             \"value\": 0.8320673712021136\n",
-       "         }\n",
-       "     ]\n",
-       " }]"
-      ]
-     },
-     "execution_count": 37,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "s140.evaluate(['sts', ])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "anaconda-cloud": {},
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.7.3"
-  },
-  "toc": {
-   "colors": {
-    "hover_highlight": "#DAA520",
-    "running_highlight": "#FF0000",
-    "selected_highlight": "#FFD700"
-   },
-   "moveMenuLeft": true,
-   "nav_menu": {
-    "height": "68px",
-    "width": "252px"
-   },
-   "navigate_menu": true,
-   "number_sections": true,
-   "sideBar": true,
-   "threshold": 4,
-   "toc_cell": false,
-   "toc_section_display": "block",
-   "toc_window_display": false
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 1
-}

docs/Makefile

@@ -24,7 +24,6 @@ I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 help:
 	@echo "Please use \`make <target>' where <target> is one of"
 	@echo "  html       to make standalone HTML files"
-	@echo "  entr       to watch for changes and continuously make HTML files"
 	@echo "  dirhtml    to make HTML files named index.html in directories"
 	@echo "  singlehtml to make a single large HTML file"
 	@echo "  pickle     to make pickle files"
@@ -50,9 +49,6 @@ help:
 clean:
 	rm -rf $(BUILDDIR)/*
 
-entr:
-	while true; do ag -g rst | entr -d make html; done 
-
 html:
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo

docs/SenpyClientUse.ipynb

@@ -0,0 +1,317 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:05:31.465571Z",
+     "start_time": "2017-04-10T19:05:31.458282+02:00"
+    },
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "# Client"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "The built-in senpy client allows you to query any Senpy endpoint. We will illustrate how to use it with the public demo endpoint, and then show you how to spin up your own endpoint using docker."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "Demo Endpoint\n",
+    "-------------"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "To start using senpy, simply create a new Client and point it to your endpoint. In this case, the latest version of Senpy at GSI."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:12.827640Z",
+     "start_time": "2017-04-10T19:29:12.818617+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "from senpy.client import Client\n",
+    "\n",
+    "c = Client('http://latest.senpy.cluster.gsi.dit.upm.es/api')\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "Now, let's use that client analyse some queries:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:14.011657Z",
+     "start_time": "2017-04-10T19:29:13.701808+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "r = c.analyse('I like sugar!!', algorithm='sentiment140')\n",
+    "r"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:08:19.616754Z",
+     "start_time": "2017-04-10T19:08:19.610767+02:00"
+    },
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "As you can see, that gave us the full JSON result. A more concise way to print it would be:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:14.854213Z",
+     "start_time": "2017-04-10T19:29:14.842068+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "for entry in r.entries:\n",
+    "      print('{} -> {}'.format(entry['text'], entry['sentiments'][0]['marl:hasPolarity']))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "We can also obtain a list of available plugins with the client:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:16.245198Z",
+     "start_time": "2017-04-10T19:29:16.056545+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "c.plugins()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "Or, more concisely:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:17.663275Z",
+     "start_time": "2017-04-10T19:29:17.484623+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "c.plugins().keys()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "Local Endpoint\n",
+    "--------------"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "To run your own instance of senpy, just create a docker container with the latest Senpy image. Using `--default-plugins` you will get some extra plugins to start playing with the API."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:20.637539Z",
+     "start_time": "2017-04-10T19:29:19.938322+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "!docker run -ti --name 'SenpyEndpoint' -d -p 6000:5000 gsiupm/senpy:0.8.6 --host 0.0.0.0 --default-plugins"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "To use this endpoint:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:21.263976Z",
+     "start_time": "2017-04-10T19:29:21.260595+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "c_local = Client('http://127.0.0.1:6000/api')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "That's all! After you are done with your analysis, stop the docker container:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-04-10T17:29:33.226686Z",
+     "start_time": "2017-04-10T19:29:22.392121+02:00"
+    },
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "!docker stop SenpyEndpoint\n",
+    "!docker rm SenpyEndpoint"
+   ]
+  }
+ ],
+ "metadata": {
+  "anaconda-cloud": {},
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.0"
+  },
+  "toc": {
+   "colors": {
+    "hover_highlight": "#DAA520",
+    "running_highlight": "#FF0000",
+    "selected_highlight": "#FFD700"
+   },
+   "moveMenuLeft": true,
+   "nav_menu": {
+    "height": "68px",
+    "width": "252px"
+   },
+   "navigate_menu": true,
+   "number_sections": true,
+   "sideBar": true,
+   "threshold": 4,
+   "toc_cell": false,
+   "toc_section_display": "block",
+   "toc_window_display": false
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}

docs/SenpyClientUse.rst

@@ -0,0 +1,106 @@
+
+Client
+======
+
+Demo Endpoint
+-------------
+
+Import Client and send a request
+
+.. code:: python
+
+    from senpy.client import Client
+    
+    c = Client('http://latest.senpy.cluster.gsi.dit.upm.es/api')
+    r = c.analyse('I like Pizza', algorithm='sentiment140')
+
+Print response
+
+.. code:: python
+
+    for entry in r.entries:
+          print('{} -> {}'.format(entry['text'], entry['sentiments'][0]['marl:hasPolarity']))
+
+
+.. parsed-literal::
+
+    I like Pizza -> marl:Positive
+
+
+Obtain a list of available plugins
+
+.. code:: python
+
+    for plugin in c.request('/plugins')['plugins']:
+        print(plugin['name'])
+
+
+.. parsed-literal::
+
+    emoRand
+    rand
+    sentiment140
+
+
+Local Endpoint
+--------------
+
+Run a docker container with Senpy image and default plugins
+
+.. code::
+
+    docker run -ti --name 'SenpyEndpoint' -d -p 5000:5000 gsiupm/senpy:0.8.6 --host 0.0.0.0 --default-plugins
+
+
+.. parsed-literal::
+
+    a0157cd98057072388bfebeed78a830da7cf0a796f4f1a3fd9188f9f2e5fe562
+
+
+Import client and send a request to localhost
+
+.. code:: python
+
+    c_local = Client('http://127.0.0.1:5000/api')
+    r = c_local.analyse('Hello world', algorithm='sentiment140')
+
+Print response
+
+.. code:: python
+
+    for entry in r.entries:
+          print('{} -> {}'.format(entry['text'], entry['sentiments'][0]['marl:hasPolarity']))
+
+
+.. parsed-literal::
+
+    Hello world -> marl:Neutral
+
+
+Obtain a list of available plugins deployed locally
+
+.. code:: python
+
+    c_local.plugins().keys()
+
+
+.. parsed-literal::
+
+    rand
+    sentiment140
+    emoRand
+
+
+Stop the docker container
+
+.. code:: python
+
+    !docker stop SenpyEndpoint
+    !docker rm SenpyEndpoint
+
+
+.. parsed-literal::
+
+    SenpyEndpoint
+    SenpyEndpoint
+

docs/about.rst

@@ -0,0 +1,11 @@
+About
+--------
+
+If you use Senpy in your research, please cite `Senpy: A Pragmatic Linked Sentiment Analysis Framework <http://gsi.dit.upm.es/index.php/es/investigacion/publicaciones?view=publication&task=show&id=417>`__ (`BibTex <http://gsi.dit.upm.es/index.php/es/investigacion/publicaciones?controller=publications&task=export&format=bibtex&id=417>`__):
+
+.. code-block:: text
+
+	Sánchez-Rada, J. F., Iglesias, C. A., Corcuera, I., & Araque, Ó. (2016, October).
+	Senpy: A Pragmatic Linked Sentiment Analysis Framework.
+	In Data Science and Advanced Analytics (DSAA),
+	2016 IEEE International Conference on (pp. 735-742). IEEE.

docs/api.rst

@@ -25,7 +25,7 @@ NIF API
         "@context":"http://127.0.0.1/api/contexts/Results.jsonld",
         "@id":"_:Results_11241245.22",
         "@type":"results"
-        "activities": [
+        "analysis": [
           "plugins/sentiment-140_0.1" 
         ],
         "entries": [
@@ -73,7 +73,7 @@ NIF API
 .. http:get:: /api/plugins
 
    Returns a list of installed plugins. 
-   **Example request and response**:
+   **Example request**:
 
    .. sourcecode:: http
 
@@ -82,6 +82,10 @@ NIF API
       Accept: application/json, text/javascript
 
 
+   **Example response**:
+
+   .. sourcecode:: http
+
       {
         "@id": "plugins/sentiment-140_0.1", 
         "@type": "sentimentPlugin", 
@@ -139,14 +143,19 @@ NIF API
 .. http:get:: /api/plugins/<pluginname>
 
    Returns the information of a specific plugin.
-   **Example request and response**:
+   **Example request**:
 
    .. sourcecode:: http
 
-      GET /api/plugins/sentiment-random/ HTTP/1.1
+      GET /api/plugins/rand/ HTTP/1.1
       Host: localhost
       Accept: application/json, text/javascript
 
+
+   **Example response**:
+
+   .. sourcecode:: http
+
       {
         "@context": "http://127.0.0.1/api/contexts/ExamplePlugin.jsonld", 
         "@id": "plugins/ExamplePlugin_0.1", 

docs/apischema.rst

@@ -1,6 +1,5 @@
-API and vocabularies
+API and Examples
-####################
+################
-
 .. toctree::
 
    vocabularies.rst

docs/bad-examples/results/example-analysis-as-id-FAIL.json

@@ -2,7 +2,7 @@
   "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
   "@id": "me:Result1",
   "@type": "results",
-  "activities": [
+  "analysis": [
     "me:SAnalysis1",
     "me:SgAnalysis1",
     "me:EmotionAnalysis1",

docs/bad-examples/results/example-basic-FAIL.json

@@ -2,7 +2,7 @@
   "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
   "@id": "http://example.com#NIFExample",
   "@type": "results",
-  "activities": [
+  "analysis": [
   ],
   "entries": [
     {

docs/commandline.rst

@@ -0,0 +1,9 @@
+Command line
+============
+
+This video shows how to analyse text directly on the command line using the senpy tool.
+
+.. image:: https://asciinema.org/a/9uwef1ghkjk062cw2t4mhzpyk.png
+   :width: 100%
+   :target: https://asciinema.org/a/9uwef1ghkjk062cw2t4mhzpyk
+   :alt: CLI demo

docs/conf.py

@@ -38,8 +38,6 @@ extensions = [
     'sphinxcontrib.httpdomain',
     'sphinx.ext.coverage',
     'sphinx.ext.autosectionlabel',
-    'nbsphinx',
-    'sphinx.ext.mathjax',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -56,7 +54,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Senpy'
-copyright = u'2019, J. Fernando Sánchez'
+copyright = u'2016, J. Fernando Sánchez'
 description = u'A framework for sentiment and emotion analysis services'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -81,9 +79,7 @@ language = None
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['_build', '**.ipynb_checkpoints']
+exclude_patterns = ['_build']
-
-
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.
@@ -130,7 +126,6 @@ html_theme_options = {
     'github_user': 'gsi-upm',
     'github_repo': 'senpy',
     'github_banner': True,
-    'sidebar_collapse': True,
 }
 
 
@@ -291,12 +286,3 @@ texinfo_documents = [
 
 # If true, do not generate a @detailmenu in the "Top" node's menu.
 #texinfo_no_detailmenu = False
-
-nbsphinx_prolog = """
-.. note:: This is an `auto-generated <https://nbsphinx.readthedocs.io>`_ static view of a Jupyter notebook.
-
-    To run the code examples in your computer, you may download the original notebook from the repository: https://github.com/gsi-upm/senpy/tree/master/docs/{{ env.doc2path(env.docname, base=None) }}
-
-
-----
-"""

docs/conversion.rst

@@ -1,152 +1,93 @@
-Automatic Model Conversion
+Conversion
---------------------------
+----------
 
-Senpy includes support for emotion and sentiment conversion.
+Senpy includes experimental support for emotion/sentiment conversion plugins.
-When a user requests a specific model, senpy will choose a strategy to convert the model that the service usually outputs and the model requested by the user.
-
-Out of the box, senpy can convert from the `emotionml:pad` (pleasure-arousal-dominance) dimensional model to `emoml:big6` (Ekman's big-6) categories, and vice versa.
-This specific conversion uses a series of dimensional centroids (`emotionml:pad`) for each emotion category (`emotionml:big6`).
-A dimensional value is converted to a category by looking for the nearest centroid.
-The centroids are calculated according to this article:
-
-.. code-block:: text
-
-    Kim, S. M., Valitutti, A., & Calvo, R. A. (2010, June).
-    Evaluation of unsupervised emotion models to textual affect recognition.
-    In Proceedings of the NAACL HLT 2010 Workshop on Computational Approaches to Analysis and Generation of Emotion in Text (pp. 62-70).
-    Association for Computational Linguistics.
-
-
-
-It is possible to add new conversion strategies by `Developing a conversion plugin`_.
 
 
 Use
 ===
 
-Consider the following query to an emotion service:  http://senpy.gsi.upm.es/api/emotion-anew?i=good
+Consider the original query: http://127.0.0.1:5000/api/?i=hello&algo=emoRand
 
-The requested plugin (emotion-random) returns emotions using the VAD space (FSRE dimensions in EmotionML):
+The requested plugin (emoRand) returns emotions using Ekman's model (or big6 in EmotionML):
 
 .. code:: json
 
 
-   [
+          ... rest of the document ...
-     {
+          {
-       "@type": "EmotionSet",
+            "@type": "emotionSet",
-       "onyx:hasEmotion": [
+            "onyx:hasEmotion": {
-         {
+                "@type": "emotion",
-           "@type": "Emotion",
+                "onyx:hasEmotionCategory": "emoml:big6anger"
-           "emoml:pad-dimensions_arousal": 5.43,
+            },
-           "emoml:pad-dimensions_dominance": 6.41,
+            "prov:wasGeneratedBy": "plugins/emoRand_0.1"
-           "emoml:pad-dimensions_pleasure": 7.47,
+          }
-           "prov:wasGeneratedBy": "prefix:Analysis_1562744784.8789825"
-         }
-       ],
-       "prov:wasGeneratedBy": "prefix:Analysis_1562744784.8789825"
-     }
-  ]
-  
 
           
 
-To get the equivalent of these emotions in Ekman's categories (i.e., Ekman's Big 6 in EmotionML), we'd do this:
+To get these emotions in VAD space (FSRE dimensions in EmotionML), we'd do this:
 
-http://senpy.gsi.upm.es/api/emotion-anew?i=good&emotion-model=emoml:big6
+http://127.0.0.1:5000/api/?i=hello&algo=emoRand&emotionModel=emoml:fsre-dimensions
 
 This call, provided there is a valid conversion plugin from Ekman's to VAD, would return something like this:
 
 .. code:: json
 
-   [
+
-      {
+          ... rest of the document ...
-        "@type": "EmotionSet",
-        "onyx:hasEmotion": [
           {
-            "@type": "Emotion",
+            "@type": "emotionSet",
-            "onyx:algorithmConfidence": 4.4979,
+            "onyx:hasEmotion": {
-            "onyx:hasEmotionCategory": "emoml:big6happiness"
+                "@type": "emotion",
+                "onyx:hasEmotionCategory": "emoml:big6anger"
+                },
+            "prov:wasGeneratedBy": "plugins/emoRand_0.1"
+          }, {
+            "@type": "emotionSet",
+            "onyx:hasEmotion": {
+                "@type": "emotion",
+                "A": 7.22,
+                "D": 6.28,
+                "V": 8.6
+            },
+            "prov:wasGeneratedBy": "plugins/Ekman2VAD_0.1"
+
           }
-        ],
-        "prov:wasDerivedFrom": {
-          "@id": "Emotions0",
-          "@type": "EmotionSet",
-          "onyx:hasEmotion": [
-            {
-              "@id": "Emotion0",
-              "@type": "Emotion",
-              "emoml:pad-dimensions_arousal": 5.43,
-              "emoml:pad-dimensions_dominance": 6.41,
-              "emoml:pad-dimensions_pleasure": 7.47,
-              "prov:wasGeneratedBy": "prefix:Analysis_1562745220.1553965"
-            }
-          ],
-          "prov:wasGeneratedBy": "prefix:Analysis_1562745220.1553965"
-        },
-        "prov:wasGeneratedBy": "prefix:Analysis_1562745220.1570725"
-      }
-    ]
 
 
 That is called a *full* response, as it simply adds the converted emotion alongside.
 It is also possible to get the original emotion nested within the new converted emotion, using the `conversion=nested` parameter:
 
-http://senpy.gsi.upm.es/api/emotion-anew?i=good&emotion-model=emoml:big6&conversion=nested
-
 .. code:: json
 
-   [
-        {
-          "@type": "EmotionSet",
-          "onyx:hasEmotion": [
-            {
-              "@type": "Emotion",
-              "onyx:algorithmConfidence": 4.4979,
-              "onyx:hasEmotionCategory": "emoml:big6happiness"
-            }
-          ],
-          "prov:wasDerivedFrom": {
-            "@id": "Emotions0",
-            "@type": "EmotionSet",
-            "onyx:hasEmotion": [
-              {
-                "@id": "Emotion0",
-                "@type": "Emotion",
-                "emoml:pad-dimensions_arousal": 5.43,
-                "emoml:pad-dimensions_dominance": 6.41,
-                "emoml:pad-dimensions_pleasure": 7.47,
-                "prov:wasGeneratedBy": "prefix:Analysis_1562744962.896306"
-              }
-            ],
-            "prov:wasGeneratedBy": "prefix:Analysis_1562744962.896306"
-          },
-          "prov:wasGeneratedBy": "prefix:Analysis_1562744962.8978968"
-        }
-  ]
 
+          ... rest of the document ...
+          {
+            "@type": "emotionSet",
+            "onyx:hasEmotion": {
+                "@type": "emotion",
+                "onyx:hasEmotionCategory": "emoml:big6anger"
+                },
+            "prov:wasGeneratedBy": "plugins/emoRand_0.1"
+            "onyx:wasDerivedFrom": {
+                "@type": "emotionSet",
+                "onyx:hasEmotion": {
+                    "@type": "emotion",
+                    "A": 7.22,
+                    "D": 6.28,
+                    "V": 8.6
+                },
+                "prov:wasGeneratedBy": "plugins/Ekman2VAD_0.1"
+             }
+
+          }
 
 
 Lastly, `conversion=filtered` would only return the converted emotions.
 
-
-.. code:: json
-
-   [
-      {
-        "@type": "EmotionSet",
-        "onyx:hasEmotion": [
-          {
-            "@type": "Emotion",
-            "onyx:algorithmConfidence": 4.4979,
-            "onyx:hasEmotionCategory": "emoml:big6happiness"
-          }
-        ],
-        "prov:wasGeneratedBy": "prefix:Analysis_1562744925.7322266"
-      }
-   ]
-
 Developing a conversion plugin
-==============================
+================================
 
 Conversion plugins are discovered by the server just like any other plugin.
 The difference is the slightly different API, and the need to specify the `source` and `target` of the conversion.
@@ -165,6 +106,7 @@ For instance, an emotion conversion plugin needs the following:
 
 
 
+
 .. code:: python
 
 
@@ -172,6 +114,3 @@ For instance, an emotion conversion plugin needs the following:
 
               def convert(self, emotionSet, fromModel, toModel, params):
                   pass
-
-
-More implementation details are shown in the `centroids plugin <https://github.com/gsi-upm/senpy/blob/master/senpy/plugins/postprocessing/emotion/centroids.py>`_.

docs/demo.rst

@@ -1,13 +1,16 @@
 Demo
 ----
 
-There is a demo available on http://senpy.gsi.upm.es/, where you can test a live instance of Senpy, with several open source plugins.
+There is a demo available on http://senpy.cluster.gsi.dit.upm.es/, where you can test a serie of different plugins.
-You can use the playground (a web interface) or the HTTP API.
+You can use the playground (a web interface) or make HTTP requests to the service API.
 
-.. image:: playground-0.20.png
+.. image:: senpy-playground.png
-  :target: http://senpy.gsi.upm.es
+  :height: 400px
   :width: 800px
+  :scale: 100 %
   :align: center
 
+Plugins Demo
+============
 
-The source code and description of the plugins used in the demo are available here: https://github.com/gsi-upm/senpy-plugins-community/.
+The source code and description of the plugins used in the demo is available here: https://lab.cluster.gsi.dit.upm.es/senpy/senpy-plugins-community/.

docs/development.rst

@@ -1,25 +0,0 @@
-Developing new services
------------------------
-
-Developing web services can be hard.
-A text analysis service must implement all the typical features, such as: extraction of parameters, validation, format conversion, visualization...
-
-Senpy implements all the common blocks, so developers can focus on what really matters: great analysis algorithms that solve real problems.
-Among other things, Senpy takes care of these tasks:
-
-  * Interfacing with the user: parameter validation, error handling.
-  * Formatting: JSON-LD, Turtle/n-triples input and output, or simple text input
-  * Linked Data: senpy results are semantically annotated, using a series of well established vocabularies, and sane default URIs.
-  * User interface: a web UI where users can explore your service and test different settings
-  * A client to interact with the service. Currently only available in Python.
-
-You only need to provide the algorithm to turn a piece of text into an annotation
-Sharing your sentiment analysis with the world has never been easier!
-
-.. toctree::
-    :maxdepth: 1
-
-    server-cli
-    plugins-quickstart
-    plugins-faq
-    plugins-definition

docs/examples.rst

@@ -1,6 +1,5 @@
 Examples
---------
+------
-
 All the examples in this page use the :download:`the main schema <_static/schemas/definitions.json>`.
 
 Simple NIF annotation
@@ -18,7 +17,6 @@ Sentiment Analysis
 .....................
 Description
 ,,,,,,,,,,,
-
 This annotation corresponds to the sentiment analysis of an input. The example shows the sentiment represented according to Marl format.
 The sentiments detected are contained in the Sentiments array with their related part of the text.
 
@@ -26,7 +24,20 @@ Representation
 ,,,,,,,,,,,,,,
 
 .. literalinclude:: examples/results/example-sentiment.json
-   :emphasize-lines: 5-11,20-30
+   :emphasize-lines: 5-10,25-33
+   :language: json-ld
+
+Suggestion Mining
+.................
+Description
+,,,,,,,,,,,
+The suggestions schema represented below shows the suggestions detected in the text. Within it, we can find the NIF fields highlighted that corresponds to the text of the detected suggestion. 
+
+Representation
+,,,,,,,,,,,,,,
+
+.. literalinclude:: examples/results/example-suggestion.json
+   :emphasize-lines: 5-8,22-27
    :language: json-ld
 
 Emotion Analysis
@@ -40,6 +51,28 @@ Representation
 
 .. literalinclude:: examples/results/example-emotion.json
    :language: json-ld
-   :emphasize-lines: 5-11,22-36
+   :emphasize-lines: 5-8,25-37
 
+Named Entity Recognition
+........................
+Description
+,,,,,,,,,,,
+The Named Entity Recognition is represented as follows. In this particular case, it can be seen within the entities array the entities recognised. For the example input, Microsoft and Windows Phone are the ones detected. 
+Representation
+,,,,,,,,,,,,,,
 
+.. literalinclude:: examples/results/example-ner.json
+   :emphasize-lines: 5-8,19-34
+   :language: json-ld
+
+Complete example
+................
+Description
+,,,,,,,,,,,
+This example covers all of the above cases, integrating all the annotations in the same document.
+
+Representation
+,,,,,,,,,,,,,,
+
+.. literalinclude:: examples/results/example-complete.json
+   :language: json-ld

docs/examples/results/example-analysis-as-id.json

@@ -2,7 +2,7 @@
   "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
   "@id": "me:Result1",
   "@type": "results",
-  "activities": [
+  "analysis": [
       {
           "@id": "_:SAnalysis1_Activity",
           "@type": "marl:SentimentAnalysis",

docs/examples/results/example-basic.json

@@ -2,7 +2,7 @@
     "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
     "@id": "me:Result1",
     "@type": "results",
-    "activities": [ ],
+    "analysis": [ ],
     "entries": [
         {
             "@id": "http://example.org#char=0,40",

docs/examples/results/example-complete.json

@@ -2,7 +2,7 @@
     "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
     "@id": "me:Result1",
     "@type": "results",
-    "activities": [
+    "analysis": [
         {
             "@id": "_:SAnalysis1_Activity",
             "@type": "marl:SentimentAnalysis",

docs/examples/results/example-emotion.json

@@ -2,7 +2,7 @@
   "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
   "@id": "me:Result1",
   "@type": "results",
-  "activities": [
+  "analysis": [
     {
       "@id": "me:EmotionAnalysis1_Activity",
       "@type": "me:EmotionAnalysis1",
@@ -17,6 +17,10 @@
         "nif:Context"
       ],
       "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+      ],
+      "suggestions": [
+      ],
       "sentiments": [
       ],
       "emotions": [

docs/examples/results/example-moral.json

@@ -1,42 +0,0 @@
-{
-  "@context": "http://senpy.gsi.upm.es/api/contexts/YXBpP2FsZ289ZW1",
-  "@id": ":Result1",
-  "@type": "results",
-  "activities": [
-    {
-      "@id": ":MoralAnalysis1_Activity",
-      "@type": "amor:MoralValueAnalysis",
-      "prov:wasAssociatedWith": ":MoralAnalysis1",
-      "amor:usedMoralValueModel": "amor-mft:MoralFoundationsTheory",
-      "amor:analysed": "news1",
-      "amor:usedMLModel": ":model1",
-      "prov:generated": ":annotation3"
-    }
-  ],
-  "entries": [
-    {
-      "@id": "http://micro.blog/status1",
-      "@type": [
-        "nif:RFC5147String",
-        "nif:Context"
-      ],
-      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
-      "morality": [
-        {
-          "@id": ":annotation3",
-	  "@type": "amor:MoralValueAnnotation",
-	  "nif:beginIndex": 80,
-          "nif:endIndex": 97,
-	  "amor:hasMoralValueCategory": "mft:Authority",
-	  "amor:confidence": 0.75,
-	  "amor-mft:hasPolarityIntensity": 0.2,
-	  "amor:annotated": "http://micro.blog/status1",
-          "nif:anchorOf": "You'll be awesome.",
-          "prov:wasGeneratedBy": ":MoralAnalysis1_Activity"
-        }
-      ],
-      "emotions": [
-      ]
-    }
-  ]
-}

docs/examples/results/example-ner.json

@@ -2,7 +2,7 @@
   "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
   "@id": "me:Result1",
   "@type": "results",
-  "activities": [
+  "analysis": [
     {
       "@id": "_:NER1_Activity",
       "@type": "me:NERAnalysis",

docs/examples/results/example-pad.json

@@ -2,12 +2,12 @@
 "@context": [
              "http://mixedemotions-project.eu/ns/context.jsonld",
              {
-               "emovoc": "http://www.gsi.upm.es/ontologies/onyx/vocabularies/emotionml/ns#"
+               "emovoc": "http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/emotionml/ns#"
              }
   ],
   "@id": "me:Result1",
   "@type": "results",
-  "activities": [
+  "analysis": [
     {
       "@id": "me:HesamsAnalysis_Activity",
       "@type": "onyx:EmotionAnalysis",

docs/examples/results/example-sentiment.json

@@ -2,7 +2,7 @@
   "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
   "@id": "me:Result1",
   "@type": "results",
-  "activities": [
+  "analysis": [
     {
       "@id": "_:SAnalysis1_Activity",
       "@type": "marl:SentimentAnalysis",
@@ -17,6 +17,10 @@
         "nif:Context"
       ],
       "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+      ],
+      "suggestions": [
+      ],
       "sentiments": [
         {
           "@id": "http://micro.blog/status1#char=80,97",
@@ -28,7 +32,7 @@
           "prov:wasGeneratedBy": "_:SAnalysis1_Activity"
         }
       ],
-      "emotions": [
+      "emotionSets": [
       ]
     }
   ]

docs/examples/results/example-suggestion.json

@@ -2,7 +2,7 @@
   "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
   "@id": "me:Result1",
   "@type": "results",
-  "activities": [
+  "analysis": [
     {
       "@id": "_:SgAnalysis1_Activity",
       "@type": "me:SuggestionAnalysis",

docs/index.rst

@@ -1,106 +1,35 @@
 Welcome to Senpy's documentation!
 =================================
-
 .. image:: https://readthedocs.org/projects/senpy/badge/?version=latest
-  :target: http://senpy.readthedocs.io/en/latest/
+   :target: http://senpy.readthedocs.io/en/latest/
 .. image:: https://badge.fury.io/py/senpy.svg
-  :target: https://badge.fury.io/py/senpy
+   :target: https://badge.fury.io/py/senpy
-.. image:: https://travis-ci.org/gsi-upm/senpy.svg
+.. image:: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/badges/master/build.svg
-  :target: https://github.com/gsi-upm/senpy/senpy/tree/master
+   :target: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/commits/master
+.. image:: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/badges/master/coverage.svg
+   :target: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/commits/master
 .. image:: https://img.shields.io/pypi/l/requests.svg
-  :target: https://lab.gsi.upm.es/senpy/senpy/
+   :target: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/
-
-Senpy is a framework to build sentiment and emotion analysis services.
-It provides functionalities for:
-
-- developing sentiment and emotion classifier and exposing them as an HTTP service
-- requesting sentiment and emotion analysis from different providers (i.e. Vader, Sentimet140, ...) using the same interface (:doc:`apischema`). In this way, applications do not depend on the API offered for these services.
-- combining services that use different sentiment model (e.g. polarity between [-1, 1] or [0,1] or emotion models (e.g. Ekkman or VAD)
-- evaluating sentiment algorithms with well known datasets
-
-
-Using senpy services is as simple as sending an HTTP request with your favourite tool or library.
-Let's analyze the sentiment of the text "Senpy is awesome".
-
-We can call the `Sentiment140 <http://www.sentiment140.com/>`_ service with an HTTP request using curl:
-
-
-.. code:: shell
-   :emphasize-lines: 14,18
-
-   $ curl "http://senpy.gsi.upm.es/api/sentiment140" \
-         --data-urlencode "input=Senpy is awesome"
-
-   {
-     "@context": "http://senpy.gsi.upm.es/api/contexts/YXBpL3NlbnRpbWVudDE0MD8j",
-     "@type": "Results",
-     "entries": [
-       {
-         "@id": "prefix:",
-         "@type": "Entry",
-         "marl:hasOpinion": [
-           {
-             "@type": "Sentiment",
-             "marl:hasPolarity": "marl:Positive",
-             "prov:wasGeneratedBy": "prefix:Analysis_1554389334.6431913"
-           }
-         ],
-         "nif:isString": "Senpy is awesome",
-         "onyx:hasEmotionSet": []
-       }
-     ]
-   }
-
-
-Congratulations, you’ve used your first senpy service!
-You can observe the result: the polarity is positive (marl:Positive). The reason of this prefix is that Senpy follows a linked data approach.
-
-You can analyze the same sentence using a different sentiment service (e.g. Vader) and requesting a different format (e.g. turtle):
 
 
 
-.. code:: shell
+Senpy is a framework for sentiment and emotion analysis services.
+Services built with senpy are interchangeable and easy to use because they share a common :doc:`apischema`.
+It also simplifies service development.
 
-   $ curl "http://senpy.gsi.upm.es/api/sentiment-vader" \
+.. image:: senpy-architecture.png
-         --data-urlencode "input=Senpy is awesome" \
+  :width: 100%
-         --data-urlencode "outformat=turtle"
+  :align: center
-
-   @prefix : <http://www.gsi.upm.es/onto/senpy/ns#> .
-   @prefix endpoint: <http://senpy.gsi.upm.es/api/> .
-   @prefix marl: <http://www.gsi.upm.es/ontologies/marl/ns#> .
-   @prefix nif: <http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#> .
-   @prefix prefix: <http://senpy.invalid/> .
-   @prefix prov: <http://www.w3.org/ns/prov#> .
-   @prefix senpy: <http://www.gsi.upm.es/onto/senpy/ns#> .
-
-   prefix: a senpy:Entry ;
-       nif:isString "Senpy is awesome" ;
-       marl:hasOpinion [ a senpy:Sentiment ;
-               marl:hasPolarity "marl:Positive" ;
-               marl:polarityValue 6.72e-01 ;
-               prov:wasGeneratedBy prefix:Analysis_1562668175.9808676 ] .
-
-   [] a senpy:Results ;
-       prov:used prefix: .
-
-As you see, Vader returns also the polarity value (0.67) in addition to the category (positive).
-
-If you are interested in consuming Senpy services, read :doc:`Quickstart`.
-To get familiar with the concepts behind Senpy, and what it can offer for service developers, check out :doc:`development`.
-:doc:`apischema` contains information about the semantic models and vocabularies used by Senpy.
 
 .. toctree::
-  :caption: Learn more about senpy:
+   :caption: Learn more about senpy:
-  :maxdepth: 2
+   :maxdepth: 2
-  :hidden:
 
-  senpy
+   senpy
-  demo
+   installation
-  Quickstart.ipynb
+   demo
-  installation
+   usage
-  conversion
+   apischema
-  Evaluation.ipynb
+   plugins
-  apischema
+   conversion
-  development
+   about
-  publications
-  projects

docs/installation.rst

@@ -1,10 +1,10 @@
 Installation
 ------------
-The stable version can be used in two ways: as a system/user library through pip, or from a docker image.
+The stable version can be used in two ways: as a system/user library through pip, or as a docker image.
 
-Using docker is recommended because the image is self-contained, reproducible and isolated from the system, which means:
+The docker image is the recommended way because it is self-contained and isolated from the system, which means:
 
-  * It can be downloaded and run with just one simple command
+  * Downloading and using it is just one command
   * All dependencies are included
   * It is OS-independent (MacOS, Windows, GNU/Linux)
   * Several versions may coexist in the same machine without additional virtual environments
@@ -17,39 +17,42 @@ Through PIP
 
 .. code:: bash
 
-   pip install senpy
-
-   # Or with --user if you get permission errors:
-
    pip install --user senpy
+
    
-..
+Alternatively, you can use the development version:
-   Alternatively, you can use the development version:
+ 
+.. code:: bash
 
-   .. code:: bash
+   git clone git@github.com:gsi-upm/senpy
-
+   cd senpy
-      git clone git@github.com:gsi-upm/senpy
+   pip install --user .
-      cd senpy
-      pip install --user .
-
-Each version is automatically tested on GNU/Linux, macOS and Windows 10.
-If you have trouble with the installation, please file an `issue on GitHub <https://github.com/gsi-upm/senpy/issues>`_.
 
+If you want to install senpy globally, use sudo instead of the ``--user`` flag.
 
 Docker Image
 ************
-
+Build the image or use the pre-built one:   
-The base image of senpy comes with some built-in plugins that you can use:   
 
 .. code:: bash
 
-   docker run -ti -p 5000:5000 gsiupm/senpy --host 0.0.0.0
+   docker run -ti -p 5000:5000 gsiupm/senpy --host 0.0.0.0 --default-plugins
 
-To use your custom plugins, you can add volume to the container: 
+To add custom plugins, use a docker volume: 
     
 .. code:: bash
 
-   docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --host 0.0.0.0 --plugins-folder /plugins
+   docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --host 0.0.0.0 --default-plugins -f /plugins
+ 
+
+Python 2
+........
+
+There is a Senpy version for python2 too:
+    
+.. code:: bash
+
+   docker run -ti -p 5000:5000 gsiupm/senpy:python2.7 --host 0.0.0.0 --default-plugins
 
 
 Alias
@@ -59,7 +62,7 @@ If you are using the docker approach regularly, it is advisable to use a script
 
 .. code:: bash
 
-   alias senpy='docker run --rm -ti -p 5000:5000 -v $PWD:/senpy-plugins gsiupm/senpy'
+   alias senpy='docker run --rm -ti -p 5000:5000 -v $PWD:/senpy-plugins gsiupm/senpy --default-plugins'
 
 
 Now, you may run senpy from any folder in your computer like so:

docs/plugins-definition.rst

@@ -9,21 +9,20 @@ Lastly, it is also possible to add new plugins programmatically.
 
 .. contents:: :local:
 
-..
+What is a plugin?
-   What is a plugin?
+=================
-   =================
 
-   A plugin is a program that, given a text, will add annotations to it.
+A plugin is a program that, given a text, will add annotations to it.
-   In practice, a plugin consists of at least two files:
+In practice, a plugin consists of at least two files:
 
-   - Definition file: a `.senpy` file that describes the plugin (e.g. what input parameters it accepts, what emotion model it uses).
+- Definition file: a `.senpy` file that describes the plugin (e.g. what input parameters it accepts, what emotion model it uses).
-   - Python module: the actual code that will add annotations to each input.
+- Python module: the actual code that will add annotations to each input.
 
-   This separation allows us to deploy plugins that use the same code but employ different parameters.
+This separation allows us to deploy plugins that use the same code but employ different parameters.
-   For instance, one could use the same classifier and processing in several plugins, but train with different datasets.
+For instance, one could use the same classifier and processing in several plugins, but train with different datasets.
-   This scenario is particularly useful for evaluation purposes.
+This scenario is particularly useful for evaluation purposes.
 
-   The only limitation is that the name of each plugin needs to be unique.
+The only limitation is that the name of each plugin needs to be unique.
 
 Definition files
 ================
@@ -110,3 +109,5 @@ Now, in a file named ``helloworld.py``:
                       sentiment['marl:hasPolarity'] = 'marl:Negative'
                   entry.sentiments.append(sentiment)
                   yield entry
+
+The complete code of the example plugin is available `here <https://lab.cluster.gsi.dit.upm.es/senpy/plugin-prueba>`__.

docs/plugins-quickstart.rst

@@ -1,87 +0,0 @@
-Quickstart for service developers
-=================================
- 
-This document contains the minimum to get you started with developing new services using Senpy.
-
-For an example of conversion plugins, see :doc:`conversion`.
-For a description of definition files, see :doc:`plugins-definition`.
-
-A more step-by-step tutorial with slides is available `here <https://lab.gsi.upm.es/senpy/senpy-tutorial>`__ 
-
-.. contents:: :local:
-
-Installation
-############
-
-First of all, you need to install the package.
-See :doc:`installation` for instructions.
-Once installed, the `senpy` command should be available. 
-
-Architecture
-############
-
-The main component of a sentiment analysis service is the algorithm itself. However, for the algorithm to work, it needs to get the appropriate parameters from the user, format the results according to the defined API, interact with the user whn errors occur or more information is needed, etc.
-
-Senpy proposes a modular and dynamic architecture that allows:
-
-* Implementing different algorithms in a extensible way, yet offering a common interface.
-* Offering common services that facilitate development, so developers can focus on implementing new and better algorithms.
-
-The framework consists of two main modules: Senpy core, which is the building block of the service, and Senpy plugins, which consist of the analysis algorithm. The next figure depicts a simplified version of the processes involved in an analysis with the Senpy framework.
-
-.. image:: senpy-architecture.png
-  :width: 100%
-  :align: center
-
-
-What is a plugin?
-#################
-
-A plugin is a python object that can process entries.
-Given an entry, it will modify it, add annotations to it, or generate new entries.
-
-
-What is an entry?
-#################
-
-Entries are objects that can be annotated.
-In general, they will be a piece of text.
-By default, entries are `NIF contexts <http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core/nif-core.html>`_ represented in JSON-LD format.
-It is a dictionary/JSON object that looks like this:
-
-  .. code:: python
-
-            {
-               "@id": "<unique identifier or blank node name>",
-               "nif:isString": "input text",
-               "sentiments": [ {
-                     ...
-               }
-               ],
-               ...
-            }
-
-Annotations are added to the object like this:
-
-.. code:: python
-
-   entry = Entry()
-   entry.vocabulary__annotationName = 'myvalue'
-   entry['vocabulary:annotationName'] = 'myvalue'
-   entry['annotationNameURI'] = 'myvalue'
-
-Where vocabulary is one of the prefixes defined in the default senpy context, and annotationURI is a full URI.
-The value may be any valid JSON-LD dictionary.
-For simplicity, senpy includes a series of models by default in the ``senpy.models`` module.
-
-Plugins Code
-############
-
-The basic methods in a plugin are:
-
-* analyse_entry: called in every user requests. It takes two parameters: ``Entry``, the entry object, and ``params``, the parameters supplied by the user. It should yield one or more ``Entry`` objects.
-* activate: used to load memory-hungry resources. For instance, to train a classifier.
-* deactivate: used to free up resources when the plugin is no longer needed.
-
-Plugins are loaded asynchronously, so don't worry if the activate method takes too long. The plugin will be marked as activated once it is finished executing the method.
-

docs/plugins.rst

@@ -1,18 +1,61 @@
-F.A.Q.
+Developing new plugins
-======
+----------------------
+This document contains the minimum to get you started with developing new analysis plugin.
+For an example of conversion plugins, see :doc:`conversion`.
+For a description of definition files, see :doc:`plugins-definition`.
+
+A more step-by-step tutorial with slides is available `here <https://lab.cluster.gsi.dit.upm.es/senpy/senpy-tutorial>`__ 
 
 .. contents:: :local:
 
+What is a plugin?
+=================
+
+A plugin is a python object that can process entries. Given an entry, it will modify it, add annotations to it, or generate new entries.
+
+
+What is an entry?
+=================
+
+Entries are objects that can be annotated.
+In general, they will be a piece of text.
+By default, entries are `NIF contexts <http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core/nif-core.html>`_ represented in JSON-LD format.
+It is a dictionary/JSON object that looks like this:
+
+  .. code:: python
+
+            {
+               "@id": "<unique identifier or blank node name>",
+               "nif:isString": "input text",
+               "sentiments": [ {
+                     ...
+               }
+               ],
+               ...
+            }
+
+Annotations are added to the object like this:
+
+.. code:: python
+
+   entry = Entry()
+   entry.vocabulary__annotationName = 'myvalue'
+   entry['vocabulary:annotationName'] = 'myvalue'
+   entry['annotationNameURI'] = 'myvalue'
+
+Where vocabulary is one of the prefixes defined in the default senpy context, and annotationURI is a full URI.
+The value may be any valid JSON-LD dictionary.
+For simplicity, senpy includes a series of models by default in the ``senpy.models`` module.
 
 
 What are annotations?
-#####################
+=====================
 They are objects just like entries.
-Senpy ships with several default annotations, including ``Sentiment`` and ``Emotion``.
+Senpy ships with several default annotations, including: ``Sentiment``, ``Emotion``, ``EmotionSet``...jk bb
 
 
 What's a plugin made of?
-########################
+========================
 
 When receiving a query, senpy selects what plugin or plugins should process each entry, and in what order.
 It also makes sure the every entry and the parameters provided by the user meet the plugin requirements.
@@ -22,33 +65,45 @@ Hence, two parts are necessary: 1) the code that will process the entry, and 2)
 In practice, this is what a plugin looks like, tests included:
 
 
-.. literalinclude:: ../example-plugins/rand_plugin.py
+.. literalinclude:: ../senpy/plugins/example/rand_plugin.py
-   :emphasize-lines: 21-28
+   :emphasize-lines: 5-11
    :language: python
 
 
 The lines highlighted contain some information about the plugin.
 In particular, the following information is mandatory:
 
-* A unique name for the class. In our example, sentiment-random.
+* A unique name for the class. In our example, Rand.
 * The subclass/type of plugin. This is typically either `SentimentPlugin` or `EmotionPlugin`. However, new types of plugin can be created for different annotations. The only requirement is that these new types inherit from `senpy.Analysis`
 * A description of the plugin. This can be done simply by adding a doc to the class.
 * A version, which should get updated.
 * An author name.
 
 
+Plugins Code
+============
+
+The basic methods in a plugin are:
+
+* analyse_entry: called in every user requests. It takes two parameters: ``Entry``, the entry object, and ``params``, the parameters supplied by the user. It should yield one or more ``Entry`` objects.
+* activate: used to load memory-hungry resources. For instance, to train a classifier.
+* deactivate: used to free up resources when the plugin is no longer needed.
+
+Plugins are loaded asynchronously, so don't worry if the activate method takes too long. The plugin will be marked as activated once it is finished executing the method.
+
+
 How does senpy find modules?
-############################
+============================
 
 Senpy looks for files of two types:
 
 * Python files of the form `senpy_<NAME>.py` or `<NAME>_plugin.py`. In these files, it will look for: 1) Instances that inherit from `senpy.Plugin`, or subclasses of `senpy.Plugin` that can be initialized without a configuration file. i.e. classes that contain all the required attributes for a plugin.
-* Plugin definition files (see :doc:`plugins-definition`)
+* Plugin definition files (see :doc:`advanced-plugins`)
 
-How can I define additional parameters for my plugin?
+Defining additional parameters
-#####################################################
+==============================
 
-Your plugin may ask for additional parameters from users by using the attribute ``extra_params`` in your plugin definition.
+Your plugin may ask for additional parameters from the users of the service by using the attribute ``extra_params`` in your plugin definition.
 It takes a dictionary, where the keys are the name of the argument/parameter, and the value has the following fields:
 
 * aliases: the different names which can be used in the request to use the parameter.
@@ -69,16 +124,15 @@ It takes a dictionary, where the keys are the name of the argument/parameter, an
 
 
 
-How should I load external data and files
+Loading data and files
-#########################################
+======================
 
 Most plugins will need access to files (dictionaries, lexicons, etc.).
 These files are usually heavy or under a license that does not allow redistribution.
 For this reason, senpy has a `data_folder` that is separated from the source files.
 The location of this folder is controlled programmatically or by setting the `SENPY_DATA` environment variable.
-You can use the `self.path(filepath)` function to get the path of a given `filepath` within the data folder.
 
-Plugins have a convenience function `self.open` which will automatically look for the file if it exists, or open a new one if it doesn't:
+Plugins have a convenience function `self.open` which will automatically prepend the data folder to relative paths:
 
 
 .. code:: python
@@ -90,7 +144,7 @@ Plugins have a convenience function `self.open` which will automatically look fo
               file_in_data = <FILE PATH>
               file_in_sources = <FILE PATH>
 
-              def on activate(self):
+              def activate(self):
                   with self.open(self.file_in_data) as f:
                       self._classifier = train_from_file(f)
                   file_in_source = os.path.join(self.get_folder(), self.file_in_sources)
@@ -101,8 +155,8 @@ Plugins have a convenience function `self.open` which will automatically look fo
 It is good practice to specify the paths of these files in the plugin configuration, so the same code can be reused with different resources.
 
 
-Can I build a docker image for my plugin?
+Docker image
-#########################################
+============
 
 Add the following dockerfile to your project to generate a docker image with your plugin:
 
@@ -133,7 +187,7 @@ And you can run it with:
    docker run -p 5000:5000 gsiupm/exampleplugin
 
 
-If the plugin uses non-source files (:ref:`How should I load external data and files`), the recommended way is to use `SENPY_DATA` folder.
+If the plugin uses non-source files (:ref:`loading data and files`), the recommended way is to use `SENPY_DATA` folder.
 Data can then be mounted in the container or added to the image.
 The former is recommended for open source plugins with licensed resources, whereas the latter is the most convenient and can be used for private images.
 
@@ -150,15 +204,17 @@ Adding data to the image:
    FROM gsiupm/senpy:1.0.1
    COPY data /
 
+F.A.Q.
+======
 What annotations can I use?
-###########################
+???????????????????????????
 
 You can add almost any annotation to an entry.
 The most common use cases are covered in the :doc:`apischema`.
 
 
 Why does the analyse function yield instead of return?
-######################################################
+??????????????????????????????????????????????????????
 
 This is so that plugins may add new entries to the response or filter some of them.
 For instance, a chunker may split one entry into several.
@@ -166,7 +222,7 @@ On the other hand, a conversion plugin may leave out those entries that do not c
 
 
 If I'm using a classifier, where should I train it?
-###################################################
+???????????????????????????????????????????????????
 
 Training a classifier can be time time consuming. To avoid running the training unnecessarily, you can use ShelfMixin to store the classifier. For instance:
 
@@ -200,7 +256,7 @@ A corrupt shelf prevents the plugin from loading.
 If you do not care about the data in the shelf, you can force your plugin to remove the corrupted file and load anyway, set the  'force_shelf' to True in your plugin and start it again.
 
 How can I turn an external service into a plugin?
-#################################################
+?????????????????????????????????????????????????
 
 This example ilustrate how to implement a plugin that accesses the Sentiment140 service.
 
@@ -236,8 +292,8 @@ This example ilustrate how to implement a plugin that accesses the Sentiment140
                   yield entry
 
 
-How can I activate a DEBUG mode for my plugin?
+Can I activate a DEBUG mode for my plugin?
-###############################################
+???????????????????????????????????????????
 
 You can activate the DEBUG mode by the command-line tool using the option -d.
 
@@ -253,6 +309,6 @@ Additionally, with the ``--pdb`` option you will be dropped into a pdb post mort
    python -m pdb yourplugin.py
 
 Where can I find more code examples?
-####################################
+????????????????????????????????????
 
 See: `<http://github.com/gsi-upm/senpy-plugins-community>`_.

docs/projects.rst

@@ -1,49 +0,0 @@
-Projects using Senpy
---------------------
-
-Are you using Senpy in your work?, we would love to hear from you!
-Here is a list of on-going and past projects that have benefited from senpy:
-
-
-MixedEmotions
-,,,,,,,,,,,,,
-
-`MixedEmotions <https://mixedemotions-project.eu/>`_ develops innovative multilingual multi-modal Big Data analytics applications.
-The analytics relies on a common toolbox for multi-modal sentiment and emotion analysis.
-The NLP parts of the toolbox are based on senpy and its API.
-
-The toolbox is featured in this publication:
-
-.. code-block:: text
-
-    Buitelaar, P., Wood, I. D., Arcan, M., McCrae, J. P., Abele, A., Robin, C., … Tummarello, G. (2018). 
-    MixedEmotions: An Open-Source Toolbox for Multi-Modal Emotion Analysis.
-    IEEE Transactions on Multimedia.
-
-EuroSentiment
-,,,,,,,,,,,,,
-
-The aim of the EUROSENTIMENT project was to create a pool for multilingual language resources and services for Sentiment Analysis.
-
-The EuroSentiment project was the main motivation behind the development of Senpy, and some early versions were used:
-
-.. code-block:: text
-
-    Sánchez-Rada, J. F., Vulcu, G., Iglesias, C. A., & Buitelaar, P. (2014).
-    EUROSENTIMENT: Linked Data Sentiment Analysis.
-    Proceedings of the ISWC 2014 Posters & Demonstrations Track
-    13th International Semantic Web Conference (ISWC 2014) (Vol. 1272, pp. 145–148).
-
-
-SoMeDi
-,,,,,,
-`SoMeDi <https://itea3.org/project/somedi.html>`_ is an ITEA3 project to research machine learning and artificial intelligence techniques that can be used to turn digital interaction data into Digital Interaction Intelligence and approaches that can be used to effectively enter and act in social media, and to automate this process.
-SoMeDi exploits senpy's interoperability of services in their customizable data enrichment and NLP workflows.
-
-TRIVALENT
-,,,,,,,,,
-
-`TRIVALENT <https://trivalent-project.eu/>`_ is an EU funded project which aims to a better understanding of root causes of the phenomenon of violent radicalisation in Europe in order to develop appropriate countermeasures, ranging from early detection methodologies to techniques of counter-narrative.
-
-In addition to sentiment and emotion analysis services, trivalent provides other types of senpy services such as radicalism and writing style analysis.
-

docs/publications.rst

@@ -1,36 +0,0 @@
-Publications
-============
-
-
-And if you use Senpy in your research, please cite `Senpy: A Pragmatic Linked Sentiment Analysis Framework <http://gsi.upm.es/index.php/es/investigacion/publicaciones?view=publication&task=show&id=417>`__ (`BibTex <http://gsi.upm.es/index.php/es/investigacion/publicaciones?controller=publications&task=export&format=bibtex&id=417>`__):
-
-.. code-block:: text
-
-	Sánchez-Rada, J. F., Iglesias, C. A., Corcuera, I., & Araque, Ó. (2016, October).
-	Senpy: A Pragmatic Linked Sentiment Analysis Framework.
-	In Data Science and Advanced Analytics (DSAA),
-	2016 IEEE International Conference on (pp. 735-742). IEEE.
-
-
-Senpy uses Onyx for emotion representation, first introduced in:
-
-.. code-block:: text
-
-    Sánchez-Rada, J. F., & Iglesias, C. A. (2016).
-    Onyx: A linked data approach to emotion representation.
-    Information Processing & Management, 52(1), 99-114.
-
-Senpy uses Marl for sentiment representation, which was presented in:
-
-.. code-block:: text
-
-    Westerski, A., Iglesias Fernandez, C. A., & Tapia Rico, F. (2011).
-    Linked opinions: Describing sentiments on the structured web of data.
-
-The representation models, formats and challenges are partially covered in a chapter of the book Sentiment Analysis in Social Networks:
-
-.. code-block:: text
-
-    Iglesias, C. A., Sánchez-Rada, J. F., Vulcu, G., & Buitelaar, P. (2017).
-    Linked Data Models for Sentiment and Emotion Analysis in Social Networks.
-    In Sentiment Analysis in Social Networks (pp. 49-69).

docs/requirements.txt

@@ -1,3 +1,2 @@
 sphinxcontrib-httpdomain>=1.4
-ipykernel
 nbsphinx

docs/senpy.rst

@@ -1,27 +1,54 @@
 What is Senpy?
 --------------
 
-Senpy is a framework for sentiment and emotion analysis services.
+Senpy is a framework for text analysis using Linked Data. There are three main applications of Senpy so far: sentiment and emotion analysis, user profiling and entity recoginition. Annotations and Services are compliant with NIF (NLP Interchange Format).
-Its goal is to produce analysis services that are interchangeable and fully interoperable.
+
+Senpy aims at providing a framework where analysis modules can be integrated easily as plugins, and providing a core functionality for managing tasks such as data validation, user interaction, formatting, logging, translation to linked data, etc. 
+
+The figure below summarizes the typical features in a text analysis service.
+Senpy implements all the common blocks, so developers can focus on what really matters: great analysis algorithms that solve real problems.
+
+.. image:: senpy-framework.png
+  :width: 60%
+  :align: center
+
+
+Senpy for end users
+===================
+
+All services built using senpy share a common interface.
+This allows users to use them (almost) interchangeably.
+Senpy comes with a :ref:`built-in client`.
+
+
+Senpy for service developers
+============================
+
+Senpy is a framework that turns your sentiment or emotion analysis algorithm into a full blown semantic service.
+Senpy takes care of:
+
+  * Interfacing with the user: parameter validation, error handling.
+  * Formatting: JSON-LD, Turtle/n-triples input and output, or simple text input
+  * Linked Data: senpy results are semantically annotated, using a series of well established vocabularies, and sane default URIs.
+  * User interface: a web UI where users can explore your service and test different settings
+  * A client to interact with the service. Currently only available in Python.
+
+Sharing your sentiment analysis with the world has never been easier!
+
+Check out the :doc:`plugins` if you have developed an analysis algorithm (e.g. sentiment analysis) and you want to publish it as a service.
+
+Architecture
+============
+
+The main component of a sentiment analysis service is the algorithm itself. However, for the algorithm to work, it needs to get the appropriate parameters from the user, format the results according to the defined API, interact with the user whn errors occur or more information is needed, etc.
+
+Senpy proposes a modular and dynamic architecture that allows:
+
+* Implementing different algorithms in a extensible way, yet offering a common interface.
+* Offering common services that facilitate development, so developers can focus on implementing new and better algorithms.
+
+The framework consists of two main modules: Senpy core, which is the building block of the service, and Senpy plugins, which consist of the analysis algorithm. The next figure depicts a simplified version of the processes involved in an analysis with the Senpy framework.
 
 .. image:: senpy-architecture.png
   :width: 100%
   :align: center
-
-All services built using senpy share a common interface.
-This allows users to use them (almost) interchangeably, with the same API and tools, simply by pointing to a different URL or changing a parameter.
-The common schema also makes it easier to evaluate the performance of different algorithms and services.
-In fact, Senpy has a built-in evaluation API you can use to compare results with different algorithms.
-
-Services can also use the common interface to communicate with each other.
-And higher level features can be built on top of these services, such as automatic fusion of results, emotion model conversion, and service discovery.
-
-These benefits are not limited to new services.
-The community has developed wrappers for some proprietary and commercial services (such as sentiment140 and Meaning Cloud), so you can consult them as.
-Senpy comes with a built-in client in the client package.
-
-
-To achieve this goal, Senpy uses a Linked Data principled approach, based on the NIF (NLP Interchange Format) specification, and open vocabularies such as Marl and Onyx.
-You can learn more about this in :doc:`vocabularies`.
-
-Check out :doc:`development` if you have developed an analysis algorithm (e.g. sentiment analysis) and you want to publish it as a service.

docs/server.rst

@@ -1,18 +1,14 @@
-Command line tool
+Server
-=================
+======
-
-Basic usage
------------
 
 The senpy server is launched via the `senpy` command:
 
 .. code:: text
 
-    usage: senpy [-h] [--level logging_level] [--log-format log_format] [--debug]
+    usage: senpy [-h] [--level logging_level] [--debug] [--default-plugins]
-                [--no-default-plugins] [--host HOST] [--port PORT]
+                [--host HOST] [--port PORT] [--plugins-folder PLUGINS_FOLDER]
-                [--plugins-folder PLUGINS_FOLDER] [--install]
+                [--only-install] [--only-list] [--data-folder DATA_FOLDER]
-                [--test] [--no-run] [--data-folder DATA_FOLDER]
+                [--threaded] [--version]
-                [--no-threaded] [--no-deps] [--version] [--allow-fail]
 
     Run a Senpy server
 
@@ -20,24 +16,20 @@ The senpy server is launched via the `senpy` command:
       -h, --help            show this help message and exit
       --level logging_level, -l logging_level
                             Logging level
-      --log-format log_format
-                            Logging format
       --debug, -d           Run the application in debug mode
-      --no-default-plugins  Do not load the default plugins
+      --default-plugins     Load the default plugins
       --host HOST           Use 0.0.0.0 to accept requests from any host.
       --port PORT, -p PORT  Port to listen on.
       --plugins-folder PLUGINS_FOLDER, -f PLUGINS_FOLDER
                             Where to look for plugins.
-      --install, -i         Install plugin dependencies before launching the server.
+      --only-install, -i    Do not run a server, only install plugin dependencies
-      --test, -t            Test all plugins before launching the server
+      --only-list, --list   Do not run a server, only list plugins found
-      --no-run 	            Do not launch the server
       --data-folder DATA_FOLDER, --data DATA_FOLDER
                             Where to look for data. It be set with the SENPY_DATA
                             environment variable as well.
-      --no-threaded         Run the server without threading
+      --threaded            Run a threaded server
-      --no-deps, -n         Skip installing dependencies
       --version, -v         Output the senpy version and exit
-      --allow-fail, --fail  Do not exit if some plugins fail to activate
+
 
 
 When launched, the server will recursively look for plugins in the specified plugins folder (the current working directory by default).
@@ -48,9 +40,9 @@ Let's run senpy with the default plugins:
 
 .. code:: bash
 
-    senpy -f .
+    senpy -f . --default-plugins
 
-Now open your browser and go to `http://localhost:5000 <http://localhost:5000>`_, where you should be greeted by the senpy playground:
+Now go to `http://localhost:5000 <http://localhost:5000>`_, you should be greeted by the senpy playground:
 
 .. image:: senpy-playground.png
    :width: 100%
@@ -59,9 +51,9 @@ Now open your browser and go to `http://localhost:5000 <http://localhost:5000>`_
 The playground is a user-friendly way to test your plugins, but you can always use the service directly:  `http://localhost:5000/api?input=hello <http://localhost:5000/api?input=hello>`_.
 
 
-By default, senpy will listen only on `127.0.0.1`.
+By default, senpy will listen only on the `127.0.0.1` address.
-That means you can only access the API from your PC (i.e. localhost).
+That means you can only access the API from your (or localhost).
-You can listen on a different address using the `--host` flag (e.g., 0.0.0.0, to allow any computer to access it).
+You can listen on a different address using the `--host` flag (e.g., 0.0.0.0).
 The default port is 5000.
 You can change it with the `--port` flag. 
 
@@ -72,14 +64,3 @@ For instance, to accept connections on port 6000 on any interface:
     senpy --host 0.0.0.0 --port 6000
 
 For more options, see the `--help` page.
-
-Sentiment analysis in the command line
---------------------------------------
-
-Although the main use of senpy is to publish services, the tool can also be used locally to analyze text in the command line.
-This is a short video demonstration:
-
-.. image:: https://asciinema.org/a/9uwef1ghkjk062cw2t4mhzpyk.png
-   :width: 100%
-   :target: https://asciinema.org/a/9uwef1ghkjk062cw2t4mhzpyk
-   :alt: CLI demo

docs/usage.rst

@@ -0,0 +1,15 @@
+Usage
+-----
+
+First of all, you need to install the package.
+See :doc:`installation` for instructions.
+Once installed, the `senpy` command should be available. 
+
+.. toctree::
+   :maxdepth: 1
+
+   server
+   SenpyClientUse
+   commandline
+
+

docs/vocabularies.rst

@@ -13,9 +13,9 @@ An overview of the vocabularies and their use can be found in [4].
 
 [1] Guidelines for developing NIF-based NLP services, Final Community Group Report 22 December 2015 Available at: https://www.w3.org/2015/09/bpmlod-reports/nif-based-nlp-webservices/
 
-[2] Marl Ontology Specification, available at http://www.gsi.upm.es/ontologies/marl/
+[2] Marl Ontology Specification, available at http://www.gsi.dit.upm.es/ontologies/marl/
 
-[3] Onyx Ontology Specification, available at http://www.gsi.upm.es/ontologies/onyx/
+[3] Onyx Ontology Specification, available at http://www.gsi.dit.upm.es/ontologies/onyx/
 
 [4] Iglesias, C. A., Sánchez-Rada, J. F., Vulcu, G., & Buitelaar, P. (2017). Linked Data Models for Sentiment and Emotion Analysis in Social Networks. In Sentiment Analysis in Social Networks (pp. 49-69).
 

example-plugins/README.md

@@ -1,6 +1,6 @@
 This is a collection of plugins that exemplify certain aspects of plugin development with senpy.
 
-The first series of plugins are the `basic` ones.
+The first series of plugins the `basic` ones.
 Their starting point is a classification function defined in `basic.py`.
 They all include testing and running them as a script will run all tests.
 In ascending order of customization, the plugins are:
@@ -19,5 +19,5 @@ In rest of the plugins show advanced topics:
 
 All of the plugins in this folder include a set of test cases and they are periodically tested with the latest version of senpy.
 
-Additioanlly, for an example of stand-alone plugin that can be tested and deployed with docker, take a look at: lab.gsi.upm.es/senpy/plugin-example
+Additioanlly, for an example of stand-alone plugin that can be tested and deployed with docker, take a look at: lab.cluster.gsi.dit.upm.es/senpy/plugin-example
  bbm

example-plugins/async_plugin.py

@@ -0,0 +1,37 @@
+from senpy import AnalysisPlugin
+
+import multiprocessing
+
+
+def _train(process_number):
+    return process_number
+
+
+class Async(AnalysisPlugin):
+    '''An example of an asynchronous module'''
+    author = '@balkian'
+    version = '0.2'
+    sync = False
+
+    def _do_async(self, num_processes):
+        pool = multiprocessing.Pool(processes=num_processes)
+        values = sorted(pool.map(_train, range(num_processes)))
+
+        return values
+
+    def activate(self):
+        self.value = self._do_async(4)
+
+    def analyse_entry(self, entry, params):
+        values = self._do_async(2)
+        entry.async_values = values
+        yield entry
+
+    test_cases = [
+        {
+            'input': 'any',
+            'expected': {
+                'async_values': [0, 1]
+            }
+        }
+    ]

example-plugins/basic.py

@@ -1,21 +1,5 @@
 #!/usr/local/bin/python
-# -*- coding: utf-8 -*-
+# coding: utf-8
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
 
 emoticons = {
     'pos': [':)', ':]', '=)', ':D'],
@@ -23,19 +7,17 @@ emoticons = {
 }
 
 emojis = {
-    'pos': [u'😁', u'😂', u'😃', u'😄', u'😆', u'😅', u'😄', u'😍'],
+    'pos': ['😁', '😂', '😃', '😄', '😆', '😅', '😄' '😍'],
-    'neg': [u'😢', u'😡', u'😠', u'😞', u'😖', u'😔', u'😓', u'😒']
+    'neg': ['😢', '😡', '😠', '😞', '😖', '😔', '😓', '😒']
 }
 
 
 def get_polarity(text, dictionaries=[emoticons, emojis]):
     polarity = 'marl:Neutral'
-    print('Input for get_polarity', text)
     for dictionary in dictionaries:
         for label, values in dictionary.items():
             for emoticon in values:
                 if emoticon and emoticon in text:
                     polarity = label
                     break
-    print('Polarity', polarity)
     return polarity

example-plugins/basic_analyse_entry_plugin.py

@@ -1,20 +1,5 @@
 #!/usr/local/bin/python
-# -*- coding: utf-8 -*-
+# coding: utf-8
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
 
 from senpy import easy_test, models, plugins
 
@@ -24,8 +9,8 @@ import basic
 class BasicAnalyseEntry(plugins.SentimentPlugin):
     '''Equivalent to Basic, implementing the analyse_entry method'''
 
-    author: str = '@balkian'
+    author = '@balkian'
-    version: str = '0.1'
+    version = '0.1'
 
     mappings = {
         'pos': 'marl:Positive',
@@ -33,17 +18,17 @@ class BasicAnalyseEntry(plugins.SentimentPlugin):
         'default': 'marl:Neutral'
     }
 
-    def analyse_entry(self, entry, activity):
+    def analyse_entry(self, entry, params):
         polarity = basic.get_polarity(entry.text)
 
         polarity = self.mappings.get(polarity, self.mappings['default'])
 
         s = models.Sentiment(marl__hasPolarity=polarity)
-        s.prov(activity)
+        s.prov(self)
         entry.sentiments.append(s)
         yield entry
 
-    test_cases: list[dict] = [{
+    test_cases = [{
         'input': 'Hello :)',
         'polarity': 'marl:Positive'
     }, {

example-plugins/basic_box_plugin.py

@@ -1,20 +1,5 @@
 #!/usr/local/bin/python
-# -*- coding: utf-8 -*-
+# coding: utf-8
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
 
 from senpy import easy_test, SentimentBox
 
@@ -24,18 +9,20 @@ import basic
 class BasicBox(SentimentBox):
     ''' A modified version of Basic that also does converts annotations manually'''
 
-    author: str = '@balkian'
+    author = '@balkian'
-    version: str = '0.1'
+    version = '0.1'
 
-    def predict_one(self, features, **kwargs):
+    mappings = {
-        output = basic.get_polarity(features[0])
+        'pos': 'marl:Positive',
-        if output == 'pos':
+        'neg': 'marl:Negative',
-            return [1, 0, 0]
+        'default': 'marl:Neutral'
-        if output == 'neg':
+    }
-            return [0, 0, 1]
-        return [0, 1, 0]
 
-    test_cases: list[dict] = [{
+    def predict_one(self, input):
+        output = basic.get_polarity(input)
+        return self.mappings.get(output, self.mappings['default'])
+
+    test_cases = [{
         'input': 'Hello :)',
         'polarity': 'marl:Positive'
     }, {

example-plugins/basic_plugin.py

@@ -1,52 +1,37 @@
 #!/usr/local/bin/python
-# -*- coding: utf-8 -*-
+# coding: utf-8
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
 
-
+from senpy import easy_test, SentimentBox, MappingMixin
-from senpy import easy_test, SentimentBox
 
 import basic
 
 
-class Basic(SentimentBox):
+class Basic(MappingMixin, SentimentBox):
     '''Provides sentiment annotation using a lexicon'''
 
-    author: str = '@balkian'
+    author = '@balkian'
-    version: str = '0.1'
+    version = '0.1'
 
-    def predict_one(self, features, **kwargs):
+    mappings = {
-        output = basic.get_polarity(features[0])
+        'pos': 'marl:Positive',
-        if output == 'pos':
+        'neg': 'marl:Negative',
-            return [1, 0, 0]
+        'default': 'marl:Neutral'
-        if output == 'neu':
+    }
-            return [0, 1, 0]
-        return [0, 0, 1]
 
-    test_cases: list[dict] = [{
+    def predict_one(self, input):
-        'input': u'Hello :)',
+        return basic.get_polarity(input)
+
+    test_cases = [{
+        'input': 'Hello :)',
         'polarity': 'marl:Positive'
     }, {
-        'input': u'So sad :(',
+        'input': 'So sad :(',
         'polarity': 'marl:Negative'
     }, {
-        'input': u'Yay! Emojis  😁',
+        'input': 'Yay! Emojis  😁',
         'polarity': 'marl:Positive'
     }, {
-        'input': u'But no emoticons 😢',
+        'input': 'But no emoticons 😢',
         'polarity': 'marl:Negative'
     }]
 

example-plugins/configurable_plugin.py

@@ -1,21 +1,5 @@
 #!/usr/local/bin/python
-# -*- coding: utf-8 -*-
+# coding: utf-8
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
 
 from senpy import easy_test, models, plugins
 
@@ -25,14 +9,14 @@ import basic
 class Dictionary(plugins.SentimentPlugin):
     '''Sentiment annotation using a configurable lexicon'''
 
-    author: str = '@balkian'
+    author = '@balkian'
-    version: str = '0.2'
+    version = '0.2'
 
     dictionaries = [basic.emojis, basic.emoticons]
 
     mappings = {'pos': 'marl:Positive', 'neg': 'marl:Negative'}
 
-    def analyse_entry(self, entry, *args, **kwargs):
+    def analyse_entry(self, entry, params):
         polarity = basic.get_polarity(entry.text, self.dictionaries)
         if polarity in self.mappings:
             polarity = self.mappings[polarity]
@@ -42,7 +26,7 @@ class Dictionary(plugins.SentimentPlugin):
         entry.sentiments.append(s)
         yield entry
 
-    test_cases: list[dict] = [{
+    test_cases = [{
         'input': 'Hello :)',
         'polarity': 'marl:Positive'
     }, {
@@ -61,7 +45,7 @@ class EmojiOnly(Dictionary):
     '''Sentiment annotation with a basic lexicon of emojis'''
     dictionaries = [basic.emojis]
 
-    test_cases: list[dict] = [{
+    test_cases = [{
         'input': 'Hello :)',
         'polarity': 'marl:Neutral'
     }, {
@@ -80,7 +64,7 @@ class EmoticonsOnly(Dictionary):
     '''Sentiment annotation with a basic lexicon of emoticons'''
     dictionaries = [basic.emoticons]
 
-    test_cases: list[dict] = [{
+    test_cases = [{
         'input': 'Hello :)',
         'polarity': 'marl:Positive'
     }, {
@@ -102,7 +86,7 @@ class Salutes(Dictionary):
         'marl:Negative': ['Good bye', ]
     }]
 
-    test_cases: list[dict] = [{
+    test_cases = [{
         'input': 'Hello :)',
         'polarity': 'marl:Positive'
     }, {

example-plugins/dummy_plugin.py

@@ -1,33 +1,17 @@
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
 from senpy import AnalysisPlugin, easy
 
 
 class Dummy(AnalysisPlugin):
     '''This is a dummy self-contained plugin'''
-    author: str = '@balkian'
+    author = '@balkian'
-    version: str = '0.1'
+    version = '0.1'
 
     def analyse_entry(self, entry, params):
-        entry.text = entry.text[::-1]
+        entry['nif:isString'] = entry['nif:isString'][::-1]
         entry.reversed = entry.get('reversed', 0) + 1
         yield entry
 
-    test_cases: list[dict] = [{
+    test_cases = [{
         'entry': {
             'nif:isString': 'Hello',
         },

example-plugins/dummy_required_plugin.py

@@ -1,27 +1,11 @@
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
 from senpy import AnalysisPlugin, easy
 
 
 class DummyRequired(AnalysisPlugin):
     '''This is a dummy self-contained plugin'''
-    author: str = '@balkian'
+    author = '@balkian'
-    version: str = '0.1'
+    version = '0.1'
-    extra_params: dict = {
+    extra_params = {
         'example': {
             'description': 'An example parameter',
             'required': True,
@@ -30,11 +14,11 @@ class DummyRequired(AnalysisPlugin):
     }
 
     def analyse_entry(self, entry, params):
-        entry.text = entry.text[::-1]
+        entry['nif:isString'] = entry['nif:isString'][::-1]
         entry.reversed = entry.get('reversed', 0) + 1
         yield entry
 
-    test_cases: list[dict] = [{
+    test_cases = [{
         'entry': {
             'nif:isString': 'Hello',
         },

example-plugins/emorand_plugin.py

@@ -1,49 +0,0 @@
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
-import random
-
-from senpy.plugins import EmotionPlugin
-from senpy.models import EmotionSet, Emotion, Entry
-
-
-class EmoRand(EmotionPlugin):
-    '''A sample plugin that returns a random emotion annotation'''
-    name: str = 'emotion-random'
-    author: str = '@balkian'
-    version: str = '0.1'
-    url: str = "https://github.com/gsi-upm/senpy-plugins-community"
-    usesEmotionModel: str = "emoml:big6"
-
-    def analyse_entry(self, entry, activity):
-        category = "emoml:big6happiness"
-        number = max(-1, min(1, random.gauss(0, 0.5)))
-        if number > 0:
-            category = "emoml:big6anger"
-        emotionSet = EmotionSet()
-        emotion = Emotion({"onyx:hasEmotionCategory": category})
-        emotionSet.onyx__hasEmotion.append(emotion)
-        emotionSet.prov(activity)
-        entry.emotions.append(emotionSet)
-        yield entry
-
-    def test(self):
-        params = dict()
-        results = list()
-        for i in range(100):
-            res = next(self.analyse_entry(Entry(nif__isString="Hello"), self.activity(params)))
-            res.validate()
-            results.append(res.emotions[0]['onyx:hasEmotion'][0]['onyx:hasEmotionCategory'])

example-plugins/moral/example.ttl

@@ -1,63 +0,0 @@
-@prefix : <http://www.gsi.upm.es/ontologies/amor/examples#> .
-@prefix amor: <http://www.gsi.upm.es/ontologies/amor/ns#> .
-@prefix amor-bhv: <http://www.gsi.upm.es/ontologies/amor/models/bhv/ns#> .
-@prefix amor-mft: <http://www.gsi.upm.es/ontologies/amor/models/mft/ns#> .
-@prefix bhv: <http://www.gsi.upm.es/ontologies/bhv#> .
-@prefix mft: <http://www.gsi.upm.es/ontologies/mft/ns#> .
-@prefix mls: <http://www.w3.org/ns/mls#> .
-@prefix owl: <http://www.w3.org/2002/07/owl#> .
-@prefix prov: <http://www.w3.org/ns/prov#> .
-@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
-@prefix schema: <http://schema.org/> .
-
-
-:news1 a owl:NamedIndividual, schema:NewsArticle ;
-  schema:articleBody "Director Comey says the probe into last year's US election would assess if crimes were committed."^^xsd:string ;
-  schema:datePublished "2017-03-20T20:30:54+00:00"^^schema:Date ;
-  schema:headline "Trump Russia claims: FBI's Comey confirms investigation of election 'interference'"^^xsd:string ;
-  schema:image <http://ichef-1.bbci.co.uk/news/560/media/images/75306000/jpg/_75306515_line976.jpg>,
-    <http://ichef.bbci.co.uk/news/560/cpsprodpb/8AB9/production/_95231553_comey2.jpg>,
-    <http://ichef.bbci.co.uk/news/560/cpsprodpb/17519/production/_95231559_committee.jpg>,
-    <http://ichef.bbci.co.uk/news/560/cpsprodpb/CC81/production/_95235325_f704a6dc-c017-4971-aac3-04c03eb097fb.jpg>,
-    <http://ichef-1.bbci.co.uk/news/560/cpsprodpb/11AA1/production/_95235327_c0b59f9e-316e-4641-aa7e-3fec6daea62b.jpg>,
-    <http://ichef.bbci.co.uk/news/560/cpsprodpb/0F99/production/_95239930_trumptweet.png>,
-    <http://ichef-1.bbci.co.uk/news/560/cpsprodpb/10DFA/production/_95241196_mediaitem95241195.jpg>,
-    <http://ichef.bbci.co.uk/news/560/cpsprodpb/2CA0/production/_95242411_comey.jpg>,
-    <http://ichef.bbci.co.uk/news/560/cpsprodpb/11318/production/_95242407_mediaitem95242406.jpg>,
-    <http://ichef-1.bbci.co.uk/news/560/cpsprodpb/BCED/production/_92856384_line976.jpg>,
-    <http://ichef-1.bbci.co.uk/news/560/cpsprodpb/12B64/production/_95244667_mediaitem95244666.jpg> ;
-  schema:mainEntityOfPage <http://www.bbc.com/news/world-us-canada-39324587> ;
-  schema:publisher :bbc ;
-  schema:url <http://www.bbc.com/news/world-us-canada-39324587> .
-
-:bbc a schema:Organization ;
-  schema:logo <http://www.bbc.co.uk/news/special/2015/newsspec_10857/bbc_news_logo.png?cb=1> ;
-  schema:name "BBC News"^^xsd:string .
-
-
-:robot1 a prov:SoftwareAgent .
-
-:model1 a mls:Model .
-
-:logisticRegression a mls:Algorithm ;
-  rdfs:label "Logistic Regression"@en ,
-    "Regresión Logística"@es .
-
-:run1 a mls:Run ;
-    mls:executes :wekaLogistic ;
-    mls:hasInput :credit-a ;
-    mls:hasOutput :model1 ;
-    mls:realizes :logisticRegression .
-
-:analysis3 a amor:MoralValueAnalysis ;
-    prov:wasAssociatedWith :robot1 ;
-    amor:usedMoralValueModel amor-mft:MoralFoundationsTheory ;
-    amor:analysed :news1 ;
-    amor:usedMLModel :model1 ;
-    prov:generated :annotation3 .
-
-:annotation3 a amor:MoralValueAnnotation ;
-    amor:hasMoralValueCategory mft:Authority ;
-    amor:confidence "0.75"^^xsd:float ;
-    amor-mft:hasPolarityIntensity "0.2"^^xsd:float ;
-    amor:annotated :news1 .

example-plugins/moral/moral_plugin.py

@@ -1,10 +0,0 @@
-from senpy.ns import amor, amor_bhv, amor_mft, prov
-from senpy.plugins import MoralityPlugin
-
-
-class DummyMoralityPlugin(MoralityPlugin):
-    moralValueModel: str = amor_mft["MoralFoundationTHeory"]
-
-    def annotate(self, entry, activity, **kwargs):
-        yield MoralAnnotation(amor_bhv['Conservation'],
-                              confidence=1.8)

example-plugins/mynoop.py

@@ -1,19 +1,3 @@
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
 import noop
 from senpy.plugins import SentimentPlugin
 
@@ -21,7 +5,7 @@ from senpy.plugins import SentimentPlugin
 class NoOp(SentimentPlugin):
     '''This plugin does nothing. Literally nothing.'''
 
-    version: str = 0
+    version = 0
 
     def analyse_entry(self, entry, *args, **kwargs):
         yield entry

example-plugins/mynoop.senpy

@@ -1,4 +1,3 @@
 module: mynoop
-optional: true
 requirements:
  - noop

example-plugins/parameterized_plugin.py

@@ -1,21 +1,5 @@
 #!/usr/local/bin/python
-# -*- coding: utf-8 -*-
+# coding: utf-8
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
 
 from senpy import easy_test, models, plugins
 
@@ -24,10 +8,11 @@ import basic
 
 class ParameterizedDictionary(plugins.SentimentPlugin):
     '''This is a basic self-contained plugin'''
-    author: str = '@balkian'
-    version: str = '0.2'
 
-    extra_params: dict = {
+    author = '@balkian'
+    version = '0.2'
+
+    extra_params = {
         'positive-words': {
             'description': 'Comma-separated list of words that are considered positive',
             'aliases': ['positive'],
@@ -40,8 +25,7 @@ class ParameterizedDictionary(plugins.SentimentPlugin):
         }
     }
 
-    def analyse_entry(self, entry, activity):
+    def analyse_entry(self, entry, params):
-        params = activity.params
         positive_words = params['positive-words'].split(',')
         negative_words = params['negative-words'].split(',')
         dictionary = {
@@ -51,11 +35,11 @@ class ParameterizedDictionary(plugins.SentimentPlugin):
         polarity = basic.get_polarity(entry.text, [dictionary])
 
         s = models.Sentiment(marl__hasPolarity=polarity)
-        s.prov(activity)
+        s.prov(self)
         entry.sentiments.append(s)
         yield entry
 
-    test_cases: list[dict] = [
+    test_cases = [
         {
             'input': 'Hello :)',
             'polarity': 'marl:Positive',

example-plugins/rand_plugin.py

@@ -1,54 +0,0 @@
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
-import random
-from senpy import SentimentPlugin, Sentiment, Entry
-
-
-class RandSent(SentimentPlugin):
-    '''A sample plugin that returns a random sentiment annotation'''
-    name: str = 'sentiment-random'
-    author: str = "@balkian"
-    version: str = '0.1'
-    url: str = "https://github.com/gsi-upm/senpy-plugins-community"
-    maxPolarityValue: float = 1
-    minPolarityValue: float = -1
-
-    def analyse_entry(self, entry, activity):
-        polarity_value = max(-1, min(1, random.gauss(0.2, 0.2)))
-        polarity = "marl:Neutral"
-        if polarity_value > 0:
-            polarity = "marl:Positive"
-        elif polarity_value < 0:
-            polarity = "marl:Negative"
-        sentiment = Sentiment(marl__hasPolarity=polarity,
-                              marl__polarityValue=polarity_value)
-        sentiment.prov(activity)
-        entry.sentiments.append(sentiment)
-        yield entry
-
-    def test(self):
-        '''Run several random analyses.'''
-        params = dict()
-        results = list()
-        for i in range(50):
-            activity = self.activity(params)
-            res = next(self.analyse_entry(Entry(nif__isString="Hello"),
-                                          activity))
-            res.validate()
-            results.append(res.sentiments[0]['marl:hasPolarity'])
-        assert 'marl:Positive' in results
-        assert 'marl:Negative' in results

example-plugins/sklearn/mydata.py

@@ -1,19 +1,3 @@
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
 '''
 Create a dummy dataset.
 Messages with a happy emoticon are labelled positive

example-plugins/sklearn/mypipeline.py

@@ -1,19 +1,3 @@
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
 from sklearn.pipeline import Pipeline
 from sklearn.feature_extraction.text import CountVectorizer
 from sklearn.model_selection import train_test_split
@@ -31,7 +15,7 @@ pipeline = Pipeline([('cv', count_vec),
                     ('clf', clf3)])
 
 pipeline.fit(X_train, y_train)
-print('Feature names: {}'.format(count_vec.get_feature_names_out()))
+print('Feature names: {}'.format(count_vec.get_feature_names()))
 print('Class count: {}'.format(clf3.class_count_))
 
 

example-plugins/sklearn/pipeline_plugin.py

@@ -1,38 +1,27 @@
-#
+from senpy import SentimentBox, MappingMixin, easy_test
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
-from senpy import SentimentBox, easy_test
 
 from mypipeline import pipeline
 
 
-class PipelineSentiment(SentimentBox):
+class PipelineSentiment(MappingMixin, SentimentBox):
-    '''This is a pipeline plugin that wraps a classifier defined in another module
+    '''
-(mypipeline).'''
+    This is a pipeline plugin that wraps a classifier defined in another module
-    author: str = '@balkian'
+    (mypipeline).
-    version: str = 0.1
+    '''
+    author = '@balkian'
+    version = 0.1
     maxPolarityValue = 1
     minPolarityValue = -1
 
-    def predict_one(self, features, **kwargs):
+    mappings = {
-        if pipeline.predict(features) > 0:
+        1: 'marl:Positive',
-            return [1, 0, 0]
+        -1: 'marl:Negative'
-        return [0, 0, 1]
+    }
 
-    test_cases: list[dict] = [
+    def predict_one(self, input):
+        return pipeline.predict([input, ])[0]
+
+    test_cases = [
         {
             'input': 'The sentiment for senpy should be positive :)',
             'polarity': 'marl:Positive'

example-plugins/sleep_plugin.py

@@ -1,29 +1,13 @@
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
 from senpy.plugins import AnalysisPlugin
 from time import sleep
 
 
 class Sleep(AnalysisPlugin):
     '''Dummy plugin to test async'''
-    author: str = "@balkian"
+    author = "@balkian"
-    version: str = "0.2"
+    version = "0.2"
     timeout = 0.05
-    extra_params: dict = {
+    extra_params = {
         "timeout": {
             "@id": "timeout_sleep",
             "aliases": ["timeout", "to"],
@@ -32,8 +16,7 @@ class Sleep(AnalysisPlugin):
         }
     }
 
-    def __init__(self, **kwargs):
+    def activate(self, *args, **kwargs):
-        super().__init__(**kwargs)
         sleep(self.timeout)
 
     def analyse_entry(self, entry, params):

extra-requirements.txt

@@ -1,6 +1 @@
-gsitk>0.1.9.1
+gsitk
-flask_cors==3.0.10
-lxml==4.9.3
-pandas==2.1.1
-textblob==0.17.1
-git+https://github.com/uob-vil/pattern.git#pattern

k8s/senpy-deployment.yaml

@@ -1,24 +1,22 @@
 ---
-apiVersion: apps/v1
+apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
   name: senpy-latest
 spec:
   replicas: 1
-  selector:
-    matchLabels:
-      app: senpy-latest
   template:
     metadata:
       labels:
-        app: senpy-latest
         role: senpy-latest
+        app: test
     spec:
       containers:
       - name: senpy-latest
         image: $IMAGEWTAG
         imagePullPolicy: Always
-        args: ["--enable-cors"]
+        args:
+          - "--default-plugins"
         resources:
           limits:
             memory: "512Mi"
@@ -26,11 +24,3 @@ spec:
         ports:
           - name: web
             containerPort: 5000
-        volumeMounts:
-          - name: senpy-data
-            mountPath: /senpy-data
-            subPath: data
-      volumes:
-        - name: senpy-data
-          persistentVolumeClaim:
-            claimName: pvc-senpy

k8s/senpy-ingress.yaml

@@ -1,29 +1,14 @@
 ---
-apiVersion: networking.k8s.io/v1
+apiVersion: extensions/v1beta1
 kind: Ingress
 metadata:
   name: senpy-ingress
-  labels:
-    app: senpy-latest
 spec:
   rules:
-  - host: senpy-latest.gsi.upm.es
+  - host: latest.senpy.cluster.gsi.dit.upm.es
     http:
       paths:
       - path: /
-        pathType: Prefix
         backend:
-          service:
+          serviceName: senpy-latest
-            name: senpy-latest
+          servicePort: 5000
-            port:
-              number: 5000
-  - host: latest.senpy.gsi.upm.es
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: senpy-latest
-            port:
-              number: 5000

k8s/senpy-svc.yaml

@@ -3,12 +3,10 @@ apiVersion: v1
 kind: Service
 metadata:
   name: senpy-latest
-  labels:
-    app: senpy-latest
 spec:
   type: ClusterIP
   ports:
     - port: 5000
       protocol: TCP
   selector:
-    app: senpy-latest
+    role: senpy-latest

requirements.txt

@@ -7,10 +7,9 @@ future
 jsonschema
 jsonref
 PyYAML
-rdflib==6.1.1
+rdflib
+rdflib-jsonld
 numpy
 scipy
-scikit-learn>=0.20
+scikit-learn
 responses
-jmespath
-deprecation==2.1.0

senpy/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
-#
+#    Copyright 2014 J. Fernando Sánchez Rada - Grupo de Sistemas Inteligentes
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
+#                                                       DIT, UPM
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -14,7 +14,6 @@
 #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
-#
 """
 Sentiment analysis server in Python
 """

senpy/__main__.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
+#    Copyright 2014 J. Fernando Sánchez Rada - Grupo de Sistemas Inteligentes
+#                                                       DIT, UPM
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -22,8 +23,6 @@ the server.
 from flask import Flask
 from senpy.extensions import Senpy
 from senpy.utils import easy_test
-from senpy.plugins import list_dependencies
-from senpy import config
 
 import logging
 import os
@@ -41,19 +40,8 @@ def main():
         '-l',
         metavar='logging_level',
         type=str,
-        default="INFO",
+        default="WARN",
         help='Logging level')
-    parser.add_argument(
-        '--no-proxy-fix',
-        action='store_true',
-        default=False,
-        help='Do not assume senpy will be running behind a proxy (e.g., nginx)')
-    parser.add_argument(
-        '--log-format',
-        metavar='log_format',
-        type=str,
-        default='%(asctime)s %(levelname)-10s %(name)-30s \t %(message)s',
-        help='Logging format')
     parser.add_argument(
         '--debug',
         '-d',
@@ -61,10 +49,10 @@ def main():
         default=False,
         help='Run the application in debug mode')
     parser.add_argument(
-        '--no-default-plugins',
+        '--default-plugins',
         action='store_true',
         default=False,
-        help='Do not load the default plugins')
+        help='Load the default plugins')
     parser.add_argument(
         '--host',
         type=str,
@@ -80,24 +68,19 @@ def main():
         '--plugins-folder',
         '-f',
         type=str,
-        action='append',
+        default='.',
         help='Where to look for plugins.')
     parser.add_argument(
-        '--install',
+        '--only-install',
         '-i',
         action='store_true',
         default=False,
-        help='Install plugin dependencies before running.')
+        help='Do not run a server, only install plugin dependencies')
     parser.add_argument(
-        '--dependencies',
+        '--only-test',
         action='store_true',
         default=False,
-        help='List plugin dependencies')
+        help='Do not run a server, just test all plugins')
-    parser.add_argument(
-        '--strict',
-        action='store_true',
-        default=config.strict,
-        help='Fail if optional plugins cannot be loaded.')
     parser.add_argument(
         '--test',
         '-t',
@@ -105,10 +88,11 @@ def main():
         default=False,
         help='Test all plugins before launching the server')
     parser.add_argument(
-        '--no-run',
+        '--only-list',
+        '--list',
         action='store_true',
         default=False,
-        help='Do not launch the server.')
+        help='Do not run a server, only list plugins found')
     parser.add_argument(
         '--data-folder',
         '--data',
@@ -116,10 +100,10 @@ def main():
         default=None,
         help='Where to look for data. It be set with the SENPY_DATA environment variable as well.')
     parser.add_argument(
-        '--no-threaded',
+        '--threaded',
-        action='store_true',
+        action='store_false',
-        default=False,
+        default=True,
-        help='Run a single-threaded server')
+        help='Run a threaded server')
     parser.add_argument(
         '--no-deps',
         '-n',
@@ -137,102 +121,51 @@ def main():
         '--fail',
         action='store_true',
         default=False,
-        help='Do not exit if some plugins can not be instantiated')
+        help='Do not exit if some plugins fail to activate')
-    parser.add_argument(
-        '--enable-cors',
-        '--cors',
-        action='store_true',
-        default=False,
-        help='Enable CORS for all domains (requires flask-cors to be installed)')
     args = parser.parse_args()
-    print('Senpy version {}'.format(senpy.__version__))
-    print(sys.version)
     if args.version:
+        print('Senpy version {}'.format(senpy.__version__))
+        print(sys.version)
         exit(1)
     rl = logging.getLogger()
     rl.setLevel(getattr(logging, args.level))
-    logger_handler = rl.handlers[0]
-
-    # First, generic formatter:
-    logger_handler.setFormatter(logging.Formatter(args.log_format))
-
     app = Flask(__name__)
     app.debug = args.debug
-
+    sp = Senpy(app, args.plugins_folder,
-    sp = Senpy(app,
+               default_plugins=args.default_plugins,
-               plugin_folder=None,
-               default_plugins=not args.no_default_plugins,
-               install=args.install,
-               strict=args.strict,
                data_folder=args.data_folder)
-
+    if args.only_list:
-    folders = list(args.plugins_folder) if args.plugins_folder else []
+        plugins = sp.plugins()
-    if not folders:
+        maxname = max(len(x.name) for x in plugins)
-        folders.append(".")
+        maxversion = max(len(x.version) for x in plugins)
-    for p in folders:
+        print('Found {} plugins:'.format(len(plugins)))
-        sp.add_folder(p)
+        for plugin in plugins:
-
+            import inspect
-    plugins = sp.plugins(plugin_type=None)
+            fpath = inspect.getfile(plugin.__class__)
-    maxname = max(len(x.name) for x in plugins)
+            print('\t{: <{maxname}} @ {: <{maxversion}} -> {}'.format(plugin.name,
-    maxversion = max(len(str(x.version)) for x in plugins)
+                                                                      plugin.version,
-    print('Found {} plugins:'.format(len(plugins)))
+                                                                      fpath,
-    for plugin in plugins:
+                                                                      maxname=maxname,
-        import inspect
+                                                                      maxversion=maxversion))
-        fpath = inspect.getfile(plugin.__class__)
-        print('\t{: <{maxname}} @ {: <{maxversion}} -> {}'.format(plugin.name,
-                                                                  plugin.version,
-                                                                  fpath,
-                                                                  maxname=maxname,
-                                                                  maxversion=maxversion))
-    if args.dependencies:
-        print('Listing dependencies')
-        missing = []
-        installed = []
-        for plug in sp.plugins():
-            inst, miss, nltkres = list_dependencies(plug)
-            if not any([inst, miss, nltkres]):
-                continue
-            print(f'Plugin: {plug.id}')
-            for m in miss:
-                missing.append(f'{m} # {plug.id}')
-            for i in inst:
-                installed.append(f'{i} # {plug.id}')
-        if installed:
-            print('Installed packages:')
-            for i in installed:
-                print(f'\t{i}')
-        if missing:
-            print('Missing packages:')
-            for m in missing:
-                print(f'\t{m}')
-
-    if args.install:
-        sp.install_deps()
-
-    if args.test:
-        easy_test(sp.plugins(), debug=args.debug)
-
-    if args.no_run:
         return
-
+    if not args.no_deps:
+        sp.install_deps()
+    if args.only_install:
+        return
+    sp.activate_all(allow_fail=args.allow_fail)
+    if args.test or args.only_test:
+        easy_test(sp.plugins(), debug=args.debug)
+        if args.only_test:
+            return
     print('Senpy version {}'.format(senpy.__version__))
     print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,
                                                             args.port))
-    if args.enable_cors:
+    app.run(args.host,
-        from flask_cors import CORS
+            args.port,
-        CORS(app)
+            threaded=args.threaded,
+            debug=app.debug)
+    sp.deactivate_all()
 
-    if not args.no_proxy_fix:
-        from werkzeug.middleware.proxy_fix import ProxyFix
-        app.wsgi_app = ProxyFix(app.wsgi_app)
-
-    try:
-        app.run(args.host,
-                args.port,
-                threaded=not args.no_threaded,
-                debug=app.debug)
-    except KeyboardInterrupt:
-        print('Bye!')
 
 if __name__ == '__main__':
     main()

senpy/api.py

@@ -1,52 +1,28 @@
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
 from future.utils import iteritems
-from .models import Results, Entry, from_string
+from .models import Analysis, Error, Results, Entry, from_string
-from .errors import Error, InvalidParams
 import logging
 logger = logging.getLogger(__name__)
 
 boolean = [True, False]
 
-processors = {
-    'string_to_tuple': lambda p: p if isinstance(p, (tuple, list)) else tuple(p.split(','))
-}
-
 API_PARAMS = {
     "algorithm": {
         "aliases": ["algorithms", "a", "algo"],
         "required": True,
         "default": 'default',
-        "processor": 'string_to_tuple',
         "description": ("Algorithms that will be used to process the request."
                         "It may be a list of comma-separated names."),
     },
     "expanded-jsonld": {
         "@id": "expanded-jsonld",
-        "description": "use JSON-LD expansion to get full URIs",
+        "aliases": ["expanded"],
-        "aliases": ["expanded", "expanded_jsonld"],
         "options": boolean,
         "required": True,
         "default": False
     },
-    "with-parameters": {
+    "with_parameters": {
         "aliases": ['withparameters',
-                    'with_parameters'],
+                    'with-parameters'],
-        "description": "include initial parameters in the response",
         "options": boolean,
         "default": False,
         "required": True
@@ -55,67 +31,9 @@ API_PARAMS = {
         "@id": "outformat",
         "aliases": ["o"],
         "default": "json-ld",
-        "description": """The data can be semantically formatted (JSON-LD, turtle or n-triples),
-given as a list of comma-separated fields (see the fields option) or constructed from a Jinja2
-template (see the template option).""",
         "required": True,
         "options": ["json-ld", "turtle", "ntriples"],
     },
-    "template": {
-        "@id": "template",
-        "required": False,
-        "description": """Jinja2 template for the result. The input data for the template will
-be the results as a dictionary.
-For example:
-
-Consider the results before templating:
-
-```
-[{
-    "@type": "entry",
-    "onyx:hasEmotionSet": [],
-    "nif:isString": "testing the template",
-    "marl:hasOpinion": [
-        {
-            "@type": "sentiment",
-            "marl:hasPolarity": "marl:Positive"
-        }
-    ]
-}]
-```
-
-
-And the template:
-
-```
-{% for entry in entries %}
-{{ entry["nif:isString"] | upper }},{{entry.sentiments[0]["marl:hasPolarity"].split(":")[1]}}
-{% endfor %}
-```
-
-The final result would be:
-
-```
-TESTING THE TEMPLATE,Positive
-```
-"""
-
-    },
-    "fields": {
-        "@id": "fields",
-        "required": False,
-        "description": """A jmespath selector, that can be used to extract a new dictionary, array or value
-from the results.
-jmespath is a powerful query language for json and/or dictionaries.
-It allows you to change the structure (and data) of your objects through queries.
-
-e.g., the following expression gets a list of `[emotion label, intensity]` for each entry:
-`entries[]."onyx:hasEmotionSet"[]."onyx:hasEmotion"[]["onyx:hasEmotionCategory","onyx:hasEmotionIntensity"]`
-
-For more information, see: https://jmespath.org
-
-"""
-    },
     "help": {
         "@id": "help",
         "description": "Show additional help to know more about the possible parameters",
@@ -126,39 +44,20 @@ For more information, see: https://jmespath.org
     },
     "verbose": {
         "@id": "verbose",
-        "description": "Show all properties in the result",
+        "description": "Show all help, including the common API parameters, or only plugin-related info",
         "aliases": ["v"],
         "required": True,
         "options": boolean,
-        "default": False
+        "default": True
     },
-    "aliases": {
+    "emotionModel": {
-        "@id": "aliases",
-        "description": "Replace JSON properties with their aliases",
-        "aliases": [],
-        "required": True,
-        "options": boolean,
-        "default": False
-    },
-    "emotion-model": {
         "@id": "emotionModel",
-        "description": """Emotion model to use in the response.
+        "aliases": ["emoModel"],
-Senpy will try to convert the output to this model automatically.
-
-Examples: `wna:liking` and `emoml:big6`.
-        """,
-        "aliases": ["emoModel", "emotionModel"],
         "required": False
     },
     "conversion": {
         "@id": "conversion",
-        "description": """How to show the elements that have (not) been converted.
+        "description": "How to show the elements that have (not) been converted",
-
-* full: converted and original elements will appear side-by-side
-* filtered: only converted elements will be shown
-* nested: converted elements will be shown, and they will include a link to the original element
-(using `prov:wasGeneratedBy`).
-""",
         "required": True,
         "options": ["filtered", "nested", "full"],
         "default": "full"
@@ -168,10 +67,9 @@ Examples: `wna:liking` and `emoml:big6`.
 EVAL_PARAMS = {
     "algorithm": {
         "aliases": ["plug", "p", "plugins", "algorithms", 'algo', 'a', 'plugin'],
-        "description": "Plugins to evaluate",
+        "description": "Plugins to be evaluated",
         "required": True,
-        "help": "See plugins at /plugins",
+        "help": "See activated plugins in /plugins"
-        "processor": API_PARAMS['algorithm']['processor']
     },
     "dataset": {
         "aliases": ["datasets", "data", "d"],
@@ -182,19 +80,18 @@ EVAL_PARAMS = {
 }
 
 PLUGINS_PARAMS = {
-    "plugin-type": {
+    "plugin_type": {
         "@id": "pluginType",
         "description": 'What kind of plugins to list',
-        "aliases": ["pluginType", "plugin_type"],
+        "aliases": ["pluginType"],
         "required": True,
         "default": 'analysisPlugin'
     }
 }
 
 WEB_PARAMS = {
-    "in-headers": {
+    "inHeaders": {
-        "aliases": ["headers", "inheaders", "inHeaders", "in-headers", "in_headers"],
+        "aliases": ["headers"],
-        "description": "Only include the JSON-LD context in the headers",
         "required": True,
         "default": False,
         "options": boolean
@@ -202,8 +99,8 @@ WEB_PARAMS = {
 }
 
 CLI_PARAMS = {
-    "plugin-folder": {
+    "plugin_folder": {
-        "aliases": ["folder", "plugin_folder"],
+        "aliases": ["folder"],
         "required": True,
         "default": "."
     },
@@ -218,7 +115,6 @@ NIF_PARAMS = {
     },
     "intype": {
         "@id": "intype",
-        "description": "input type",
         "aliases": ["t"],
         "required": False,
         "default": "direct",
@@ -226,7 +122,6 @@ NIF_PARAMS = {
     },
     "informat": {
         "@id": "informat",
-        "description": "input format",
         "aliases": ["f"],
         "required": False,
         "default": "text",
@@ -234,20 +129,17 @@ NIF_PARAMS = {
     },
     "language": {
         "@id": "language",
-        "description": "language of the input",
         "aliases": ["l"],
         "required": False,
     },
     "prefix": {
         "@id": "prefix",
-        "description": "prefix to use for new entities",
         "aliases": ["p"],
         "required": True,
         "default": "",
     },
     "urischeme": {
         "@id": "urischeme",
-        "description": "scheme for NIF URIs",
         "aliases": ["u"],
         "required": False,
         "default": "RFC5147String",
@@ -278,24 +170,26 @@ def parse_params(indict, *specs):
                 if alias in indict and alias != param:
                     outdict[param] = indict[alias]
                     del outdict[alias]
-                    break
+                    continue
             if param not in outdict:
                 if "default" in options:
                     # We assume the default is correct
                     outdict[param] = options["default"]
                 elif options.get("required", False):
                     wrong_params[param] = spec[param]
-                continue
+            elif "options" in options:
-            if 'processor' in options:
-                outdict[param] = processors[options['processor']](outdict[param])
-            if "options" in options:
                 if options["options"] == boolean:
-                    outdict[param] = str(outdict[param]).lower() in ['true', '1', '']
+                    outdict[param] = str(outdict[param]).lower() in ['true', '1']
                 elif outdict[param] not in options["options"]:
                     wrong_params[param] = spec[param]
     if wrong_params:
         logger.debug("Error parsing: %s", wrong_params)
-        raise InvalidParams(wrong_params)
+        message = Error(
+            status=400,
+            message='Missing or invalid parameters',
+            parameters=outdict,
+            errors=wrong_params)
+        raise message
     return outdict
 
 
@@ -311,9 +205,10 @@ def get_all_params(plugins, *specs):
 def get_extra_params(plugins):
     '''Get a list of possible parameters given a list of plugins'''
     params = {}
-    extra_params: dict = {}
+    extra_params = {}
     for plugin in plugins:
-        for k, v in plugin.extra_params.items():
+        this_params = plugin.get('extra_params', {})
+        for k, v in this_params.items():
             if k not in extra_params:
                 extra_params[k] = {}
             extra_params[k][plugin.name] = v
@@ -357,7 +252,7 @@ def get_extra_params(plugins):
     return params
 
 
-def parse_analyses(params, plugins):
+def parse_analysis(params, plugins):
     '''
     Parse the given parameters individually for each plugin, and get a list of the parameters that
     belong to each of the plugins. Each item can then be used in the plugin.analyse_entries method.
@@ -367,7 +262,7 @@ def parse_analyses(params, plugins):
         if not plugin:
             continue
         this_params = filter_params(params, plugin, i)
-        parsed = parse_params(this_params, plugin.extra_params)
+        parsed = parse_params(this_params, plugin.get('extra_params', {}))
         analysis = plugin.activity(parsed)
         analysis_list.append(analysis)
     return analysis_list
@@ -409,7 +304,7 @@ def parse_call(params):
     params = parse_params(params, NIF_PARAMS)
     if params['informat'] == 'text':
         results = Results()
-        entry = Entry(text=params['input'], id='prefix:')  # Use @base
+        entry = Entry(nif__isString=params['input'], id='#')  # Use @base
         results.entries.append(entry)
     elif params['informat'] == 'json-ld':
         results = from_string(params['input'], cls=Results)

senpy/blueprints.py

@@ -1,33 +1,29 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
+# Copyright 2014 J. Fernando Sánchez Rada - Grupo de Sistemas Inteligentes
+# DIT, UPM
 #
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#    Licensed under the Apache License, Version 2.0 (the "License");
+# http://www.apache.org/licenses/LICENSE-2.0
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
 #
-#        http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
-#
-#    Unless required by applicable law or agreed to in writing, software
 #    distributed under the License is distributed on an "AS IS" BASIS,
 #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
-#
 """
 Blueprints for Senpy
 """
 from flask import (Blueprint, request, current_app, render_template, url_for,
                    jsonify, redirect)
-from .models import BASE_CONTEXT, Help, Plugins, Datasets
+from .models import Error, Response, Help, Plugins, read_schema, dump_schema, Datasets
-from .errors import Error
 from . import api
 from .version import __version__
 from functools import wraps
 
-from .gsitk_compat import GSITK_AVAILABLE, datasets
-
 import logging
 import json
 import base64
@@ -67,44 +63,44 @@ def get_params(req):
     return indict
 
 
-def encode_url(url=None):
+def encoded_url(url=None, base=None):
     code = ''
     if not url:
-        url = request.parameters.get('prefix', request.full_path[1:] + '#')
+        if request.method == 'GET':
-    return code or base64.urlsafe_b64encode(url.encode()).decode()
+            url = request.full_path[1:]  # Remove the first slash
+        else:
+            hash(frozenset(tuple(request.parameters.items())))
+            code = 'hash:{}'.format(hash)
 
+    code = code or base64.urlsafe_b64encode(url.encode()).decode()
 
-def url_for_code(code, base=None):
+    if base:
-    # if base:
+        return base + code
-    #     return base + code
+    return url_for('api.decode', code=code, _external=True)
-    # return url_for('api.decode', code=code, _external=True)
-    # This was producing unique yet very long URIs, which wasn't ideal for visualization.
-    return 'http://senpy.invalid/'
 
 
 def decoded_url(code, base=None):
-    path = base64.urlsafe_b64decode(code.encode()).decode()
+    if code.startswith('hash:'):
-    if path[:4] == 'http':
+        raise Exception('Can not decode a URL for a POST request')
-        return path
     base = base or request.url_root
+    path = base64.urlsafe_b64decode(code.encode()).decode()
     return base + path
 
 
 @demo_blueprint.route('/')
 def index():
-    # ev = str(get_params(request).get('evaluation', True))
+    ev = str(get_params(request).get('evaluation', False))
-    # evaluation_enabled = ev.lower() not in ['false', 'no', 'none']
+    evaluation_enabled = ev.lower() not in ['false', 'no', 'none']
-    evaluation_enabled = GSITK_AVAILABLE
 
     return render_template("index.html",
                            evaluation=evaluation_enabled,
                            version=__version__)
 
 
-@api_blueprint.route('/contexts/<code>')
+@api_blueprint.route('/contexts/<entity>.jsonld')
-def context(code=''):
+def context(entity="context"):
-    context = BASE_CONTEXT
+    context = Response._context
-    context['@base'] = url_for('api.decode', code=code, _external=True)
+    context['@vocab'] = url_for('ns.index', _external=True)
     context['endpoint'] = url_for('api.api_root', _external=True)
     return jsonify({"@context": context})
 
@@ -114,115 +110,46 @@ def decode(code):
     try:
         return redirect(decoded_url(code))
     except Exception:
-        return to_flask(Error('invalid URL'))
+        return Error('invalid URL').flask()
 
 
 @ns_blueprint.route('/')  # noqa: F811
 def index():
-    context = BASE_CONTEXT.copy()
+    context = Response._context.copy()
     context['endpoint'] = url_for('api.api_root', _external=True)
     return jsonify({"@context": context})
 
 
-# from .models import read_schema, dump_schema
+@api_blueprint.route('/schemas/<schema>')
-#@api_blueprint.route('/schemas/<schema>')
+def schema(schema="definitions"):
-#def schema(schema="definitions"):
+    try:
-#    try:
+        return dump_schema(read_schema(schema))
-#        return dump_schema(read_schema(schema))
+    except Exception as ex:  # Should be FileNotFoundError, but it's missing from py2
-#    except Exception as ex:  # Should be FileNotFoundError, but it's missing from py2
+        return Error(message="Schema not found: {}".format(ex), status=404).flask()
-#        return Error(message="Schema not found: {}".format(ex), status=404).flask()
-
-def to_flask(resp,
-             in_headers=False,
-             headers=None,
-             outformat='json-ld',
-             **kwargs):
-    """
-    Return the values and error to be used in flask.
-    So far, it returns a fixed context. We should store/generate different
-    contexts if the plugin adds more aliases.
-    """
-    headers = headers or {}
-    if isinstance(resp, Error):
-        status = resp.code
-        resp = resp.to_response()
-    else:
-        status = 200
-
-    kwargs["with_context"] = not in_headers
-    content, mimetype = resp.serialize(format=outformat,
-                                       with_mime=True,
-                                       **kwargs)
-
-    if outformat == 'json-ld' and in_headers:
-        headers.update({
-            "Link":
-            ('<%s>;'
-                'rel="http://www.w3.org/ns/json-ld#context";'
-                ' type="application/ld+json"' % kwargs.get('context_uri'))
-        })
-    return FlaskResponse(
-        response=content,
-        status=status,
-        headers=headers,
-        mimetype=mimetype)
 
 
 def basic_api(f):
     default_params = {
-        'in-headers': False,
+        'inHeaders': False,
         'expanded-jsonld': False,
         'outformat': None,
-        'with-parameters': True,
+        'with_parameters': True,
     }
 
     @wraps(f)
     def decorated_function(*args, **kwargs):
         raw_params = get_params(request)
-        # logger.info('Getting request: {}'.format(raw_params))
+        logger.info('Getting request: {}'.format(raw_params))
-        logger.debug('Getting request. Params: {}'.format(raw_params))
         headers = {'X-ORIGINAL-PARAMS': json.dumps(raw_params)}
         params = default_params
 
-        mime = request.accept_mimetypes\
-                      .best_match(MIMETYPES.keys(),
-                                  DEFAULT_MIMETYPE)
-
-        mimeformat = MIMETYPES.get(mime, DEFAULT_FORMAT)
-        outformat = mimeformat
-
         try:
             params = api.parse_params(raw_params, api.WEB_PARAMS, api.API_PARAMS)
-            outformat = params.get('outformat', mimeformat)
             if hasattr(request, 'parameters'):
                 request.parameters.update(params)
             else:
                 request.parameters = params
             response = f(*args, **kwargs)
-
-            if 'parameters' in response and not params['with-parameters']:
-                del response.parameters
-
-            logger.debug('Response: {}'.format(response))
-
-            prefix = params.get('prefix')
-            code = encode_url(prefix)
-
-            return to_flask(response,
-                            in_headers=params['in-headers'],
-                            headers=headers,
-                            prefix=prefix or url_for_code(code),
-                            base=prefix,
-                            context_uri=url_for('api.context',
-                                                code=code,
-                                                _external=True),
-                            outformat=outformat,
-                            expanded=params['expanded-jsonld'],
-                            template=params.get('template'),
-                            verbose=params['verbose'],
-                            aliases=params['aliases'],
-                            fields=params.get('fields'))
-
         except (Exception) as ex:
             if current_app.debug or current_app.config['TESTING']:
                 raise
@@ -232,51 +159,61 @@ def basic_api(f):
             response = ex
             response.parameters = raw_params
             logger.exception(ex)
-            return to_flask(response,
+
-                            outformat=outformat,
+        if 'parameters' in response and not params['with_parameters']:
-                            expanded=params['expanded-jsonld'],
+            del response.parameters
-                            verbose=params.get('verbose', True))
+
+        logger.info('Response: {}'.format(response))
+        mime = request.accept_mimetypes\
+                      .best_match(MIMETYPES.keys(),
+                                  DEFAULT_MIMETYPE)
+
+        mimeformat = MIMETYPES.get(mime, DEFAULT_FORMAT)
+        outformat = params['outformat'] or mimeformat
+
+        return response.flask(
+            in_headers=params['inHeaders'],
+            headers=headers,
+            prefix=params.get('prefix', encoded_url()),
+            context_uri=url_for('api.context',
+                                entity=type(response).__name__,
+                                _external=True),
+            outformat=outformat,
+            expanded=params['expanded-jsonld'])
 
     return decorated_function
 
 
-@api_blueprint.route('/', defaults={'plugins': None}, methods=['POST', 'GET'], strict_slashes=False)
+@api_blueprint.route('/', defaults={'plugin': None}, methods=['POST', 'GET'])
-@api_blueprint.route('/<path:plugins>', methods=['POST', 'GET'], strict_slashes=False)
+@api_blueprint.route('/<path:plugin>', methods=['POST', 'GET'])
 @basic_api
-def api_root(plugins):
+def api_root(plugin):
-    if plugins:
+    if plugin:
         if request.parameters['algorithm'] != api.API_PARAMS['algorithm']['default']:
             raise Error('You cannot specify the algorithm with a parameter and a URL variable.'
                         ' Please, remove one of them')
-        plugins = plugins.replace('+', ',').replace('/', ',')
+        request.parameters['algorithm'] =  tuple(plugin.replace('+', '/').split('/'))
-        plugins = api.processors['string_to_tuple'](plugins)
-    else:
-        plugins = request.parameters['algorithm']
 
-    print(plugins)
+    params = request.parameters
+    plugin = request.parameters['algorithm']
 
     sp = current_app.senpy
-    plugins = sp.get_plugins(plugins)
+    plugins = sp.get_plugins(plugin)
 
     if request.parameters['help']:
-        apis = [api.WEB_PARAMS, api.API_PARAMS, api.NIF_PARAMS]
+        apis = []
-        # Verbose is set to False as default, but we want it to default to
+        if request.parameters['verbose']:
-        # True for help. This checks the original value, to make sure it wasn't
+            apis.append(api.BUILTIN_PARAMS)
-        # set by default.
-        if not request.parameters['verbose'] and get_params(request).get('verbose'):
-            apis = []
-        if request.parameters['algorithm'] == ['default', ]:
-            plugins = []
         allparameters = api.get_all_params(plugins, *apis)
         response = Help(valid_parameters=allparameters)
         return response
     req = api.parse_call(request.parameters)
-    analyses = api.parse_analyses(req.parameters, plugins)
+    analysis = api.parse_analysis(req.parameters, plugins)
-    results = current_app.senpy.analyse(req, analyses)
+    results = current_app.senpy.analyse(req, analysis)
     return results
 
 
-@api_blueprint.route('/evaluate', methods=['POST', 'GET'], strict_slashes=False)
+@api_blueprint.route('/evaluate/', methods=['POST', 'GET'])
 @basic_api
 def evaluate():
     if request.parameters['help']:
@@ -289,26 +226,28 @@ def evaluate():
         return response
 
 
-@api_blueprint.route('/plugins', methods=['POST', 'GET'], strict_slashes=False)
+@api_blueprint.route('/plugins/', methods=['POST', 'GET'])
 @basic_api
 def plugins():
     sp = current_app.senpy
     params = api.parse_params(request.parameters, api.PLUGINS_PARAMS)
-    ptype = params.get('plugin-type')
+    ptype = params.get('plugin_type')
-    plugins = list(sp.analysis_plugins(plugin_type=ptype))
+    plugins = list(sp.plugins(plugin_type=ptype))
     dic = Plugins(plugins=plugins)
     return dic
 
 
-@api_blueprint.route('/plugins/<plugin>', methods=['POST', 'GET'], strict_slashes=False)
+@api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
 @basic_api
 def plugin(plugin):
     sp = current_app.senpy
     return sp.get_plugin(plugin)
 
 
-@api_blueprint.route('/datasets', methods=['POST', 'GET'], strict_slashes=False)
+@api_blueprint.route('/datasets/', methods=['POST', 'GET'])
 @basic_api
-def get_datasets():
+def datasets():
+    sp = current_app.senpy
+    datasets = sp.datasets
     dic = Datasets(datasets=list(datasets.values()))
     return dic

senpy/cli.py

@@ -1,22 +1,5 @@
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-from __future__ import print_function
-
 import sys
-from .errors import Error
+from .models import Error
 from .extensions import Senpy
 from . import api
 
@@ -44,11 +27,16 @@ def main_function(argv):
                               api.CLI_PARAMS,
                               api.API_PARAMS,
                               api.NIF_PARAMS)
-    plugin_folder = params['plugin-folder']
+    plugin_folder = params['plugin_folder']
-    default_plugins = not params.get('no-default-plugins', False)
+    default_plugins = params.get('default-plugins', False)
     sp = Senpy(default_plugins=default_plugins, plugin_folder=plugin_folder)
     request = api.parse_call(params)
     algos = sp.get_plugins(request.parameters.get('algorithm', None))
+    if algos:
+        for algo in algos:
+            sp.activate_plugin(algo.name)
+    else:
+        sp.activate_all()
     res = sp.analyse(request)
     return res
 
@@ -60,7 +48,7 @@ def main():
         res = main_function(sys.argv[1:])
         print(res.serialize())
     except Error as err:
-        print(err.serialize(), file=sys.stderr)
+        print(err.serialize())
         sys.exit(2)
 
 

senpy/client.py

@@ -1,19 +1,3 @@
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
 import requests
 import logging
 from . import models

senpy/config.py

@@ -1,7 +0,0 @@
-import os
-
-strict = os.environ.get('SENPY_STRICT', '').lower() not in ["", "false", "f"]
-data_folder = os.environ.get('SENPY_DATA', None)
-if data_folder:
-    data_folder = os.path.abspath(data_folder)
-testing = os.environ.get('SENPY_TESTING', "") != ""

senpy/errors.py

@@ -1,23 +0,0 @@
-from .models import ErrorResponse
-
-class Error(Exception):
-    def __init__(self, message='Generic senpy exception', errors=[]):
-        Exception.__init__(self, message)
-        self.message = message
-        self.errors = errors
-
-    def toResponse(self) -> ErrorResponse:
-        return ErrorResponse(self.message)
-
-    def __str__(self):
-        if not hasattr(self, 'errors'):
-            return self.message
-        return '{}:\n\t{}'.format(self.message, self.errors)
-
-    def __hash__(self):
-        return Exception.__hash__(self)
-
-
-class InvalidParams(Error):
-    def __init__(self, wrong_params):
-        super().__init__(message='Wrong parameters:\n\t{}'.format(wrong_params))

senpy/extensions.py

@@ -1,30 +1,12 @@
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
 """
 Main class for Senpy.
-It orchestrates plugin discovery, creation and analysis.
+It orchestrates plugin (de)activation and analysis.
 """
 from future import standard_library
 standard_library.install_aliases()
 
-from . import config
 from . import plugins, api
-from .models import AggregatedEvaluation
+from .models import Error, AggregatedEvaluation
-from .errors import Error
-from .plugins import AnalysisPlugin
 from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
 
 from threading import Thread
@@ -44,13 +26,10 @@ class Senpy(object):
 
     def __init__(self,
                  app=None,
-                 plugin_folders=[".", ],
+                 plugin_folder=".",
                  data_folder=None,
-                 install=False,
-                 strict=None,
                  default_plugins=False):
 
-
         default_data = os.path.join(os.getcwd(), 'senpy_data')
         self.data_folder = data_folder or os.environ.get('SENPY_DATA', default_data)
         try:
@@ -62,26 +41,19 @@ class Senpy(object):
                 raise
 
         self._default = None
-        self.strict = strict if strict is not None else config.strict
-        self.strict = True # TODO: remove this after tests pass
-        self.install = install
         self._plugins = {}
-        self.plugin_folders = plugin_folders
+        if plugin_folder:
-        for folder in plugin_folders:
+            self.add_folder(plugin_folder)
-            self._add_folder(folder)
 
         if default_plugins:
-            self._add_folder('plugins', from_root=True)
+            self.add_folder('plugins', from_root=True)
         else:
             # Add only conversion plugins
-            self._add_folder(os.path.join('plugins', 'postprocessing'),
+            self.add_folder(os.path.join('plugins', 'postprocessing'),
                             from_root=True)
         self.app = app
         if app is not None:
             self.init_app(app)
-        self._conversion_candidates = {}
-        if self.install:
-            self.install_deps()
 
     def init_app(self, app):
         """ Initialise a flask app to add plugins to its context """
@@ -101,20 +73,15 @@ class Senpy(object):
         app.register_blueprint(demo_blueprint, url_prefix="/")
 
     def add_plugin(self, plugin):
-        name = plugin.name.lower()
+        self._plugins[plugin.name.lower()] = plugin
-        assert name
-        self._plugins[name] = plugin
-        self._conversion_candidates = {}
 
     def delete_plugin(self, plugin):
         del self._plugins[plugin.name.lower()]
 
-    def plugins(self, plugin_type=None, **kwargs):
+    def plugins(self, plugin_type=None, is_activated=True, **kwargs):
         """ Return the plugins registered for a given application. Filtered by criteria  """
-        return sorted(plugins.pfilter(self._plugins,
+        return list(plugins.pfilter(self._plugins, plugin_type=plugin_type,
-                                      plugin_type=plugin_type,
+                                    is_activated=is_activated, **kwargs))
-                                      **kwargs),
-                      key=lambda x: x.id)
 
     def get_plugin(self, name, default=None):
         if name == 'default':
@@ -137,7 +104,7 @@ class Senpy(object):
 
         msg = ("Plugin not found: '{}'\n"
                "Make sure it is ACTIVATED\n"
-               "Valid algorithms: {}").format(name,
+                "Valid algorithms: {}").format(name,
                                               self._plugins.keys())
         raise Error(message=msg, status=404)
 
@@ -148,43 +115,60 @@ class Senpy(object):
             pass  # Assume it is a tuple or a list
         return tuple(self.get_plugin(n) for n in name)
 
-    def analysis_plugins(self, **kwargs):
+    @property
+    def analysis_plugins(self):
         """ Return only the analysis plugins that are active"""
-        candidates = self.plugins(**kwargs)
+        return self.plugins(plugin_type='analysisPlugin', is_activated=True)
-        return list(plugins.pfilter(candidates, plugin_type=AnalysisPlugin))
 
-    def _add_folder(self, folder, from_root=False):
+    def add_folder(self, folder, from_root=False):
         """ Find plugins in this folder and add them to this instance """
         if from_root:
             folder = os.path.join(os.path.dirname(__file__), folder)
         logger.debug("Adding folder: %s", folder)
         if os.path.isdir(folder):
             new_plugins = plugins.from_folder([folder],
-                                               data_folder=self.data_folder,
+                                              data_folder=self.data_folder)
-                                               strict=self.strict)
             for plugin in new_plugins:
                 self.add_plugin(plugin)
         else:
             raise AttributeError("Not a folder or does not exist: %s", folder)
 
-    def _process(self, req, pending):
+    # def check_analysis_request(self, analysis):
+    #     '''Check if the analysis request can be fulfilled'''
+    #     if not self.plugins():
+    #         raise Error(
+    #             status=404,
+    #             message=("No plugins found."
+    #                      " Please install one."))
+    #     for a in analysis:
+    #         algo = a.algorithm
+    #         if algo == 'default' and not self.default_plugin:
+    #             raise Error(
+    #                 status=404,
+    #                 message="No default plugin found, and None provided")
+    #         else:
+    #             self.get_plugin(algo)
+
+
+    def _process(self, req, pending, done=None):
         """
         Recursively process the entries with the first plugin in the list, and pass the results
         to the rest of the plugins.
         """
+        done = done or []
         if not pending:
             return req
 
         analysis = pending[0]
         results = analysis.run(req)
-        results.activities.append(analysis)
+        results.analysis.append(analysis)
-        return self._process(results, pending[1:])
+        done += analysis
+        return self._process(results, pending[1:], done)
 
     def install_deps(self):
-        logger.info('Installing dependencies')
+        plugins.install_deps(*self.plugins())
-        plugins.install_deps(*self._plugins.values())
 
-    def analyse(self, request, analyses=None):
+    def analyse(self, request, analysis=None):
         """
         Main method that analyses a request, either from CLI or HTTP.
         It takes a processed request, provided by the user, as returned
@@ -195,17 +179,18 @@ class Senpy(object):
                 status=404,
                 message=("No plugins found."
                          " Please install one."))
-        if analyses is None:
+        if analysis is None:
+            params = str(request)
             plugins = self.get_plugins(request.parameters['algorithm'])
-            analyses = api.parse_analyses(request.parameters, plugins)
+            analysis = api.parse_analysis(request.parameters, plugins)
         logger.debug("analysing request: {}".format(request))
-        results = self._process(request, analyses)
+        results = self._process(request, analysis)
         logger.debug("Got analysis result: {}".format(results))
-        results = self.postprocess(results, analyses)
+        results = self.postprocess(results)
         logger.debug("Returning post-processed result: {}".format(results))
         return results
 
-    def convert_emotions(self, resp, analyses):
+    def convert_emotions(self, resp):
         """
         Conversion of all emotions in a response **in place**.
         In addition to converting from one model to another, it has
@@ -213,126 +198,128 @@ class Senpy(object):
         Needless to say, this is far from an elegant solution, but it works.
         @todo refactor and clean up
         """
+        plugins = resp.analysis
 
-        logger.debug("Converting emotions")
         if 'parameters' not in resp:
-            logger.debug("NO PARAMETERS")
             return resp
 
         params = resp['parameters']
-        toModel = params.get('emotion-model', None)
+        toModel = params.get('emotionModel', None)
         if not toModel:
-            logger.debug("NO tomodel PARAMETER")
             return resp
 
         logger.debug('Asked for model: {}'.format(toModel))
         output = params.get('conversion', None)
-
+        candidates = {}
+        for plugin in plugins:
+            try:
+                fromModel = plugin.get('onyx:usesEmotionModel', None)
+                candidates[plugin.id] = next(self._conversion_candidates(fromModel, toModel))
+                logger.debug('Analysis plugin {} uses model: {}'.format(
+                    plugin.id, fromModel))
+            except StopIteration:
+                e = Error(('No conversion plugin found for: '
+                           '{} -> {}'.format(fromModel, toModel)),
+                          status=404)
+                e.original_response = resp
+                e.parameters = params
+                raise e
         newentries = []
         done = []
         for i in resp.entries:
-
             if output == "full":
                 newemotions = copy.deepcopy(i.emotions)
             else:
                 newemotions = []
             for j in i.emotions:
-                activity = j['prov:wasGeneratedBy']
+                plugname = j['prov:wasGeneratedBy']
-                act = resp.activity(activity)
+                candidate = candidates[plugname]
-                if not act:
+                done.append({'plugin': candidate, 'parameters': params})
-                    raise Error('Could not find the emotion model for {}'.format(activity))
-                fromModel = act.plugin['onyx:usesEmotionModel']
-                if toModel == fromModel:
-                    continue
-                candidate = self._conversion_candidate(fromModel, toModel)
-                if not candidate:
-                    e = Error(('No conversion plugin found for: '
-                              '{} -> {}'.format(fromModel, toModel)),
-                              status=404)
-                    e.original_response = resp
-                    e.parameters = params
-                    raise e
-
-                analysis = candidate.activity(params)
-                done.append(analysis)
                 for k in candidate.convert(j, fromModel, toModel, params):
-                    k.prov__wasGeneratedBy = analysis.id
+                    k.prov__wasGeneratedBy = candidate.id
                     if output == 'nested':
                         k.prov__wasDerivedFrom = j
                     newemotions.append(k)
             i.emotions = newemotions
             newentries.append(i)
         resp.entries = newentries
-        resp.activities.extend(done)
         return resp
 
-    def _conversion_candidate(self, fromModel, toModel):
+    def _conversion_candidates(self, fromModel, toModel):
-        if not self._conversion_candidates:
+        candidates = self.plugins(plugin_type=plugins.EmotionConversion)
-            candidates = {}
+        for candidate in candidates:
-            for conv in self.plugins(plugin_type=plugins.EmotionConversion):
+            for pair in candidate.onyx__doesConversion:
-                for pair in conv.onyx__doesConversion:
+                logging.debug(pair)
-                    logging.debug(pair)
+                if candidate.can_convert(fromModel, toModel):
-                    key = (pair['onyx:conversionFrom'], pair['onyx:conversionTo'])
+                    yield candidate
-                    if key not in candidates:
-                        candidates[key] = []
-                    candidates[key].append(conv)
-            self._conversion_candidates = candidates
 
-        key = (fromModel, toModel)
+    def postprocess(self, response):
-        if key not in self._conversion_candidates:
-            return None
-        return self._conversion_candidates[key][0]
-
-    def postprocess(self, response, analyses):
         '''
         Transform the results from the analysis plugins.
         It has some pre-defined post-processing like emotion conversion,
         and it also allows plugins to auto-select themselves.
         '''
 
-        response = self.convert_emotions(response, analyses)
+        response = self.convert_emotions(response)
 
         for plug in self.plugins(plugin_type=plugins.PostProcessing):
-            if plug.check(response, response.activities):
+            if plug.check(response, response.analysis):
-                activity = plug.activity(response.parameters)
+                response = plug.process(response)
-                response = plug.process(response, activity)
         return response
 
     def _get_datasets(self, request):
+        if not self.datasets:
+            raise Error(
+                status=404,
+                message=("No datasets found."
+                         " Please verify DatasetManager"))
         datasets_name = request.parameters.get('dataset', None).split(',')
         for dataset in datasets_name:
-            if dataset not in gsitk_compat.datasets:
+            if dataset not in self.datasets:
                 logger.debug(("The dataset '{}' is not valid\n"
                               "Valid datasets: {}").format(
-                                  dataset, gsitk_compat.datasets.keys()))
+                                  dataset, self.datasets.keys()))
                 raise Error(
                     status=404,
                     message="The dataset '{}' is not valid".format(dataset))
-        return datasets_name
+        dm = gsitk_compat.DatasetManager()
+        datasets = dm.prepare_datasets(datasets_name)
+        return datasets
+
+    @property
+    def datasets(self):
+        self._dataset_list = {}
+        dm = gsitk_compat.DatasetManager()
+        for item in dm.get_datasets():
+            for key in item:
+                if key in self._dataset_list:
+                    continue
+                properties = item[key]
+                properties['@id'] = key
+                self._dataset_list[key] = properties
+        return self._dataset_list
 
     def evaluate(self, params):
         logger.debug("evaluating request: {}".format(params))
         results = AggregatedEvaluation()
         results.parameters = params
         datasets = self._get_datasets(results)
-        plugs = []
+        plugins = []
-        for plugname in params['algorithm']:
+        for plugname in params.algorithm:
-            plugs = self.get_plugins(plugname)
+            plugins = self.get_plugin(plugname)
-        for plug in plugs:
-            if not isinstance(plug, plugins.Evaluable):
-                raise Exception('Plugin {} can not be evaluated', plug.id)
 
-        for eval in plugins.evaluate(plugs, datasets):
+        for eval in plugins.evaluate(plugins, datasets):
             results.evaluations.append(eval)
-        if 'with-parameters' not in results.parameters:
+        if 'with_parameters' not in results.parameters:
             del results.parameters
         logger.debug("Returning evaluation result: {}".format(results))
         return results
 
     @property
     def default_plugin(self):
-        if not self._default:
+        if not self._default or not self._default.is_activated:
-            candidates = self.analysis_plugins()
+            candidates = self.plugins(
+                plugin_type='analysisPlugin', is_activated=True)
             if len(candidates) > 0:
                 self._default = candidates[0]
             else:
@@ -343,10 +330,88 @@ class Senpy(object):
     @default_plugin.setter
     def default_plugin(self, value):
         if isinstance(value, plugins.Plugin):
+            if not value.is_activated:
+                raise AttributeError('The default plugin has to be activated.')
             self._default = value
 
         else:
             self._default = self._plugins[value.lower()]
 
+    def activate_all(self, sync=True, allow_fail=False):
+        ps = []
+        for plug in self._plugins.keys():
+            try:
+                self.activate_plugin(plug, sync=sync)
+            except Exception as ex:
+                if not allow_fail:
+                    raise
+                logger.error('Could not activate {}: {}'.format(plug, ex))
+        return ps
+
+    def deactivate_all(self, sync=True):
+        ps = []
+        for plug in self._plugins.keys():
+            ps.append(self.deactivate_plugin(plug, sync=sync))
+        return ps
+
+    def _set_active(self, plugin, active=True, *args, **kwargs):
+        ''' We're using a variable in the plugin itself to activate/deactivate plugins.\
+        Note that plugins may activate themselves by setting this variable.
+        '''
+        plugin.is_activated = active
+
+    def _activate(self, plugin):
+        success = False
+        with plugin._lock:
+            if plugin.is_activated:
+                return
+            plugin.activate()
+            msg = "Plugin activated: {}".format(plugin.name)
+            logger.info(msg)
+            success = True
+            self._set_active(plugin, success)
+        return success
+
+    def activate_plugin(self, plugin_name, sync=True):
+        plugin_name = plugin_name.lower()
+        if plugin_name not in self._plugins:
+            raise Error(
+                message="Plugin not found: {}".format(plugin_name), status=404)
+        plugin = self._plugins[plugin_name]
+
+        logger.info("Activating plugin: {}".format(plugin.name))
+
+        if sync or not getattr(plugin, 'async', True) or getattr(
+                plugin, 'sync', False):
+            return self._activate(plugin)
+        else:
+            th = Thread(target=partial(self._activate, plugin))
+            th.start()
+            return th
+
+    def _deactivate(self, plugin):
+        with plugin._lock:
+            if not plugin.is_activated:
+                return
+            plugin.deactivate()
+            logger.info("Plugin deactivated: {}".format(plugin.name))
+
+    def deactivate_plugin(self, plugin_name, sync=True):
+        plugin_name = plugin_name.lower()
+        if plugin_name not in self._plugins:
+            raise Error(
+                message="Plugin not found: {}".format(plugin_name), status=404)
+        plugin = self._plugins[plugin_name]
+
+        self._set_active(plugin, False)
+
+        if sync or not getattr(plugin, 'async', True) or not getattr(
+                plugin, 'sync', False):
+            self._deactivate(plugin)
+        else:
+            th = Thread(target=partial(self._deactivate, plugin))
+            th.start()
+            return th
+
     def teardown(self, exception):
         pass

senpy/gsitk_compat.py

@@ -1,21 +1,4 @@
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
 import logging
-import os
 
 from pkg_resources import parse_version, get_distribution, DistributionNotFound
 
@@ -33,35 +16,16 @@ def raise_exception(*args, **kwargs):
 try:
     gsitk_distro = get_distribution("gsitk")
     GSITK_VERSION = parse_version(gsitk_distro.version)
-
+    GSITK_AVAILABLE = GSITK_VERSION > parse_version("0.1.9.1")  # Earlier versions have a bug
-    if not os.environ.get('DATA_PATH'):
+except DistributionNotFound:
-        os.environ['DATA_PATH'] = os.environ.get('SENPY_DATA', 'senpy_data')
-
-    from gsitk.datasets.datasets import DatasetManager
-    from gsitk.evaluation.evaluation import Evaluation as Eval  # noqa: F401
-    from gsitk.evaluation.evaluation import EvalPipeline  # noqa: F401
-    from sklearn.pipeline import Pipeline
-    modules = locals()
-    GSITK_AVAILABLE = True
-    datasets = {}
-    manager = DatasetManager()
-
-    for item in manager.get_datasets():
-        for key in item:
-            if key in datasets:
-                continue
-            properties = item[key]
-            properties['@id'] = key
-            datasets[key] = properties
-
-    def prepare(ds, *args, **kwargs):
-        return manager.prepare_datasets(ds, *args, **kwargs)
-
-
-except (DistributionNotFound, ImportError) as err:
-    logger.debug('Error importing GSITK: {}'.format(err))
-    logger.warning(IMPORTMSG)
     GSITK_AVAILABLE = False
     GSITK_VERSION = ()
-    DatasetManager = Eval = Pipeline = prepare = raise_exception
+
-    datasets = {}
+if GSITK_AVAILABLE:
+    from gsitk.datasets.datasets import DatasetManager
+    from gsitk.evaluation.evaluation import Evaluation as Eval
+    from sklearn.pipeline import Pipeline
+    modules = locals()
+else:
+    logger.warning(IMPORTMSG)
+    DatasetManager = Eval = Pipeline = raise_exception

senpy/meta.py

@@ -0,0 +1,258 @@
+'''
+Meta-programming for the models.
+'''
+import os
+import json
+import jsonschema
+import inspect
+import copy
+
+from abc import ABCMeta
+from collections import MutableMapping, namedtuple
+
+
+class BaseMeta(ABCMeta):
+    '''
+    Metaclass for models. It extracts the default values for the fields in
+    the model.
+
+    For instance, instances of the following class wouldn't need to mark
+    their version or description on initialization:
+
+    .. code-block:: python
+
+       class MyPlugin(Plugin):
+           version=0.3
+           description='A dull plugin'
+
+
+    Note that these operations could be included in the __init__ of the
+    class, but it would be very inefficient.
+    '''
+    _subtypes = {}
+
+    def __new__(mcs, name, bases, attrs, **kwargs):
+        register_afterwards = False
+        defaults = {}
+
+        attrs = mcs.expand_with_schema(name, attrs)
+        if 'schema' in attrs:
+            register_afterwards = True
+        for base in bases:
+            if hasattr(base, '_defaults'):
+                defaults.update(getattr(base, '_defaults'))
+
+        info, rest = mcs.split_attrs(attrs)
+
+        for i in list(info.keys()):
+            if isinstance(info[i], _Alias):
+                fget, fset, fdel = make_property(info[i].indict)
+                rest[i] = property(fget=fget, fset=fset, fdel=fdel)
+            else:
+                defaults[i] = info[i]
+
+        rest['_defaults'] = defaults
+
+        cls = super(BaseMeta, mcs).__new__(mcs, name, tuple(bases), rest)
+
+        if register_afterwards:
+            mcs.register(cls, defaults['@type'])
+        return cls
+
+    @classmethod
+    def register(mcs, rsubclass, rtype=None):
+        mcs._subtypes[rtype or rsubclass.__name__] = rsubclass
+
+    @staticmethod
+    def expand_with_schema(name, attrs):
+        if 'schema' in attrs:  # Schema specified by name
+            schema_file = '{}.json'.format(attrs['schema'])
+        elif 'schema_file' in attrs:
+            schema_file = attrs['schema_file']
+            del attrs['schema_file']
+        else:
+            return attrs
+
+        if '/' not in 'schema_file':
+            thisdir = os.path.dirname(os.path.realpath(__file__))
+            schema_file = os.path.join(thisdir,
+                                       'schemas',
+                                       schema_file)
+
+        schema_path = 'file://' + schema_file
+
+        with open(schema_file) as f:
+            schema = json.load(f)
+
+        resolver = jsonschema.RefResolver(schema_path, schema)
+        if '@type' not in attrs:
+            attrs['@type'] = "".join((name[0].lower(), name[1:]))
+        attrs['_schema_file'] = schema_file
+        attrs['schema'] = schema
+        attrs['_validator'] = jsonschema.Draft4Validator(schema, resolver=resolver)
+
+        schema_defaults = BaseMeta.get_defaults(attrs['schema'])
+        attrs.update(schema_defaults)
+
+        return attrs
+
+    @staticmethod
+    def is_func(v):
+        return inspect.isroutine(v) or inspect.ismethod(v) or \
+            inspect.ismodule(v) or isinstance(v, property)
+
+    @staticmethod
+    def is_internal(k):
+        return k[0] == '_' or k == 'schema' or k == 'data'
+
+    @staticmethod
+    def get_key(key):
+        if key[0] != '_':
+            key = key.replace("__", ":", 1)
+        return key
+
+    @staticmethod
+    def split_attrs(attrs):
+        '''
+        Extract the attributes of the class.
+
+        This allows adding default values in the class definition.
+        e.g.:
+        '''
+        isattr = {}
+        rest = {}
+        for key, value in attrs.items():
+            if not (BaseMeta.is_internal(key)) and (not BaseMeta.is_func(value)):
+                isattr[key] = value
+            else:
+                rest[key] = value
+        return isattr, rest
+
+    @staticmethod
+    def get_defaults(schema):
+        temp = {}
+        for obj in [
+                schema,
+        ] + schema.get('allOf', []):
+            for k, v in obj.get('properties', {}).items():
+                if 'default' in v and k not in temp:
+                    temp[k] = v['default']
+        return temp
+
+
+def make_property(key):
+
+    def fget(self):
+        return self[key]
+
+    def fdel(self):
+        del self[key]
+
+    def fset(self, value):
+        self[key] = value
+
+    return fget, fset, fdel
+
+
+class CustomDict(MutableMapping, object):
+    '''
+    A dictionary whose elements can also be accessed as attributes. Since some
+    characters are not valid in the dot-notation, the attribute names also
+    converted. e.g.:
+
+    > d = CustomDict()
+    > d.key = d['ns:name'] = 1
+    > d.key == d['key']
+    True
+    > d.ns__name == d['ns:name']
+    '''
+
+    _defaults = {}
+    _map_attr_key = {'id': '@id'}
+
+    def __init__(self, *args, **kwargs):
+        super(CustomDict, self).__init__()
+        for k, v in self._defaults.items():
+            self[k] = copy.copy(v)
+        for arg in args:
+            self.update(arg)
+        for k, v in kwargs.items():
+            self[self._attr_to_key(k)] = v
+        return self
+
+    def serializable(self):
+        def ser_or_down(item):
+            if hasattr(item, 'serializable'):
+                return item.serializable()
+            elif isinstance(item, dict):
+                temp = dict()
+                for kp in item:
+                    vp = item[kp]
+                    temp[kp] = ser_or_down(vp)
+                return temp
+            elif isinstance(item, list) or isinstance(item, set):
+                return list(ser_or_down(i) for i in item)
+            else:
+                return item
+
+        return ser_or_down(self.as_dict())
+
+    def __getitem__(self, key):
+        key = self._key_to_attr(key)
+        return self.__dict__[key]
+
+    def __setitem__(self, key, value):
+        '''Do not insert data directly, there might be a property in that key. '''
+        key = self._key_to_attr(key)
+        return setattr(self, key, value)
+
+    def as_dict(self):
+        return {self._attr_to_key(k): v for k, v in self.__dict__.items()
+                if not self._internal_key(k)}
+
+    def __iter__(self):
+        return (k for k in self.__dict__ if not self._internal_key(k))
+
+    def __len__(self):
+        return len(self.__dict__)
+
+    def __delitem__(self, key):
+        del self.__dict__[key]
+
+    def update(self, other):
+        for k, v in other.items():
+            self[k] = v
+
+    def _attr_to_key(self, key):
+        key = key.replace("__", ":", 1)
+        key = self._map_attr_key.get(key, key)
+        return key
+
+    def _key_to_attr(self, key):
+        if self._internal_key(key):
+            return key
+        key = key.replace(":", "__", 1)
+        return key
+
+    def __getattr__(self, key):
+        try:
+            return self.__dict__[self._attr_to_key(key)]
+        except KeyError:
+            raise AttributeError
+
+    @staticmethod
+    def _internal_key(key):
+        return key[0] == '_'
+
+    def __str__(self):
+        return json.dumps(self.serializable(), sort_keys=True, indent=4)
+
+    def __repr__(self):
+        return json.dumps(self.serializable(), sort_keys=True, indent=4)
+
+
+_Alias = namedtuple('Alias', 'indict')
+
+
+def alias(key):
+    return _Alias(key)

senpy/models.py

@@ -1,101 +1,197 @@
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
 '''
 Senpy Models.
+
+This implementation should mirror the JSON schema definition.
+For compatibility with Py3 and for easier debugging, this new version drops
+introspection and adds all arguments to the models.
 '''
+from __future__ import print_function
+from future import standard_library
+standard_library.install_aliases()
 
+from future.utils import with_metaclass
+from past.builtins import basestring
 
-from typing import *
+import time
-
+import copy
-import os
-import logging
-import jmespath
 import json
+import os
+import jsonref
+from flask import Response as FlaskResponse
+from pyld import jsonld
 
-from textwrap import dedent
+import logging
-
-from pydantic import BaseModel as PydanticModel
-from pydantic import computed_field, field_validator, ConfigDict, Field
-from pydantic_core import from_json
 
 logging.getLogger('rdflib').setLevel(logging.WARN)
 logger = logging.getLogger(__name__)
 
-
+from rdflib import Graph
-CONTEXT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
-                            'schemas',
-                            'context.jsonld')
-
-with open(CONTEXT_PATH) as f:
-    BASE_CONTEXT = dict(json.loads(f.read()))
 
 
-class BaseModel(PydanticModel):
+from .meta import BaseMeta, CustomDict, alias
+
+DEFINITIONS_FILE = 'definitions.json'
+CONTEXT_PATH = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld')
+
+
+def get_schema_path(schema_file, absolute=False):
+    if absolute:
+        return os.path.realpath(schema_file)
+    else:
+        return os.path.join(
+            os.path.dirname(os.path.realpath(__file__)), 'schemas',
+            schema_file)
+
+
+def read_schema(schema_file, absolute=False):
+    schema_path = get_schema_path(schema_file, absolute)
+    schema_uri = 'file://{}'.format(schema_path)
+    with open(schema_path) as f:
+        return jsonref.load(f, base_uri=schema_uri)
+
+
+def dump_schema(schema):
+    return jsonref.dumps(schema)
+
+
+def load_context(context):
+    logging.debug('Loading context: {}'.format(context))
+    if not context:
+        return context
+    elif isinstance(context, list):
+        contexts = []
+        for c in context:
+            contexts.append(load_context(c))
+        return contexts
+    elif isinstance(context, dict):
+        return dict(context)
+    elif isinstance(context, basestring):
+        try:
+            with open(context) as f:
+                return dict(json.loads(f.read()))
+        except IOError:
+            return context
+    else:
+        raise AttributeError('Please, provide a valid context')
+
+
+base_context = load_context(CONTEXT_PATH)
+
+
+def register(rsubclass, rtype=None):
+    BaseMeta.register(rsubclass, rtype)
+
+
+class BaseModel(with_metaclass(BaseMeta, CustomDict)):
     '''
     Entities of the base model are a special kind of dictionary that emulates
-    a JSON-LD object.
+    a JSON-LD object. The structure of the dictionary is checked via JSON-schema.
-    The basic attributes should be set in the class, but additional values can 
+    For convenience, the values can also be accessed as attributes
-    also be provided/set via attributes (à la Javascript). e.g.:
+    (a la Javascript). e.g.:
 
     >>> myobject.key == myobject['key']
     True
-    >>> myobject['ns:name'] = 'Peter'
+    >>> myobject.ns__name == myobject['ns:name']
     True
+
+    Additionally, subclasses of this class can specify default values for their
+    instances. These defaults are inherited by subclasses. e.g.:
+
+    >>> class NewModel(BaseModel):
+    ...     mydefault = 5
+    >>> n1 = NewModel()
+    >>> n1['mydefault'] == 5
+    True
+    >>> n1.mydefault = 3
+    >>> n1['mydefault'] = 3
+    True
+    >>> n2 = NewModel()
+    >>> n2 == 5
+    True
+    >>> class SubModel(NewModel):
+            pass
+    >>> subn = SubModel()
+    >>> subn.mydefault == 5
+    True
+
+    Lastly, every subclass that also specifies a schema will get registered, so it
+    is possible to deserialize JSON and get the right type.
+    i.e. to recover an instance of the original class from a plain JSON.
+
     '''
-    model_config = ConfigDict(extra='ignore', ignored_types=(object, ))
+
-    context: Optional[Union[str, Dict]] = Field(default=None, serialization_alias="@context")
+    # schema_file = DEFINITIONS_FILE
-    wasGeneratedBy: Optional[str] = Field(default=None, serialization_alias="prov:wasGeneratedBy")
+    _context = base_context["@context"]
+
+    def __init__(self, *args, **kwargs):
+        auto_id = kwargs.pop('_auto_id', False)
+
+        super(BaseModel, self).__init__(*args, **kwargs)
+
+        if auto_id:
+            self.id
+
+        if '@type' not in self:
+            logger.warning('Created an instance of an unknown model')
 
     @property
     def id(self):
         if '@id' not in self:
-            self['@id'] = 'prefix:{}_{}'.format(type(self).__name__, time.time())
+            self['@id'] = '_:{}_{}'.format(type(self).__name__, time.time())
         return self['@id']
 
     @id.setter
     def id(self, value):
         self['@id'] = value
 
-    def serialize(self, format='json-ld', with_mime=False,
+    def flask(self,
-                  template=None, prefix=None, fields=None, **kwargs):
+              in_headers=False,
-        js = self.jsonld(prefix=prefix, **kwargs)
+              headers=None,
-        if template is not None:
+              outformat='json-ld',
-            rtemplate = Environment(loader=BaseLoader).from_string(template)
+              **kwargs):
-            content = rtemplate.render(**self)
+        """
-            mimetype = 'text'
+        Return the values and error to be used in flask.
-        elif fields is not None:
+        So far, it returns a fixed context. We should store/generate different
-            # Emulate field selection by constructing a template
+        contexts if the plugin adds more aliases.
-            content = json.dumps(jmespath.search(fields, js))
+        """
-            mimetype = 'text'
+        headers = headers or {}
-        elif format == 'json-ld':
+        kwargs["with_context"] = not in_headers
-            content = json.dumps(js, indent=2, sort_keys=True)
+        content, mimetype = self.serialize(format=outformat,
+                                           with_mime=True,
+                                           **kwargs)
+
+        if outformat == 'json-ld' and in_headers:
+            headers.update({
+                "Link":
+                ('<%s>;'
+                    'rel="http://www.w3.org/ns/json-ld#context";'
+                    ' type="application/ld+json"' % kwargs.get('context_uri'))
+            })
+        return FlaskResponse(
+            response=content,
+            status=self.get('status', 200),
+            headers=headers,
+            mimetype=mimetype)
+
+    def serialize(self, format='json-ld', with_mime=False, **kwargs):
+        js = self.jsonld(**kwargs)
+        content = json.dumps(js, indent=2, sort_keys=True)
+        if format == 'json-ld':
             mimetype = "application/json"
         elif format in ['turtle', 'ntriples']:
-            content = json.dumps(js, indent=2, sort_keys=True)
             logger.debug(js)
-            context = [self._context, {'prefix': prefix, '@base': prefix}]
+            base = kwargs.get('prefix')
             g = Graph().parse(
                 data=content,
                 format='json-ld',
-                prefix=prefix,
+                base=base,
-                context=context)
+                context=[self._context,
+                         {'@base': base}])
             logger.debug(
                 'Parsing with prefix: {}'.format(kwargs.get('prefix')))
             content = g.serialize(format=format,
-                                  prefix=prefix)
+                                  base=base).decode('utf-8')
             mimetype = 'text/{}'.format(format)
         else:
             raise Error('Unknown outformat: {}'.format(format))
@@ -108,25 +204,14 @@ class BaseModel(PydanticModel):
                with_context=False,
                context_uri=None,
                prefix=None,
-               base=None,
+               expanded=False):
-               expanded=False,
-               **kwargs):
 
-        result = self.serializable(**kwargs)
+        result = self.serializable()
 
         if expanded:
             result = jsonld.expand(
-                result,
+                result, options={'base': prefix,
-                options={
+                                 'expandContext': self._context})[0]
-                    'expandContext': [
-                        self.context,
-                        {
-                            'prefix': prefix,
-                            'endpoint': prefix
-                        }
-                    ]
-                }
-            )[0]
         if not with_context:
             try:
                 del result['@context']
@@ -150,177 +235,168 @@ class BaseModel(PydanticModel):
         self['prov:wasGeneratedBy'] = another.id
 
 
+def subtypes():
+    return BaseMeta._subtypes
+
+
+def from_dict(indict, cls=None):
+    if not cls:
+        target = indict.get('@type', None)
+        cls = BaseModel
+        try:
+            cls = subtypes()[target]
+        except KeyError:
+            pass
+    outdict = dict()
+    for k, v in indict.items():
+        if k == '@context':
+            pass
+        elif isinstance(v, dict):
+            v = from_dict(indict[k])
+        elif isinstance(v, list):
+            v = v[:]
+            for ix, v2 in enumerate(v):
+                if isinstance(v2, dict):
+                    v[ix] = from_dict(v2)
+        outdict[k] = copy.copy(v)
+    return cls(**outdict)
+
+
+def from_string(string, **kwargs):
+    return from_dict(json.loads(string), **kwargs)
+
+
+def from_json(injson):
+    indict = json.loads(injson)
+    return from_dict(indict)
+
+
 class Entry(BaseModel):
-    text: str = Field(serialization_alias="nif:isString")
+    schema = 'entry'
 
-class Activity(BaseModel):
+    text = alias('nif:isString')
-    plugin: str
-    params: Dict[str, Any]
-    analyzed: List[str] = []
 
 
-class Results(BaseModel):
+class Sentiment(BaseModel):
-    activities: List[Activity] = []
+    schema = 'sentiment'
-    entries: List[Entry] = []
+
-    parameters: Dict[str, str] = {}
+    polarity = alias('marl:hasPolarity')
+    polarityValue = alias('marl:hasPolarityValue')
+
+
+class Error(BaseModel, Exception):
+    schema = 'error'
+
+    def __init__(self, message='Generic senpy exception', *args, **kwargs):
+        Exception.__init__(self, message)
+        super(Error, self).__init__(*args, **kwargs)
+        self.message = message
+
+    def __str__(self):
+        if not hasattr(self, 'errors'):
+            return self.message
+        return '{}:\n\t{}'.format(self.message, self.errors)
+
+    def __hash__(self):
+        return Exception.__hash__(self)
+
+
+# Add the remaining schemas programmatically
+
+def _class_from_schema(name, schema=None, schema_file=None, base_classes=None):
+    base_classes = base_classes or []
+    base_classes.append(BaseModel)
+    attrs = {}
+    if schema:
+        attrs['schema'] = schema
+    elif schema_file:
+        attrs['schema_file'] = schema_file
+    else:
+        attrs['schema'] = name
+    name = "".join((name[0].upper(), name[1:]))
+    return BaseMeta(name, base_classes, attrs)
+
+
+def _add_class_from_schema(*args, **kwargs):
+    generatedClass = _class_from_schema(*args, **kwargs)
+    globals()[generatedClass.__name__] = generatedClass
+    del generatedClass
+
+
+for i in [
+        'aggregatedEvaluation',
+        'dataset',
+        'datasets',
+        'emotion',
+        'emotionConversion',
+        'emotionConversionPlugin',
+        'emotionAnalysis',
+        'emotionModel',
+        'emotionPlugin',
+        'emotionSet',
+        'evaluation',
+        'entity',
+        'help',
+        'metric',
+        'parameter',
+        'plugins',
+        'response',
+        'results',
+        'sentimentPlugin',
+        'suggestion',
+        'topic',
+
+]:
+    _add_class_from_schema(i)
 
 
 class Analysis(BaseModel):
-    plugin: 'Plugin'
+    schema = 'analysis'
-    params: dict[str, object] = {}
+
+    parameters  = alias('prov:used')
+
+    @property
+    def params(self):
+        outdict = {}
+        outdict['algorithm'] = self.algorithm
+        for param in self.parameters:
+            outdict[param['name']] = param['value']
+        return outdict
+
+    @params.setter
+    def params(self, value):
+        for k, v in value.items():
+            for param in self.parameters:
+                if param.name == k:
+                    param.value = v
+                    break
+            else:
+                self.parameters.append(Parameter(name=k, value=v))
+
+    @property
+    def algorithm(self):
+        return self['prov:wasAssociatedWith']
+
+    @property
+    def plugin(self):
+        return self._plugin
+
+    @plugin.setter
+    def plugin(self, value):
+        self._plugin = value
+        self['prov:wasAssociatedWith'] = value.id
 
     def run(self, request):
-        return self.plugin.process(request, self)
+        return self.plugin.process(request, self.params)
-
-
-class MoralityAnalysis(Analysis):
-    usedMoralValueModel: str
-    usedMLModel: Optional[str] = None
-
-
-class Annotation(BaseModel):
-    annotated: Optional[str] = None
-    wasGeneratedBy: Optional[str] = None
-
-    def add_provenance(self, entry, activity):
-        self.annotated = entry.id
-        self.wasGeneratedBy = activity.id
-        self[prov['wasGeneratedBy']] = activity.id
-        activity.analysed.append(entry.id)
-
-
-class MoralAnnotation(Annotation):
-    def __init__(self, category, confidence=None, intensity=None,
-                 entry=None, activity=None):
-        super().__init__()
-        if confidence is not None:
-            self[amor['confidence']] = confidence
-        if intensity is not None:
-            self[amor_mft['hasPolarityIntensity']] = intensity
-        if entry and activity:
-             self.add_provenance(entry, activity)
-        elif entry or activity:
-            raise Exception()
-
-    @classmethod
-    def from_label(cls, label):
-        return cls(label.category, label.confidence, label.intensity)
 
 
 class Plugin(BaseModel):
-    name: str = Field(default='', validate_default=True)
+    schema = 'plugin'
-    version: Optional[str] = 'dev'
-    author: Optional[str] = None
-    extra_params: Dict[str, Any] = {}
-    description: str = Field(default="", json_schema_extra=dict(hidden=True))
 
-    @field_validator("description", mode="before")
+    def activity(self, parameters):
-    @classmethod
+        '''Generate a prov:Activity from this plugin and the '''
-    def dedentdoc(cls, doc: str) -> str:
+        a = Analysis()
-        doc = doc or cls.get('__doc__', None)
+        a.plugin = self
-        return dedent(doc)
+        a.params = parameters
-
-    @field_validator("name", mode="before")
-    @classmethod
-    def clsname(cls, name: str) -> str:
-        name = name or cls.__name__.capitalize()
-        return name
-
-    @computed_field
-    @property
-    def id(self) -> str:
-        return 'endpoint:plugins/{}_{}'.format(self.name, self.version)
-
-    def activity(self, params, *, cls=Analysis):
-        '''Generate an Analysis (prov:Activity) from this plugin and the given parameters'''
-        a = cls(plugin=self, params=params)
         return a
- 
 
-class Plugins(BaseModel):
-    plugins: List[Plugin] = []
-
-class Emotion:
-    hasEmotionIntensity: str = Field(serialization_alias="onyx:hasEmotionIntensity")
-    hasEmotionCategory: str = Field(serialization_alias="onyx:hasEmotionCategory")
-
-class Sentiment:
-    pass
-
-class SentimentPlugin(Plugin):
-    pass
-
-class Suggestion:
-    pass
-
-class Topic:
-    pass
-
-
-class Context:
-    pass
-
-class Dataset:
-    pass
-
-class Datasets:
-    pass
-
-class Definitions:
-    pass
-
-class Dimensions:
-    pass
-
-class EmotionAnalysis:
-    pass
-
-class EmotionConversion:
-    pass
-
-class EmotionConversionPlugin(Plugin):
-    pass
-
-class Emotion:
-    pass
-
-class EmotionModel:
-    pass
-
-class EmotionPlugin(Plugin):
-    pass
-
-class EmotionSet:
-    pass
-
-class Entity:
-    pass
-
-class ErrorResponse(BaseModel):
-    code: Optional[int] = 500
-    message: str
-
-
-class Metric:
-    pass
-
-class Evaluation:
-    pass
-
-class AggregatedEvaluation:
-    pass
-
-class Help:
-    pass
-
-class Parameter:
-    pass
-
-
-
-def from_dict(c, cls):
-    return cls.model_validate(d)
-
-
-def from_string(s, cls):
-    return from_dict(from_json(s, allow_partial=True))

senpy/ns.py

@@ -1,7 +0,0 @@
-from rdflib import Namespace
-
-amor = Namespace('http://www.gsi.upm.es/ontologies/amor/ns#')
-amor_bhv = Namespace('http://www.gsi.upm.es/ontologies/amor-bhv/ns#')
-amor_mft = Namespace('http://www.gsi.upm.es/ontologies/amor-mft/ns#')
-prov = Namespace('http://www.w3.org/ns/prov#')
-emoml = Namespace('http://www.gsi.upm.es/ontologies/onyx/vocabularies/emotionml/ns#')

senpy/plugins/__init__.py

@@ -1,22 +1,7 @@
-#!/usr/local/bin/python
+from future import standard_library
-# -*- coding: utf-8 -*-
+standard_library.install_aliases()
-#
-#    Copyright 2014 Grupo de Sistemas Inteligentes (GSI) DIT, UPM
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
 
-from typing import *
+from future.utils import with_metaclass
 from functools import partial
 
 import os.path
@@ -26,32 +11,60 @@ import pickle
 import logging
 import pprint
 
-import deprecation
-
 import inspect
 import sys
 import subprocess
 import importlib
 import yaml
 import threading
-import multiprocessing
-import pkg_resources
 from nltk import download
-from sklearn.base import TransformerMixin, BaseEstimator
-from itertools import product
 
 from .. import models, utils
-from ..errors import Error
 from .. import api
 from .. import gsitk_compat
 from .. import testing
-from .. import config
-from .. import ns
 
 logger = logging.getLogger(__name__)
 
 
-class Plugin(models.Plugin):
+class PluginMeta(models.BaseMeta):
+    _classes = {}
+
+    def __new__(mcs, name, bases, attrs, **kwargs):
+        plugin_type = []
+        if hasattr(bases[0], 'plugin_type'):
+            plugin_type += bases[0].plugin_type
+        plugin_type.append(name)
+        alias = attrs.get('name', name)
+        attrs['plugin_type'] = plugin_type
+        attrs['name'] = alias
+        if 'description' not in attrs:
+            doc = attrs.get('__doc__', None)
+            if doc:
+                attrs['description'] = doc
+            else:
+                logger.warning(
+                    ('Plugin {} does not have a description. '
+                     'Please, add a short summary to help other developers'
+                     ).format(name))
+        cls = super(PluginMeta, mcs).__new__(mcs, name, bases, attrs)
+
+        if alias in mcs._classes:
+            if os.environ.get('SENPY_TESTING', ""):
+                raise Exception(
+                    ('The type of plugin {} already exists. '
+                     'Please, choose a different name').format(name))
+            else:
+                logger.warning('Overloading plugin class: {}'.format(alias))
+        mcs._classes[alias] = cls
+        return cls
+
+    @classmethod
+    def for_type(cls, ptype):
+        return cls._classes[ptype]
+
+
+class Plugin(with_metaclass(PluginMeta, models.Plugin)):
     '''
     Base class for all plugins in senpy.
     A plugin must provide at least these attributes:
@@ -64,25 +77,24 @@ class Plugin(models.Plugin):
 
     '''
 
-    data_folder: str = "."
+    def __init__(self, info=None, data_folder=None, **kwargs):
-    test_cases: List[Dict] = []
-
-
-    def __init__(self, *, data_folder=None, **data):
         """
         Provides a canonical name for plugins and serves as base for other
         kinds of plugins.
         """
-        super().__init__(**data)
+        logger.debug("Initialising {}".format(info))
+        super(Plugin, self).__init__(**kwargs)
+        if info:
+            self.update(info)
+        self.validate()
+        self.id = 'endpoint:plugins/{}_{}'.format(self['name'],
+                                                  self['version'])
+        self.is_activated = False
+        self._lock = threading.Lock()
         self._directory = os.path.abspath(
             os.path.dirname(inspect.getfile(self.__class__)))
 
-        if not data_folder:
+        data_folder = data_folder or os.getcwd()
-            data_folder = config.data_folder
-        if not data_folder:
-            data_folder = os.getcwd()
-
-        data_folder = os.path.abspath(data_folder)
         subdir = os.path.join(data_folder, self.name)
 
         self._data_paths = [
@@ -102,30 +114,45 @@ class Plugin(models.Plugin):
     def log(self):
         return self._log
 
+    def validate(self):
+        missing = []
+        for x in ['name', 'description', 'version']:
+            if x not in self:
+                missing.append(x)
+        if missing:
+            raise models.Error(
+                'Missing configuration parameters: {}'.format(missing))
+
     def get_folder(self):
         return os.path.dirname(inspect.getfile(self.__class__))
 
-    def process(self, request, activity, **kwargs):
+    def activate(self):
+        pass
+
+    def deactivate(self):
+        pass
+
+    def process(self, request, parameters, **kwargs):
         """
         An implemented plugin should override this method.
-        Here, we assume that a process_entries method exists.
+        Here, we assume that a process_entries method exists."""
-        """
         newentries = list(
-            self.process_entries(request.entries, activity))
+            self.process_entries(request.entries, parameters))
         request.entries = newentries
         return request
 
-    def process_entries(self, entries, activity):
+    def process_entries(self, entries, parameters):
         for entry in entries:
-            self.log.debug('Processing entry with plugin %s: %s', self, entry)
+            self.log.debug('Processing entry with plugin {}: {}'.format(
-            results = self.process_entry(entry, activity)
+                self, entry))
+            results = self.process_entry(entry, parameters)
             if inspect.isgenerator(results):
                 for result in results:
                     yield result
-            elif results:
+            else:
                 yield results
 
-    def process_entry(self, entry, activity):
+    def process_entry(self, entry, parameters):
         """
         This base method is here to adapt plugins which only
         implement the *process* function.
@@ -146,12 +173,9 @@ class Plugin(models.Plugin):
             test_cases = self.test_cases
         for case in test_cases:
             try:
-                fmt = 'case: {}'.format(case.get('name', case))
-                if 'name' in case:
-                    self.log.info('Test case: {}'.format(case['name']))
-                self.log.debug('Test case:\n\t{}'.format(
-                    pprint.pformat(fmt)))
                 self.test_case(case)
+                self.log.debug('Test case passed:\n{}'.format(
+                    pprint.pformat(case)))
             except Exception as ex:
                 self.log.warning('Test case failed:\n{}'.format(
                     pprint.pformat(case)))
@@ -160,7 +184,7 @@ class Plugin(models.Plugin):
     def test_case(self, case, mock=testing.MOCK_REQUESTS):
         if 'entry' not in case and 'input' in case:
             entry = models.Entry(_auto_id=False)
-            entry.text = case['input']
+            entry.nif__isString = case['input']
             case['entry'] = entry
         entry = models.Entry(case['entry'])
         given_parameters = case.get('params', case.get('parameters', {}))
@@ -176,9 +200,7 @@ class Plugin(models.Plugin):
                 entry,
             ]
 
-            activity = self.activity(parameters)
+            method = partial(self.process, request, parameters)
-
-            method = partial(self.process, request, activity)
 
             if mock:
                 res = method()
@@ -190,30 +212,24 @@ class Plugin(models.Plugin):
                 expected = [expected]
             utils.check_template(res.entries, expected)
             res.validate()
-        except Error:
+        except models.Error:
             if should_fail:
                 return
             raise
         assert not should_fail
 
     def find_file(self, fname):
-        tried = []
         for p in self._data_paths:
-            alternative = os.path.abspath(os.path.join(p, fname))
+            alternative = os.path.join(p, fname)
             if os.path.exists(alternative):
                 return alternative
-            tried.append(alternative)
+        raise IOError('File does not exist: {}'.format(fname))
-        raise IOError(f'File does not exist: {fname}. Tried: {tried}')
-
-    def path(self, fpath):
-        if not os.path.isabs(fpath):
-            fpath = os.path.join(self.data_folder, fpath)
-        return fpath
 
     def open(self, fpath, mode='r'):
         if 'w' in mode:
             # When writing, only use absolute paths or data_folder
-            fpath = self.path(fpath)
+            if not os.path.isabs(fpath):
+                fpath = os.path.join(self.data_folder, fpath)
         else:
             fpath = self.find_file(fpath)
 
@@ -227,41 +243,34 @@ class Plugin(models.Plugin):
 SenpyPlugin = Plugin
 
 
-class Analyser(Plugin):
+class Analysis(Plugin):
     '''
     A subclass of Plugin that analyses text and provides an annotation.
     '''
 
-    @deprecation.deprecated(details="Use process instead")
+    def analyse(self, request, parameters):
-    def analyse(self, request, activity):
+        return super(Analysis, self).process(request, parameters)
-        return super(Analyser, self).process(request, activity)
 
-    @deprecation.deprecated(details="Use process_entries instead")
+    def analyse_entries(self, entries, parameters):
-    def analyse_entries(self, entries, activity):
+        for i in super(Analysis, self).process_entries(entries, parameters):
-        for i in super(Analyser, self).process_entries(entries, activity):
             yield i
 
-    def process(self, request, activity, **kwargs):
+    def process(self, request, parameters, **kwargs):
-        return self.analyse(request, activity)
+        return self.analyse(request, parameters)
 
-    def process_entries(self, entries, activity):
+    def process_entries(self, entries, parameters):
-        for i in self.analyse_entries(entries, activity):
+        for i in self.analyse_entries(entries, parameters):
             yield i
 
-    def process_entry(self, entry, activity, **kwargs):
+    def process_entry(self, entry, parameters, **kwargs):
         if hasattr(self, 'analyse_entry'):
-            for i in self.analyse_entry(entry, activity):
+            for i in self.analyse_entry(entry, parameters):
                 yield i
         else:
-            super(Analyser, self).process_entry(entry, activity, **kwargs)
+            super(Analysis, self).process_entry(entry, parameters, **kwargs)
 
 
-AnalysisPlugin = Analyser
+AnalysisPlugin = Analysis
-
-
-class Transformation(AnalysisPlugin):
-    '''Empty'''
-    pass
 
 
 class Conversion(Plugin):
@@ -288,98 +297,31 @@ class Conversion(Plugin):
 ConversionPlugin = Conversion
 
 
-class Evaluable(Plugin):
+class SentimentPlugin(Analysis, models.SentimentPlugin):
-    '''
-    Common class for plugins that can be evaluated with GSITK.
-
-    They should implement the methods below.
-    '''
-
-    def as_pipe(self):
-        raise Exception('Implement the as_pipe function')
-
-    def evaluate_func(self, X, activity=None):
-        raise Exception('Implement the evaluate_func function')
-
-    def evaluate(self, *args, **kwargs):
-        return evaluate([self], *args, **kwargs)
-
-
-class SentimentPlugin(Analyser, Evaluable):
     '''
     Sentiment plugins provide sentiment annotation (using Marl)
     '''
-    minPolarityValue: float = 0
+    minPolarityValue = 0
-    maxPolarityValue: float = 1
+    maxPolarityValue = 1
 
     def test_case(self, case):
         if 'polarity' in case:
             expected = case.get('expected', {})
             s = models.Sentiment(_auto_id=False)
             s.marl__hasPolarity = case['polarity']
-            if 'marl:hasOpinion' not in expected:
+            if 'sentiments' not in expected:
-                expected['marl:hasOpinion'] = []
+                expected['sentiments'] = []
-            expected['marl:hasOpinion'].append(s)
+            expected['sentiments'].append(s)
             case['expected'] = expected
         super(SentimentPlugin, self).test_case(case)
 
-    def normalize(self, value, minValue, maxValue):
-        nv = minValue + (value - self.minPolarityValue) * (
-            self.maxPolarityValue - self.minPolarityValue) / (maxValue - minValue)
-        return nv
 
-    def as_pipe(self):
+class EmotionPlugin(Analysis, models.EmotionPlugin):
-        pipe = gsitk_compat.Pipeline([('senpy-plugin', ScikitWrapper(self))])
-        pipe.name = self.id
-        return pipe
-
-    def evaluate_func(self, X, activity=None):
-        if activity is None:
-            parameters = api.parse_params({},
-                                          self.extra_params)
-            activity = self.activity(parameters)
-        entries = []
-        for feat in X:
-            if isinstance(feat, list):
-                feat = ' '.join(feat)
-            entries.append(models.Entry(text=feat))
-        labels = []
-        for e in self.process_entries(entries, activity):
-            sent = e.sentiments[0].polarity
-            label = -1
-            if sent == 'marl:Positive':
-                label = 1
-            elif sent == 'marl:Negative':
-                label = -1
-            labels.append(label)
-        return labels
-
-
-class EmotionPlugin(Analyser, models.EmotionPlugin):
     '''
     Emotion plugins provide emotion annotation (using Onyx)
     '''
-    minEmotionValue: float = 0
+    minEmotionValue = 0
-    maxEmotionValue: float = 1
+    maxEmotionValue = 1
-
-
-class AnnotationPlugin(AnalysisPlugin):
-    def process_entry(self, entry, activity, **kwargs):
-        for annotation in self.annotate(entry):
-             annotation.add_provenance(entry, activity)
-        yield entry
-
-    def annotate(self, entry, **kwargs):
-        raise NotImplemented("this should be implemented in subclasses")
-
-
-class MoralityPlugin(AnnotationPlugin):
-    moralValueModel: str = ns.amor_mft['MoralFoundationTHeory']
-
-    def activity(self, parameters=None):
-        return models.MoralityAnalysis(plugin=self,
-                                       usedMLModel=None,
-                                       usedMoralValueModel=self.model)
 
 
 class EmotionConversion(Conversion):
@@ -403,67 +345,69 @@ EmotionConversionPlugin = EmotionConversion
 
 
 class PostProcessing(Plugin):
-    '''
-    A plugin that converts the output of other plugins (post-processing).
-    '''
     def check(self, request, plugins):
         '''Should this plugin be run for this request?'''
         return False
 
 
-class Box(Analyser):
+class Box(AnalysisPlugin):
     '''
     Black box plugins delegate analysis to a function.
-    The flow is like this:
+    The flow is like so:
 
     .. code-block::
 
-                   entries --> to_features() --> predict_many() --> to_entry() --> entries'
+                   entry --> input() --> predict_one() --> output() --> entry'
 
 
-    In other words: their ``to_features`` method converts a query (entry and a set of parameters)
+    In other words: their ``input`` method convers a query (entry and a set of parameters) into
-    into the input to the `predict_one` method, which only uses an array of features.
+    the input to the box method. The ``output`` method convers the results given by the box into
-    The ``to_entry`` method converts the results given by the box into an entry that senpy can
+    an entry that senpy can handle.
-    handle.
     '''
 
-    def to_features(self, entry, activity=None):
+    def input(self, entry, params=None):
         '''Transforms a query (entry+param) into an input for the black box'''
         return entry
 
-    def to_entry(self, features, entry=None, activity=None):
+    def output(self, output, entry=None, params=None):
         '''Transforms the results of the black box into an entry'''
-        return entry
+        return output
 
-    def predict_one(self, features, activity=None):
+    def predict_one(self, input):
         raise NotImplementedError(
             'You should define the behavior of this plugin')
 
-    def predict_many(self, features, activity=None):
+    def process_entries(self, entries, params):
-        results = []
-        for feat in features:
-            results.append(self.predict_one(features=feat, activity=activity))
-        return results
-
-    def process_entry(self, entry, activity):
-        for i in self.process_entries([entry], activity):
-            yield i
-
-    def process_entries(self, entries, activity):
-        features = []
         for entry in entries:
-            features.append(self.to_features(entry=entry, activity=activity))
+            input = self.input(entry=entry, params=params)
-        results = self.predict_many(features=features, activity=activity)
+            results = self.predict_one(input=input)
+            yield self.output(output=results, entry=entry, params=params)
 
-        for (result, entry) in zip(results, entries):
+    def fit(self, X=None, y=None):
-            yield self.to_entry(features=result, entry=entry, activity=activity)
+        return self
+
+    def transform(self, X):
+        return [self.predict_one(x) for x in X]
+
+    def predict(self, X):
+        return self.transform(X)
+
+    def fit_transform(self, X, y):
+        self.fit(X, y)
+        return self.transform(X)
+
+    def as_pipe(self):
+        pipe = gsitk_compat.Pipeline([('plugin', self)])
+        pipe.name = self.name
+        return pipe
 
 
 class TextBox(Box):
     '''A black box plugin that takes only text as input'''
 
-    def to_features(self, entry, activity):
+    def input(self, entry, params):
-        return [entry.text]
+        entry = super(TextBox, self).input(entry, params)
+        return entry['nif:isString']
 
 
 class SentimentBox(TextBox, SentimentPlugin):
@@ -471,35 +415,17 @@ class SentimentBox(TextBox, SentimentPlugin):
     A box plugin where the output is only a polarity label or a tuple (polarity, polarityValue)
     '''
 
-    classes: List[str] = ['marl:Positive', 'marl:Neutral', 'marl:Negative']
+    def output(self, output, entry, **kwargs):
-    binary: bool = True
+        s = models.Sentiment()
-
+        try:
-    def to_entry(self, features, entry, activity, **kwargs):
+            label, value = output
-
+        except ValueError:
-        if len(features) != len(self.classes):
+            label, value = output, None
-            raise Error('The number of features ({}) does not match the classes '
+        s.prov(self)
-                        '(plugin.classes ({})'.format(len(features), len(self.classes)))
+        s.polarity = label
-
+        if value is not None:
-        minValue = activity.param('marl:minPolarityValue', 0)
+            s.polarityValue = value
-        maxValue = activity.param('marl:minPolarityValue', 1)
+        entry.sentiments.append(s)
-        activity['marl:minPolarityValue'] = minValue
-        activity['marl:maxPolarityValue'] = maxValue
-
-        for k, v in zip(self.classes, features):
-            s = models.Sentiment()
-            if self.binary:
-                if not v:  # Carry on if the value is 0
-                    continue
-                s['marl:hasPolarity'] = k
-            else:
-                if v is not None:
-                    s['marl:hasPolarity'] = k
-                    nv = self.normalize(v, minValue, maxValue)
-                    s['marl:polarityValue'] = nv
-            s.prov(activity)
-
-            entry.sentiments.append(s)
-
         return entry
 
 
@@ -508,23 +434,14 @@ class EmotionBox(TextBox, EmotionPlugin):
     A box plugin where the output is only an a tuple of emotion labels
     '''
 
-    EMOTIONS: List[str] = []
+    def output(self, output, entry, **kwargs):
-    with_intensity: bool = True
+        if not isinstance(output, list):
-
+            output = [output]
-    def to_entry(self, features, entry, activity, **kwargs):
         s = models.EmotionSet()
-
-        if len(features) != len(self.EMOTIONS):
-            raise Exception(('The number of classes in the plugin and the number of features '
-                             'do not match'))
-
-        for label, intensity in zip(self.EMOTIONS, features):
-            e = models.Emotion(onyx__hasEmotionCategory=label)
-            if self.with_intensity:
-                e.onyx__hasEmotionIntensity = intensity
-            s.onyx__hasEmotion.append(e)
-        s.prov(activity)
         entry.emotions.append(s)
+        for label in output:
+            e = models.Emotion(onyx__hasEmotionCategory=label)
+            s.append(e)
         return entry
 
 
@@ -537,15 +454,11 @@ class MappingMixin(object):
     def mappings(self, value):
         self._mappings = value
 
-    def to_entry(self, features, entry, activity):
+    def output(self, output, entry, params):
-        features = list(features)
+        output = self.mappings.get(output, self.mappings.get(
-        for i, feat in enumerate(features):
+            'default', output))
-            features[i] = self.mappings.get(feat,
+        return super(MappingMixin, self).output(
-                                            self.mappings.get('default',
+            output=output, entry=entry, params=params)
-                                                              feat))
-        return super(MappingMixin, self).to_entry(features=features,
-                                                  entry=entry,
-                                                  activity=activity)
 
 
 class ShelfMixin(object):
@@ -585,20 +498,14 @@ class ShelfMixin(object):
     def shelf_file(self, value):
         self._shelf_file = value
 
-    def save(self, ignore_errors=False):
+    def save(self):
-        try:
+        self.log.debug('Saving pickle')
-            self.log.debug('Saving pickle')
+        if hasattr(self, '_sh') and self._sh is not None:
-            if hasattr(self, '_sh') and self._sh is not None:
+            with self.open(self.shelf_file, 'wb') as f:
-                with self.open(self.shelf_file, 'wb') as f:
+                pickle.dump(self._sh, f)
-                    pickle.dump(self._sh, f)
-        except Exception as ex:
-            self.log.warning("Could not save shelf state. Check folder permissions for: "
-                             f" {self.shelf_file}. Error: { ex }")
-            if not ignore_errors:
-                raise
 
 
-def pfilter(plugins, plugin_type=Analyser, **kwargs):
+def pfilter(plugins, plugin_type=Analysis, **kwargs):
     """ Filter plugins by different criteria """
     if isinstance(plugins, models.Plugins):
         plugins = plugins.plugins
@@ -615,13 +522,10 @@ def pfilter(plugins, plugin_type=Analyser, **kwargs):
             logger.debug('Class: {}'.format(pclass))
             candidates = filter(lambda x: isinstance(x, pclass), plugins)
         except KeyError:
-            raise Error('{} is not a valid type'.format(plugin_type))
+            raise models.Error('{} is not a valid type'.format(plugin_type))
     else:
         candidates = plugins
 
-    if 'name' in kwargs:
-        kwargs['name'] = kwargs['name'].lower()
-
     logger.debug(candidates)
 
     def matches(plug):
@@ -645,58 +549,33 @@ def load_module(name, root=None):
 
 def _log_subprocess_output(process):
     for line in iter(process.stdout.readline, b''):
-        logger.info('%s', line.decode())
+        logger.info('%r', line)
     for line in iter(process.stderr.readline, b''):
-        logger.error('%s', line.decode())
+        logger.error('%r', line)
 
 
-def missing_requirements(reqs):
-    queue = []
-    pool = multiprocessing.Pool(4)
-    for req in reqs:
-        res = pool.apply_async(pkg_resources.get_distribution, (req,))
-        queue.append((req, res))
-    missing = []
-    installed = []
-    for req, job in queue:
-        try:
-            installed.append(job.get(1))
-        except Exception:
-            missing.append(req)
-    return installed, missing
-
-def list_dependencies(*plugins):
-    '''List all dependencies (python and nltk) for the given list of plugins'''
-    nltk_resources = set()
-    missing = []
-    installed = []
-    for info in plugins:
-        reqs = info.get('requirements', [])
-        if reqs:
-            inst, miss= missing_requirements(reqs)
-            installed += inst
-            missing += miss
-        nltk_resources |= set(info.get('nltk_resources', []))
-    return installed, missing, nltk_resources
-
 def install_deps(*plugins):
-    _, requirements, nltk_resources = list_dependencies(*plugins)
     installed = False
-    if requirements:
+    nltk_resources = set()
-        logger.info('Installing requirements: ' + str(requirements))
+    for info in plugins:
-        pip_args = [sys.executable, '-m', 'pip', 'install']
+        requirements = info.get('requirements', [])
-        for req in requirements:
+        if requirements:
-            pip_args.append(req)
+            pip_args = [sys.executable, '-m', 'pip', 'install']
-        process = subprocess.Popen(
+            for req in requirements:
-            pip_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                pip_args.append(req)
-        _log_subprocess_output(process)
+            logger.info('Installing requirements: ' + str(requirements))
-        exitcode = process.wait()
+            process = subprocess.Popen(
-        installed = True
+                pip_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        if exitcode != 0:
+            _log_subprocess_output(process)
-            raise Error(
+            exitcode = process.wait()
-                "Dependencies not properly installed: {}".format(pip_args))
+            installed = True
-    installed_nltk = download(list(nltk_resources))
+            if exitcode != 0:
-    return installed or installed_nltk
+                raise models.Error(
+                    "Dependencies not properly installed: {}".format(pip_args))
+        nltk_resources |= set(info.get('nltk_resources', []))
+
+    installed |= download(list(nltk_resources))
+    return installed
 
 
 is_plugin_file = re.compile(r'.*\.senpy$|senpy_[a-zA-Z0-9_]+\.py$|'
@@ -713,7 +592,7 @@ def find_plugins(folders):
                 yield fpath
 
 
-def from_path(fpath, **kwargs):
+def from_path(fpath, install_on_fail=False, **kwargs):
     logger.debug("Loading plugin from {}".format(fpath))
     if fpath.endswith('.py'):
         # We asume root is the dir of the file, and module is the name of the file
@@ -723,18 +602,18 @@ def from_path(fpath, **kwargs):
             yield instance
     else:
         info = parse_plugin_info(fpath)
-        yield from_info(info, **kwargs)
+        yield from_info(info, install_on_fail=install_on_fail, **kwargs)
+
 
 def from_folder(folders, loader=from_path, **kwargs):
     plugins = []
     for fpath in find_plugins(folders):
         for plugin in loader(fpath, **kwargs):
-            if plugin:
+            plugins.append(plugin)
-                plugins.append(plugin)
     return plugins
 
 
-def from_info(info, root=None, strict=False, **kwargs):
+def from_info(info, root=None, install_on_fail=True, **kwargs):
     if any(x not in info for x in ('module', )):
         raise ValueError('Plugin info is not valid: {}'.format(info))
     module = info["module"]
@@ -745,17 +624,15 @@ def from_info(info, root=None, strict=False, **kwargs):
     fun = partial(one_from_module, module, root=root, info=info, **kwargs)
     try:
         return fun()
-    except (ImportError, LookupError) as ex:
+    except (ImportError, LookupError):
-        if strict or str(info.get("optional", "false")).lower() not in ["true", "t"]:
+        install_deps(info)
-            raise
+        return fun()
-        print(f"Could not import plugin: { info }: {ex}", file=sys.stderr)
-        return FailedPlugin(info, fun)
 
 
 def parse_plugin_info(fpath):
     logger.debug("Parsing plugin info: {}".format(fpath))
     with open(fpath, 'r') as f:
-        info = yaml.load(f, Loader=yaml.FullLoader)
+        info = yaml.load(f)
     info['_path'] = fpath
     return info
 
@@ -773,7 +650,7 @@ def from_module(module, **kwargs):
 def one_from_module(module, root, info, **kwargs):
     if '@type' in info:
         cls = PluginMeta.from_type(info['@type'])
-        return cls(**info, **kwargs)
+        return cls(info=info, **kwargs)
     instance = next(
         from_module(module=module, root=root, info=info, **kwargs), None)
     if not instance:
@@ -806,42 +683,19 @@ def _from_module_name(module, root, info=None, **kwargs):
 
 def _from_loaded_module(module, info=None, **kwargs):
     for cls in _classes_in_module(module):
-        yield cls(**(info or {}), **kwargs)
+        yield cls(info=info, **kwargs)
     for instance in _instances_in_module(module):
         yield instance
 
 
-cached_evs = {}
-
-
 def evaluate(plugins, datasets, **kwargs):
-    for plug in plugins:
+    ev = gsitk_compat.Eval(
-        if not hasattr(plug, 'as_pipe'):
+        tuples=None,
-            raise Error('Plugin {} cannot be evaluated'.format(plug.name))
+        datasets=datasets,
-
+        pipelines=[plugin.as_pipe() for plugin in plugins])
-    if not isinstance(datasets, dict):
+    ev.evaluate()
-        datasets = gsitk_compat.prepare(datasets, download=True)
+    results = ev.results
-
+    evaluations = evaluations_to_JSONLD(results, **kwargs)
-    tuples = list(product(plugins, datasets))
-    missing = []
-    for (p, d) in tuples:
-        if (p.id, d) not in cached_evs:
-            pipe = p.as_pipe()
-            missing.append(gsitk_compat.EvalPipeline(pipe, d))
-    if missing:
-        ev = gsitk_compat.Eval(tuples=missing, datasets=datasets)
-        ev.evaluate()
-        results = ev.results
-        new_ev = evaluations_to_JSONLD(results, **kwargs)
-        for ev in new_ev:
-            dataset = ev.evaluatesOn
-            model = ev.evaluates
-            cached_evs[(model, dataset)] = ev
-    evaluations = []
-    logger.debug('%s. Cached evs: %s', tuples, cached_evs)
-    for (p, d) in tuples:
-        logger.debug('Adding %s, %s', d, p)
-        evaluations.append(cached_evs[(p.id, d)])
     return evaluations
 
 
@@ -854,12 +708,12 @@ def evaluations_to_JSONLD(results, flatten=False):
     metric_names = ['accuracy', 'precision_macro', 'recall_macro',
                     'f1_macro', 'f1_weighted', 'f1_micro', 'f1_macro']
 
-    for index, row in results.fillna('Not Available').iterrows():
+    for index, row in results.iterrows():
         evaluation = models.Evaluation()
         if row.get('CV', True):
             evaluation['@type'] = ['StaticCV', 'Evaluation']
         evaluation.evaluatesOn = row['Dataset']
-        evaluation.evaluates = row['Model'].rstrip('__' + row['Dataset'])
+        evaluation.evaluates = row['Model']
         i = 0
         if flatten:
             metric = models.Metric()
@@ -870,28 +724,10 @@ def evaluations_to_JSONLD(results, flatten=False):
             # We should probably discontinue this representation
             for name in metric_names:
                 metric = models.Metric()
+                metric['@id'] = 'Metric' + str(i)
                 metric['@type'] = name.capitalize()
                 metric.value = row[name]
                 evaluation.metrics.append(metric)
                 i += 1
         evaluations.append(evaluation)
     return evaluations
-
-
-class ScikitWrapper(BaseEstimator, TransformerMixin):
-    def __init__(self, plugin=None, **data):
-        super().__init__(**data)
-        self.plugin = plugin
-
-    def fit(self, X=None, y=None):
-        return self
-
-    def transform(self, X):
-        return self.plugin.evaluate_func(X, None)
-
-    def predict(self, X):
-        return self.transform(X)
-
-    def fit_transform(self, X, y):
-        self.fit(X, y)
-        return self.transform(X)

senpy/plugins/emotion/anew/README.md

@@ -1,60 +0,0 @@
-# Plugin emotion-anew 
-
-This plugin consists on an **emotion classifier** that detects six possible emotions:
-- Anger : general-dislike.
-- Fear : negative-fear.
-- Disgust : shame.
-- Joy : gratitude, affective, enthusiasm, love, joy, liking.
-- Sadness : ingrattitude, daze, humlity, compassion, despair, anxiety, sadness.
-- Neutral: not detected a particulary emotion. 
-
-The plugin uses **ANEW lexicon** dictionary to calculate VAD (valence-arousal-dominance) of the sentence and determinate which emotion is closer to this value. To do this comparision, it is defined that each emotion has a centroid, calculated according to this article: http://www.aclweb.org/anthology/W10-0208. 
-
-The plugin is going to look for the words in the sentence that appear in the ANEW dictionary and calculate the average VAD score for the sentence. Once this score is calculated, it is going to seek the emotion that is closest to this value.
-
-The response of this plugin uses [Onyx ontology](https://www.gsi.dit.upm.es/ontologies/onyx/) developed at GSI UPM, to express the information.
-
-## Installation
-
-* Download
-```
-git clone https://lab.cluster.gsi.dit.upm.es/senpy/emotion-anew.git
-```
-* Get data
-```
-cd emotion-anew
-git submodule update --init --recursive
-```
-* Run
-```
-docker run -p 5000:5000 -v $PWD:/plugins gsiupm/senpy:python2.7 -f /plugins
-```
-
-## Data format
-
-`data/Corpus/affective-isear.tsv` contains data from ISEAR Databank: http://emotion-research.net/toolbox/toolboxdatabase.2006-10-13.2581092615 
-
-##Usage
-
-Params accepted:
-- Language: English (en) and Spanish (es).
-- Input: input text to analyse.
-
-
-Example request: 
-```
-http://senpy.cluster.gsi.dit.upm.es/api/?algo=emotion-anew&language=en&input=I%20love%20Madrid
-```
-
-Example respond: This plugin follows the standard for the senpy plugin response. For more information, please visit [senpy documentation](http://senpy.readthedocs.io). Specifically, NIF API section.
-# Known issues
-
-- To obtain Anew dictionary you can download from here: <https://github.com/hcorona/SMC2015/blob/master/resources/ANEW2010All.txt> 
-
-- This plugin only supports **Python2**
-
-
-![alt GSI Logo][logoGSI]
-
-[logoES]: https://www.gsi.dit.upm.es/ontologies/onyx/img/eurosentiment_logo.png "EuroSentiment logo"
-[logoGSI]: http://www.gsi.dit.upm.es/images/stories/logos/gsi.png "GSI Logo"

senpy/plugins/emotion/anew/emotion-anew.senpy

@@ -1,12 +0,0 @@
----
-module: emotion_anew_plugin
-optional: true
-requirements:
-- numpy
-- pandas
-- nltk
-- scipy
-- scikit-learn
-- textblob
-- 'git+https://github.com/uob-vil/pattern.git'
-- lxml

senpy/plugins/emotion/anew/emotion_anew_plugin.py

@@ -1,270 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import re
-import nltk
-import csv
-import sys
-import os
-import unicodedata
-import string
-import xml.etree.ElementTree as ET
-import math
-
-from sklearn.svm import LinearSVC
-from sklearn.feature_extraction import DictVectorizer
-
-from nltk import bigrams
-from nltk import trigrams
-from nltk.corpus import stopwords
-
-from pattern.en import parse as parse_en
-from pattern.es import parse as parse_es
-from senpy.plugins import EmotionPlugin, SenpyPlugin
-from senpy.models import Results, EmotionSet, Entry, Emotion
-
-
-### BEGIN WORKAROUND FOR PATTERN
-# See: https://github.com/clips/pattern/issues/308
-
-import os.path
-
-import pattern.text
-
-from pattern.helpers import decode_string
-from codecs import BOM_UTF8
-
-BOM_UTF8 = BOM_UTF8.decode("utf-8")
-decode_utf8 = decode_string
-
-MODEL = "emoml:pad-dimensions_"
-VALENCE = f"{MODEL}_valence"
-AROUSAL = f"{MODEL}_arousal"
-DOMINANCE = f"{MODEL}_dominance"
-
-def _read(path, encoding="utf-8", comment=";;;"):
-    """Returns an iterator over the lines in the file at the given path,
-    strippping comments and decoding each line to Unicode.
-    """
-    if path:
-        if isinstance(path, str) and os.path.exists(path):
-            # From file path.
-            f = open(path, "r", encoding="utf-8")
-        elif isinstance(path, str):
-            # From string.
-            f = path.splitlines()
-        else:
-            # From file or buffer.
-            f = path
-        for i, line in enumerate(f):
-            line = line.strip(BOM_UTF8) if i == 0 and isinstance(line, str) else line
-            line = line.strip()
-            line = decode_utf8(line, encoding)
-            if not line or (comment and line.startswith(comment)):
-                continue
-            yield line
-
-
-pattern.text._read = _read
-## END WORKAROUND
-
-
-class ANEW(EmotionPlugin):
-    description = "This plugin consists on an emotion classifier using ANEW lexicon dictionary. It averages the VAD (valence-arousal-dominance) value of each word in the text that is also in the ANEW dictionary. To obtain a categorical value (e.g., happy) use the emotion conversion API (e.g., `emotion-model=emoml:big6`)."
-    author: str = "@icorcuera"
-    version: str = "0.5.2"
-    name = "emotion-anew"
-
-    extra_params: dict = {
-        "language": {
-            "description": "language of the input",
-            "aliases": ["language", "l"],
-            "required": True,
-            "options": ["es","en"],
-            "default": "en"
-        }
-    }
-
-    anew_path_es = "Dictionary/Redondo(2007).csv"
-    anew_path_en = "Dictionary/ANEW2010All.txt"
-    onyx__usesEmotionModel = MODEL
-    nltk_resources = ['stopwords']
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self._stopwords = stopwords.words('english')
-        dictionary={}
-        dictionary['es'] = {}
-        with self.open(self.anew_path_es,'r') as tabfile:
-            reader = csv.reader(tabfile, delimiter='\t')
-            for row in reader:
-                dictionary['es'][row[2]]={}
-                dictionary['es'][row[2]]['V']=row[3]
-                dictionary['es'][row[2]]['A']=row[5]
-                dictionary['es'][row[2]]['D']=row[7]
-        dictionary['en'] = {}
-        with self.open(self.anew_path_en,'r') as tabfile:
-            reader = csv.reader(tabfile, delimiter='\t')
-            for row in reader:
-                dictionary['en'][row[0]]={}
-                dictionary['en'][row[0]]['V']=row[2]
-                dictionary['en'][row[0]]['A']=row[4]
-                dictionary['en'][row[0]]['D']=row[6]
-        self._dictionary = dictionary
-
-    def _my_preprocessor(self, text):
-
-        regHttp = re.compile('(http://)[a-zA-Z0-9]*.[a-zA-Z0-9/]*(.[a-zA-Z0-9]*)?')
-        regHttps = re.compile('(https://)[a-zA-Z0-9]*.[a-zA-Z0-9/]*(.[a-zA-Z0-9]*)?')
-        regAt = re.compile('@([a-zA-Z0-9]*[*_/&%#@$]*)*[a-zA-Z0-9]*')
-        text = re.sub(regHttp, '', text)
-        text = re.sub(regAt, '', text)
-        text = re.sub('RT : ', '', text)
-        text = re.sub(regHttps, '', text)
-        text = re.sub('[0-9]', '', text)
-        text = self._delete_punctuation(text)
-        return text
-
-    def _delete_punctuation(self, text):
-
-        exclude = set(string.punctuation)
-        s = ''.join(ch for ch in text if ch not in exclude)
-        return s
-
-    def _extract_ngrams(self, text, lang):
-        unigrams_lemmas = []
-        unigrams_words = []
-        pos_tagged = []
-        if lang == 'es':
-            sentences = list(parse_es(text, lemmata=True).split())
-        else:
-            sentences = list(parse_en(text, lemmata=True).split())
-
-        for sentence in sentences:
-            for token in sentence:
-                if token[0].lower() not in self._stopwords:
-                    unigrams_words.append(token[0].lower())
-                    unigrams_lemmas.append(token[4])
-                    pos_tagged.append(token[1])
-
-        return unigrams_lemmas,unigrams_words,pos_tagged
-
-    def _find_ngrams(self, input_list, n):
-        return zip(*[input_list[i:] for i in range(n)])
-
-    def _extract_features(self, tweet,dictionary,lang):
-        feature_set={}
-        ngrams_lemmas,ngrams_words,pos_tagged = self._extract_ngrams(tweet,lang)
-        pos_tags={'NN':'NN', 'NNS':'NN', 'JJ':'JJ', 'JJR':'JJ', 'JJS':'JJ', 'RB':'RB', 'RBR':'RB',
-         'RBS':'RB', 'VB':'VB', 'VBD':'VB', 'VGB':'VB', 'VBN':'VB', 'VBP':'VB', 'VBZ':'VB'}
-        totalVAD=[0,0,0]
-        matches=0
-        for word in range(len(ngrams_lemmas)):
-            VAD=[]
-            if ngrams_lemmas[word] in dictionary:
-                matches+=1
-                totalVAD = [totalVAD[0]+float(dictionary[ngrams_lemmas[word]]['V']),
-                            totalVAD[1]+float(dictionary[ngrams_lemmas[word]]['A']),
-                            totalVAD[2]+float(dictionary[ngrams_lemmas[word]]['D'])]
-            elif ngrams_words[word] in dictionary:
-                matches+=1
-                totalVAD = [totalVAD[0]+float(dictionary[ngrams_words[word]]['V']),
-                            totalVAD[1]+float(dictionary[ngrams_words[word]]['A']),
-                            totalVAD[2]+float(dictionary[ngrams_words[word]]['D'])]
-        if matches==0:
-            emotion='neutral'
-        else:
-            totalVAD=[totalVAD[0]/matches,totalVAD[1]/matches,totalVAD[2]/matches]
-        feature_set['V'] = totalVAD[0]
-        feature_set['A'] = totalVAD[1]
-        feature_set['D'] = totalVAD[2]
-        return feature_set
-
-    def analyse_entry(self, entry, activity):
-        params = activity.params
-
-        text_input = entry.text
-
-        text = self._my_preprocessor(text_input)
-        dictionary = self._dictionary[params['language']]
-
-        feature_set=self._extract_features(text, dictionary, params['language'])
-
-        emotions = EmotionSet()
-        emotions.id = "Emotions0"
-
-        emotion1 = Emotion(id="Emotion0")
-        emotion1[VALENCE] = feature_set['V']
-        emotion1[AROUSAL] = feature_set['A']
-        emotion1[DOMINANCE] = feature_set['D']
-
-        emotion1.prov(activity)
-        emotions.prov(activity)
-
-        emotions.onyx__hasEmotion.append(emotion1)
-        entry.emotions = [emotions, ]
-
-        yield entry
-
-    test_cases: list[dict] = [
-        {
-            'name': 'anger with VAD=(2.12, 6.95, 5.05)',
-            'input': 'I hate you',
-            'expected': {
-                'onyx:hasEmotionSet': [{
-                    'onyx:hasEmotion': [{
-                        AROUSAL: 6.95,
-                        DOMINANCE: 5.05,
-                        VALENCE: 2.12,
-                    }]
-                }]
-            }
-        }, {
-            'input': 'i am sad',
-            'expected': {
-                'onyx:hasEmotionSet': [{
-                    'onyx:hasEmotion': [{
-                        f"{MODEL}_arousal": 4.13,
-
-                    }]
-                }]
-            }
-        }, {
-            'name': 'joy',
-            'input': 'i am happy with my marks',
-            'expected': {
-                'onyx:hasEmotionSet': [{
-                    'onyx:hasEmotion': [{
-                        AROUSAL: 6.49,
-                        DOMINANCE: 6.63,
-                        VALENCE: 8.21,
-                    }]
-                }]
-            }
-        }, {
-            'name': 'negative-feat',
-            'input': 'This movie is scary',
-            'expected': {
-                'onyx:hasEmotionSet': [{
-                    'onyx:hasEmotion': [{
-                    AROUSAL: 5.8100000000000005,
-                    DOMINANCE: 4.33,
-                    VALENCE: 5.050000000000001,
-
-                    }]
-                }]
-            }
-        }, {
-            'name': 'negative-fear',
-            'input': 'this cake is disgusting' ,
-            'expected': {
-                'onyx:hasEmotionSet': [{
-                    'onyx:hasEmotion': [{
-                        AROUSAL: 5.09,
-                        DOMINANCE: 4.4,
-                        VALENCE: 5.109999999999999,
-
-                    }]
-                }]
-            }
-        }
-    ]

senpy/plugins/emotion/depechemood_plugin.py

@@ -1,178 +0,0 @@
-#!/usr/local/bin/python
-# coding: utf-8
-
-from future import standard_library
-standard_library.install_aliases()
-
-import os
-import re
-import sys
-import string
-import numpy as np
-from six.moves import urllib
-import nltk
-
-from nltk.corpus import stopwords
-
-from senpy import EmotionBox, models
-
-
-def ignore(dchars):
-    deletechars = "".join(dchars)
-    tbl = str.maketrans("", "", deletechars)
-    ignore = lambda s: s.translate(tbl)
-    return ignore
-
-
-class DepecheMood(EmotionBox):
-    '''
-    Plugin that uses the DepecheMood emotion lexicon.
-
-    DepecheMood is an emotion lexicon automatically generated from news articles where users expressed their associated emotions. It contains two languages (English and Italian), as well as three types of word representations (token, lemma and lemma#PoS). For English, the lexicon contains 165k tokens, while the Italian version contains 116k. Unsupervised techniques can be applied to generate simple but effective baselines. To learn more, please visit https://github.com/marcoguerini/DepecheMood and http://www.depechemood.eu/
-    '''
-
-    author: str = 'Oscar Araque'
-    name: str = 'emotion-depechemood'
-    version: str = '0.1'
-    requirements = ['pandas']
-    optional = True
-    usesEmotionModel: str = 'wna:WNAModel'
-
-    nltk_resources = ["stopwords"]
-
-
-    EMOTIONS =  ['wna:negative-fear',
-                 'wna:amusement',
-                 'wna:anger',
-                 'wna:annoyance',
-                 'wna:indifference',
-                 'wna:joy',
-                 'wna:awe',
-                 'wna:sadness']
-
-    DM_EMOTIONS = ['AFRAID', 'AMUSED', 'ANGRY', 'ANNOYED', 'DONT_CARE', 'HAPPY', 'INSPIRED', 'SAD',]
-    LEXICON_URL = "https://github.com/marcoguerini/DepecheMood/raw/master/DepecheMood%2B%2B/DepecheMood_english_token_full.tsv"
-
-    def __init__(self, **data):
-        super().__init__(**data)
-        self._denoise = ignore(set(string.punctuation)|set('«»'))
-        self._stop_words = []
-        self._lex = self.download_lex()
-        self._lex_vocab = set(list(self._lex.keys()))
-        self._stop_words = stopwords.words('english') + ['']
-
-    def clean_str(self, string):
-        string = re.sub(r"[^A-Za-z0-9().,!?\'\`]", " ", string)
-        string = re.sub(r"[0-9]+", " num ", string)
-        string = re.sub(r"\'s", " \'s", string)
-        string = re.sub(r"\'ve", " \'ve", string)
-        string = re.sub(r"n\'t", " n\'t", string)
-        string = re.sub(r"\'re", " \'re", string)
-        string = re.sub(r"\'d", " \'d", string)
-        string = re.sub(r"\'ll", " \'ll", string)
-        string = re.sub(r"\.", " . ", string)
-        string = re.sub(r",", " , ", string)
-        string = re.sub(r"!", " ! ", string)
-        string = re.sub(r"\(", " ( ", string)
-        string = re.sub(r"\)", " ) ", string)
-        string = re.sub(r"\?", " ? ", string)
-        string = re.sub(r"\s{2,}", " ", string)
-        return string.strip().lower()
-
-    def preprocess(self, text):
-        if text is None:
-            return None
-        tokens = self._denoise(self.clean_str(text)).split(' ')
-        tokens = [tok for tok in tokens if tok not in self._stop_words]
-        return tokens   
-
-    def estimate_emotion(self, tokens, emotion):
-        s = []
-        for tok in tokens:
-            s.append(self._lex[tok][emotion])
-        dividend = np.sum(s) if np.sum(s) > 0 else 0
-        divisor = len(s) if len(s) > 0 else 1
-        S = np.sum(s) / divisor
-        return S
-
-    def estimate_all_emotions(self, tokens):
-        S = []
-        intersection = set(tokens) & self._lex_vocab
-        for emotion in self.DM_EMOTIONS:
-            s = self.estimate_emotion(intersection, emotion)
-            S.append(s)
-        return S
-
-    def download_lex(self, file_path='DepecheMood_english_token_full.tsv', freq_threshold=10):
-
-        import pandas as pd
-
-        try:
-            file_path = self.find_file(file_path)
-        except IOError:
-            file_path = self.path(file_path)
-            filename, _ = urllib.request.urlretrieve(self.LEXICON_URL, file_path)
-
-        lexicon = pd.read_csv(file_path, sep='\t', index_col=0)
-        lexicon = lexicon[lexicon['freq'] >= freq_threshold]
-        lexicon.drop('freq', axis=1, inplace=True)
-        lexicon = lexicon.T.to_dict()
-        return lexicon
-
-    def predict_one(self, features, **kwargs):
-        tokens = self.preprocess(features[0])
-        estimation = self.estimate_all_emotions(tokens)
-        return estimation
-
-    test_cases: list[dict] = [
-        {
-            'entry': {
-                'nif:isString': 'My cat is very happy',
-            },
-            'expected': {
-                'onyx:hasEmotionSet': [
-                    {
-                        'onyx:hasEmotion': [
-                            {
-                             'onyx:hasEmotionCategory': 'wna:negative-fear',
-                             'onyx:hasEmotionIntensity': 0.05278117640010922
-                            },
-                            {
-                                'onyx:hasEmotionCategory': 'wna:amusement',
-                                'onyx:hasEmotionIntensity': 0.2114806151413433,
-                            },
-                            {
-                                'onyx:hasEmotionCategory': 'wna:anger',
-                                'onyx:hasEmotionIntensity': 0.05726119426520887
-                            },
-                            {
-                                'onyx:hasEmotionCategory': 'wna:annoyance',
-                                'onyx:hasEmotionIntensity': 0.12295990731053638,
-                            },
-                            {
-                                'onyx:hasEmotionCategory': 'wna:indifference',
-                                'onyx:hasEmotionIntensity': 0.1860159893608025,
-                            },
-                            {
-                                'onyx:hasEmotionCategory': 'wna:joy',
-                                'onyx:hasEmotionIntensity': 0.12904050973724163,
-                            },
-                            {
-                                'onyx:hasEmotionCategory': 'wna:awe',
-                                'onyx:hasEmotionIntensity': 0.17973650399862967,
-                            },
-                            {
-                                'onyx:hasEmotionCategory': 'wna:sadness',
-                                'onyx:hasEmotionIntensity': 0.060724103786128455,
-                            },
-                        ]
-                    }
-                ]
-            }
-        }
-    ]
-
-
-if __name__ == '__main__':
-    from senpy.utils import easy_test
-    easy_test(debug=False)

senpy/plugins/emotion/wnaffect/Dockerfile.template

@@ -1,3 +0,0 @@
-FROM gsiupm/senpy:python{{PYVERSION}}
-
-MAINTAINER manuel.garcia-amado.sancho@alumnos.upm.es