Fixed version.sh

.gitignore

@@ -6,5 +6,3 @@ build
 README.html
 __pycache__
 VERSION
-Dockerfile-*
-Dockerfile

.gitlab-ci.yml

@@ -1,98 +1,80 @@
-# Uncomment if you want to use docker-in-docker
+image: docker:latest
-# image: gsiupm/dockermake:latest
+
-# services:
+
-# - docker:dind
 # When using dind, it's wise to use the overlayfs driver for
 # improved performance.
+variables:
+  DOCKER_DRIVER: overlay
+  DOCKERFILE: Dockerfile
+
+before_script:
+  - sh version.sh > senpy/VERSION
 
 stages:
   - test
-  - push
+  - images
-  - deploy
+  - release
-  - clean
-
-before_script:
-  - docker login -u $HUB_USER -p $HUB_PASSWORD
 
 .test: &test_definition
+  variables:
+    PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
+  cache:
+    paths:
+      - .eggs/
+      - "$CI_PROJECT_DIR/pip-cache"
+      - .venv
+    key: "$CI_PROJECT_NAME"
   stage: test
   script:
-    - make -e test-$PYTHON_VERSION
+    - pip install --use-wheel -U pip setuptools virtualenv
+    - virtualenv .venv/$PYTHON_VERSION
+    - source .venv/$PYTHON_VERSION/bin/activate
+    - pip install --use-wheel -r requirements.txt
+    - pip install --use-wheel -r test-requirements.txt
+    - py.test --cov=senpy --cov-report term-missing
+    - python 
     
 test-3.5:
   <<: *test_definition
-  variables:
+  image: "python:3.5"
-    PYTHON_VERSION: "3.5"
 
 test-2.7:
   <<: *test_definition
-  variables:
+  image: "python:2.7"
-    PYTHON_VERSION: "2.7"
+
 
 .image: &image_definition
-  stage: push
+  variables:
+    PYTHON_VERSION: "3.5"
+  before_script:
+    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
   script:
-    - make -e push-$PYTHON_VERSION
+    - docker build -f Dockerfile-3.5 . -t $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG-$PYTHON_VERSION
+    - docker push $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG-$PYTHON_VERSION
+  stage: images
   only:
     - tags
     - triggers
 
-push-3.5:
+image-3.5:
   <<: *image_definition
   variables:
     PYTHON_VERSION: "3.5"
 
-push-2.7:
+image-2.7:
   <<: *image_definition
   variables:
     PYTHON_VERSION: "2.7"
 
-push-latest:
+image-latest:
-  <<: *image_definition
+  stage: release
-  variables:
+  before_script:
-    PYTHON_VERSION: latest
+    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
+  script:
+    - docker build -f Dockerfile . -t $CI_REGISTRY_IMAGE
+    - docker tag $CI_REGISTRY_IMAGE $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG
+    - docker push $CI_REGISTRY_IMAGE
+    - docker push $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG
   only:
     - master
-    - triggers
+    - triggers
-
-push-github:
-  stage: deploy
-  script:
-    - make -e push-github
-  only:
-    - master
-    - triggers
-
-deploy_pypi:
-  stage: deploy
-  script:   # Configure the PyPI credentials, then push the package, and cleanup the creds.
-    - echo "[server-login]" >> ~/.pypirc
-    - echo "username=" ${PYPI_USER} >> ~/.pypirc
-    - echo "password=" ${PYPI_PASSWORD} >> ~/.pypirc
-    - make pip_upload
-    - echo "" > ~/.pypirc && rm ~/.pypirc  # If the above fails, this won't run.
-  only:
-    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
-  except:
-    - branches
-
-deploy:
-  stage: deploy
-  environment: test
-  script:
-    - make -e deploy
-  only:
-    - master
-
-clean_docker :
-  stage: clean
-  script:
-    - make -e clean
-  when: manual
-
-cleanup_pypirc:
-   stage: clean
-   when: always   # this is important; run even if preceding stages failed.
-   script:
-    - rm -vf ~/.pypirc  # we don't want to leave these around, but GitLab may clean up anyway.
-    - docker logout

.pre-commit-config.yaml

@@ -1,5 +0,0 @@
--   repo: git://github.com/pre-commit/pre-commit-hooks
-    sha: e626cd57090d8df0be21e4df0f4e55cc3511d6ab
-    hooks:
-    -   id: flake8
-    -   id: check-json

.travis.yml

@@ -1,13 +1,8 @@
-sudo: required
-
-services:
-  - docker
-
 language: python
-
+python:
-env:
+  - "2.7"
-  - PYV=2.7
+  - "3.4"
-  - PYV=3.4
+  - "3.5"
-  - PYV=3.5
+install: "pip install -r requirements.txt"
 # run nosetests - Tests
-script: make test-$PYV
+script: nosetests

Dockerfile

@@ -0,0 +1 @@
+Dockerfile-3.5

Dockerfile-2.7

@@ -0,0 +1,21 @@
+from python:2.7
+
+RUN mkdir /cache/
+ENV PIP_CACHE_DIR=/cache/
+
+WORKDIR /usr/src/app
+ADD requirements.txt /usr/src/app/
+RUN pip install --use-wheel -r requirements.txt
+ADD . /usr/src/app/
+RUN pip install .
+
+
+VOLUME /data/
+
+RUN mkdir /senpy-plugins/
+
+WORKDIR /senpy-plugins/
+ONBUILD ADD . /senpy-plugins/
+ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
+
+ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

Dockerfile-3.4

@@ -0,0 +1,21 @@
+from python:3.4
+
+RUN mkdir /cache/
+ENV PIP_CACHE_DIR=/cache/
+
+WORKDIR /usr/src/app
+ADD requirements.txt /usr/src/app/
+RUN pip install --use-wheel -r requirements.txt
+ADD . /usr/src/app/
+RUN pip install .
+
+
+VOLUME /data/
+
+RUN mkdir /senpy-plugins/
+
+WORKDIR /senpy-plugins/
+ONBUILD ADD . /senpy-plugins/
+ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
+
+ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

Dockerfile-3.5

@@ -0,0 +1,21 @@
+FROM python:3.5
+
+RUN mkdir /cache/
+ENV PIP_CACHE_DIR=/cache/
+
+WORKDIR /usr/src/app
+ADD requirements.txt /usr/src/app/
+RUN pip install --use-wheel -r requirements.txt
+ADD . /usr/src/app/
+RUN pip install .
+
+
+VOLUME /data/
+
+RUN mkdir /senpy-plugins/
+
+WORKDIR /senpy-plugins/
+ONBUILD ADD . /senpy-plugins/
+ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
+
+ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

Dockerfile.deps

@@ -0,0 +1,33 @@
+from python:2.7
+
+RUN apt-get update
+RUN apt-get -y install git
+RUN mkdir -p /senpy-plugins
+
+RUN apt-get -y install python-numpy
+RUN apt-get -y install python-scipy
+RUN apt-get -y install python-sklearn
+RUN apt-get -y install python-gevent
+RUN apt-get -y install libopenblas-dev
+RUN apt-get -y install gfortran
+RUN apt-get -y install libxml2-dev libxslt1-dev python-dev
+
+#RUN pip install --upgrade pip
+
+ADD id_rsa /root/.ssh/id_rsa
+RUN chmod 700 /root/.ssh/id_rsa
+RUN echo "Host github.com\n\tStrictHostKeyChecking no\n" >> /root/.ssh/config
+
+RUN git clone https://github.com/gsi-upm/senpy /usr/src/app/
+RUN git clone git@github.com:gsi-upm/senpy-plugins-enterprise /senpy-plugins/enterprise
+RUN git clone https://github.com/gsi-upm/senpy-plugins-community /senpy-plugins/community
+
+RUN pip install /usr/src/app
+RUN pip install --no-use-wheel -r /senpy-plugins/enterprise/requirements.txt
+RUN python -m nltk.downloader stopwords
+RUN python -m nltk.downloader punkt
+RUN python -m nltk.downloader maxent_treebank_pos_tagger
+RUN python -m nltk.downloader wordnet
+
+WORKDIR /senpy-plugins
+ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]

Dockerfile.template

@@ -1,22 +1,21 @@
 from python:{{PYVERSION}}
 
-MAINTAINER J. Fernando Sánchez <jf.sanchez@upm.es>
+RUN mkdir /cache/
+ENV PIP_CACHE_DIR=/cache/
+
+WORKDIR /usr/src/app
+ADD requirements.txt /usr/src/app/
+RUN pip install --use-wheel -r requirements.txt
+ADD . /usr/src/app/
+RUN pip install .
 
-RUN mkdir /cache/ /senpy-plugins /data/
 
 VOLUME /data/
 
-ENV PIP_CACHE_DIR=/cache/ SENPY_DATA=/data
+RUN mkdir /senpy-plugins/
 
-ONBUILD COPY . /senpy-plugins/
+WORKDIR /senpy-plugins/
+ONBUILD ADD . /senpy-plugins/
 ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
-ONBUILD WORKDIR /senpy-plugins/
 
-
+ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]
-WORKDIR /usr/src/app
-COPY test-requirements.txt requirements.txt /usr/src/app/
-RUN pip install --use-wheel -r test-requirements.txt -r requirements.txt
-COPY . /usr/src/app/
-RUN pip install --no-index --no-deps --editable .
-
-ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

Makefile

@@ -1,43 +1,25 @@
-NAME=senpy
+PYVERSIONS=3.5 3.4 2.7
-VERSION=$(shell git describe --tags --dirty 2>/dev/null)
-GITHUB_REPO=git@github.com:gsi-upm/senpy.git
-
-IMAGENAME=gsiupm/senpy
-IMAGEWTAG=$(IMAGENAME):$(VERSION)
-
-PYVERSIONS=3.5 2.7
 PYMAIN=$(firstword $(PYVERSIONS))
-
+NAME=senpy
-DEVPORT=5000
+REPO=gsiupm
-
+VERSION=$(shell ./version.sh)
 TARNAME=$(NAME)-$(VERSION).tar.gz 
-action="test-${PYMAIN}"
+IMAGENAME=$(REPO)/$(NAME):$(VERSION)
-
+TEST_COMMAND=gitlab-runner exec docker --cache-dir=/tmp/gitlabrunner --docker-volumes /tmp/gitlabrunner:/tmp/gitlabrunner --env CI_PROJECT_NAME=$(NAME)
-KUBE_CA_PEM_FILE=""
-KUBE_URL=""
-KUBE_TOKEN=""
-KUBE_NAMESPACE=$(NAME)
-KUBECTL=docker run --rm -v $(KUBE_CA_PEM_FILE):/tmp/ca.pem -v $$PWD:/tmp/cwd/ -i lachlanevenson/k8s-kubectl --server="$(KUBE_URL)" --token="$(KUBE_TOKEN)" --certificate-authority="/tmp/ca.pem" -n $(KUBE_NAMESPACE)
-CI_REGISTRY=docker.io
-CI_REGISTRY_USER=gitlab
-CI_BUILD_TOKEN=""
-CI_COMMIT_REF_NAME=master
-
-
 
 all: build run
 
-.FORCE:
+FORCE:
 
-version: .FORCE
+version: FORCE
 	@echo $(VERSION) > $(NAME)/VERSION
 	@echo $(VERSION)
 
 yapf:
-	yapf -i -r $(NAME)
+	yapf -i -r senpy
 	yapf -i -r tests
 
-init:
+dev:
 	pip install --user pre-commit
 	pre-commit install
 
@@ -52,32 +34,29 @@ quick_build: $(addprefix build-, $(PYMAIN))
 
 build: $(addprefix build-, $(PYVERSIONS))
 
-build-%: version Dockerfile-%
+build-%: Dockerfile-%
-	docker build -t '$(IMAGEWTAG)-python$*' --cache-from $(IMAGENAME):python$* -f Dockerfile-$* .;
+	docker build -t '$(IMAGENAME)-python$*' -f Dockerfile-$* .;
 
 quick_test: $(addprefix test-,$(PYMAIN))
 
-dev-%:
+test: $(addprefix test-,$(PYVERSIONS))
-	@docker start $(NAME)-dev$* || (\
+
+debug-%:
+	@docker start $(NAME)-debug || (\
 		$(MAKE) build-$*; \
-		docker run -d -w /usr/src/app/ -p $(DEVPORT):5000 -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti --name $(NAME)-dev$* '$(IMAGEWTAG)-python$*'; \
+		docker run -d -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -p 5000:5000 -ti --name $(NAME)-debug '$(IMAGENAME)-python$*'; \
+		docker exec -ti $(NAME)-debug pip install -r test-requirements.txt; \
 	)\
 
-	docker exec -ti $(NAME)-dev$* bash
+	docker attach $(NAME)-debug
 
-dev: dev-$(PYMAIN)
+debug: debug-$(PYMAIN)
-
-test-all: $(addprefix test-,$(PYVERSIONS))
 
 test-%:
-	docker run --rm --entrypoint /usr/local/bin/python -w /usr/src/app $(IMAGENAME):python$*  setup.py test
+	$(TEST_COMMAND) test-$*
 
-test: test-$(PYMAIN)
+dist/$(TARNAME):
-
-dist/$(TARNAME): version
 	docker run --rm -ti -v $$PWD:/usr/src/app/ -w /usr/src/app/ python:$(PYMAIN) python setup.py sdist;
-	docker run --rm -ti -v $$PWD:/usr/src/app/ -w /usr/src/app/ python:$(PYMAIN) chmod -R a+rwx dist;
-
 
 sdist: dist/$(TARNAME)
 
@@ -86,14 +65,20 @@ pip_test-%: sdist
 
 pip_test: $(addprefix pip_test-,$(PYVERSIONS))
 
-pip_upload: pip_test 
+upload-%: test-%
-	python setup.py sdist upload ;
+	docker push '$(IMAGENAME)-python$*'
+
+upload: test $(addprefix upload-,$(PYVERSIONS))
+	docker tag '$(IMAGENAME)-python$(PYMAIN)' '$(IMAGENAME)'
+	docker tag '$(IMAGENAME)-python$(PYMAIN)' '$(REPO)/$(NAME)'
+	docker push '$(IMAGENAME)'
+	docker push '$(REPO)/$(NAME)'
 
 clean:
-	@docker ps -a | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true
+	@docker ps -a | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true
-	@docker images | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true
+	@docker images | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true
-	@docker stop $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true
+	@docker rmi $(NAME)-debug 2>/dev/null || true
-	@docker rm $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true
+
 
 git_commit:
 	git commit -a
@@ -101,47 +86,17 @@ git_commit:
 git_tag:
 	git tag ${VERSION}
 
-git_push:
+upload_git:
 	git push --tags origin master
 
+pip_upload:
+	python setup.py sdist upload ;
+
+pip_test: $(addprefix pip_test-,$(PYVERSIONS))
+
 run-%: build-%
-	docker run --rm -p $(DEVPORT):5000 -ti '$(IMAGEWTAG)-python$(PYMAIN)' --default-plugins
+	docker run --rm -p 5000:5000 -ti '$(IMAGENAME)-python$(PYMAIN)' --default-plugins
 
 run: run-$(PYMAIN)
 
-push-latest: $(addprefix push-latest-,$(PYVERSIONS))
+.PHONY: test test-% build-% build test pip_test run yapf dev
-	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
-	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGENAME)'
-	docker push '$(IMAGENAME):latest'
-	docker push '$(IMAGEWTAG)'
-
-push-latest-%: build-%
-	docker tag $(IMAGENAME):$(VERSION)-python$* $(IMAGENAME):python$*
-	docker push $(IMAGENAME):$(VERSION)-python$*
-	docker push $(IMAGENAME):python$*
-
-push-%: build-%
-	docker push $(IMAGENAME):$(VERSION)-python$*
-
-push: $(addprefix push-,$(PYVERSIONS))
-	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
-	docker push  $(IMAGENAME):$(VERSION)
-
-push-github:
-	$(eval KEY_FILE := $(shell mktemp))
-	@echo "$$GITHUB_DEPLOY_KEY" > $(KEY_FILE)
-	@git remote rm github-deploy || true
-	git remote add github-deploy $(GITHUB_REPO)
-	@GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git push github-deploy $(CI_COMMIT_REF_NAME)
-	rm $(KEY_FILE)
-
-ci:
-	gitlab-runner exec docker --docker-volumes /var/run/docker.sock:/var/run/docker.sock --env CI_PROJECT_NAME=$(NAME) ${action}
-
-deploy:
-	@$(KUBECTL) delete secret $(CI_REGISTRY) || true
-	@$(KUBECTL) create secret docker-registry $(CI_REGISTRY) --docker-server=$(CI_REGISTRY) --docker-username=$(CI_REGISTRY_USER) --docker-email=$(CI_REGISTRY_USER) --docker-password=$(CI_BUILD_TOKEN)
-	@$(KUBECTL) apply -f /tmp/cwd/k8s/
-
-
-.PHONY: test test-% test-all build-% build test pip_test run yapf push-main push-% dev ci version .FORCE deploy

README.rst

@@ -23,7 +23,7 @@ Through PIP
 
 .. code:: bash
 
-   pip install -U --user senpy
+   pip install --user senpy
 
    
 Alternatively, you can use the development version:
@@ -38,56 +38,9 @@ If you want to install senpy globally, use sudo instead of the ``--user`` flag.
 
 Docker Image
 ************
-Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy --default-plugins``.
+Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy --host 0.0.0.0 --default-plugins``.
 
-To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --default-plugins -f /plugins``
+To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --host 0.0.0.0 --default-plugins -f /plugins``
-
-
-Developing
-----------
-
-Developing/debugging
-********************
-This command will run the senpy container using the latest image available, mounting your current folder so you get your latest code:
-
-.. code:: bash
-
-
-    # Python 3.5
-    make dev
-    # Python 2.7
-    make dev-2.7
-
-Building a docker image
-***********************
-
-.. code:: bash
-
-
-    # Python 3.5
-    make build-3.5
-    # Python 2.7
-    make build-2.7
-
-Testing
-*******
-
-.. code:: bash
-
-
-    make test
-
-Running
-*******
-This command will run the senpy server listening on localhost:5000
-
-.. code:: bash
-
-
-    # Python 3.5
-    make run-3.5
-    # Python 2.7
-    make run-2.7
 
 Usage
 -----
@@ -96,14 +49,12 @@ However, the easiest and recommended way is to just use the command-line tool to
 
 .. code:: bash
 
-
    senpy
 
 or, alternatively:
 
 .. code:: bash
 
-
     python -m senpy
 
 

docs/api.rst

@@ -1,5 +1,5 @@
 NIF API
--------
+=======
 .. http:get:: /api
 
    Basic endpoint for sentiment/emotion analysis.
@@ -22,32 +22,38 @@ NIF API
       Content-Type: text/javascript
 
       {
-        "@context":"http://127.0.0.1/api/contexts/Results.jsonld",
+        "@context": [
-        "@id":"_:Results_11241245.22",
+          "http://127.0.0.1/static/context.jsonld",
-        "@type":"results"
+      ],
-        "analysis": [
+      "analysis": [
-          "plugins/sentiment-140_0.1" 
+        {
-        ],
+          "@id": "SentimentAnalysisExample",
-        "entries": [
+          "@type": "marl:SentimentAnalysis",
-          {  
+          "dc:language": "en", 
-            "@id": "_:Entry_11241245.22"
+          "marl:maxPolarityValue": 10.0,
-            "@type":"entry",
+          "marl:minPolarityValue": 0.0
-            "emotions": [],
+        }
-            "entities": [],
+      ],
-            "sentiments": [
+      "domain": "wndomains:electronics", 
-              {  
+      "entries": [
-                "@id": "Sentiment0",  
+        {
-                "@type": "sentiment", 
+          "opinions": [
-                "marl:hasPolarity": "marl:Negative",
+            {
-                "marl:polarityValue": 0,
+              "prov:generatedBy": "SentimentAnalysisExample",
-                "prefix": ""
+              "marl:polarityValue": 7.8, 
-              }
+              "marl:hasPolarity": "marl:Positive",
-            ],
+              "marl:describesObject": "http://www.gsi.dit.upm.es",
-            "suggestions": [],
+            }
-            "text": "This text makes me sad.\nwhilst this text makes me happy and surprised at the same time.\nI cannot believe it!",
+          ],
-            "topics": []
+          "nif:isString": "I love GSI",
-          }
+          "strings": [
-        ]
+            {
+              "nif:anchorOf": "GSI",
+              "nif:taIdentRef": "http://www.gsi.dit.upm.es"
+            }
+          ]
+        }
+       ]
       }
 
    :query i input: No default. Depends on informat and intype
@@ -56,7 +62,6 @@ NIF API
    :query o outformat: one of `turtle` (default), `text`, `json-ld`
    :query p prefix: prefix for the URIs
    :query algo algorithm: algorithm/plugin to use for the analysis. For a list of options, see :http:get:`/api/plugins`. If not provided, the default plugin will be used (:http:get:`/api/plugins/default`).
-   :query algo emotionModel: desired emotion model in the results. If the requested algorithm does not use that emotion model, there are conversion plugins specifically for this. If none of the plugins match, an error will be returned, which includes the results *as is*.
 
    :reqheader Accept: the response content type depends on
                       :mailheader:`Accept` header
@@ -64,7 +69,6 @@ NIF API
                             header of request
    :statuscode 200: no error
    :statuscode 404: service not found
-   :statuscode 400: error while processing the request
 
 .. http:post:: /api
 
@@ -86,59 +90,56 @@ NIF API
 
    .. sourcecode:: http
 
-      {
+        {
-        "@id": "plugins/sentiment-140_0.1", 
+            "@context": {
-        "@type": "sentimentPlugin", 
+                   ...
-        "author": "@balkian", 
+            }, 
-        "description": "Sentiment classifier using rule-based classification for English and Spanish. This plugin uses sentiment140 data to perform classification. For more information: http://help.sentiment140.com/for-students/", 
+            "sentiment140": {
-        "extra_params": {
+                "name": "sentiment140", 
-          "language": {
+                "is_activated": true, 
-            "@id": "lang_sentiment140", 
+                "version": "0.1", 
-            "aliases": [
+                "extra_params": {
-              "language", 
+                    "@id": "extra_params_sentiment140_0.1", 
-              "l"
+                    "language": {
-            ], 
+                        "required": false, 
-            "options": [
+                        "@id": "lang_sentiment140", 
-              "es", 
+                        "options": [
-              "en", 
+                            "es", 
-              "auto"
+                            "en", 
-            ], 
+                            "auto"
-            "required": false
+                        ], 
-          }
+                        "aliases": [
-        }, 
+                            "language", 
-        "is_activated": true, 
+                            "l"
-        "maxPolarityValue": 1.0, 
+                        ]
-        "minPolarityValue": 0.0, 
+                    }
-        "module": "sentiment-140", 
+                }, 
-        "name": "sentiment-140", 
+                "@id": "sentiment140_0.1"
-        "requirements": {}, 
+            }, 
-        "version": "0.1"
+            "rand": {
-      }, 
+                "name": "rand", 
-      {
+                "is_activated": true, 
-        "@id": "plugins/ExamplePlugin_0.1", 
+                "version": "0.1", 
-        "@type": "sentimentPlugin", 
+                "extra_params": {
-        "author": "@balkian", 
+                    "@id": "extra_params_rand_0.1", 
-        "custom_attribute": "42", 
+                    "language": {
-        "description": "I am just an example", 
+                        "required": false, 
-        "extra_params": {
+                        "@id": "lang_rand", 
-          "parameter": {
+                        "options": [
-            "@id": "parameter", 
+                            "es", 
-            "aliases": [
+                            "en", 
-              "parameter", 
+                            "auto"
-              "param"
+                        ], 
-            ], 
+                        "aliases": [
-            "default": 42, 
+                            "language", 
-            "required": true
+                            "l"
-          }
+                        ]
-        }, 
+                    }
-        "is_activated": true, 
+                }, 
-        "maxPolarityValue": 1.0, 
+                "@id": "rand_0.1"
-        "minPolarityValue": 0.0, 
+            }
-        "module": "example", 
+        }
-        "name": "ExamplePlugin", 
+
-        "requirements": "noop", 
-        "version": "0.1"
-      }
 
 .. http:get:: /api/plugins/<pluginname>
 
@@ -147,7 +148,7 @@ NIF API
 
    .. sourcecode:: http
 
-      GET /api/plugins/rand/ HTTP/1.1
+      GET /api/plugins/rand HTTP/1.1
       Host: localhost
       Accept: application/json, text/javascript
 
@@ -157,61 +158,51 @@ NIF API
    .. sourcecode:: http
 
       {
-        "@context": "http://127.0.0.1/api/contexts/ExamplePlugin.jsonld", 
+          "@id": "rand_0.1",
-        "@id": "plugins/ExamplePlugin_0.1", 
+          "extra_params": {
-        "@type": "sentimentPlugin", 
+              "@id": "extra_params_rand_0.1",
-        "author": "@balkian", 
+              "language": {
-        "custom_attribute": "42", 
+                  "@id": "lang_rand",
-        "description": "I am just an example", 
+                  "aliases": [
-        "extra_params": {
+                      "language",
-          "parameter": {
+                      "l"
-            "@id": "parameter", 
+                  ],
-            "aliases": [
+                  "options": [
-              "parameter", 
+                      "es",
-              "param"
+                      "en",
-            ], 
+                      "auto"
-            "default": 42, 
+                  ],
-            "required": true
+                  "required": false
-          }
+              }
-        }, 
+          },
-        "is_activated": true, 
+          "is_activated": true,
-        "maxPolarityValue": 1.0, 
+          "name": "rand",
-        "minPolarityValue": 0.0, 
+          "version": "0.1"
-        "module": "example", 
-        "name": "ExamplePlugin", 
-        "requirements": "noop", 
-        "version": "0.1"
       }
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
 .. http:get:: /api/plugins/default
 
    Return the information about the default plugin.
 
+.. http:get:: /api/plugins/<pluginname>/{de}activate
+
+   {De}activate a plugin.
+
+   **Example request**:
+
+   .. sourcecode:: http
+
+      GET /api/plugins/rand/deactivate HTTP/1.1
+      Host: localhost
+      Accept: application/json, text/javascript
+
+
+   **Example response**:
+
+   .. sourcecode:: http
+
+       {
+          "@context": {}, 
+          "message": "Ok"
+       }

docs/apischema.rst

@@ -1,7 +0,0 @@
-API and Schema
-##############
-.. toctree::
-
-   vocabularies.rst
-   api.rst
-   schema.rst

docs/architecture.rst

@@ -1,15 +0,0 @@
-Architecture
-============
-
-The main component of a sentiment analysis service is the algorithm itself. However, for the algorithm to work, it needs to get the appropriate parameters from the user, format the results according to the defined API, interact with the user whn errors occur or more information is needed, etc.
-
-Senpy proposes a modular and dynamic architecture that allows:
-
-* Implementing different algorithms in a extensible way, yet offering a common interface.
-* Offering common services that facilitate development, so developers can focus on implementing new and better algorithms.
-
-The framework consists of two main modules: Senpy core, which is the building block of the service, and Senpy plugins, which consist of the analysis algorithm. The next figure depicts a simplified version of the processes involved in an analysis with the Senpy framework.
-
-.. image:: senpy-architecture.png
-  :width: 100%
-  :align: center

docs/bad-examples/results/example-analysis-as-id-FAIL.json

@@ -1,78 +0,0 @@
-{
-  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
-  "@id": "me:Result1",
-  "@type": "results",
-  "analysis": [
-    "me:SAnalysis1",
-    "me:SgAnalysis1",
-    "me:EmotionAnalysis1",
-    "me:NER1",
-    {
-      "@type": "analysis",
-      "@id": "wrong"
-    }
-  ],
-  "entries": [
-    {
-      "@id": "http://micro.blog/status1",
-      "@type": [
-        "nif:RFC5147String",
-        "nif:Context"
-      ],
-      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
-      "entities": [
-        {
-          "@id": "http://micro.blog/status1#char=5,13",
-          "nif:beginIndex": 5,
-          "nif:endIndex": 13,
-          "nif:anchorOf": "Microsoft",
-          "me:references": "http://dbpedia.org/page/Microsoft",
-          "prov:wasGeneratedBy": "me:NER1"
-        },
-        {
-          "@id": "http://micro.blog/status1#char=25,37",
-          "nif:beginIndex": 25,
-          "nif:endIndex": 37,
-          "nif:anchorOf": "Windows Phone",
-          "me:references": "http://dbpedia.org/page/Windows_Phone",
-          "prov:wasGeneratedBy": "me:NER1"
-        }
-      ],
-      "suggestions": [
-        {
-          "@id": "http://micro.blog/status1#char=16,77",
-          "nif:beginIndex": 16,
-          "nif:endIndex": 77,
-          "nif:anchorOf": "put your Windows Phone on your newest #open technology program",
-          "prov:wasGeneratedBy": "me:SgAnalysis1"
-        }
-      ],
-      "sentiments": [
-        {
-          "@id": "http://micro.blog/status1#char=80,97",
-          "nif:beginIndex": 80,
-          "nif:endIndex": 97,
-          "nif:anchorOf": "You'll be awesome.",
-          "marl:hasPolarity": "marl:Positive",
-          "marl:polarityValue": 0.9,
-          "prov:wasGeneratedBy": "me:SAnalysis1"
-        }
-      ],
-      "emotions": [
-        {
-          "@id": "http://micro.blog/status1#char=0,109",
-          "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
-          "prov:wasGeneratedBy": "me:EAnalysis1",
-          "onyx:hasEmotion": [
-            {
-              "onyx:hasEmotionCategory": "wna:liking"
-            },
-            {
-              "onyx:hasEmotionCategory": "wna:excitement"
-            }
-          ]
-        }
-      ]
-    }
-  ]
-}

docs/conf.py

@@ -37,7 +37,6 @@ extensions = [
     'sphinx.ext.todo',
     'sphinxcontrib.httpdomain',
     'sphinx.ext.coverage',
-    'sphinx.ext.autosectionlabel'
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -55,21 +54,20 @@ master_doc = 'index'
 # General information about the project.
 project = u'Senpy'
 copyright = u'2016, J. Fernando Sánchez'
-description = u'A framework for sentiment and emotion analysis services'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-# with open('../senpy/VERSION') as f:
+with open('../senpy/VERSION') as f:
-#     version = f.read().strip()
+    version = f.read().strip()
 # The full version, including alpha/beta/rc tags.
-# release = version
+release = version
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
-language = None
+#language = None
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
@@ -106,14 +104,14 @@ pygments_style = 'sphinx'
 #keep_warnings = False
 
 
-html_theme = 'alabaster'
 # -- Options for HTML output ----------------------------------------------
-# if not on_rtd:  # only import and set the theme if we're building docs locally
+if not on_rtd:  # only import and set the theme if we're building docs locally
-#     import sphinx_rtd_theme
+    import sphinx_rtd_theme
-#     html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+    html_theme = 'sphinx_rtd_theme'
+    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
-# else:
+else:
-#     html_theme = 'default'
+    html_theme = 'default'
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
@@ -121,13 +119,7 @@ html_theme = 'alabaster'
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-html_theme_options = {
+#html_theme_options = {}
-    'logo': 'header.png',
-    'github_user': 'gsi-upm',
-    'github_repo': 'senpy',
-    'github_banner': True,
-}
-
 
 # Add any paths that contain custom themes here, relative to this directory.
 #html_theme_path = []
@@ -167,13 +159,7 @@ html_static_path = ['_static']
 #html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
-html_sidebars = {
+#html_sidebars = {}
-    '**': [
-        'about.html',
-        'navigation.html',
-        'searchbox.html',
-        ]
-    }
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.

docs/conversion.rst

@@ -1,116 +0,0 @@
-Conversion
-----------
-
-Senpy includes experimental support for emotion/sentiment conversion plugins.
-
-
-Use
-===
-
-Consider the original query: http://127.0.0.1:5000/api/?i=hello&algo=emoRand
-
-The requested plugin (emoRand) returns emotions using Ekman's model (or big6 in EmotionML):
-
-.. code:: json
-
-
-          ... rest of the document ...
-          {
-            "@type": "emotionSet",
-            "onyx:hasEmotion": {
-                "@type": "emotion",
-                "onyx:hasEmotionCategory": "emoml:big6anger"
-            },
-            "prov:wasGeneratedBy": "plugins/emoRand_0.1"
-          }
-
-          
-
-To get these emotions in VAD space (FSRE dimensions in EmotionML), we'd do this:
-
-http://127.0.0.1:5000/api/?i=hello&algo=emoRand&emotionModel=emoml:fsre-dimensions
-
-This call, provided there is a valid conversion plugin from Ekman's to VAD, would return something like this:
-
-.. code:: json
-
-
-          ... rest of the document ...
-          {
-            "@type": "emotionSet",
-            "onyx:hasEmotion": {
-                "@type": "emotion",
-                "onyx:hasEmotionCategory": "emoml:big6anger"
-                },
-            "prov:wasGeneratedBy": "plugins/emoRand_0.1"
-          }, {
-            "@type": "emotionSet",
-            "onyx:hasEmotion": {
-                "@type": "emotion",
-                "A": 7.22,
-                "D": 6.28,
-                "V": 8.6
-            },
-            "prov:wasGeneratedBy": "plugins/Ekman2VAD_0.1"
-
-          }
-
-
-That is called a *full* response, as it simply adds the converted emotion alongside.
-It is also possible to get the original emotion nested within the new converted emotion, using the `conversion=nested` parameter:
-
-.. code:: json
-
-
-          ... rest of the document ...
-          {
-            "@type": "emotionSet",
-            "onyx:hasEmotion": {
-                "@type": "emotion",
-                "onyx:hasEmotionCategory": "emoml:big6anger"
-                },
-            "prov:wasGeneratedBy": "plugins/emoRand_0.1"
-            "onyx:wasDerivedFrom": {
-                "@type": "emotionSet",
-                "onyx:hasEmotion": {
-                    "@type": "emotion",
-                    "A": 7.22,
-                    "D": 6.28,
-                    "V": 8.6
-                },
-                "prov:wasGeneratedBy": "plugins/Ekman2VAD_0.1"
-             }
-
-          }
-
-
-Lastly, `conversion=filtered` would only return the converted emotions.
-
-Developing a conversion plugin
-================================
-
-Conversion plugins are discovered by the server just like any other plugin.
-The difference is the slightly different API, and the need to specify the `source` and `target` of the conversion.
-For instance, an emotion conversion plugin needs the following:
-
-
-.. code:: yaml
-          
-
-          ---
-          onyx:doesConversion:
-            - onyx:conversionFrom: emoml:big6
-              onyx:conversionTo: emoml:fsre-dimensions
-            - onyx:conversionFrom: emoml:fsre-dimensions
-              onyx:conversionTo: emoml:big6
-
-
-
-
-.. code:: python
-
-
-          class MyConversion(EmotionConversionPlugin):
-
-              def convert(self, emotionSet, fromModel, toModel, params):
-                  pass

docs/demo.rst

@@ -1,8 +1,7 @@
 Demo
 ----
 
-There is a demo available on http://senpy.cluster.gsi.dit.upm.es/, where you can test a serie of different plugins.
+There is a demo available on http://senpy.demos.gsi.dit.upm.es/, where you can a serie of different plugins. You can use them in the playground or make a directly requests to the service.
-You can use the playground (a web interface) or make HTTP requests to the service API.
 
 .. image:: senpy-playground.png
   :height: 400px
@@ -13,4 +12,64 @@ You can use the playground (a web interface) or make HTTP requests to the servic
 Plugins Demo
 ============
 
-The source code and description of the plugins used in the demo is available here: https://lab.cluster.gsi.dit.upm.es/senpy/senpy-plugins-community/.
+The next plugins are available at the demo:
+
+* emoTextAnew extracts the VAD (valence-arousal-dominance) of a sentence by matching words from the ANEW dictionary.
+* emoTextWordnetAffect based on the hierarchy of WordnetAffect to calculate the emotion of the sentence.
+* vaderSentiment utilizes the software from vaderSentiment to calculate the sentiment of a sentence.
+* sentiText is a software developed during the TASS 2015 competition, it has been adapted for English and Spanish.
+
+emoTextANEW plugin
+******************
+
+This plugin is going to used the ANEW lexicon dictionary to calculate de VAD (valence-arousal-dominance) of the sentence and the determinate which emotion is closer to this value.
+
+Each emotion has a centroid, which it has been approximated using the formula described in this article:
+
+http://www.aclweb.org/anthology/W10-0208
+
+The plugin is going to look for the words in the sentence that appear in the ANEW dictionary and calculate the average VAD score for the sentence. Once this score is calculated, it is going to seek the emotion that is closest to this value.
+
+emoTextWAF plugin
+*****************
+
+This plugin uses WordNet-Affect (http://wndomains.fbk.eu/wnaffect.html) to calculate the percentage of each emotion. The emotions that are going to be used are: anger, fear, disgust, joy and sadness. It is has been used a emotion mapping enlarge the emotions:
+
+* anger : general-dislike
+* fear : negative-fear
+* disgust : shame
+* joy : gratitude, affective, enthusiasm, love, joy, liking
+* sadness : ingrattitude, daze, humlity, compassion, despair, anxiety, sadness
+
+sentiText plugin
+****************
+
+This plugin is based in the classifier developed for the TASS 2015 competition. It has been developed for Spanish and English. The different phases that has this plugin when it is activated:
+
+* Train both classifiers (English and Spanish).
+* Initialize resources (dictionaries,stopwords,etc.).
+* Extract bag of words,lemmas and chars.
+
+Once the plugin is activated, the features that are going to be extracted for the classifiers are:
+
+* Matches with the bag of words extracted from the train corpus.
+* Sentiment score of the sentences extracted from the dictionaries (lexicons and emoticons).
+* Identify negations and intensifiers in the sentences.
+* Complementary features such as exclamation and interrogation marks, eloganted and caps words, hashtags, etc.
+
+The plugin has a preprocessor, which is focues on Twitter corpora, that is going to be used for cleaning the text to simplify the feature extraction.
+
+There is more information avaliable in the next article.
+
+Aspect based Sentiment Analysis of Spanish Tweets, Oscar Araque and Ignacio Corcuera-Platas and Constantino Román-Gómez and Carlos A. Iglesias and J. Fernando Sánchez-Rada. http://gsi.dit.upm.es/es/investigacion/publicaciones?view=publication&task=show&id=37
+
+vaderSentiment plugin
+*********************
+
+For developing this plugin, it has been used the module vaderSentiment, which is described in the paper: VADER: A Parsimonious Rule-based Model for Sentiment Analysis of Social Media Text C.J. Hutto and Eric Gilbert Eighth International Conference on Weblogs and Social Media (ICWSM-14). Ann Arbor, MI, June 2014.
+
+If you use this plugin in your research, please cite the above paper
+
+For more information about the functionality, check the official repository
+
+https://github.com/cjhutto/vaderSentiment

docs/examples/results/example-analysis-as-id.json

@@ -1,74 +0,0 @@
-{
-  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
-  "@id": "me:Result1",
-  "@type": "results",
-  "analysis": [
-    "me:SAnalysis1",
-    "me:SgAnalysis1",
-    "me:EmotionAnalysis1",
-    "me:NER1"
-  ],
-  "entries": [
-    {
-      "@id": "http://micro.blog/status1",
-      "@type": [
-        "nif:RFC5147String",
-        "nif:Context"
-      ],
-      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
-      "entities": [
-        {
-          "@id": "http://micro.blog/status1#char=5,13",
-          "nif:beginIndex": 5,
-          "nif:endIndex": 13,
-          "nif:anchorOf": "Microsoft",
-          "me:references": "http://dbpedia.org/page/Microsoft",
-          "prov:wasGeneratedBy": "me:NER1"
-        },
-        {
-          "@id": "http://micro.blog/status1#char=25,37",
-          "nif:beginIndex": 25,
-          "nif:endIndex": 37,
-          "nif:anchorOf": "Windows Phone",
-          "me:references": "http://dbpedia.org/page/Windows_Phone",
-          "prov:wasGeneratedBy": "me:NER1"
-        }
-      ],
-      "suggestions": [
-        {
-          "@id": "http://micro.blog/status1#char=16,77",
-          "nif:beginIndex": 16,
-          "nif:endIndex": 77,
-          "nif:anchorOf": "put your Windows Phone on your newest #open technology program",
-          "prov:wasGeneratedBy": "me:SgAnalysis1"
-        }
-      ],
-      "sentiments": [
-        {
-          "@id": "http://micro.blog/status1#char=80,97",
-          "nif:beginIndex": 80,
-          "nif:endIndex": 97,
-          "nif:anchorOf": "You'll be awesome.",
-          "marl:hasPolarity": "marl:Positive",
-          "marl:polarityValue": 0.9,
-          "prov:wasGeneratedBy": "me:SAnalysis1"
-        }
-      ],
-      "emotions": [
-        {
-          "@id": "http://micro.blog/status1#char=0,109",
-          "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
-          "prov:wasGeneratedBy": "me:EAnalysis1",
-          "onyx:hasEmotion": [
-            {
-              "onyx:hasEmotionCategory": "wna:liking"
-            },
-            {
-              "onyx:hasEmotionCategory": "wna:excitement"
-            }
-          ]
-        }
-      ]
-    }
-  ]
-}

docs/index.rst

@@ -1,28 +1,19 @@
+.. Senpy documentation master file, created by
+   sphinx-quickstart on Tue Feb 24 08:57:32 2015.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
 Welcome to Senpy's documentation!
 =================================
 
-With Senpy, you can easily turn your sentiment or emotion analysis algorithm into a full blown semantic service.
+Contents:
-Sharing your sentiment analysis with the world has never been easier.
-Senpy provides:
-
-* Parameter validation, error handling
-* Formatting: JSON-LD, Turtle/n-triples input and output, or simple text input
-* Linked Data. Results are semantically annotated, using a series of well established vocabularies, and sane default URIs.
-* A web UI where users can explore your service and test different settings
-* A client to interact with any senpy service
-* A command line tool
-
-
 
 .. toctree::
-   :caption: Learn more about senpy
-   :maxdepth: 2
-
    senpy
    installation
    usage
-   apischema
+   api
+   schema
    plugins
-   conversion
    demo
-   research.rst
+   :maxdepth: 2

docs/installation.rst

@@ -22,35 +22,6 @@ If you want to install senpy globally, use sudo instead of the ``--user`` flag.
 
 Docker Image
 ************
-Build the image or use the pre-built one:   
+Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy --host 0.0.0.0 --default-plugins``.
-
-.. code:: bash
-
-   docker run -ti -p 5000:5000 gsiupm/senpy --host 0.0.0.0 --default-plugins
-
-To add custom plugins, use a docker volume: 
-    
-.. code:: bash
-
-   docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --host 0.0.0.0 --default-plugins -f /plugins
-
-Alias
-.....
-
-If you are using the docker approach regularly, it is advisable to use a script or an alias to simplify your executions:
-
-.. code:: bash
-
-   alias senpy='docker run --rm -ti -p 5000:5000 -v $PWD:/senpy-plugins gsiupm/senpy --default-plugins'
- 
-
-Python 2
-........
-
-There is a Senpy version for python2 too:
-    
-.. code:: bash
-
-   docker run -ti -p 5000:5000 gsiupm/senpy:python2.7 --host 0.0.0.0 --default-plugins
-
 
+To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --host 0.0.0.0 --default-plugins -f /plugins``

docs/plugins.rst

@@ -1,35 +1,26 @@
 Developing new plugins
 ----------------------
-This document describes how to develop a new analysis plugin. For an example of conversion plugins, see :doc:`conversion`.
+Each plugin represents a different analysis process.There are two types of files that are needed by senpy for loading a plugin:
 
-A more step-by-step tutorial with slides is available `here <https://lab.cluster.gsi.dit.upm.es/senpy/senpy-tutorial>`__ 
+- Definition file, has the ".senpy" extension.
+- Code file, is a python file.
 
-What is a plugin?
+This separation will allow us to deploy plugins that use the same code but employ different parameters.
-=================
-
-A plugin is a program that, given a text, will add annotations to it.
-In practice, a plugin consists of at least two files:
-
-- Definition file: a `.senpy` file that describes the plugin (e.g. what input parameters it accepts, what emotion model it uses).
-- Python module: the actual code that will add annotations to each input.
-
-This separation allows us to deploy plugins that use the same code but employ different parameters.
 For instance, one could use the same classifier and processing in several plugins, but train with different datasets.
 This scenario is particularly useful for evaluation purposes.
 
 The only limitation is that the name of each plugin needs to be unique.
 
-Plugin Definition files
+Plugins Definitions
-=======================
+===================
 
 The definition file contains all the attributes of the plugin, and can be written in YAML or JSON.
-When the server is launched, it will recursively search for definition files in the plugin folder (the current folder, by default).
 The most important attributes are:
 
 * **name**: unique name that senpy will use internally to identify the plugin.
 * **module**: indicates the module that contains the plugin code, which will be automatically loaded by senpy.
 * **version**
-* extra_params: to add parameters to the senpy API when this plugin is requested. Those parameters may be required, and have aliased names. For instance:
+* extra_params: used to specify parameters that the plugin accepts that are not already part of the senpy API. Those parameters may be required, and have aliased names. For instance:
 
   .. code:: yaml
 
@@ -75,28 +66,10 @@ The basic methods in a plugin are:
 * __init__
 * activate: used to load memory-hungry resources
 * deactivate: used to free up resources
-* analyse_entry: called in every user requests. It takes two parameters: ``Entry``, the entry object, and ``params``, the parameters supplied by the user. It should yield one or more ``Entry`` objects.
+* analyse_entry: called in every user requests. It takes in the parameters supplied by a user and should yield one or more ``Entry`` objects.
 
 Plugins are loaded asynchronously, so don't worry if the activate method takes too long. The plugin will be marked as activated once it is finished executing the method.
 
-Entries
-=======
-
-Entries are objects that can be annotated.
-By default, entries are `NIF contexts <http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core/nif-core.html>`_ represented in JSON-LD format.
-Annotations are added to the object like this:
-
-.. code:: python
-
-   entry = Entry()
-   entry.vocabulary__annotationName = 'myvalue'
-   entry['vocabulary:annotationName'] = 'myvalue'
-   entry['annotationNameURI'] = 'myvalue'
-
-Where vocabulary is one of the prefixes defined in the default senpy context, and annotationURI is a full URI.
-The value may be any valid JSON-LD dictionary.
-For simplicity, senpy includes a series of models by default in the ``senpy.models`` module.
-
 
 Example plugin
 ==============
@@ -142,13 +115,6 @@ Now, in a file named ``helloworld.py``:
 
 F.A.Q.
 ======
-What annotations can I use?
-???????????????????????????
-
-You can add almost any annotation to an entry.
-The most common use cases are covered in the :doc:`schema`.
-
-
 Why does the analyse function yield instead of return?
 ??????????????????????????????????????????????????????
 
@@ -183,11 +149,7 @@ Training a classifier can be time time consuming. To avoid running the training
               def deactivate(self):
                   self.close()
 
-You can specify a 'shelf_file' in your .senpy file. By default the ShelfMixin creates a file based on the plugin name and stores it in that plugin's folder.
+You can speficy a 'shelf_file' in your .senpy file. By default the ShelfMixin creates a file based on the plugin name and stores it in that plugin's folder.
-
-Shelves may get corrupted if the plugin exists unexpectedly.
-A corrupt shelf prevents the plugin from loading.
-If you do not care about the pickle, you can force your plugin to remove the corrupted file and load anyway, set the  'force_shelf' to True in your .senpy file.
 
 I want to implement my service as a plugin, How i can do it?
 ????????????????????????????????????????????????????????????

docs/research.rst

@@ -1,11 +0,0 @@
-Research
---------
-
-If you use Senpy in your research, please cite `Senpy: A Pragmatic Linked Sentiment Analysis Framework <http://gsi.dit.upm.es/index.php/es/investigacion/publicaciones?view=publication&task=show&id=417>`__ (`BibTex <http://gsi.dit.upm.es/index.php/es/investigacion/publicaciones?controller=publications&task=export&format=bibtex&id=417>`__):
-
-.. code-block:: text
-
-	Sánchez-Rada, J. F., Iglesias, C. A., Corcuera, I., & Araque, Ó. (2016, October).
-	Senpy: A Pragmatic Linked Sentiment Analysis Framework.
-	In Data Science and Advanced Analytics (DSAA),
-	2016 IEEE International Conference on (pp. 735-742). IEEE.

docs/schema.rst

@@ -1,74 +1,74 @@
-Schema
+Schema Examples
-------
+===============
 All the examples in this page use the :download:`the main schema <_static/schemas/definitions.json>`.
 
 Simple NIF annotation
-.....................
+---------------------
 Description
-,,,,,,,,,,,
+...........
 This example covers the basic example in the NIF documentation: `<http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core/nif-core.html>`_.
 
 Representation
-,,,,,,,,,,,,,,
+..............
-.. literalinclude:: examples/results/example-basic.json
+.. literalinclude:: examples/example-basic.json
    :language: json-ld
 
 Sentiment Analysis
-.....................
+---------------------
 Description
-,,,,,,,,,,,
+...........
 
 Representation
-,,,,,,,,,,,,,,
+..............
 
-.. literalinclude:: examples/results/example-sentiment.json
+.. literalinclude:: examples/example-sentiment.json
    :emphasize-lines: 5-10,25-33
    :language: json-ld
 
 Suggestion Mining
-.................
+-----------------
 Description
-,,,,,,,,,,,
+...........
 
 Representation
-,,,,,,,,,,,,,,
+..............
 
-.. literalinclude:: examples/results/example-suggestion.json
+.. literalinclude:: examples/example-suggestion.json
    :emphasize-lines: 5-8,22-27
    :language: json-ld
 
 Emotion Analysis
-................
+----------------
 Description
-,,,,,,,,,,,
+...........
 
 Representation
-,,,,,,,,,,,,,,
+..............
 
-.. literalinclude:: examples/results/example-emotion.json
+.. literalinclude:: examples/example-emotion.json
    :language: json-ld
    :emphasize-lines: 5-8,25-37
 
 Named Entity Recognition
-........................
+------------------------
 Description
-,,,,,,,,,,,
+...........
 
 Representation
-,,,,,,,,,,,,,,
+..............
 
-.. literalinclude:: examples/results/example-ner.json
+.. literalinclude:: examples/example-ner.json
    :emphasize-lines: 5-8,19-34
    :language: json-ld
 
 Complete example
-................
+----------------
 Description
-,,,,,,,,,,,
+...........
 This example covers all of the above cases, integrating all the annotations in the same document.
 
 Representation
-,,,,,,,,,,,,,,
+..............
 
-.. literalinclude:: examples/results/example-complete.json
+.. literalinclude:: examples/example-complete.json
    :language: json-ld

docs/senpy.rst

@@ -1,32 +1,35 @@
 What is Senpy?
 --------------
 
-Senpy is a framework that turns your sentiment or emotion analysis algorithm into a full blown semantic service.
+Senpy is an open source reference implementation of a linked data model for sentiment and emotion analysis services based on the vocabularies NIF, Marl and Onyx. 
-Senpy takes care of:
 
-  * Interfacing with the user: parameter validation, error handling.
+The overall goal of the reference implementation Senpy is easing the adoption of the proposed linked data model for sentiment and emotion analysis services, so that services from different providers become interoperable. With this aim, the design of the reference implementation has focused on its extensibility and reusability. 
-  * Formatting: JSON-LD, Turtle/n-triples input and output, or simple text input
-  * Linked Data: senpy results are semantically annotated, using a series of well established vocabularies, and sane default URIs.
-  * User interface: a web UI where users can explore your service and test different settings
-  * A client to interact with the service. Currently only available in Python.
 
-Sharing your sentiment analysis with the world has never been easier!
+A modular approach allows organizations to replace individual components with custom ones developed in-house. Furthermore, organizations can benefit from reusing prepackages modules that provide advanced functionalities, such as algorithms for sentiment and emotion analysis, linked data publication or emotion and sentiment mapping between different providers.
 
-Senpy for service developers
+Specifications
-============================
+==============
 
-Check out the :doc:`plugins` if you have developed an analysis algorithm (e.g. sentiment analysis) and you want to publish it as a service.
+The model used in Senpy is based on the following specifications:
 
-Senpy for end users
+* Marl, a vocabulary designed to annotate and describe subjetive opinions expressed on the web or in information systems.
-===================
+* Onyx, which is built one the same principles as Marl to annotate and describe emotions, and provides interoperability with Emotion Markup Language.
+* NIF 2.0, which defines a semantic format and APO for improving interoperability among natural language processing services
 
-All services built using senpy share a common interface.
+Architecture
-This allows users to use them (almost) interchangeably.
+============
-Senpy comes with a :ref:`built-in client`.
 
+The main component of a sentiment analysis service is the algorithm itself. However, for the algorithm to work, it needs to get the appropriate parameters from the user, format the results according to the defined API, interact with the user whn errors occur or more information is needed, etc.
 
-.. toctree::
+Senpy proposes a modular and dynamic architecture that allows:
-    :caption: Interested? Check out senpy's:
- 
-    architecture
 
+* Implementing different algorithms in a extensible way, yet offering a common interface.
+* Offering common services that facilitate development, so developers can focus on implementing new and better algorithms.
+
+The framework consists of two main modules: Senpy core, which is the building block of the service, and Senpy plugins, which consist of the analysis algorithm. The next figure depicts a simplified version of the processes involved in an analysis with the Senpy framework.
+
+.. image:: senpy-architecture.png
+  :height: 400px
+  :width: 800px
+  :scale: 100 %
+  :align: center

docs/usage.rst

@@ -1,9 +1,20 @@
 Usage
 -----
 
-First of all, you need to install the package.
+The easiest and recommended way is to just use the command-line tool to load your plugins and launch the server.
-See :doc:`installation` for installation instructions.
+
-Once installed, the `senpy` command should be available. 
+.. code:: bash
+
+   senpy
+
+Or, alternatively:
+
+.. code:: bash
+
+    python -m senpy
+
+
+This will create a server with any modules found in the current path.
 
 Useful command-line options
 ===========================
@@ -12,19 +23,19 @@ In case you want to load modules, which are located in different folders under t
 
 .. code:: bash
 
-    senpy -f .
+    python -m senpy -f .
 
-The default port used by senpy is 5000, but you can change it using the `--port` flag.
+The default port used by senpy is 5000, but you can change it using the option `--port`.
 
 .. code:: bash
 
-    senpy --port 8080
+    python -m senpy --port 8080
 
 Also, the host can be changed where senpy is deployed. The default value is `127.0.0.1`.
 
 .. code:: bash
 
-    senpy --host 0.0.0.0
+    python -m senpy --host 0.0.0.0
 
 For more options, see the `--help` page.
 
@@ -37,37 +48,28 @@ Once the server is launched, there is a basic endpoint in the server, which prov
 
 In case you want to know the different endpoints of the server, there is more information available in the NIF API section_.
 
-CLI demo
+Video example
-========
+=============
 
 This video shows how to use senpy through command-line tool.
 
-.. image:: https://asciinema.org/a/9uwef1ghkjk062cw2t4mhzpyk.png
+https://asciinema.org/a/9uwef1ghkjk062cw2t4mhzpyk
-   :width: 100%
-   :target: https://asciinema.org/a/9uwef1ghkjk062cw2t4mhzpyk
-   :alt: CLI demo
 
+Request example in python
+=========================
 
-Built-in client
+This example shows how to make a request to a plugin.
-===============
-
-This example shows how to make a request to the default plugin:
 
 .. code:: python
 
-    from senpy.client import Client
+    import requests
+    import json
 
-    c = Client('http://127.0.0.1:5000/api/')
+    r = requests.get('http://127.0.0.1:5000/api/?algo=rand&i=Testing')
-    r = c.analyse('hello world')
+    response = r.content.decode('utf-8')
-
+    response_json = json.loads(response)
-    for entry in r.entries:
-          print('{} -> {}'.format(entry.text, entry.emotions))
 
 
 
 .. _section: http://senpy.readthedocs.org/en/latest/api.html
 
-
-Conversion
-==========
-See :doc:`conversion`

docs/vocabularies.rst

@@ -1,8 +0,0 @@
-Vocabularies and model
-======================
-
-The model used in Senpy is based on the following vocabularies:
-
-* Marl, a vocabulary designed to annotate and describe subjetive opinions expressed on the web or in information systems.
-* Onyx, which is built one the same principles as Marl to annotate and describe emotions, and provides interoperability with Emotion Markup Language.
-* NIF 2.0, which defines a semantic format and APO for improving interoperability among natural language processing services

k8s/README.md

@@ -1,7 +0,0 @@
-Deploy senpy to a kubernetes cluster.
-
-Usage:
-
-```
-kubectl apply -f . -n senpy
-```

k8s/senpy-deployment.yaml

@@ -1,25 +0,0 @@
----
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: senpy-latest
-spec:
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        role: senpy-latest
-        app: test
-    spec:
-      containers:
-      - name: senpy-latest
-        image: gsiupm/senpy:latest
-        args:
-          - "--default-plugins"
-        resources:
-          limits:
-            memory: "512Mi"
-            cpu: "1000m"
-        ports:
-          - name: web
-            containerPort: 5000

k8s/senpy-ingress.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: extensions/v1beta1
-kind: Ingress
-metadata:
-  name: senpy-ingress
-spec:
-  rules:
-  - host: latest.senpy.cluster.gsi.dit.upm.es
-    http:
-      paths:
-      - path: /
-        backend:
-          serviceName: senpy-latest
-          servicePort: 5000

k8s/senpy-svc.yaml

@@ -1,12 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: senpy-latest
-spec:
-  type: ClusterIP
-  ports:
-    - port: 5000
-      protocol: TCP
-  selector:
-    role: senpy-latest

requirements.txt

@@ -1,6 +1,6 @@
 Flask>=0.10.1
 requests>=2.4.1
-tornado>=4.4.3
+gevent>=1.1rc4
 PyLD>=0.6.5
 six
 future

senpy/__init__.py

@@ -17,6 +17,7 @@
 """
 Sentiment analysis server in Python
 """
+from __future__ import print_function
 from .version import __version__
 
 import logging

senpy/__main__.py

@@ -22,19 +22,35 @@ the server.
 
 from flask import Flask
 from senpy.extensions import Senpy
-from tornado.wsgi import WSGIContainer
+from gevent.wsgi import WSGIServer
-from tornado.httpserver import HTTPServer
+from gevent.monkey import patch_all
-from tornado.ioloop import IOLoop
-
-
 import logging
 import os
+import sys
 import argparse
 import senpy
 
+patch_all(thread=False)
+
 SERVER_PORT = os.environ.get("PORT", 5000)
 
 
+def info(type, value, tb):
+    if hasattr(sys, 'ps1') or not sys.stderr.isatty():
+        #  we are in interactive mode or we don't have a tty-like
+        #  device, so we call the default hook
+        sys.__excepthook__(type, value, tb)
+    else:
+        import traceback
+        import pdb
+        # we are NOT in interactive mode, print the exception...
+        traceback.print_exception(type, value, tb)
+        print
+        # ...then start the debugger in post-mortem mode.
+        # pdb.pm() # deprecated
+        pdb.post_mortem(tb)  # more "modern"
+
+
 def main():
     parser = argparse.ArgumentParser(description='Run a Senpy server')
     parser.add_argument(
@@ -78,41 +94,28 @@ def main():
         action='store_true',
         default=False,
         help='Do not run a server, only install plugin dependencies')
-    parser.add_argument(
-        '--version',
-        '-v',
-        action='store_true',
-        default=False,
-        help='Output the senpy version and exit')
     args = parser.parse_args()
-    if args.version:
-        print('Senpy version {}'.format(senpy.__version__))
-        exit(1)
     logging.basicConfig()
     rl = logging.getLogger()
     rl.setLevel(getattr(logging, args.level))
     app = Flask(__name__)
     app.debug = args.debug
+    if args.debug:
+        sys.excepthook = info
     sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins)
     if args.only_install:
         sp.install_deps()
         return
     sp.activate_all()
-    print('Senpy version {}'.format(senpy.__version__))
+    http_server = WSGIServer((args.host, args.port), app)
-    print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,
+    try:
-                                                            args.port))
+        print('Senpy version {}'.format(senpy.__version__))
-    if not app.debug:
+        print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,
-        http_server = HTTPServer(WSGIContainer(app))
+                                                                args.port))
-        http_server.listen(args.port, address=args.host)
+        http_server.serve_forever()
-        try:
+    except KeyboardInterrupt:
-            IOLoop.instance().start()
+        print('Bye!')
-        except KeyboardInterrupt:
+    http_server.stop()
-            print('Bye!')
-        http_server.stop()
-    else:
-        app.run(args.host,
-                args.port,
-                debug=True)
     sp.deactivate_all()
 
 

senpy/api.py

@@ -26,13 +26,6 @@ API_PARAMS = {
         "aliases": ["emotionModel", "emoModel"],
         "required": False
     },
-    "plugin_type": {
-        "@id": "pluginType",
-        "description": 'What kind of plugins to list',
-        "aliases": ["pluginType", "plugin_type"],
-        "required": True,
-        "default": "analysisPlugin"
-    },
     "conversion": {
         "@id": "conversion",
         "description": "How to show the elements that have (not) been converted",
@@ -70,7 +63,7 @@ NIF_PARAMS = {
         "aliases": ["f", "informat"],
         "required": False,
         "default": "text",
-        "options": ["turtle", "text", "json-ld"],
+        "options": ["turtle", "text"],
     },
     "intype": {
         "@id": "intype",

senpy/blueprints.py

@@ -25,7 +25,6 @@ from .version import __version__
 from functools import wraps
 
 import logging
-import json
 
 logger = logging.getLogger(__name__)
 
@@ -75,7 +74,7 @@ def basic_api(f):
     @wraps(f)
     def decorated_function(*args, **kwargs):
         raw_params = get_params(request)
-        headers = {'X-ORIGINAL-PARAMS': json.dumps(raw_params)}
+        headers = {'X-ORIGINAL-PARAMS': raw_params}
         # Get defaults
         web_params = parse_params({}, spec=WEB_PARAMS)
         api_params = parse_params({}, spec=API_PARAMS)
@@ -93,9 +92,6 @@ def basic_api(f):
             response = f(*args, **kwargs)
         except Error as ex:
             response = ex
-            logger.error(ex)
-            if current_app.debug:
-                raise
 
         in_headers = web_params['inHeaders'] != "0"
         expanded = api_params['expanded-jsonld']
@@ -125,9 +121,7 @@ def api():
 @basic_api
 def plugins():
     sp = current_app.senpy
-    ptype = request.params.get('plugin_type')
+    dic = Plugins(plugins=list(sp.plugins.values()))
-    plugins = sp.filter_plugins(plugin_type=ptype)
-    dic = Plugins(plugins=list(plugins.values()))
     return dic
 
 

senpy/client.py

@@ -1,7 +1,6 @@
 import requests
 import logging
 from . import models
-from .plugins import default_plugin_type
 
 logger = logging.getLogger(__name__)
 
@@ -13,16 +12,13 @@ class Client(object):
     def analyse(self, input, method='GET', **kwargs):
         return self.request('/', method=method, input=input, **kwargs)
 
-    def plugins(self, ptype=default_plugin_type):
-        resp = self.request(path='/plugins', plugin_type=ptype).plugins
-        return {p.name: p for p in resp}
-
     def request(self, path=None, method='GET', **params):
         url = '{}{}'.format(self.endpoint, path)
         response = requests.request(method=method, url=url, params=params)
         try:
             resp = models.from_dict(response.json())
             resp.validate(resp)
+            return resp
         except Exception as ex:
             logger.error(('There seems to be a problem with the response:\n'
                           '\tURL: {url}\n'
@@ -37,6 +33,3 @@ class Client(object):
                               code=response.status_code,
                               content=response.content))
             raise ex
-        if isinstance(resp, models.Error):
-            raise resp
-        return resp

senpy/extensions.py

@@ -5,35 +5,26 @@ It orchestrates plugin (de)activation and analysis.
 from future import standard_library
 standard_library.install_aliases()
 
-from . import plugins
+from .plugins import SentimentPlugin, SenpyPlugin
-from .plugins import SenpyPlugin
+from .models import Error, Entry, Results
-from .models import Error, Entry, Results, from_string
 from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
 from .api import API_PARAMS, NIF_PARAMS, parse_params
 
 from threading import Thread
 
 import os
-import copy
 import fnmatch
 import inspect
 import sys
-import importlib
+import imp
 import logging
 import traceback
 import yaml
-import subprocess
+import pip
 
 logger = logging.getLogger(__name__)
 
 
-def log_subprocess_output(process):
-    for line in iter(process.stdout.readline, b''):
-        logger.info('%r', line)
-    for line in iter(process.stderr.readline, b''):
-        logger.error('%r', line)
-
-
 class Senpy(object):
     """ Default Senpy extension for Flask """
 
@@ -85,105 +76,74 @@ class Senpy(object):
         else:
             logger.debug("Not a folder: %s", folder)
 
-    def _find_plugins(self, params):
+    def _find_plugin(self, params):
-        if not self.analysis_plugins:
+        api_params = parse_params(params, spec=API_PARAMS)
+        algo = None
+        if "algorithm" in api_params and api_params["algorithm"]:
+            algo = api_params["algorithm"]
+        elif self.plugins:
+            algo = self.default_plugin and self.default_plugin.name
+        if not algo:
             raise Error(
                 status=404,
                 message=("No plugins found."
-                         " Please install one."))
+                         " Please install one.").format(algo))
-        api_params = parse_params(params, spec=API_PARAMS)
+        if algo not in self.plugins:
-        algos = None
+            logger.debug(("The algorithm '{}' is not valid\n"
-        if "algorithm" in api_params and api_params["algorithm"]:
+                          "Valid algorithms: {}").format(algo,
-            algos = api_params["algorithm"].split(',')
+                                                         self.plugins.keys()))
-        elif self.default_plugin:
-            algos = [self.default_plugin.name, ]
-        else:
             raise Error(
                 status=404,
-                message="No default plugin found, and None provided")
+                message="The algorithm '{}' is not valid".format(algo))
 
-        plugins = list()
+        if not self.plugins[algo].is_activated:
-        for algo in algos:
+            logger.debug("Plugin not activated: {}".format(algo))
-            if algo not in self.plugins:
+            raise Error(
-                logger.debug(("The algorithm '{}' is not valid\n"
+                status=400,
-                              "Valid algorithms: {}").format(algo,
+                message=("The algorithm '{}'"
-                                                             self.plugins.keys()))
+                         " is not activated yet").format(algo))
-                raise Error(
+        return self.plugins[algo]
-                    status=404,
-                    message="The algorithm '{}' is not valid".format(algo))
 
-            if not self.plugins[algo].is_activated:
+    def _get_params(self, params, plugin):
-                logger.debug("Plugin not activated: {}".format(algo))
-                raise Error(
-                    status=400,
-                    message=("The algorithm '{}'"
-                             " is not activated yet").format(algo))
-            plugins.append(self.plugins[algo])
-        return plugins
-
-    def _get_params(self, params, plugin=None):
         nif_params = parse_params(params, spec=NIF_PARAMS)
-        if plugin:
+        extra_params = plugin.get('extra_params', {})
-            extra_params = plugin.get('extra_params', {})
+        specific_params = parse_params(params, spec=extra_params)
-            specific_params = parse_params(params, spec=extra_params)
+        nif_params.update(specific_params)
-            nif_params.update(specific_params)
         return nif_params
 
     def _get_entries(self, params):
+        entry = None
         if params['informat'] == 'text':
-            results = Results()
             entry = Entry(text=params['input'])
-            results.entries.append(entry)
-        elif params['informat'] == 'json-ld':
-            results = from_string(params['input'], cls=Results)
         else:
-            raise NotImplemented('Informat {} is not implemented'.format(params['informat']))
+            raise NotImplemented('Only text input format implemented')
-        return results
+        yield entry
-
-    def _process_entries(self, entries, plugins, nif_params):
-        if not plugins:
-            for i in entries:
-                yield i
-            return
-        plugin = plugins[0]
-        specific_params = self._get_params(nif_params, plugin)
-        results = plugin.analyse_entries(entries, specific_params)
-        for i in self._process_entries(results, plugins[1:], nif_params):
-            yield i
-
-    def _process_response(self, resp, plugins, nif_params):
-        entries = resp.entries
-        resp.entries = []
-        for plug in plugins:
-            resp.analysis.append(plug.id)
-        for i in self._process_entries(entries, plugins, nif_params):
-            resp.entries.append(i)
-        return resp
 
     def analyse(self, **api_params):
-        """
-        Main method that analyses a request, either from CLI or HTTP.
-        It uses a dictionary of parameters, provided by the user.
-        """
         logger.debug("analysing with params: {}".format(api_params))
-        plugins = self._find_plugins(api_params)
+        plugin = self._find_plugin(api_params)
-        nif_params = self._get_params(api_params)
+        nif_params = self._get_params(api_params, plugin)
-        resp = self._get_entries(nif_params)
+        resp = Results()
         if 'with_parameters' in api_params:
             resp.parameters = nif_params
         try:
-            resp = self._process_response(resp, plugins, nif_params)
+            entries = []
-            self.convert_emotions(resp, plugins, nif_params)
+            for i in self._get_entries(nif_params):
+                entries += list(plugin.analyse_entry(i, nif_params))
+            resp.entries = entries
+            self.convert_emotions(resp, plugin, nif_params)
+            resp.analysis.append(plugin.id)
             logger.debug("Returning analysis result: {}".format(resp))
-        except (Error, Exception) as ex:
+        except Error as ex:
-            if not isinstance(ex, Error):
-                ex = Error(message=str(ex), status=500)
             logger.exception('Error returning analysis result')
-            raise ex
+            resp = ex
+        except Exception as ex:
+            logger.exception('Error returning analysis result')
+            resp = Error(message=str(ex), status=500)
         return resp
 
     def _conversion_candidates(self, fromModel, toModel):
-        candidates = self.filter_plugins(plugin_type='emotionConversionPlugin')
+        candidates = self.filter_plugins(**{'@type': 'emotionConversionPlugin'})
         for name, candidate in candidates.items():
             for pair in candidate.onyx__doesConversion:
                 logging.debug(pair)
@@ -193,42 +153,37 @@ class Senpy(object):
                     # logging.debug('Found candidate: {}'.format(candidate))
                     yield candidate
 
-    def convert_emotions(self, resp, plugins, params):
+    def convert_emotions(self, resp, plugin, params):
         """
-        Conversion of all emotions in a response **in place**.
+        Conversion of all emotions in a response.
         In addition to converting from one model to another, it has
         to include the conversion plugin to the analysis list.
         Needless to say, this is far from an elegant solution, but it works.
         @todo refactor and clean up
         """
+        fromModel = plugin.get('onyx:usesEmotionModel', None)
         toModel = params.get('emotionModel', None)
+        output = params.get('conversion', None)
+        logger.debug('Asked for model: {}'.format(toModel))
+        logger.debug('Analysis plugin uses model: {}'.format(fromModel))
+
         if not toModel:
             return
-
+        try:
-        logger.debug('Asked for model: {}'.format(toModel))
+            candidate = next(self._conversion_candidates(fromModel, toModel))
-        output = params.get('conversion', None)
+        except StopIteration:
-        candidates = {}
+            e = Error(('No conversion plugin found for: '
-        for plugin in plugins:
+                       '{} -> {}'.format(fromModel, toModel)))
-            try:
+            e.original_response = resp
-                fromModel = plugin.get('onyx:usesEmotionModel', None)
+            e.parameters = params
-                candidates[plugin.id] = next(self._conversion_candidates(fromModel, toModel))
+            raise e
-                logger.debug('Analysis plugin {} uses model: {}'.format(plugin.id, fromModel))
-            except StopIteration:
-                e = Error(('No conversion plugin found for: '
-                           '{} -> {}'.format(fromModel, toModel)))
-                e.original_response = resp
-                e.parameters = params
-                raise e
         newentries = []
         for i in resp.entries:
             if output == "full":
-                newemotions = copy.deepcopy(i.emotions)
+                newemotions = i.emotions.copy()
             else:
                 newemotions = []
             for j in i.emotions:
-                plugname = j['prov:wasGeneratedBy']
-                candidate = candidates[plugname]
-                resp.analysis.append(candidate.id)
                 for k in candidate.convert(j, fromModel, toModel, params):
                     k.prov__wasGeneratedBy = candidate.id
                     if output == 'nested':
@@ -237,14 +192,13 @@ class Senpy(object):
             i.emotions = newemotions
             newentries.append(i)
         resp.entries = newentries
-        resp.analysis = list(set(resp.analysis))
+        resp.analysis.append(candidate.id)
 
     @property
     def default_plugin(self):
         candidate = self._default
         if not candidate:
-            candidates = self.filter_plugins(plugin_type='analysisPlugin',
+            candidates = self.filter_plugins(is_activated=True)
-                                             is_activated=True)
             if len(candidates) > 0:
                 candidate = list(candidates.values())[0]
         logger.debug("Default: {}".format(candidate))
@@ -298,12 +252,11 @@ class Senpy(object):
                 logger.error(msg)
                 raise Error(msg)
 
-        if sync or 'async' in plugin and not plugin.async:
+        if sync:
             act()
         else:
             th = Thread(target=act)
             th.start()
-            return th
 
     def deactivate_plugin(self, plugin_name, sync=False):
         try:
@@ -323,45 +276,31 @@ class Senpy(object):
                     "Error deactivating plugin {}: {}".format(plugin.name, ex))
                 logger.error("Trace: {}".format(traceback.format_exc()))
 
-        if sync or 'async' in plugin and not plugin.async:
+        if sync:
             deact()
         else:
             th = Thread(target=deact)
             th.start()
-            return th
 
     @classmethod
     def validate_info(cls, info):
-        return all(x in info for x in ('name', 'module', 'description', 'version'))
+        return all(x in info for x in ('name', 'module', 'version'))
 
     def install_deps(self):
         for i in self.plugins.values():
-            self._install_deps(i)
+            self._install_deps(i._info)
 
     @classmethod
     def _install_deps(cls, info=None):
         requirements = info.get('requirements', [])
         if requirements:
-            pip_args = ['pip']
+            pip_args = []
             pip_args.append('install')
             pip_args.append('--use-wheel')
             for req in requirements:
                 pip_args.append(req)
             logger.info('Installing requirements: ' + str(requirements))
-            process = subprocess.Popen(pip_args,
+            pip.main(pip_args)
-                                       stdout=subprocess.PIPE,
-                                       stderr=subprocess.PIPE)
-            log_subprocess_output(process)
-            exitcode = process.wait()
-            if exitcode != 0:
-                raise Error("Dependencies not properly installed")
-
-    @classmethod
-    def _load_module(cls, name, root):
-        sys.path.append(root)
-        tmp = importlib.import_module(name)
-        sys.path.remove(root)
-        return tmp
 
     @classmethod
     def _load_plugin_from_info(cls, info, root):
@@ -370,10 +309,11 @@ class Senpy(object):
             return None, None
         module = info["module"]
         name = info["name"]
-
+        sys.path.append(root)
+        (fp, pathname, desc) = imp.find_module(module, [root, ])
         cls._install_deps(info)
-        tmp = cls._load_module(module, root)
+        tmp = imp.load_module(module, fp, pathname, desc)
-
+        sys.path.remove(root)
         candidate = None
         for _, obj in inspect.getmembers(tmp):
             if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
@@ -419,9 +359,23 @@ class Senpy(object):
         return self._plugin_list
 
     def filter_plugins(self, **kwargs):
-        return plugins.pfilter(self.plugins, **kwargs)
+        """ Filter plugins by different criteria """
 
-    @property
+        def matches(plug):
-    def analysis_plugins(self):
+            res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
-        """ Return only the analysis plugins """
+            logger.debug(
-        return self.filter_plugins(plugin_type='analysisPlugin')
+                "matching {} with {}: {}".format(plug.name, kwargs, res))
+            return res
+
+        if not kwargs:
+            return self.plugins
+        else:
+            return {n: p for n, p in self.plugins.items() if matches(p)}
+
+    def sentiment_plugins(self):
+        """ Return only the sentiment plugins """
+        return {
+            p: plugin
+            for p, plugin in self.plugins.items()
+            if isinstance(plugin, SentimentPlugin)
+        }

senpy/models.py

@@ -214,7 +214,6 @@ class BaseModel(SenpyMixin, dict):
             temp['@type'] = getattr(self, '@type')
         except AttributeError:
             logger.warn('Creating an instance of an unknown model')
-
         super(BaseModel, self).__init__(temp)
 
     def _get_key(self, key):
@@ -237,10 +236,7 @@ class BaseModel(SenpyMixin, dict):
         self.__setitem__(self._get_key(key), value)
 
     def __delattr__(self, key):
-        try:
+        self.__delitem__(self._get_key(key))
-            object.__delattr__(self, key)
-        except AttributeError:
-            self.__delitem__(self._get_key(key))
 
     def _plain_dict(self):
         d = {k: v for (k, v) in self.items() if k[0] != "_"}
@@ -256,32 +252,13 @@ def register(rsubclass, rtype=None):
     _subtypes[rtype or rsubclass.__name__] = rsubclass
 
 
-def from_dict(indict, cls=None):
+def from_dict(indict):
-    if not cls:
+    target = indict.get('@type', None)
-        target = indict.get('@type', None)
+    if target and target in _subtypes:
-        try:
+        cls = _subtypes[target]
-            if target and target in _subtypes:
+    else:
-                cls = _subtypes[target]
+        cls = BaseModel
-            else:
+    return cls(**indict)
-                cls = BaseModel
-        except Exception:
-            cls = BaseModel
-    outdict = dict()
-    for k, v in indict.items():
-        if k == '@context':
-            pass
-        elif isinstance(v, dict):
-            v = from_dict(indict[k])
-        elif isinstance(v, list):
-            for ix, v2 in enumerate(v):
-                if isinstance(v2, dict):
-                    v[ix] = from_dict(v2)
-        outdict[k] = v
-    return cls(**outdict)
-
-
-def from_string(string, **kwargs):
-    return from_dict(json.loads(string), **kwargs)
 
 
 def from_json(injson):
@@ -331,7 +308,7 @@ for i in [
 _ErrorModel = from_schema('error')
 
 
-class Error(SenpyMixin, Exception):
+class Error(SenpyMixin, BaseException):
     def __init__(self, message, *args, **kwargs):
         super(Error, self).__init__(self, message, message)
         self._error = _ErrorModel(message=message, *args, **kwargs)
@@ -360,8 +337,5 @@ class Error(SenpyMixin, Exception):
     def __delattr__(self, key):
         delattr(self._error, key)
 
-    def __str__(self):
-        return str(self.to_JSON(with_context=False))
-
 
 register(Error, 'error')

senpy/plugins.py

@@ -3,18 +3,16 @@ standard_library.install_aliases()
 
 import inspect
 import os.path
-import os
 import pickle
 import logging
 import tempfile
 import copy
-from .. import models
+from . import models
-from ..api import API_PARAMS
 
 logger = logging.getLogger(__name__)
 
 
-class Plugin(models.Plugin):
+class SenpyPlugin(models.Plugin):
     def __init__(self, info=None):
         """
         Provides a canonical name for plugins and serves as base for other
@@ -25,24 +23,12 @@ class Plugin(models.Plugin):
                                         "information for the plugin."))
         logger.debug("Initialising {}".format(info))
         id = 'plugins/{}_{}'.format(info['name'], info['version'])
-        super(Plugin, self).__init__(id=id, **info)
+        super(SenpyPlugin, self).__init__(id=id, **info)
         self.is_activated = False
 
     def get_folder(self):
         return os.path.dirname(inspect.getfile(self.__class__))
 
-    def activate(self):
-        pass
-
-    def deactivate(self):
-        pass
-
-
-SenpyPlugin = Plugin
-
-
-class AnalysisPlugin(Plugin):
-
     def analyse(self, *args, **kwargs):
         raise NotImplemented(
             'Your method should implement either analyse or analyse_entry')
@@ -61,33 +47,30 @@ class AnalysisPlugin(Plugin):
         for i in results.entries:
             yield i
 
-    def analyse_entries(self, entries, parameters):
+    def activate(self):
-        for entry in entries:
+        pass
-            logger.debug('Analysing entry with plugin {}: {}'.format(self, entry))
+
-            for result in self.analyse_entry(entry, parameters):
+    def deactivate(self):
-                yield result
+        pass
 
 
-class ConversionPlugin(Plugin):
+class SentimentPlugin(models.SentimentPlugin, SenpyPlugin):
-    pass
-
-
-class SentimentPlugin(models.SentimentPlugin, AnalysisPlugin):
     def __init__(self, info, *args, **kwargs):
         super(SentimentPlugin, self).__init__(info, *args, **kwargs)
         self.minPolarityValue = float(info.get("minPolarityValue", 0))
         self.maxPolarityValue = float(info.get("maxPolarityValue", 1))
 
 
-class EmotionPlugin(models.EmotionPlugin, AnalysisPlugin):
+class EmotionPlugin(models.EmotionPlugin, SenpyPlugin):
     def __init__(self, info, *args, **kwargs):
         super(EmotionPlugin, self).__init__(info, *args, **kwargs)
         self.minEmotionValue = float(info.get("minEmotionValue", -1))
         self.maxEmotionValue = float(info.get("maxEmotionValue", 1))
 
 
-class EmotionConversionPlugin(models.EmotionConversionPlugin, ConversionPlugin):
+class EmotionConversionPlugin(models.EmotionConversionPlugin, SenpyPlugin):
-    pass
+    def __init__(self, info, *args, **kwargs):
+        super(EmotionConversionPlugin, self).__init__(info, *args, **kwargs)
 
 
 class ShelfMixin(object):
@@ -96,12 +79,7 @@ class ShelfMixin(object):
         if not hasattr(self, '_sh') or self._sh is None:
             self.__dict__['_sh'] = {}
             if os.path.isfile(self.shelf_file):
-                try:
+                self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb'))
-                    self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb'))
-                except (IndexError, EOFError, pickle.UnpicklingError):
-                    logger.warning('{} has a corrupted shelf file!'.format(self.id))
-                    if not self.get('force_shelf', False):
-                        raise
         return self._sh
 
     @sh.deleter
@@ -114,8 +92,8 @@ class ShelfMixin(object):
     @property
     def shelf_file(self):
         if 'shelf_file' not in self or not self['shelf_file']:
-            sd = os.environ.get('SENPY_DATA', tempfile.gettempdir())
+            self.shelf_file = os.path.join(tempfile.gettempdir(),
-            self.shelf_file = os.path.join(sd, self.name + '.p')
+                                           self.name + '.p')
         return self['shelf_file']
 
     def save(self):
@@ -123,40 +101,3 @@ class ShelfMixin(object):
         if hasattr(self, '_sh') and self._sh is not None:
             with open(self.shelf_file, 'wb') as f:
                 pickle.dump(self._sh, f)
-
-
-default_plugin_type = API_PARAMS['plugin_type']['default']
-
-
-def pfilter(plugins, **kwargs):
-    """ Filter plugins by different criteria """
-    if isinstance(plugins, models.Plugins):
-        plugins = plugins.plugins
-    elif isinstance(plugins, dict):
-        plugins = plugins.values()
-    ptype = kwargs.pop('plugin_type', default_plugin_type)
-    logger.debug('#' * 100)
-    logger.debug('ptype {}'.format(ptype))
-    if ptype:
-        try:
-            ptype = ptype[0].upper() + ptype[1:]
-            pclass = globals()[ptype]
-            logger.debug('Class: {}'.format(pclass))
-            candidates = filter(lambda x: isinstance(x, pclass),
-                                plugins)
-        except KeyError:
-            raise models.Error('{} is not a valid type'.format(ptype))
-    else:
-        candidates = plugins
-
-    logger.debug(candidates)
-
-    def matches(plug):
-        res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
-        logger.debug(
-            "matching {} with {}: {}".format(plug.name, kwargs, res))
-        return res
-
-    if kwargs:
-        candidates = filter(matches, candidates)
-    return {p.name: p for p in candidates}

senpy/plugins/conversion/centroids.py

@@ -1,80 +0,0 @@
-from senpy.plugins import EmotionConversionPlugin
-from senpy.models import EmotionSet, Emotion, Error
-
-import logging
-logger = logging.getLogger(__name__)
-
-
-class CentroidConversion(EmotionConversionPlugin):
-    def __init__(self, info):
-        if 'centroids' not in info:
-            raise Error('Centroid conversion plugins should provide '
-                        'the centroids in their senpy file')
-        if 'onyx:doesConversion' not in info:
-            if 'centroids_direction' not in info:
-                raise Error('Please, provide centroids direction')
-
-            cf, ct = info['centroids_direction']
-            info['onyx:doesConversion'] = [{
-                'onyx:conversionFrom': cf,
-                'onyx:conversionTo': ct
-            }, {
-                'onyx:conversionFrom': ct,
-                'onyx:conversionTo': cf
-            }]
-
-        if 'aliases' in info:
-            aliases = info['aliases']
-            ncentroids = {}
-            for k1, v1 in info['centroids'].items():
-                nv1 = {}
-                for k2, v2 in v1.items():
-                    nv1[aliases.get(k2, k2)] = v2
-                ncentroids[aliases.get(k1, k1)] = nv1
-            info['centroids'] = ncentroids
-        super(CentroidConversion, self).__init__(info)
-
-    def _forward_conversion(self, original):
-        """Sum the VAD value of all categories found."""
-        res = Emotion()
-        for e in original.onyx__hasEmotion:
-            category = e.onyx__hasEmotionCategory
-            if category in self.centroids:
-                for dim, value in self.centroids[category].items():
-                    try:
-                        res[dim] += value
-                    except Exception:
-                        res[dim] = value
-        return res
-
-    def _backwards_conversion(self, original):
-        """Find the closest category"""
-        dimensions = list(self.centroids.values())[0]
-
-        def distance(e1, e2):
-            return sum((e1[k] - e2.get(k, 0)) for k in dimensions)
-
-        emotion = ''
-        mindistance = 10000000000000000000000.0
-        for state in self.centroids:
-            d = distance(self.centroids[state], original)
-            if d < mindistance:
-                mindistance = d
-                emotion = state
-        result = Emotion(onyx__hasEmotionCategory=emotion)
-        return result
-
-    def convert(self, emotionSet, fromModel, toModel, params):
-
-        cf, ct = self.centroids_direction
-        logger.debug(
-            '{}\n{}\n{}\n{}'.format(emotionSet, fromModel, toModel, params))
-        e = EmotionSet()
-        if fromModel == cf and toModel == ct:
-            e.onyx__hasEmotion.append(self._forward_conversion(emotionSet))
-        elif fromModel == ct and toModel == cf:
-            for i in emotionSet.onyx__hasEmotion:
-                e.onyx__hasEmotion.append(self._backwards_conversion(i))
-        else:
-            raise Error('EMOTION MODEL NOT KNOWN')
-        yield e

senpy/plugins/conversion/emotion/ekman2fsre.senpy

@@ -1,39 +0,0 @@
----
-name: Ekman2FSRE
-module: senpy.plugins.conversion.centroids
-description: Plugin to convert emotion sets from Ekman to VAD
-version: 0.1
-# No need to specify onyx:doesConversion because centroids.py adds it automatically from centroids_direction
-centroids:
-  anger:
-    A: 6.95
-    D: 5.1
-    V: 2.7
-  disgust:
-    A: 5.3
-    D: 8.05
-    V: 2.7
-  fear:
-    A: 6.5
-    D: 3.6
-    V: 3.2
-  happiness:
-    A: 7.22
-    D: 6.28
-    V: 8.6
-  sadness:
-    A: 5.21
-    D: 2.82
-    V: 2.21
-centroids_direction:
-  - emoml:big6
-  - emoml:fsre-dimensions
-aliases: # These are aliases for any key in the centroid, to avoid repeating a long name several times
-  A: emoml:arousal
-  V: emoml:valence
-  D: emoml:dominance
-  anger: emoml:big6anger
-  disgust: emoml:big6disgust
-  fear: emoml:big6fear
-  happiness: emoml:big6happiness
-  sadness: emoml:big6sadness

senpy/plugins/conversion/emotion/ekman2vad.py

@@ -0,0 +1,56 @@
+from senpy.plugins import EmotionConversionPlugin
+from senpy.models import EmotionSet, Emotion, Error
+
+import logging
+logger = logging.getLogger(__name__)
+
+import math
+
+
+class WNA2VAD(EmotionConversionPlugin):
+
+    def _ekman_to_vad(self, ekmanSet):
+        potency = 0
+        arousal = 0
+        dominance = 0
+        for e in ekmanSet.onyx__hasEmotion:
+            category = e.onyx__hasEmotionCategory
+            centroid = self.centroids[category]
+            potency += centroid['V']
+            arousal += centroid['A']
+            dominance += centroid['D']
+        e = Emotion({'emoml:potency': potency,
+                     'emoml:arousal': arousal,
+                     'emoml:dominance': dominance})
+        return e
+
+    def _vad_to_ekman(self, VADEmotion):
+        V = VADEmotion['emoml:valence']
+        A = VADEmotion['emoml:potency']
+        D = VADEmotion['emoml:dominance']
+        emotion = ''
+        value = 10000000000000000000000.0
+        for state in self.centroids:
+            valence = V - self.centroids[state]['V']
+            arousal = A - self.centroids[state]['A']
+            dominance = D - self.centroids[state]['D']
+            new_value = math.sqrt((valence**2) +
+                                  (arousal**2) +
+                                  (dominance**2))
+            if new_value < value:
+                value = new_value
+                emotion = state
+        result = Emotion(onyx__hasEmotionCategory=emotion)
+        return result
+
+    def convert(self, emotionSet, fromModel, toModel, params):
+        logger.debug('{}\n{}\n{}\n{}'.format(emotionSet, fromModel, toModel, params))
+        e = EmotionSet()
+        if fromModel == 'emoml:big6':
+            e.onyx__hasEmotion.append(self._ekman_to_vad(emotionSet))
+        elif fromModel == 'emoml:fsre-dimensions':
+            for i in emotionSet.onyx__hasEmotion:
+                e.onyx__hasEmotion.append(self._vad_to_ekman(e))
+        else:
+            raise Error('EMOTION MODEL NOT KNOWN')
+        yield e

senpy/plugins/conversion/emotion/ekman2vad.senpy

@@ -1,39 +1,35 @@
 ---
-name: Ekman2PAD
+name: Ekman2VAD
-module: senpy.plugins.conversion.centroids
+module: ekman2vad
-description: Plugin to convert emotion sets from Ekman to VAD
+description: Plugin to convert from Ekman to VAD
 version: 0.1
-# No need to specify onyx:doesConversion because centroids.py adds it automatically from centroids_direction
+onyx:doesConversion:
+  - onyx:conversionFrom: emoml:big6
+    onyx:conversionTo: emoml:fsre-dimensions
+  - onyx:conversionFrom: emoml:fsre-dimensions
+    onyx:conversionTo: wna:WNAModel
 centroids:
-  anger:
+  emoml:big6anger:
     A: 6.95
     D: 5.1
     V: 2.7
-  disgust:
+  emoml:big6disgust:
     A: 5.3
     D: 8.05
     V: 2.7
-  fear:
+  emoml:big6fear:
     A: 6.5
     D: 3.6
     V: 3.2
-  happiness:
+  emoml:big6happiness:
     A: 7.22
     D: 6.28
     V: 8.6
-  sadness:
+  emoml:big6sadness:
     A: 5.21
     D: 2.82
     V: 2.21
-centroids_direction:
+aliases:
-  - emoml:big6
-  - emoml:pad
-aliases: # These are aliases for any key in the centroid, to avoid repeating a long name several times
   A: emoml:arousal
-  V: emoml:valence
+  V: emoml:potency
-  D: emoml:dominance
+  D: emoml:dominance
-  anger: emoml:big6anger
-  disgust: emoml:big6disgust
-  fear: emoml:big6fear
-  happiness: emoml:big6happiness
-  sadness: emoml:big6sadness

senpy/schemas/context.jsonld

@@ -37,12 +37,6 @@
       "@type": "@id",
       "@container": "@set"
     },
-    "options": {
-      "@container": "@set"
-    },
-    "plugins": {
-      "@container": "@set"
-    },
     "prov:wasGeneratedBy": {
       "@type": "@id"
     },

senpy/schemas/plugins.json

@@ -6,10 +6,11 @@
       "properties": {
         "plugins": {
           "type": "array",
-          "default": [],
           "items": {
             "$ref": "plugin.json"
           }
+        },
+        "@type": {
         }
       }
     }

senpy/schemas/results.json

@@ -18,16 +18,10 @@
           "type": "string"
         },
         "analysis": {
-          "default": [],
           "type": "array",
+          "default": [],
           "items": {
-            "anyOf": [
+            "$ref": "analysis.json"
-              {
-                "$ref": "analysis.json"
-              },{
-                "type": "string"
-              }
-            ]
           }
         },
         "entries": {

senpy/templates/index.html

@@ -47,7 +47,7 @@
                     This website is the senpy Playground, which allows you to test the instance of senpy in this server. It provides a user-friendly interface to the functions exposed by the senpy API.
                     </p>
                   <p>
-                    Once you get comfortable with the parameters and results, you are encouraged to issue your own requests to the API endpoint. You can find examples of API URL's when you try out a plugin with the "Analyse!" button on the "Test it" tab.
+                    Once you get comfortable with the parameters and results, you are encouraged to issue your own requests to the API endpoint, which should be <a href="/api">here</a>.
                   </p>
                   <p>
                     These are some of the things you can do with the API:

senpy/version.py

@@ -8,12 +8,8 @@ DEFAULT_FILE = os.path.join(ROOT, 'VERSION')
 
 
 def read_version(versionfile=DEFAULT_FILE):
-    try:
+    with open(versionfile) as f:
-        with open(versionfile) as f:
+        return f.read().strip()
-            return f.read().strip()
-    except IOError:
-        logger.error('Running an unknown version of senpy. Be careful!.')
-        return '0.0'
 
 
 __version__ = read_version()

setup.cfg

@@ -9,6 +9,4 @@ test=pytest
 ignore = E402
 max-line-length = 100
 [bdist_wheel]
-universal=1
+universal=1
-[tool:pytest]
-addopts = --cov=senpy --cov-report term-missing

test-requirements.txt

@@ -1,3 +1,3 @@
+pytest
 mock
 pytest-cov
-pytest

tests/plugins/async_plugin/asyncplugin.py

@@ -1,23 +0,0 @@
-from senpy.plugins import AnalysisPlugin
-
-import multiprocessing
-
-
-def _train(process_number):
-    return process_number
-
-
-class AsyncPlugin(AnalysisPlugin):
-    def _do_async(self, num_processes):
-        pool = multiprocessing.Pool(processes=num_processes)
-        values = pool.map(_train, range(num_processes))
-
-        return values
-
-    def activate(self):
-        self.value = self._do_async(4)
-
-    def analyse_entry(self, entry, params):
-        values = self._do_async(2)
-        entry.async_values = values
-        yield entry

tests/plugins/async_plugin/asyncplugin.senpy

@@ -1,8 +0,0 @@
----
-name: Async
-module: asyncplugin
-description: I am async
-author: "@balkian"
-version: '0.1'
-async: true
-extra_params: {}

tests/plugins/dummy_plugin/dummy.py

@@ -4,5 +4,4 @@ from senpy.plugins import SentimentPlugin
 class DummyPlugin(SentimentPlugin):
     def analyse_entry(self, entry, params):
         entry.text = entry.text[::-1]
-        entry.reversed = entry.get('reversed', 0) + 1
         yield entry

tests/plugins/sleep_plugin/sleep.py

@@ -1,8 +1,8 @@
-from senpy.plugins import AnalysisPlugin
+from senpy.plugins import SenpyPlugin
 from time import sleep
 
 
-class SleepPlugin(AnalysisPlugin):
+class SleepPlugin(SenpyPlugin):
     def activate(self, *args, **kwargs):
         sleep(self.timeout)
 

tests/test_api.py

@@ -1,68 +0,0 @@
-import logging
-
-logger = logging.getLogger(__name__)
-
-from unittest import TestCase
-from senpy.api import parse_params, API_PARAMS, NIF_PARAMS, WEB_PARAMS
-from senpy.models import Error
-
-
-class APITest(TestCase):
-
-    def test_api_params(self):
-        """The API should not define any required parameters without a default"""
-        parse_params({}, spec=API_PARAMS)
-
-    def test_web_params(self):
-        """The WEB should not define any required parameters without a default"""
-        parse_params({}, spec=WEB_PARAMS)
-
-    def test_basic(self):
-        a = {}
-        try:
-            parse_params(a, spec=NIF_PARAMS)
-            raise AssertionError()
-        except Error:
-            pass
-        a = {'input': 'hello'}
-        p = parse_params(a, spec=NIF_PARAMS)
-        assert 'input' in p
-        b = {'i': 'hello'}
-        p = parse_params(b, spec=NIF_PARAMS)
-        assert 'input' in p
-
-    def test_plugin(self):
-        query = {}
-        plug_params = {
-            'hello': {
-                'aliases': ['hello', 'hiya'],
-                'required': True
-            }
-        }
-        try:
-            parse_params(query, spec=plug_params)
-            raise AssertionError()
-        except Error:
-            pass
-        query['hello'] = 'world'
-        p = parse_params(query, spec=plug_params)
-        assert 'hello' in p
-        assert p['hello'] == 'world'
-        del query['hello']
-
-        query['hiya'] = 'dlrow'
-        p = parse_params(query, spec=plug_params)
-        assert 'hello' in p
-        assert 'hiya' in p
-        assert p['hello'] == 'dlrow'
-
-    def test_default(self):
-        spec = {
-            'hello': {
-                'required': True,
-                'default': 1
-            }
-        }
-        p = parse_params({}, spec=spec)
-        assert 'hello' in p
-        assert p['hello'] == 1

tests/test_blueprints.py

@@ -19,7 +19,6 @@ def parse_resp(resp):
 class BlueprintsTest(TestCase):
     def setUp(self):
         self.app = Flask("test_extensions")
-        self.app.debug = False
         self.client = self.app.test_client()
         self.senpy = Senpy()
         self.senpy.init_app(self.app)

tests/test_client.py

@@ -4,21 +4,18 @@ try:
 except ImportError:
     from mock import patch
 
-import json
-
 from senpy.client import Client
-from senpy.models import Results, Plugins, Error
+from senpy.models import Results, Error
-from senpy.plugins import AnalysisPlugin, default_plugin_type
 
 
 class Call(dict):
     def __init__(self, obj):
-        self.obj = obj.serialize()
+        self.obj = obj.jsonld()
         self.status_code = 200
         self.content = self.json()
 
     def json(self):
-        return json.loads(self.obj)
+        return self.obj
 
 
 class ModelsTest(TestCase):
@@ -37,29 +34,10 @@ class ModelsTest(TestCase):
             url=endpoint + '/', method='GET', params={'input': 'hello'})
         error = Call(Error('Nothing'))
         with patch('requests.request', return_value=error) as patched:
-            try:
+            resp = client.analyse(input='hello', algorithm='NONEXISTENT')
-                client.analyse(input='hello', algorithm='NONEXISTENT')
+            assert isinstance(resp, Error)
-                raise Exception('Exceptions should be raised. This is not golang')
-            except Error:
-                pass
         patched.assert_called_with(
             url=endpoint + '/',
             method='GET',
             params={'input': 'hello',
                     'algorithm': 'NONEXISTENT'})
-
-    def test_plugins(self):
-        endpoint = 'http://dummy/'
-        client = Client(endpoint)
-        plugins = Plugins()
-        p1 = AnalysisPlugin({'name': 'AnalysisP1', 'version': 0, 'description': 'No'})
-        plugins.plugins = [p1, ]
-        success = Call(plugins)
-        with patch('requests.request', return_value=success) as patched:
-            response = client.plugins()
-            assert isinstance(response, dict)
-            assert len(response) == 1
-            assert "AnalysisP1" in response
-        patched.assert_called_with(
-            url=endpoint + '/plugins', method='GET',
-            params={'plugin_type': default_plugin_type})

tests/test_extensions.py

@@ -1,6 +1,5 @@
 from __future__ import print_function
 import os
-from copy import deepcopy
 import logging
 
 try:
@@ -10,7 +9,7 @@ except ImportError:
 
 from functools import partial
 from senpy.extensions import Senpy
-from senpy.models import Error, Results, Entry, EmotionSet, Emotion, Plugin
+from senpy.models import Error
 from flask import Flask
 from unittest import TestCase
 
@@ -43,7 +42,6 @@ class ExtensionsTest(TestCase):
         info = {
             'name': 'TestPip',
             'module': 'dummy',
-            'description': None,
             'requirements': ['noop'],
             'version': 0
         }
@@ -53,7 +51,6 @@ class ExtensionsTest(TestCase):
         assert module
         import noop
         dir(noop)
-        self.senpy.install_deps()
 
     def test_installing(self):
         """ Enabling a plugin """
@@ -61,19 +58,6 @@ class ExtensionsTest(TestCase):
         assert len(self.senpy.plugins) >= 3
         assert self.senpy.plugins["Sleep"].is_activated
 
-    def test_installing_nonexistent(self):
-        """ Fail if the dependencies cannot be met """
-        info = {
-            'name': 'TestPipFail',
-            'module': 'dummy',
-            'description': None,
-            'requirements': ['IAmMakingThisPackageNameUpToFail'],
-            'version': 0
-        }
-        root = os.path.join(self.dir, 'plugins', 'dummy_plugin')
-        with self.assertRaises(Error):
-            name, module = self.senpy._load_plugin_from_info(info, root=root)
-
     def test_disabling(self):
         """ Disabling a plugin """
         self.senpy.deactivate_all(sync=True)
@@ -109,49 +93,19 @@ class ExtensionsTest(TestCase):
         assert r2.analysis[0] == "plugins/Dummy_0.1"
         assert r1.entries[0].text == 'input'
 
-    def test_analyse_jsonld(self):
-        """ Using a plugin with JSON-LD input"""
-        js_input = '''{
-        "@id": "prueba",
-        "@type": "results",
-        "entries": [
-          {"@id": "entry1",
-           "text": "tupni",
-           "@type": "entry"
-          }
-        ]
-        }'''
-        r1 = self.senpy.analyse(algorithm="Dummy",
-                                input=js_input,
-                                informat="json-ld",
-                                output="tuptuo")
-        r2 = self.senpy.analyse(input="tupni", output="tuptuo")
-        assert r1.analysis[0] == "plugins/Dummy_0.1"
-        assert r2.analysis[0] == "plugins/Dummy_0.1"
-        assert r1.entries[0].text == 'input'
-
     def test_analyse_error(self):
         mm = mock.MagicMock()
-        mm.id = 'magic_mock'
+        mm.analyse_entry.side_effect = Error('error on analysis', status=900)
-        mm.analyse_entries.side_effect = Error('error on analysis', status=500)
         self.senpy.plugins['MOCK'] = mm
-        try:
+        resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
-            self.senpy.analyse(input='nothing', algorithm='MOCK')
+        assert resp['message'] == 'error on analysis'
-            assert False
+        assert resp['status'] == 900
-        except Error as ex:
-            assert ex['message'] == 'error on analysis'
-            assert ex['status'] == 500
-
         mm.analyse.side_effect = Exception('generic exception on analysis')
-        mm.analyse_entries.side_effect = Exception(
+        mm.analyse_entry.side_effect = Exception(
             'generic exception on analysis')
-
+        resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
-        try:
+        assert resp['message'] == 'generic exception on analysis'
-            self.senpy.analyse(input='nothing', algorithm='MOCK')
+        assert resp['status'] == 500
-            assert False
-        except Error as ex:
-            assert ex['message'] == 'generic exception on analysis'
-            assert ex['status'] == 500
 
     def test_filtering(self):
         """ Filtering plugins """
@@ -165,55 +119,3 @@ class ExtensionsTest(TestCase):
     def test_load_default_plugins(self):
         senpy = Senpy(plugin_folder=self.dir, default_plugins=True)
         assert len(senpy.plugins) > 1
-
-    def test_convert_emotions(self):
-        self.senpy.activate_all(sync=True)
-        plugin = Plugin({
-            'id': 'imaginary',
-            'onyx:usesEmotionModel': 'emoml:fsre-dimensions'
-        })
-        eSet1 = EmotionSet()
-        eSet1.prov__wasGeneratedBy = plugin['id']
-        eSet1['onyx:hasEmotion'].append(Emotion({
-            'emoml:arousal': 1,
-            'emoml:potency': 0,
-            'emoml:valence': 0
-        }))
-        response = Results({
-            'entries': [Entry({
-                'text': 'much ado about nothing',
-                'emotions': [eSet1]
-            })]
-        })
-        params = {'emotionModel': 'emoml:big6',
-                  'conversion': 'full'}
-        r1 = deepcopy(response)
-        self.senpy.convert_emotions(r1,
-                                    [plugin, ],
-                                    params)
-        assert len(r1.entries[0].emotions) == 2
-        params['conversion'] = 'nested'
-        r2 = deepcopy(response)
-        self.senpy.convert_emotions(r2,
-                                    [plugin, ],
-                                    params)
-        assert len(r2.entries[0].emotions) == 1
-        assert r2.entries[0].emotions[0]['prov:wasDerivedFrom'] == eSet1
-        params['conversion'] = 'filtered'
-        r3 = deepcopy(response)
-        self.senpy.convert_emotions(r3,
-                                    [plugin, ],
-                                    params)
-        assert len(r3.entries[0].emotions) == 1
-        r3.jsonld()
-
-    # def test_async_plugin(self):
-    #     """ We should accept multiprocessing plugins with async=False"""
-    #     thread1 = self.senpy.activate_plugin("Async", sync=False)
-    #     thread1.join(timeout=1)
-    #     assert len(self.senpy.plugins['Async'].value) == 4
-
-    #     resp = self.senpy.analyse(input='nothing', algorithm='Async')
-
-    #     assert len(resp.entries[0].async_values) == 2
-    #     self.senpy.activate_plugin("Async", sync=True)

tests/test_models.py

@@ -11,12 +11,8 @@ from senpy.models import (Emotion,
                           Entry,
                           Error,
                           Results,
-                          Sentiment,
+                          Sentiment)
-                          Plugins,
+from senpy.plugins import SenpyPlugin
-                          Plugin,
-                          from_string,
-                          from_dict)
-from senpy import plugins
 from pprint import pprint
 
 
@@ -57,8 +53,8 @@ class ModelsTest(TestCase):
         assert (received["entries"][0]["nif:isString"] != "Not testing")
 
     def test_id(self):
-        """ Adding the id after creation should overwrite the automatic ID
+        ''' Adding the id after creation should overwrite the automatic ID
-        """
+        '''
         r = Entry()
         j = r.jsonld()
         assert '@id' in j
@@ -98,32 +94,20 @@ class ModelsTest(TestCase):
         r.validate()
 
     def test_plugins(self):
-        self.assertRaises(Error, plugins.Plugin)
+        self.assertRaises(Error, SenpyPlugin)
-        p = plugins.Plugin({"name": "dummy",
+        p = SenpyPlugin({"name": "dummy", "version": 0})
-                            "version": 0,
-                            "extra_params": {
-                                "none": {
-                                    "options": ["es", ],
-                                    "required": False,
-                                    "default": "0"
-                                }
-                            }})
         c = p.jsonld()
-        assert '@type' in c
+        assert "info" not in c
-        assert c['@type'] == 'plugin'
+        assert "repo" not in c
-        assert 'info' not in c
+        assert "extra_params" in c
-        assert 'repo' not in c
+        logging.debug("Framed:")
-        assert 'extra_params' in c
-        logging.debug('Framed:')
         logging.debug(c)
         p.validate()
-        assert 'es' in c['extra_params']['none']['options']
-        assert isinstance(c['extra_params']['none']['options'], list)
 
     def test_str(self):
         """The string representation shouldn't include private variables"""
         r = Results()
-        p = plugins.Plugin({"name": "STR test", "version": 0})
+        p = SenpyPlugin({"name": "STR test", "version": 0})
         p._testing = 0
         s = str(p)
         assert "_testing" not in s
@@ -160,39 +144,6 @@ class ModelsTest(TestCase):
         g = rdflib.Graph().parse(data=t, format='turtle')
         assert len(g) == len(triples)
 
-    def test_plugin_list(self):
+    def test_convert_emotions(self):
-        """The plugin list should be of type \"plugins\""""
+        """It should be possible to convert between different emotion models"""
-        plugs = Plugins()
+        pass
-        c = plugs.jsonld()
-        assert '@type' in c
-        assert c['@type'] == 'plugins'
-
-    def test_single_plugin(self):
-        """A response with a single plugin should still return a list"""
-        plugs = Plugins()
-        p = Plugin({'id': str(1),
-                    'version': 0,
-                    'description': 'dummy'})
-        plugs.plugins.append(p)
-        assert isinstance(plugs.plugins, list)
-        js = plugs.jsonld()
-        assert isinstance(js['plugins'], list)
-
-    def test_from_string(self):
-        results = {
-            '@type': 'results',
-            '@id': 'prueba',
-            'entries': [{
-                '@id': 'entry1',
-                '@type': 'entry',
-                'text': 'TEST'
-            }]
-        }
-        recovered = from_dict(results)
-        assert isinstance(recovered, Results)
-        assert isinstance(recovered.entries[0], Entry)
-
-        string = json.dumps(results)
-        recovered = from_string(string)
-        assert isinstance(recovered, Results)
-        assert isinstance(recovered.entries[0], Entry)

tests/test_plugins.py

@@ -83,39 +83,7 @@ class PluginsTest(TestCase):
         res2 = a.analyse(input=1)
         assert res2.entries[0].nif__isString == 2
 
-    def test_corrupt_shelf(self):
+    def test_two(self):
-        ''' Reusing the values of a previous shelf '''
-
-        emptyfile = os.path.join(self.shelf_dir, "emptyfile")
-        invalidfile = os.path.join(self.shelf_dir, "invalid_file")
-        with open(emptyfile, 'w+b'), open(invalidfile, 'w+b') as inf:
-            inf.write(b'ohno')
-
-        files = {emptyfile: ['empty file', (EOFError, IndexError)],
-                 invalidfile: ['invalid file', (pickle.UnpicklingError, IndexError)]}
-
-        for fn in files:
-            with open(fn, 'rb') as f:
-                msg, error = files[fn]
-                a = ShelfDummyPlugin(info={
-                    'name': 'shelve',
-                    'version': 'test',
-                    'shelf_file': f.name
-                })
-                assert os.path.isfile(a.shelf_file)
-                print('Shelf file: %s' % a.shelf_file)
-                with self.assertRaises(error):
-                    a.sh['a'] = 'fromA'
-                    a.save()
-                del a._sh
-                assert os.path.isfile(a.shelf_file)
-                a.force_shelf = True
-                a.sh['a'] = 'fromA'
-                a.save()
-                b = pickle.load(f)
-                assert b['a'] == 'fromA'
-
-    def test_reuse_shelf(self):
         ''' Reusing the values of a previous shelf '''
         a = ShelfDummyPlugin(info={
             'name': 'shelve',

version.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+git describe --long --tags --dirty