mirror of
https://github.com/gsi-upm/senpy
synced 2025-10-19 09:48:26 +00:00
Compare commits
1 Commits
0.9.0a6
...
12-convers
| Author | SHA1 | Date | |
|---|---|---|---|
|
5493070d40 |
@@ -1,19 +1,18 @@
|
||||
# Uncomment if you want to use docker-in-docker
|
||||
# image: gsiupm/dockermake:latest
|
||||
# services:
|
||||
# - docker:dind
|
||||
image: gsiupm/dockermake:latest
|
||||
|
||||
|
||||
# When using dind, it's wise to use the overlayfs driver for
|
||||
# improved performance.
|
||||
variables:
|
||||
DOCKER_DRIVER: overlay
|
||||
DOCKERFILE: Dockerfile
|
||||
IMAGENAME: $CI_REGISTRY_IMAGE
|
||||
|
||||
stages:
|
||||
- test
|
||||
- push
|
||||
- deploy
|
||||
- clean
|
||||
|
||||
before_script:
|
||||
- make -e login
|
||||
|
||||
.test: &test_definition
|
||||
stage: test
|
||||
script:
|
||||
@@ -29,14 +28,16 @@ test-2.7:
|
||||
variables:
|
||||
PYTHON_VERSION: "2.7"
|
||||
|
||||
|
||||
.image: &image_definition
|
||||
stage: push
|
||||
before_script:
|
||||
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
|
||||
script:
|
||||
- make -e push-$PYTHON_VERSION
|
||||
only:
|
||||
- tags
|
||||
- triggers
|
||||
- fix-makefiles
|
||||
|
||||
push-3.5:
|
||||
<<: *image_definition
|
||||
@@ -56,55 +57,9 @@ push-latest:
|
||||
- master
|
||||
- triggers
|
||||
|
||||
push-github:
|
||||
stage: deploy
|
||||
script:
|
||||
- make -e push-github
|
||||
only:
|
||||
- master
|
||||
- triggers
|
||||
- fix-makefiles
|
||||
|
||||
deploy_pypi:
|
||||
stage: deploy
|
||||
script: # Configure the PyPI credentials, then push the package, and cleanup the creds.
|
||||
- echo "[server-login]" >> ~/.pypirc
|
||||
- echo "repository=https://upload.pypi.org/legacy/" >> ~/.pypirc
|
||||
- echo "username=" ${PYPI_USER} >> ~/.pypirc
|
||||
- echo "password=" ${PYPI_PASSWORD} >> ~/.pypirc
|
||||
- make pip_upload
|
||||
- echo "" > ~/.pypirc && rm ~/.pypirc # If the above fails, this won't run.
|
||||
only:
|
||||
- /^v?\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
|
||||
except:
|
||||
- branches
|
||||
|
||||
deploy:
|
||||
stage: deploy
|
||||
environment: test
|
||||
script:
|
||||
- make -e deploy
|
||||
only:
|
||||
- master
|
||||
- fix-makefiles
|
||||
|
||||
push-github:
|
||||
stage: deploy
|
||||
script:
|
||||
- make -e push-github
|
||||
only:
|
||||
- master
|
||||
- triggers
|
||||
|
||||
clean :
|
||||
stage: clean
|
||||
script:
|
||||
- make -e clean
|
||||
when: manual
|
||||
|
||||
cleanup_py:
|
||||
stage: clean
|
||||
when: always # this is important; run even if preceding stages failed.
|
||||
script:
|
||||
- rm -vf ~/.pypirc # we don't want to leave these around, but GitLab may clean up anyway.
|
||||
- docker logout
|
||||
only:
|
||||
- master
|
||||
@@ -1,27 +0,0 @@
|
||||
These makefiles are recipes for several common tasks in different types of projects.
|
||||
To add them to your project, simply do:
|
||||
|
||||
```
|
||||
git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git
|
||||
git subtree add --prefix=.makefiles/ makefiles master
|
||||
touch Makefile
|
||||
echo "include .makefiles/base.mk" >> Makefile
|
||||
```
|
||||
|
||||
Now you can take advantage of the recipes.
|
||||
For instance, to add useful targets for a python project, just add this to your Makefile:
|
||||
|
||||
```
|
||||
include .makefiles/python.mk
|
||||
```
|
||||
|
||||
You may need to set special variables like the name of your project or the python versions you're targetting.
|
||||
Take a look at each specific `.mk` file for more information, and the `Makefile` in the [senpy](https://lab.cluster.gsi.dit.upm.es/senpy/senpy) project for a real use case.
|
||||
|
||||
If you update the makefiles from your repository, make sure to push the changes for review in upstream (this repository):
|
||||
|
||||
```
|
||||
make makefiles-push
|
||||
```
|
||||
|
||||
It will automatically commit all unstaged changes in the .makefiles folder.
|
||||
@@ -1,38 +0,0 @@
|
||||
export
|
||||
NAME ?= $(shell basename $(CURDIR))
|
||||
VERSION ?= $(shell git describe --tags --dirty 2>/dev/null)
|
||||
|
||||
# Get the location of this makefile.
|
||||
MK_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
|
||||
|
||||
-include .env
|
||||
-include ../.env
|
||||
|
||||
.FORCE:
|
||||
|
||||
version: .FORCE
|
||||
@echo $(VERSION) > $(NAME)/VERSION
|
||||
@echo $(VERSION)
|
||||
|
||||
help: ## Show this help.
|
||||
@fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/\(.*:\)[^#]*##\s*\(.*\)/\1\t\2/' | column -t -s " "
|
||||
|
||||
config: ## Load config from the environment. You should run it once in every session before other tasks. Run: eval $(make config)
|
||||
@awk '{ print "export " $$0}' ../.env
|
||||
@awk '{ print "export " $$0}' .env
|
||||
@echo "# Please, run: "
|
||||
@echo "# eval \$$(make config)"
|
||||
# If you need to run a command on the key/value pairs, use this:
|
||||
# @awk '{ split($$0, a, "="); "echo " a[2] " | base64 -w 0" |& getline b64; print "export " a[1] "=" a[2]; print "export " a[1] "_BASE64=" b64}' .env
|
||||
|
||||
ci: ## Run a task using gitlab-runner. Only use to debug problems in the CI pipeline
|
||||
gitlab-runner exec shell --builds-dir '.builds' --env CI_PROJECT_NAME=$(NAME) ${action}
|
||||
|
||||
include $(MK_DIR)/makefiles.mk
|
||||
include $(MK_DIR)/docker.mk
|
||||
include $(MK_DIR)/git.mk
|
||||
|
||||
info:: ## List all variables
|
||||
env
|
||||
|
||||
.PHONY:: config help ci version .FORCE
|
||||
@@ -1,25 +0,0 @@
|
||||
IMAGEWTAG ?= $(IMAGENAME):$(VERSION)
|
||||
|
||||
docker-login: ## Log in to the registry. It will only be used in the server, or when running a CI task locally (if CI_BUILD_TOKEN is set).
|
||||
ifeq ($(CI_BUILD_TOKEN),)
|
||||
@echo "Not logging in to the docker registry" "$(CI_REGISTRY)"
|
||||
else
|
||||
@docker login -u gitlab-ci-token -p $(CI_BUILD_TOKEN) $(CI_REGISTRY)
|
||||
endif
|
||||
ifeq ($(HUB_USER),)
|
||||
@echo "Not logging in to global the docker registry"
|
||||
else
|
||||
@docker login -u $(HUB_USER) -p $(HUB_PASSWORD)
|
||||
endif
|
||||
|
||||
docker-clean: ## Remove docker credentials
|
||||
ifeq ($(HUB_USER),)
|
||||
else
|
||||
@docker logout
|
||||
endif
|
||||
|
||||
login:: docker-login
|
||||
|
||||
clean:: docker-clean
|
||||
|
||||
.PHONY:: docker-login docker-clean login clean
|
||||
@@ -1,28 +0,0 @@
|
||||
commit:
|
||||
git commit -a
|
||||
|
||||
tag:
|
||||
git tag ${VERSION}
|
||||
|
||||
git-push::
|
||||
git push --tags -u origin HEAD
|
||||
|
||||
git-pull:
|
||||
git pull --all
|
||||
|
||||
push-github: ## Push the code to github. You need to set up GITHUB_DEPLOY_KEY
|
||||
ifeq ($(GITHUB_DEPLOY_KEY),)
|
||||
else
|
||||
$(eval KEY_FILE := "$(shell mktemp)")
|
||||
@echo "$(GITHUB_DEPLOY_KEY)" > $(KEY_FILE)
|
||||
@git remote rm github-deploy || true
|
||||
git remote add github-deploy $(GITHUB_REPO)
|
||||
-@GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git fetch github-deploy $(CI_COMMIT_REF_NAME)
|
||||
@GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git push github-deploy HEAD:$(CI_COMMIT_REF_NAME)
|
||||
rm $(KEY_FILE)
|
||||
endif
|
||||
|
||||
push:: git-push
|
||||
pull:: git-pull
|
||||
|
||||
.PHONY:: commit tag push git-push git-pull push-github
|
||||
@@ -1,51 +0,0 @@
|
||||
# Deployment with Kubernetes
|
||||
|
||||
# KUBE_CA_PEM_FILE is the path of a certificate file. It automatically set by GitLab
|
||||
# if you enable Kubernetes integration in a project.
|
||||
#
|
||||
# As of this writing, Kubernetes integration can not be set on a group level, so it has to
|
||||
# be manually set in every project.
|
||||
# Alternatively, we use a custom KUBE_CA_BUNDLE environment variable, which can be set at
|
||||
# the group level. In this case, the variable contains the whole content of the certificate,
|
||||
# which we dump to a temporary file
|
||||
#
|
||||
# Check if the KUBE_CA_PEM_FILE exists. Otherwise, create it from KUBE_CA_BUNDLE
|
||||
KUBE_CA_TEMP=false
|
||||
ifndef KUBE_CA_PEM_FILE
|
||||
KUBE_CA_PEM_FILE:=$$PWD/.ca.crt
|
||||
CREATED:=$(shell echo -e "$(KUBE_CA_BUNDLE)" > $(KUBE_CA_PEM_FILE))
|
||||
endif
|
||||
KUBE_TOKEN?=""
|
||||
KUBE_NAMESPACE?=$(NAME)
|
||||
KUBECTL=docker run --rm -v $(KUBE_CA_PEM_FILE):/tmp/ca.pem -i lachlanevenson/k8s-kubectl --server="$(KUBE_URL)" --token="$(KUBE_TOKEN)" --certificate-authority="/tmp/ca.pem" -n $(KUBE_NAMESPACE)
|
||||
CI_COMMIT_REF_NAME?=master
|
||||
|
||||
info:: ## Print variables. Useful for debugging.
|
||||
@echo "#KUBERNETES"
|
||||
@echo KUBE_URL=$(KUBE_URL)
|
||||
@echo KUBE_CA_PEM_FILE=$(KUBE_CA_PEM_FILE)
|
||||
@echo KUBE_CA_BUNDLE=$$KUBE_CA_BUNDLE
|
||||
@echo KUBE_TOKEN=$(KUBE_TOKEN)
|
||||
@echo KUBE_NAMESPACE=$(KUBE_NAMESPACE)
|
||||
@echo KUBECTL=$(KUBECTL)
|
||||
|
||||
@echo "#CI"
|
||||
@echo CI_PROJECT_NAME=$(CI_PROJECT_NAME)
|
||||
@echo CI_REGISTRY=$(CI_REGISTRY)
|
||||
@echo CI_REGISTRY_USER=$(CI_REGISTRY_USER)
|
||||
@echo CI_COMMIT_REF_NAME=$(CI_COMMIT_REF_NAME)
|
||||
@echo "CREATED=$(CREATED)"
|
||||
|
||||
#
|
||||
# Deployment and advanced features
|
||||
#
|
||||
|
||||
|
||||
deploy: ## Deploy to kubernetes using the credentials in KUBE_CA_PEM_FILE (or KUBE_CA_BUNDLE ) and TOKEN
|
||||
@ls k8s/*.yaml k8s/*.yml k8s/*.tmpl 2>/dev/null || true
|
||||
@cat k8s/*.yaml k8s/*.yml k8s/*.tmpl 2>/dev/null | envsubst | $(KUBECTL) apply -f -
|
||||
|
||||
deploy-check: ## Get the deployed configuration.
|
||||
@$(KUBECTL) get deploy,pods,svc,ingress
|
||||
|
||||
.PHONY:: info deploy deploy-check
|
||||
@@ -1,17 +0,0 @@
|
||||
makefiles-remote:
|
||||
@git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git 2>/dev/null || true
|
||||
|
||||
makefiles-commit: makefiles-remote
|
||||
git add -f .makefiles
|
||||
git commit -em "Updated makefiles from ${NAME}"
|
||||
|
||||
makefiles-push:
|
||||
git subtree push --prefix=.makefiles/ makefiles $(NAME)
|
||||
|
||||
makefiles-pull: makefiles-remote
|
||||
git subtree pull --prefix=.makefiles/ makefiles master --squash
|
||||
|
||||
pull:: makefiles-pull
|
||||
push:: makefiles-push
|
||||
|
||||
.PHONY:: makefiles-remote makefiles-commit makefiles-push makefiles-pull pull push
|
||||
@@ -1,5 +0,0 @@
|
||||
init: ## Init pre-commit hooks (i.e. enforcing format checking before allowing a commit)
|
||||
pip install --user pre-commit
|
||||
pre-commit install
|
||||
|
||||
.PHONY:: init
|
||||
@@ -1,92 +0,0 @@
|
||||
PYVERSIONS ?= 2.7
|
||||
PYMAIN ?= $(firstword $(PYVERSIONS))
|
||||
TARNAME ?= $(NAME)-$(VERSION).tar.gz
|
||||
|
||||
DEVPORT ?= 6000
|
||||
|
||||
yapf: ## Format python code
|
||||
yapf -i -r $(NAME)
|
||||
yapf -i -r tests
|
||||
|
||||
dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS)) ## Generate dockerfiles for each python version
|
||||
@unlink Dockerfile >/dev/null
|
||||
ln -s Dockerfile-$(PYMAIN) Dockerfile
|
||||
|
||||
Dockerfile-%: Dockerfile.template ## Generate a specific dockerfile (e.g. Dockerfile-2.7)
|
||||
sed "s/{{PYVERSION}}/$*/" Dockerfile.template > Dockerfile-$*
|
||||
|
||||
quick_build: $(addprefix build-, $(PYMAIN))
|
||||
|
||||
build: $(addprefix build-, $(PYVERSIONS)) ## Build all images / python versions
|
||||
|
||||
build-%: version Dockerfile-% ## Build a specific version (e.g. build-2.7)
|
||||
docker build -t '$(IMAGEWTAG)-python$*' --cache-from $(IMAGENAME):python$* -f Dockerfile-$* .;
|
||||
|
||||
dev-%: ## Launch a specific development environment using docker (e.g. dev-2.7)
|
||||
@docker start $(NAME)-dev$* || (\
|
||||
$(MAKE) build-$*; \
|
||||
docker run -d -w /usr/src/app/ -p $(DEVPORT):5000 -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti --name $(NAME)-dev$* '$(IMAGEWTAG)-python$*'; \
|
||||
)\
|
||||
|
||||
docker exec -ti $(NAME)-dev$* bash
|
||||
|
||||
dev: dev-$(PYMAIN) ## Launch a development environment using docker, using the default python version
|
||||
|
||||
quick_test: test-$(PYMAIN)
|
||||
|
||||
test-%: ## Run setup.py from in an isolated container, built from the base image. (e.g. test-2.7)
|
||||
# This speeds tests up because the image has most (if not all) of the dependencies already.
|
||||
docker rm $(NAME)-test-$* || true
|
||||
docker create -ti --name $(NAME)-test-$* --entrypoint="" -w /usr/src/app/ $(IMAGENAME):python$* python setup.py test
|
||||
docker cp . $(NAME)-test-$*:/usr/src/app
|
||||
docker start -a $(NAME)-test-$*
|
||||
|
||||
test: $(addprefix test-,$(PYVERSIONS)) ## Run the tests with the main python version
|
||||
|
||||
run-%: build-%
|
||||
docker run --rm -p $(DEVPORT):5000 -ti '$(IMAGEWTAG)-python$(PYMAIN)' --default-plugins
|
||||
|
||||
run: run-$(PYMAIN)
|
||||
|
||||
# Pypy - Upload a package
|
||||
|
||||
dist/$(TARNAME): version
|
||||
python setup.py sdist;
|
||||
|
||||
sdist: dist/$(TARNAME) ## Generate the distribution file (wheel)
|
||||
|
||||
pip_test-%: sdist ## Test the distribution file using pip install and a specific python version (e.g. pip_test-2.7)
|
||||
docker run --rm -v $$PWD/dist:/dist/ python:$* pip install /dist/$(TARNAME);
|
||||
|
||||
pip_test: $(addprefix pip_test-,$(PYVERSIONS)) ## Test pip installation with the main python version
|
||||
|
||||
pip_upload: pip_test ## Upload package to pip
|
||||
python setup.py sdist upload ;
|
||||
|
||||
# Pushing to docker
|
||||
|
||||
push-latest: $(addprefix push-latest-,$(PYVERSIONS)) ## Push the "latest" tag to dockerhub
|
||||
docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
|
||||
docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGENAME)'
|
||||
docker push '$(IMAGENAME):latest'
|
||||
docker push '$(IMAGEWTAG)'
|
||||
|
||||
push-latest-%: build-% ## Push the latest image for a specific python version
|
||||
docker tag $(IMAGENAME):$(VERSION)-python$* $(IMAGENAME):python$*
|
||||
docker push $(IMAGENAME):$(VERSION)-python$*
|
||||
docker push $(IMAGENAME):python$*
|
||||
|
||||
push-%: build-% ## Push the image of the current version (tagged). e.g. push-2.7
|
||||
docker push $(IMAGENAME):$(VERSION)-python$*
|
||||
|
||||
push:: $(addprefix push-,$(PYVERSIONS)) ## Push an image with the current version for every python version
|
||||
docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
|
||||
docker push $(IMAGENAME):$(VERSION)
|
||||
|
||||
clean:: ## Clean older docker images and containers related to this project and dev environments
|
||||
@docker stop $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true
|
||||
@docker rm $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true
|
||||
@docker ps -a | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true
|
||||
@docker images | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true
|
||||
|
||||
.PHONY:: yapf dockerfiles Dockerfile-% quick_build build build-% dev-% quick-dev test quick_test push-latest push-latest-% push-% push
|
||||
@@ -7,6 +7,7 @@ language: python
|
||||
|
||||
env:
|
||||
- PYV=2.7
|
||||
- PYV=3.4
|
||||
- PYV=3.5
|
||||
# run nosetests - Tests
|
||||
script: make test-$PYV
|
||||
|
||||
@@ -17,6 +17,6 @@ WORKDIR /usr/src/app
|
||||
COPY test-requirements.txt requirements.txt /usr/src/app/
|
||||
RUN pip install --use-wheel -r test-requirements.txt -r requirements.txt
|
||||
COPY . /usr/src/app/
|
||||
RUN pip install --no-index --no-deps --editable .
|
||||
RUN pip install --no-deps --no-index .
|
||||
|
||||
ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]
|
||||
|
||||
120
Makefile
120
Makefile
@@ -1,17 +1,109 @@
|
||||
NAME=senpy
|
||||
GITHUB_REPO=git@github.com:gsi-upm/senpy.git
|
||||
|
||||
IMAGENAME=gsiupm/senpy
|
||||
|
||||
# The first version is the main one (used for quick builds)
|
||||
# See .makefiles/python.mk for more info
|
||||
PYVERSIONS=3.5 2.7
|
||||
|
||||
DEVPORT=5000
|
||||
|
||||
PYMAIN=$(firstword $(PYVERSIONS))
|
||||
NAME=senpy
|
||||
REPO=gsiupm
|
||||
VERSION=$(shell git describe --tags --dirty 2>/dev/null)
|
||||
TARNAME=$(NAME)-$(VERSION).tar.gz
|
||||
IMAGENAME=$(REPO)/$(NAME)
|
||||
IMAGEWTAG=$(IMAGENAME):$(VERSION)
|
||||
action="test-${PYMAIN}"
|
||||
GITHUB_REPO=git@github.com:gsi-upm/senpy.git
|
||||
|
||||
include .makefiles/base.mk
|
||||
include .makefiles/k8s.mk
|
||||
include .makefiles/python.mk
|
||||
all: build run
|
||||
|
||||
.FORCE:
|
||||
|
||||
version: .FORCE
|
||||
@echo $(VERSION) > $(NAME)/VERSION
|
||||
@echo $(VERSION)
|
||||
|
||||
yapf:
|
||||
yapf -i -r senpy
|
||||
yapf -i -r tests
|
||||
|
||||
init:
|
||||
pip install --user pre-commit
|
||||
pre-commit install
|
||||
|
||||
dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS))
|
||||
@unlink Dockerfile >/dev/null
|
||||
ln -s Dockerfile-$(PYMAIN) Dockerfile
|
||||
|
||||
Dockerfile-%: Dockerfile.template
|
||||
sed "s/{{PYVERSION}}/$*/" Dockerfile.template > Dockerfile-$*
|
||||
|
||||
quick_build: $(addprefix build-, $(PYMAIN))
|
||||
|
||||
build: $(addprefix build-, $(PYVERSIONS))
|
||||
|
||||
build-%: version Dockerfile-%
|
||||
docker build -t '$(IMAGEWTAG)-python$*' -f Dockerfile-$* .;
|
||||
|
||||
quick_test: $(addprefix test-,$(PYMAIN))
|
||||
|
||||
dev-%:
|
||||
@docker start $(NAME)-dev$* || (\
|
||||
$(MAKE) build-$*; \
|
||||
docker run -d -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti --name $(NAME)-dev$* '$(IMAGEWTAG)-python$*'; \
|
||||
)\
|
||||
|
||||
docker exec -ti $(NAME)-dev$* bash
|
||||
|
||||
dev: dev-$(PYMAIN)
|
||||
|
||||
test-all: $(addprefix test-,$(PYVERSIONS))
|
||||
|
||||
test-%: build-%
|
||||
docker run --rm --entrypoint /usr/local/bin/python -w /usr/src/app $(IMAGEWTAG)-python$* setup.py test
|
||||
|
||||
test: test-$(PYMAIN)
|
||||
|
||||
dist/$(TARNAME):
|
||||
docker run --rm -ti -v $$PWD:/usr/src/app/ -w /usr/src/app/ python:$(PYMAIN) python setup.py sdist;
|
||||
|
||||
sdist: dist/$(TARNAME)
|
||||
|
||||
pip_test-%: sdist
|
||||
docker run --rm -v $$PWD/dist:/dist/ -ti python:$* pip install /dist/$(TARNAME);
|
||||
|
||||
pip_test: $(addprefix pip_test-,$(PYVERSIONS))
|
||||
|
||||
clean:
|
||||
@docker ps -a | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true
|
||||
@docker images | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true
|
||||
@docker rmi $(NAME)-dev 2>/dev/null || true
|
||||
|
||||
|
||||
git_commit:
|
||||
git commit -a
|
||||
|
||||
git_tag:
|
||||
git tag ${VERSION}
|
||||
|
||||
git_push:
|
||||
git push --tags origin master
|
||||
|
||||
pip_upload:
|
||||
python setup.py sdist upload ;
|
||||
|
||||
run-%: build-%
|
||||
docker run --rm -p 5000:5000 -ti '$(IMAGEWTAG)-python$(PYMAIN)' --default-plugins
|
||||
|
||||
run: run-$(PYMAIN)
|
||||
|
||||
push-latest: build-$(PYMAIN)
|
||||
docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
|
||||
docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGENAME)'
|
||||
docker push '$(IMAGENAME):latest'
|
||||
docker push '$(IMAGEWTAG)'
|
||||
|
||||
push-%: build-%
|
||||
docker push $(IMAGENAME):$(VERSION)-python$*
|
||||
|
||||
push: $(addprefix push-,$(PYVERSIONS))
|
||||
docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
|
||||
docker push $(IMAGENAME):$(VERSION)
|
||||
|
||||
ci:
|
||||
gitlab-runner exec docker --docker-volumes /var/run/docker.sock:/var/run/docker.sock --env CI_PROJECT_NAME=$(NAME) ${action}
|
||||
|
||||
.PHONY: test test-% test-all build-% build test pip_test run yapf push-main push-% dev ci version .FORCE
|
||||
|
||||
51
README.rst
51
README.rst
@@ -23,7 +23,7 @@ Through PIP
|
||||
|
||||
.. code:: bash
|
||||
|
||||
pip install -U --user senpy
|
||||
pip install --user senpy
|
||||
|
||||
|
||||
Alternatively, you can use the development version:
|
||||
@@ -42,53 +42,6 @@ Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/s
|
||||
|
||||
To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --default-plugins -f /plugins``
|
||||
|
||||
|
||||
Developing
|
||||
----------
|
||||
|
||||
Developing/debugging
|
||||
********************
|
||||
This command will run the senpy container using the latest image available, mounting your current folder so you get your latest code:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
|
||||
# Python 3.5
|
||||
make dev
|
||||
# Python 2.7
|
||||
make dev-2.7
|
||||
|
||||
Building a docker image
|
||||
***********************
|
||||
|
||||
.. code:: bash
|
||||
|
||||
|
||||
# Python 3.5
|
||||
make build-3.5
|
||||
# Python 2.7
|
||||
make build-2.7
|
||||
|
||||
Testing
|
||||
*******
|
||||
|
||||
.. code:: bash
|
||||
|
||||
|
||||
make test
|
||||
|
||||
Running
|
||||
*******
|
||||
This command will run the senpy server listening on localhost:5000
|
||||
|
||||
.. code:: bash
|
||||
|
||||
|
||||
# Python 3.5
|
||||
make run-3.5
|
||||
# Python 2.7
|
||||
make run-2.7
|
||||
|
||||
Usage
|
||||
-----
|
||||
|
||||
@@ -96,14 +49,12 @@ However, the easiest and recommended way is to just use the command-line tool to
|
||||
|
||||
.. code:: bash
|
||||
|
||||
|
||||
senpy
|
||||
|
||||
or, alternatively:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
|
||||
python -m senpy
|
||||
|
||||
|
||||
|
||||
@@ -1,317 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2017-04-10T17:05:31.465571Z",
|
||||
"start_time": "2017-04-10T19:05:31.458282+02:00"
|
||||
},
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"source": [
|
||||
"# Client"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"collapsed": true,
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"source": [
|
||||
"The built-in senpy client allows you to query any Senpy endpoint. We will illustrate how to use it with the public demo endpoint, and then show you how to spin up your own endpoint using docker."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"source": [
|
||||
"Demo Endpoint\n",
|
||||
"-------------"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"source": [
|
||||
"To start using senpy, simply create a new Client and point it to your endpoint. In this case, the latest version of Senpy at GSI."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2017-04-10T17:29:12.827640Z",
|
||||
"start_time": "2017-04-10T19:29:12.818617+02:00"
|
||||
},
|
||||
"collapsed": false,
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from senpy.client import Client\n",
|
||||
"\n",
|
||||
"c = Client('http://latest.senpy.cluster.gsi.dit.upm.es/api')\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"source": [
|
||||
"Now, let's use that client analyse some queries:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2017-04-10T17:29:14.011657Z",
|
||||
"start_time": "2017-04-10T19:29:13.701808+02:00"
|
||||
},
|
||||
"collapsed": false,
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"r = c.analyse('I like sugar!!', algorithm='sentiment140')\n",
|
||||
"r"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2017-04-10T17:08:19.616754Z",
|
||||
"start_time": "2017-04-10T19:08:19.610767+02:00"
|
||||
},
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"source": [
|
||||
"As you can see, that gave us the full JSON result. A more concise way to print it would be:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2017-04-10T17:29:14.854213Z",
|
||||
"start_time": "2017-04-10T19:29:14.842068+02:00"
|
||||
},
|
||||
"collapsed": false,
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"for entry in r.entries:\n",
|
||||
" print('{} -> {}'.format(entry['text'], entry['sentiments'][0]['marl:hasPolarity']))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"source": [
|
||||
"We can also obtain a list of available plugins with the client:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2017-04-10T17:29:16.245198Z",
|
||||
"start_time": "2017-04-10T19:29:16.056545+02:00"
|
||||
},
|
||||
"collapsed": false,
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"c.plugins()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"source": [
|
||||
"Or, more concisely:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2017-04-10T17:29:17.663275Z",
|
||||
"start_time": "2017-04-10T19:29:17.484623+02:00"
|
||||
},
|
||||
"collapsed": false,
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"c.plugins().keys()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"source": [
|
||||
"Local Endpoint\n",
|
||||
"--------------"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"source": [
|
||||
"To run your own instance of senpy, just create a docker container with the latest Senpy image. Using `--default-plugins` you will get some extra plugins to start playing with the API."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2017-04-10T17:29:20.637539Z",
|
||||
"start_time": "2017-04-10T19:29:19.938322+02:00"
|
||||
},
|
||||
"collapsed": false,
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!docker run -ti --name 'SenpyEndpoint' -d -p 6000:5000 gsiupm/senpy:0.8.6 --host 0.0.0.0 --default-plugins"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"source": [
|
||||
"To use this endpoint:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2017-04-10T17:29:21.263976Z",
|
||||
"start_time": "2017-04-10T19:29:21.260595+02:00"
|
||||
},
|
||||
"collapsed": false,
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"c_local = Client('http://127.0.0.1:6000/api')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"source": [
|
||||
"That's all! After you are done with your analysis, stop the docker container:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2017-04-10T17:29:33.226686Z",
|
||||
"start_time": "2017-04-10T19:29:22.392121+02:00"
|
||||
},
|
||||
"collapsed": false,
|
||||
"deletable": true,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!docker stop SenpyEndpoint\n",
|
||||
"!docker rm SenpyEndpoint"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"anaconda-cloud": {},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.0"
|
||||
},
|
||||
"toc": {
|
||||
"colors": {
|
||||
"hover_highlight": "#DAA520",
|
||||
"running_highlight": "#FF0000",
|
||||
"selected_highlight": "#FFD700"
|
||||
},
|
||||
"moveMenuLeft": true,
|
||||
"nav_menu": {
|
||||
"height": "68px",
|
||||
"width": "252px"
|
||||
},
|
||||
"navigate_menu": true,
|
||||
"number_sections": true,
|
||||
"sideBar": true,
|
||||
"threshold": 4,
|
||||
"toc_cell": false,
|
||||
"toc_section_display": "block",
|
||||
"toc_window_display": false
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 1
|
||||
}
|
||||
BIN
docs/_static/header.png
vendored
BIN
docs/_static/header.png
vendored
Binary file not shown.
|
Before Width: | Height: | Size: 208 KiB |
@@ -1,11 +0,0 @@
|
||||
About
|
||||
--------
|
||||
|
||||
If you use Senpy in your research, please cite `Senpy: A Pragmatic Linked Sentiment Analysis Framework <http://gsi.dit.upm.es/index.php/es/investigacion/publicaciones?view=publication&task=show&id=417>`__ (`BibTex <http://gsi.dit.upm.es/index.php/es/investigacion/publicaciones?controller=publications&task=export&format=bibtex&id=417>`__):
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
Sánchez-Rada, J. F., Iglesias, C. A., Corcuera, I., & Araque, Ó. (2016, October).
|
||||
Senpy: A Pragmatic Linked Sentiment Analysis Framework.
|
||||
In Data Science and Advanced Analytics (DSAA),
|
||||
2016 IEEE International Conference on (pp. 735-742). IEEE.
|
||||
237
docs/api.rst
237
docs/api.rst
@@ -1,5 +1,5 @@
|
||||
NIF API
|
||||
-------
|
||||
=======
|
||||
.. http:get:: /api
|
||||
|
||||
Basic endpoint for sentiment/emotion analysis.
|
||||
@@ -22,32 +22,38 @@ NIF API
|
||||
Content-Type: text/javascript
|
||||
|
||||
{
|
||||
"@context":"http://127.0.0.1/api/contexts/Results.jsonld",
|
||||
"@id":"_:Results_11241245.22",
|
||||
"@type":"results"
|
||||
"analysis": [
|
||||
"plugins/sentiment-140_0.1"
|
||||
],
|
||||
"entries": [
|
||||
{
|
||||
"@id": "_:Entry_11241245.22"
|
||||
"@type":"entry",
|
||||
"emotions": [],
|
||||
"entities": [],
|
||||
"sentiments": [
|
||||
{
|
||||
"@id": "Sentiment0",
|
||||
"@type": "sentiment",
|
||||
"marl:hasPolarity": "marl:Negative",
|
||||
"marl:polarityValue": 0,
|
||||
"prefix": ""
|
||||
}
|
||||
],
|
||||
"suggestions": [],
|
||||
"text": "This text makes me sad.\nwhilst this text makes me happy and surprised at the same time.\nI cannot believe it!",
|
||||
"topics": []
|
||||
}
|
||||
]
|
||||
"@context": [
|
||||
"http://127.0.0.1/static/context.jsonld",
|
||||
],
|
||||
"analysis": [
|
||||
{
|
||||
"@id": "SentimentAnalysisExample",
|
||||
"@type": "marl:SentimentAnalysis",
|
||||
"dc:language": "en",
|
||||
"marl:maxPolarityValue": 10.0,
|
||||
"marl:minPolarityValue": 0.0
|
||||
}
|
||||
],
|
||||
"domain": "wndomains:electronics",
|
||||
"entries": [
|
||||
{
|
||||
"opinions": [
|
||||
{
|
||||
"prov:generatedBy": "SentimentAnalysisExample",
|
||||
"marl:polarityValue": 7.8,
|
||||
"marl:hasPolarity": "marl:Positive",
|
||||
"marl:describesObject": "http://www.gsi.dit.upm.es",
|
||||
}
|
||||
],
|
||||
"nif:isString": "I love GSI",
|
||||
"strings": [
|
||||
{
|
||||
"nif:anchorOf": "GSI",
|
||||
"nif:taIdentRef": "http://www.gsi.dit.upm.es"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
:query i input: No default. Depends on informat and intype
|
||||
@@ -86,59 +92,58 @@ NIF API
|
||||
|
||||
.. sourcecode:: http
|
||||
|
||||
{
|
||||
"@id": "plugins/sentiment-140_0.1",
|
||||
"@type": "sentimentPlugin",
|
||||
"author": "@balkian",
|
||||
"description": "Sentiment classifier using rule-based classification for English and Spanish. This plugin uses sentiment140 data to perform classification. For more information: http://help.sentiment140.com/for-students/",
|
||||
"extra_params": {
|
||||
"language": {
|
||||
"@id": "lang_sentiment140",
|
||||
"aliases": [
|
||||
"language",
|
||||
"l"
|
||||
],
|
||||
"options": [
|
||||
"es",
|
||||
"en",
|
||||
"auto"
|
||||
],
|
||||
"required": false
|
||||
}
|
||||
},
|
||||
"is_activated": true,
|
||||
"maxPolarityValue": 1.0,
|
||||
"minPolarityValue": 0.0,
|
||||
"module": "sentiment-140",
|
||||
"name": "sentiment-140",
|
||||
"requirements": {},
|
||||
"version": "0.1"
|
||||
},
|
||||
{
|
||||
"@id": "plugins/ExamplePlugin_0.1",
|
||||
"@type": "sentimentPlugin",
|
||||
"author": "@balkian",
|
||||
"custom_attribute": "42",
|
||||
"description": "I am just an example",
|
||||
"extra_params": {
|
||||
"parameter": {
|
||||
"@id": "parameter",
|
||||
"aliases": [
|
||||
"parameter",
|
||||
"param"
|
||||
],
|
||||
"default": 42,
|
||||
"required": true
|
||||
}
|
||||
},
|
||||
"is_activated": true,
|
||||
"maxPolarityValue": 1.0,
|
||||
"minPolarityValue": 0.0,
|
||||
"module": "example",
|
||||
"name": "ExamplePlugin",
|
||||
"requirements": "noop",
|
||||
"version": "0.1"
|
||||
}
|
||||
{
|
||||
"@context": {
|
||||
...
|
||||
},
|
||||
"@type": "plugins",
|
||||
"plugins": [
|
||||
{
|
||||
"name": "sentiment140",
|
||||
"is_activated": true,
|
||||
"version": "0.1",
|
||||
"extra_params": {
|
||||
"@id": "extra_params_sentiment140_0.1",
|
||||
"language": {
|
||||
"required": false,
|
||||
"@id": "lang_sentiment140",
|
||||
"options": [
|
||||
"es",
|
||||
"en",
|
||||
"auto"
|
||||
],
|
||||
"aliases": [
|
||||
"language",
|
||||
"l"
|
||||
]
|
||||
}
|
||||
},
|
||||
"@id": "sentiment140_0.1"
|
||||
}, {
|
||||
"name": "rand",
|
||||
"is_activated": true,
|
||||
"version": "0.1",
|
||||
"extra_params": {
|
||||
"@id": "extra_params_rand_0.1",
|
||||
"language": {
|
||||
"required": false,
|
||||
"@id": "lang_rand",
|
||||
"options": [
|
||||
"es",
|
||||
"en",
|
||||
"auto"
|
||||
],
|
||||
"aliases": [
|
||||
"language",
|
||||
"l"
|
||||
]
|
||||
}
|
||||
},
|
||||
"@id": "rand_0.1"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
.. http:get:: /api/plugins/<pluginname>
|
||||
|
||||
@@ -157,60 +162,30 @@ NIF API
|
||||
.. sourcecode:: http
|
||||
|
||||
{
|
||||
"@context": "http://127.0.0.1/api/contexts/ExamplePlugin.jsonld",
|
||||
"@id": "plugins/ExamplePlugin_0.1",
|
||||
"@type": "sentimentPlugin",
|
||||
"author": "@balkian",
|
||||
"custom_attribute": "42",
|
||||
"description": "I am just an example",
|
||||
"extra_params": {
|
||||
"parameter": {
|
||||
"@id": "parameter",
|
||||
"aliases": [
|
||||
"parameter",
|
||||
"param"
|
||||
],
|
||||
"default": 42,
|
||||
"required": true
|
||||
}
|
||||
},
|
||||
"is_activated": true,
|
||||
"maxPolarityValue": 1.0,
|
||||
"minPolarityValue": 0.0,
|
||||
"module": "example",
|
||||
"name": "ExamplePlugin",
|
||||
"requirements": "noop",
|
||||
"version": "0.1"
|
||||
"@id": "rand_0.1",
|
||||
"@type": "sentimentPlugin",
|
||||
"extra_params": {
|
||||
"@id": "extra_params_rand_0.1",
|
||||
"language": {
|
||||
"@id": "lang_rand",
|
||||
"aliases": [
|
||||
"language",
|
||||
"l"
|
||||
],
|
||||
"options": [
|
||||
"es",
|
||||
"en",
|
||||
"auto"
|
||||
],
|
||||
"required": false
|
||||
}
|
||||
},
|
||||
"is_activated": true,
|
||||
"name": "rand",
|
||||
"version": "0.1"
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
.. http:get:: /api/plugins/default
|
||||
|
||||
Return the information about the default plugin.
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
API and Examples
|
||||
################
|
||||
.. toctree::
|
||||
|
||||
vocabularies.rst
|
||||
api.rst
|
||||
examples.rst
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
"@type": "plugins",
|
||||
"plugins": {}
|
||||
"plugins": [
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
{
|
||||
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
|
||||
"@id": "me:Result1",
|
||||
"@type": "results",
|
||||
"analysis": [
|
||||
"me:SAnalysis1",
|
||||
"me:SgAnalysis1",
|
||||
"me:EmotionAnalysis1",
|
||||
"me:NER1",
|
||||
{
|
||||
"description": "missing @id and @type"
|
||||
}
|
||||
],
|
||||
"entries": [
|
||||
{
|
||||
"@id": "http://micro.blog/status1",
|
||||
"@type": [
|
||||
"nif:RFC5147String",
|
||||
"nif:Context"
|
||||
],
|
||||
"nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
|
||||
"entities": [
|
||||
{
|
||||
"@id": "http://micro.blog/status1#char=5,13",
|
||||
"nif:beginIndex": 5,
|
||||
"nif:endIndex": 13,
|
||||
"nif:anchorOf": "Microsoft",
|
||||
"me:references": "http://dbpedia.org/page/Microsoft",
|
||||
"prov:wasGeneratedBy": "me:NER1"
|
||||
},
|
||||
{
|
||||
"@id": "http://micro.blog/status1#char=25,37",
|
||||
"nif:beginIndex": 25,
|
||||
"nif:endIndex": 37,
|
||||
"nif:anchorOf": "Windows Phone",
|
||||
"me:references": "http://dbpedia.org/page/Windows_Phone",
|
||||
"prov:wasGeneratedBy": "me:NER1"
|
||||
}
|
||||
],
|
||||
"suggestions": [
|
||||
{
|
||||
"@id": "http://micro.blog/status1#char=16,77",
|
||||
"nif:beginIndex": 16,
|
||||
"nif:endIndex": 77,
|
||||
"nif:anchorOf": "put your Windows Phone on your newest #open technology program",
|
||||
"prov:wasGeneratedBy": "me:SgAnalysis1"
|
||||
}
|
||||
],
|
||||
"sentiments": [
|
||||
{
|
||||
"@id": "http://micro.blog/status1#char=80,97",
|
||||
"nif:beginIndex": 80,
|
||||
"nif:endIndex": 97,
|
||||
"nif:anchorOf": "You'll be awesome.",
|
||||
"marl:hasPolarity": "marl:Positive",
|
||||
"marl:polarityValue": 0.9,
|
||||
"prov:wasGeneratedBy": "me:SAnalysis1"
|
||||
}
|
||||
],
|
||||
"emotions": [
|
||||
{
|
||||
"@id": "http://micro.blog/status1#char=0,109",
|
||||
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
|
||||
"prov:wasGeneratedBy": "me:EAnalysis1",
|
||||
"onyx:hasEmotion": [
|
||||
{
|
||||
"onyx:hasEmotionCategory": "wna:liking"
|
||||
},
|
||||
{
|
||||
"onyx:hasEmotionCategory": "wna:excitement"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
Command line
|
||||
============
|
||||
|
||||
This video shows how to analyse text directly on the command line using the senpy tool.
|
||||
|
||||
.. image:: https://asciinema.org/a/9uwef1ghkjk062cw2t4mhzpyk.png
|
||||
:width: 100%
|
||||
:target: https://asciinema.org/a/9uwef1ghkjk062cw2t4mhzpyk
|
||||
:alt: CLI demo
|
||||
38
docs/conf.py
38
docs/conf.py
@@ -37,7 +37,6 @@ extensions = [
|
||||
'sphinx.ext.todo',
|
||||
'sphinxcontrib.httpdomain',
|
||||
'sphinx.ext.coverage',
|
||||
'sphinx.ext.autosectionlabel',
|
||||
]
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
@@ -55,21 +54,20 @@ master_doc = 'index'
|
||||
# General information about the project.
|
||||
project = u'Senpy'
|
||||
copyright = u'2016, J. Fernando Sánchez'
|
||||
description = u'A framework for sentiment and emotion analysis services'
|
||||
|
||||
# The version info for the project you're documenting, acts as replacement for
|
||||
# |version| and |release|, also used in various other places throughout the
|
||||
# built documents.
|
||||
#
|
||||
# The short X.Y version.
|
||||
# with open('../senpy/VERSION') as f:
|
||||
# version = f.read().strip()
|
||||
with open('../senpy/VERSION') as f:
|
||||
version = f.read().strip()
|
||||
# The full version, including alpha/beta/rc tags.
|
||||
# release = version
|
||||
release = version
|
||||
|
||||
# The language for content autogenerated by Sphinx. Refer to documentation
|
||||
# for a list of supported languages.
|
||||
language = None
|
||||
#language = None
|
||||
|
||||
# There are two options for replacing |today|: either, you set today to some
|
||||
# non-false value, then it is used:
|
||||
@@ -106,14 +104,14 @@ pygments_style = 'sphinx'
|
||||
#keep_warnings = False
|
||||
|
||||
|
||||
html_theme = 'alabaster'
|
||||
# -- Options for HTML output ----------------------------------------------
|
||||
# if not on_rtd: # only import and set the theme if we're building docs locally
|
||||
# import sphinx_rtd_theme
|
||||
# html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
|
||||
if not on_rtd: # only import and set the theme if we're building docs locally
|
||||
import sphinx_rtd_theme
|
||||
html_theme = 'sphinx_rtd_theme'
|
||||
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
|
||||
|
||||
# else:
|
||||
# html_theme = 'default'
|
||||
else:
|
||||
html_theme = 'default'
|
||||
|
||||
# The theme to use for HTML and HTML Help pages. See the documentation for
|
||||
# a list of builtin themes.
|
||||
@@ -121,13 +119,7 @@ html_theme = 'alabaster'
|
||||
# Theme options are theme-specific and customize the look and feel of a theme
|
||||
# further. For a list of options available for each theme, see the
|
||||
# documentation.
|
||||
html_theme_options = {
|
||||
'logo': 'header.png',
|
||||
'github_user': 'gsi-upm',
|
||||
'github_repo': 'senpy',
|
||||
'github_banner': True,
|
||||
}
|
||||
|
||||
#html_theme_options = {}
|
||||
|
||||
# Add any paths that contain custom themes here, relative to this directory.
|
||||
#html_theme_path = []
|
||||
@@ -167,13 +159,7 @@ html_static_path = ['_static']
|
||||
#html_use_smartypants = True
|
||||
|
||||
# Custom sidebar templates, maps document names to template names.
|
||||
html_sidebars = {
|
||||
'**': [
|
||||
'about.html',
|
||||
'navigation.html',
|
||||
'searchbox.html',
|
||||
]
|
||||
}
|
||||
#html_sidebars = {}
|
||||
|
||||
# Additional templates that should be rendered to pages, maps page names to
|
||||
# template names.
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
Demo
|
||||
----
|
||||
|
||||
There is a demo available on http://senpy.cluster.gsi.dit.upm.es/, where you can test a serie of different plugins.
|
||||
You can use the playground (a web interface) or make HTTP requests to the service API.
|
||||
There is a demo available on http://senpy.demos.gsi.dit.upm.es/, where you can a serie of different plugins. You can use them in the playground or make a directly requests to the service.
|
||||
|
||||
.. image:: senpy-playground.png
|
||||
:height: 400px
|
||||
@@ -13,4 +12,64 @@ You can use the playground (a web interface) or make HTTP requests to the servic
|
||||
Plugins Demo
|
||||
============
|
||||
|
||||
The source code and description of the plugins used in the demo is available here: https://lab.cluster.gsi.dit.upm.es/senpy/senpy-plugins-community/.
|
||||
The next plugins are available at the demo:
|
||||
|
||||
* emoTextAnew extracts the VAD (valence-arousal-dominance) of a sentence by matching words from the ANEW dictionary.
|
||||
* emoTextWordnetAffect based on the hierarchy of WordnetAffect to calculate the emotion of the sentence.
|
||||
* vaderSentiment utilizes the software from vaderSentiment to calculate the sentiment of a sentence.
|
||||
* sentiText is a software developed during the TASS 2015 competition, it has been adapted for English and Spanish.
|
||||
|
||||
emoTextANEW plugin
|
||||
******************
|
||||
|
||||
This plugin is going to used the ANEW lexicon dictionary to calculate de VAD (valence-arousal-dominance) of the sentence and the determinate which emotion is closer to this value.
|
||||
|
||||
Each emotion has a centroid, which it has been approximated using the formula described in this article:
|
||||
|
||||
http://www.aclweb.org/anthology/W10-0208
|
||||
|
||||
The plugin is going to look for the words in the sentence that appear in the ANEW dictionary and calculate the average VAD score for the sentence. Once this score is calculated, it is going to seek the emotion that is closest to this value.
|
||||
|
||||
emoTextWAF plugin
|
||||
*****************
|
||||
|
||||
This plugin uses WordNet-Affect (http://wndomains.fbk.eu/wnaffect.html) to calculate the percentage of each emotion. The emotions that are going to be used are: anger, fear, disgust, joy and sadness. It is has been used a emotion mapping enlarge the emotions:
|
||||
|
||||
* anger : general-dislike
|
||||
* fear : negative-fear
|
||||
* disgust : shame
|
||||
* joy : gratitude, affective, enthusiasm, love, joy, liking
|
||||
* sadness : ingrattitude, daze, humlity, compassion, despair, anxiety, sadness
|
||||
|
||||
sentiText plugin
|
||||
****************
|
||||
|
||||
This plugin is based in the classifier developed for the TASS 2015 competition. It has been developed for Spanish and English. The different phases that has this plugin when it is activated:
|
||||
|
||||
* Train both classifiers (English and Spanish).
|
||||
* Initialize resources (dictionaries,stopwords,etc.).
|
||||
* Extract bag of words,lemmas and chars.
|
||||
|
||||
Once the plugin is activated, the features that are going to be extracted for the classifiers are:
|
||||
|
||||
* Matches with the bag of words extracted from the train corpus.
|
||||
* Sentiment score of the sentences extracted from the dictionaries (lexicons and emoticons).
|
||||
* Identify negations and intensifiers in the sentences.
|
||||
* Complementary features such as exclamation and interrogation marks, eloganted and caps words, hashtags, etc.
|
||||
|
||||
The plugin has a preprocessor, which is focues on Twitter corpora, that is going to be used for cleaning the text to simplify the feature extraction.
|
||||
|
||||
There is more information avaliable in the next article.
|
||||
|
||||
Aspect based Sentiment Analysis of Spanish Tweets, Oscar Araque and Ignacio Corcuera-Platas and Constantino Román-Gómez and Carlos A. Iglesias and J. Fernando Sánchez-Rada. http://gsi.dit.upm.es/es/investigacion/publicaciones?view=publication&task=show&id=37
|
||||
|
||||
vaderSentiment plugin
|
||||
*********************
|
||||
|
||||
For developing this plugin, it has been used the module vaderSentiment, which is described in the paper: VADER: A Parsimonious Rule-based Model for Sentiment Analysis of Social Media Text C.J. Hutto and Eric Gilbert Eighth International Conference on Weblogs and Social Media (ICWSM-14). Ann Arbor, MI, June 2014.
|
||||
|
||||
If you use this plugin in your research, please cite the above paper
|
||||
|
||||
For more information about the functionality, check the official repository
|
||||
|
||||
https://github.com/cjhutto/vaderSentiment
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
Examples
|
||||
------
|
||||
All the examples in this page use the :download:`the main schema <_static/schemas/definitions.json>`.
|
||||
|
||||
Simple NIF annotation
|
||||
.....................
|
||||
Description
|
||||
,,,,,,,,,,,
|
||||
This example covers the basic example in the NIF documentation: `<http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core/nif-core.html>`_.
|
||||
|
||||
Representation
|
||||
,,,,,,,,,,,,,,
|
||||
.. literalinclude:: examples/results/example-basic.json
|
||||
:language: json-ld
|
||||
|
||||
Sentiment Analysis
|
||||
.....................
|
||||
Description
|
||||
,,,,,,,,,,,
|
||||
This annotation corresponds to the sentiment analysis of an input. The example shows the sentiment represented according to Marl format.
|
||||
The sentiments detected are contained in the Sentiments array with their related part of the text.
|
||||
|
||||
Representation
|
||||
,,,,,,,,,,,,,,
|
||||
|
||||
.. literalinclude:: examples/results/example-sentiment.json
|
||||
:emphasize-lines: 5-10,25-33
|
||||
:language: json-ld
|
||||
|
||||
Suggestion Mining
|
||||
.................
|
||||
Description
|
||||
,,,,,,,,,,,
|
||||
The suggestions schema represented below shows the suggestions detected in the text. Within it, we can find the NIF fields highlighted that corresponds to the text of the detected suggestion.
|
||||
|
||||
Representation
|
||||
,,,,,,,,,,,,,,
|
||||
|
||||
.. literalinclude:: examples/results/example-suggestion.json
|
||||
:emphasize-lines: 5-8,22-27
|
||||
:language: json-ld
|
||||
|
||||
Emotion Analysis
|
||||
................
|
||||
Description
|
||||
,,,,,,,,,,,
|
||||
This annotation represents the emotion analysis of an input to Senpy. The emotions are contained in the emotions section with the text that refers to following Onyx format and the emotion model defined beforehand.
|
||||
|
||||
Representation
|
||||
,,,,,,,,,,,,,,
|
||||
|
||||
.. literalinclude:: examples/results/example-emotion.json
|
||||
:language: json-ld
|
||||
:emphasize-lines: 5-8,25-37
|
||||
|
||||
Named Entity Recognition
|
||||
........................
|
||||
Description
|
||||
,,,,,,,,,,,
|
||||
The Named Entity Recognition is represented as follows. In this particular case, it can be seen within the entities array the entities recognised. For the example input, Microsoft and Windows Phone are the ones detected.
|
||||
Representation
|
||||
,,,,,,,,,,,,,,
|
||||
|
||||
.. literalinclude:: examples/results/example-ner.json
|
||||
:emphasize-lines: 5-8,19-34
|
||||
:language: json-ld
|
||||
|
||||
Complete example
|
||||
................
|
||||
Description
|
||||
,,,,,,,,,,,
|
||||
This example covers all of the above cases, integrating all the annotations in the same document.
|
||||
|
||||
Representation
|
||||
,,,,,,,,,,,,,,
|
||||
|
||||
.. literalinclude:: examples/results/example-complete.json
|
||||
:language: json-ld
|
||||
@@ -1,74 +0,0 @@
|
||||
{
|
||||
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
|
||||
"@id": "me:Result1",
|
||||
"@type": "results",
|
||||
"analysis": [
|
||||
"me:SAnalysis1",
|
||||
"me:SgAnalysis1",
|
||||
"me:EmotionAnalysis1",
|
||||
"me:NER1"
|
||||
],
|
||||
"entries": [
|
||||
{
|
||||
"@id": "http://micro.blog/status1",
|
||||
"@type": [
|
||||
"nif:RFC5147String",
|
||||
"nif:Context"
|
||||
],
|
||||
"nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
|
||||
"entities": [
|
||||
{
|
||||
"@id": "http://micro.blog/status1#char=5,13",
|
||||
"nif:beginIndex": 5,
|
||||
"nif:endIndex": 13,
|
||||
"nif:anchorOf": "Microsoft",
|
||||
"me:references": "http://dbpedia.org/page/Microsoft",
|
||||
"prov:wasGeneratedBy": "me:NER1"
|
||||
},
|
||||
{
|
||||
"@id": "http://micro.blog/status1#char=25,37",
|
||||
"nif:beginIndex": 25,
|
||||
"nif:endIndex": 37,
|
||||
"nif:anchorOf": "Windows Phone",
|
||||
"me:references": "http://dbpedia.org/page/Windows_Phone",
|
||||
"prov:wasGeneratedBy": "me:NER1"
|
||||
}
|
||||
],
|
||||
"suggestions": [
|
||||
{
|
||||
"@id": "http://micro.blog/status1#char=16,77",
|
||||
"nif:beginIndex": 16,
|
||||
"nif:endIndex": 77,
|
||||
"nif:anchorOf": "put your Windows Phone on your newest #open technology program",
|
||||
"prov:wasGeneratedBy": "me:SgAnalysis1"
|
||||
}
|
||||
],
|
||||
"sentiments": [
|
||||
{
|
||||
"@id": "http://micro.blog/status1#char=80,97",
|
||||
"nif:beginIndex": 80,
|
||||
"nif:endIndex": 97,
|
||||
"nif:anchorOf": "You'll be awesome.",
|
||||
"marl:hasPolarity": "marl:Positive",
|
||||
"marl:polarityValue": 0.9,
|
||||
"prov:wasGeneratedBy": "me:SAnalysis1"
|
||||
}
|
||||
],
|
||||
"emotions": [
|
||||
{
|
||||
"@id": "http://micro.blog/status1#char=0,109",
|
||||
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
|
||||
"prov:wasGeneratedBy": "me:EAnalysis1",
|
||||
"onyx:hasEmotion": [
|
||||
{
|
||||
"onyx:hasEmotionCategory": "wna:liking"
|
||||
},
|
||||
{
|
||||
"onyx:hasEmotionCategory": "wna:excitement"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
{
|
||||
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
|
||||
"@id": "me:Result1",
|
||||
"@type": "results",
|
||||
"analysis": [
|
||||
"me:SAnalysis1",
|
||||
"me:SgAnalysis1",
|
||||
"me:EmotionAnalysis1",
|
||||
"me:NER1",
|
||||
{
|
||||
"@type": "analysis",
|
||||
"@id": "anonymous"
|
||||
}
|
||||
],
|
||||
"entries": [
|
||||
{
|
||||
"@id": "http://micro.blog/status1",
|
||||
"@type": [
|
||||
"nif:RFC5147String",
|
||||
"nif:Context"
|
||||
],
|
||||
"nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
|
||||
"entities": [
|
||||
{
|
||||
"@id": "http://micro.blog/status1#char=5,13",
|
||||
"nif:beginIndex": 5,
|
||||
"nif:endIndex": 13,
|
||||
"nif:anchorOf": "Microsoft",
|
||||
"me:references": "http://dbpedia.org/page/Microsoft",
|
||||
"prov:wasGeneratedBy": "me:NER1"
|
||||
},
|
||||
{
|
||||
"@id": "http://micro.blog/status1#char=25,37",
|
||||
"nif:beginIndex": 25,
|
||||
"nif:endIndex": 37,
|
||||
"nif:anchorOf": "Windows Phone",
|
||||
"me:references": "http://dbpedia.org/page/Windows_Phone",
|
||||
"prov:wasGeneratedBy": "me:NER1"
|
||||
}
|
||||
],
|
||||
"suggestions": [
|
||||
{
|
||||
"@id": "http://micro.blog/status1#char=16,77",
|
||||
"nif:beginIndex": 16,
|
||||
"nif:endIndex": 77,
|
||||
"nif:anchorOf": "put your Windows Phone on your newest #open technology program",
|
||||
"prov:wasGeneratedBy": "me:SgAnalysis1"
|
||||
}
|
||||
],
|
||||
"sentiments": [
|
||||
{
|
||||
"@id": "http://micro.blog/status1#char=80,97",
|
||||
"nif:beginIndex": 80,
|
||||
"nif:endIndex": 97,
|
||||
"nif:anchorOf": "You'll be awesome.",
|
||||
"marl:hasPolarity": "marl:Positive",
|
||||
"marl:polarityValue": 0.9,
|
||||
"prov:wasGeneratedBy": "me:SAnalysis1"
|
||||
}
|
||||
],
|
||||
"emotions": [
|
||||
{
|
||||
"@id": "http://micro.blog/status1#char=0,109",
|
||||
"nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
|
||||
"prov:wasGeneratedBy": "me:EAnalysis1",
|
||||
"onyx:hasEmotion": [
|
||||
{
|
||||
"onyx:hasEmotionCategory": "wna:liking"
|
||||
},
|
||||
{
|
||||
"onyx:hasEmotionCategory": "wna:excitement"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -3,7 +3,10 @@
|
||||
"@id": "me:Result1",
|
||||
"@type": "results",
|
||||
"analysis": [
|
||||
"me:SgAnalysis1"
|
||||
{
|
||||
"@id": "me:SgAnalysis1",
|
||||
"@type": "me:SuggestionAnalysis"
|
||||
}
|
||||
],
|
||||
"entries": [
|
||||
{
|
||||
|
||||
@@ -1,35 +1,15 @@
|
||||
Welcome to Senpy's documentation!
|
||||
=================================
|
||||
.. image:: https://readthedocs.org/projects/senpy/badge/?version=latest
|
||||
:target: http://senpy.readthedocs.io/en/latest/
|
||||
.. image:: https://badge.fury.io/py/senpy.svg
|
||||
:target: https://badge.fury.io/py/senpy
|
||||
.. image:: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/badges/master/build.svg
|
||||
:target: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/commits/master
|
||||
.. image:: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/badges/master/coverage.svg
|
||||
:target: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/commits/master
|
||||
.. image:: https://img.shields.io/pypi/l/requests.svg
|
||||
:target: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/
|
||||
|
||||
|
||||
|
||||
Senpy is a framework for sentiment and emotion analysis services.
|
||||
Services built with senpy are interchangeable and easy to use because they share a common :doc:`apischema`.
|
||||
It also simplifies service development.
|
||||
|
||||
.. image:: senpy-architecture.png
|
||||
:width: 100%
|
||||
:align: center
|
||||
Contents:
|
||||
|
||||
.. toctree::
|
||||
:caption: Learn more about senpy:
|
||||
:maxdepth: 2
|
||||
|
||||
senpy
|
||||
installation
|
||||
demo
|
||||
usage
|
||||
apischema
|
||||
api
|
||||
schema
|
||||
plugins
|
||||
conversion
|
||||
about
|
||||
demo
|
||||
:maxdepth: 2
|
||||
|
||||
@@ -1,16 +1,6 @@
|
||||
Installation
|
||||
------------
|
||||
The stable version can be used in two ways: as a system/user library through pip, or as a docker image.
|
||||
|
||||
The docker image is the recommended way because it is self-contained and isolated from the system, which means:
|
||||
|
||||
* Downloading and using it is just one command
|
||||
* All dependencies are included
|
||||
* It is OS-independent (MacOS, Windows, GNU/Linux)
|
||||
* Several versions may coexist in the same machine without additional virtual environments
|
||||
|
||||
Additionally, you may create your own docker image with your custom plugins, ready to be used by others.
|
||||
|
||||
The stable version can be installed in three ways.
|
||||
|
||||
Through PIP
|
||||
***********
|
||||
@@ -32,41 +22,6 @@ If you want to install senpy globally, use sudo instead of the ``--user`` flag.
|
||||
|
||||
Docker Image
|
||||
************
|
||||
Build the image or use the pre-built one:
|
||||
Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy --host 0.0.0.0 --default-plugins``.
|
||||
|
||||
.. code:: bash
|
||||
|
||||
docker run -ti -p 5000:5000 gsiupm/senpy --host 0.0.0.0 --default-plugins
|
||||
|
||||
To add custom plugins, use a docker volume:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --host 0.0.0.0 --default-plugins -f /plugins
|
||||
|
||||
|
||||
Python 2
|
||||
........
|
||||
|
||||
There is a Senpy version for python2 too:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
docker run -ti -p 5000:5000 gsiupm/senpy:python2.7 --host 0.0.0.0 --default-plugins
|
||||
|
||||
|
||||
Alias
|
||||
.....
|
||||
|
||||
If you are using the docker approach regularly, it is advisable to use a script or an alias to simplify your executions:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
alias senpy='docker run --rm -ti -p 5000:5000 -v $PWD:/senpy-plugins gsiupm/senpy --default-plugins'
|
||||
|
||||
|
||||
Now, you may run senpy from any folder in your computer like so:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
senpy --version
|
||||
To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --host 0.0.0.0 --default-plugins -f /plugins``
|
||||
|
||||
197
docs/plugins.rst
197
docs/plugins.rst
@@ -2,36 +2,27 @@ Developing new plugins
|
||||
----------------------
|
||||
This document describes how to develop a new analysis plugin. For an example of conversion plugins, see :doc:`conversion`.
|
||||
|
||||
A more step-by-step tutorial with slides is available `here <https://lab.cluster.gsi.dit.upm.es/senpy/senpy-tutorial>`__
|
||||
Each plugin represents a different analysis process.There are two types of files that are needed by senpy for loading a plugin:
|
||||
|
||||
.. contents:: :local:
|
||||
- Definition file, has the ".senpy" extension.
|
||||
- Code file, is a python file.
|
||||
|
||||
What is a plugin?
|
||||
=================
|
||||
|
||||
A plugin is a program that, given a text, will add annotations to it.
|
||||
In practice, a plugin consists of at least two files:
|
||||
|
||||
- Definition file: a `.senpy` file that describes the plugin (e.g. what input parameters it accepts, what emotion model it uses).
|
||||
- Python module: the actual code that will add annotations to each input.
|
||||
|
||||
This separation allows us to deploy plugins that use the same code but employ different parameters.
|
||||
This separation will allow us to deploy plugins that use the same code but employ different parameters.
|
||||
For instance, one could use the same classifier and processing in several plugins, but train with different datasets.
|
||||
This scenario is particularly useful for evaluation purposes.
|
||||
|
||||
The only limitation is that the name of each plugin needs to be unique.
|
||||
|
||||
Plugin Definition files
|
||||
=======================
|
||||
Plugins Definitions
|
||||
===================
|
||||
|
||||
The definition file contains all the attributes of the plugin, and can be written in YAML or JSON.
|
||||
When the server is launched, it will recursively search for definition files in the plugin folder (the current folder, by default).
|
||||
The most important attributes are:
|
||||
|
||||
* **name**: unique name that senpy will use internally to identify the plugin.
|
||||
* **module**: indicates the module that contains the plugin code, which will be automatically loaded by senpy.
|
||||
* **version**
|
||||
* extra_params: to add parameters to the senpy API when this plugin is requested. Those parameters may be required, and have aliased names. For instance:
|
||||
* extra_params: used to specify parameters that the plugin accepts that are not already part of the senpy API. Those parameters may be required, and have aliased names. For instance:
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
@@ -77,28 +68,10 @@ The basic methods in a plugin are:
|
||||
* __init__
|
||||
* activate: used to load memory-hungry resources
|
||||
* deactivate: used to free up resources
|
||||
* analyse_entry: called in every user requests. It takes two parameters: ``Entry``, the entry object, and ``params``, the parameters supplied by the user. It should yield one or more ``Entry`` objects.
|
||||
* analyse_entry: called in every user requests. It takes in the parameters supplied by a user and should yield one or more ``Entry`` objects.
|
||||
|
||||
Plugins are loaded asynchronously, so don't worry if the activate method takes too long. The plugin will be marked as activated once it is finished executing the method.
|
||||
|
||||
Entries
|
||||
=======
|
||||
|
||||
Entries are objects that can be annotated.
|
||||
By default, entries are `NIF contexts <http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core/nif-core.html>`_ represented in JSON-LD format.
|
||||
Annotations are added to the object like this:
|
||||
|
||||
.. code:: python
|
||||
|
||||
entry = Entry()
|
||||
entry.vocabulary__annotationName = 'myvalue'
|
||||
entry['vocabulary:annotationName'] = 'myvalue'
|
||||
entry['annotationNameURI'] = 'myvalue'
|
||||
|
||||
Where vocabulary is one of the prefixes defined in the default senpy context, and annotationURI is a full URI.
|
||||
The value may be any valid JSON-LD dictionary.
|
||||
For simplicity, senpy includes a series of models by default in the ``senpy.models`` module.
|
||||
|
||||
|
||||
Example plugin
|
||||
==============
|
||||
@@ -115,7 +88,7 @@ The definition file would look like this:
|
||||
module: helloworld
|
||||
version: 0.0
|
||||
threshold: 10
|
||||
description: Hello World
|
||||
|
||||
|
||||
Now, in a file named ``helloworld.py``:
|
||||
|
||||
@@ -124,11 +97,11 @@ Now, in a file named ``helloworld.py``:
|
||||
#!/bin/env python
|
||||
#helloworld.py
|
||||
|
||||
from senpy.plugins import AnalysisPlugin
|
||||
from senpy.plugins import SenpyPlugin
|
||||
from senpy.models import Sentiment
|
||||
|
||||
|
||||
class HelloWorld(AnalysisPlugin):
|
||||
class HelloWorld(SenpyPlugin):
|
||||
|
||||
def analyse_entry(entry, params):
|
||||
'''Basically do nothing with each entry'''
|
||||
@@ -141,112 +114,15 @@ Now, in a file named ``helloworld.py``:
|
||||
entry.sentiments.append(sentiment)
|
||||
yield entry
|
||||
|
||||
The complete code of the example plugin is available `here <https://lab.cluster.gsi.dit.upm.es/senpy/plugin-prueba>`__.
|
||||
|
||||
Loading data and files
|
||||
======================
|
||||
|
||||
Most plugins will need access to files (dictionaries, lexicons, etc.).
|
||||
It is good practice to specify the paths of these files in the plugin configuration, so the same code can be reused with different resources.
|
||||
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
name: dictworld
|
||||
module: dictworld
|
||||
dictionary_path: <PATH OF THE FILE>
|
||||
|
||||
The path can be either absolute, or relative.
|
||||
|
||||
From absolute paths
|
||||
???????????????????
|
||||
|
||||
Absolute paths (such as ``/data/dictionary.csv`` are straightfoward:
|
||||
|
||||
.. code:: python
|
||||
|
||||
with open(os.path.join(self.dictionary_path) as f:
|
||||
...
|
||||
|
||||
From relative paths
|
||||
???????????????????
|
||||
Since plugins are loading dynamically, relative paths will refer to the current working directory.
|
||||
Instead, what you usually want is to load files *relative to the plugin source folder*, like so:
|
||||
|
||||
|
||||
::
|
||||
|
||||
.
|
||||
..
|
||||
plugin.senpy
|
||||
plugin.py
|
||||
dictionary.csv
|
||||
|
||||
For this, we need to first get the path of your source folder first, like so:
|
||||
|
||||
.. code:: python
|
||||
|
||||
import os
|
||||
root = os.path.realpath(__file__)
|
||||
with open(os.path.join(root, self.dictionary_path) as f:
|
||||
...
|
||||
|
||||
|
||||
Docker image
|
||||
============
|
||||
|
||||
Add the following dockerfile to your project to generate a docker image with your plugin:
|
||||
|
||||
.. code:: dockerfile
|
||||
|
||||
FROM gsiupm/senpy:0.8.8
|
||||
|
||||
This will copy your source folder to the image, and install all dependencies.
|
||||
Now, to build an image:
|
||||
|
||||
.. code:: shell
|
||||
|
||||
docker build . -t gsiupm/exampleplugin
|
||||
|
||||
And you can run it with:
|
||||
|
||||
.. code:: shell
|
||||
|
||||
docker run -p 5000:5000 gsiupm/exampleplugin
|
||||
|
||||
|
||||
If the plugin non-source files (:ref:`loading data and files`), the recommended way is to use absolute paths.
|
||||
Data can then be mounted in the container or added to the image.
|
||||
The former is recommended for open source plugins with licensed resources, whereas the latter is the most convenient and can be used for private images.
|
||||
|
||||
Mounting data:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
docker run -v $PWD/data:/data gsiupm/exampleplugin
|
||||
|
||||
Adding data to the image:
|
||||
|
||||
.. code:: dockerfile
|
||||
|
||||
FROM gsiupm/senpy:0.8.8
|
||||
COPY data /
|
||||
|
||||
F.A.Q.
|
||||
======
|
||||
What annotations can I use?
|
||||
???????????????????????????
|
||||
|
||||
You can add almost any annotation to an entry.
|
||||
The most common use cases are covered in the :doc:`apischema`.
|
||||
|
||||
|
||||
Why does the analyse function yield instead of return?
|
||||
??????????????????????????????????????????????????????
|
||||
|
||||
This is so that plugins may add new entries to the response or filter some of them.
|
||||
For instance, a `context detection` plugin may add a new entry for each context in the original entry.
|
||||
On the other hand, a conversion plugin may leave out those entries that do not contain relevant information.
|
||||
On the other hand, a conveersion plugin may leave out those entries that do not contain relevant information.
|
||||
|
||||
|
||||
If I'm using a classifier, where should I train it?
|
||||
@@ -256,9 +132,9 @@ Training a classifier can be time time consuming. To avoid running the training
|
||||
|
||||
.. code:: python
|
||||
|
||||
from senpy.plugins import ShelfMixin, AnalysisPlugin
|
||||
from senpy.plugins import ShelfMixin, SenpyPlugin
|
||||
|
||||
class MyPlugin(ShelfMixin, AnalysisPlugin):
|
||||
class MyPlugin(ShelfMixin, SenpyPlugin):
|
||||
def train(self):
|
||||
''' Code to train the classifier
|
||||
'''
|
||||
@@ -275,16 +151,12 @@ Training a classifier can be time time consuming. To avoid running the training
|
||||
def deactivate(self):
|
||||
self.close()
|
||||
|
||||
You can specify a 'shelf_file' in your .senpy file. By default the ShelfMixin creates a file based on the plugin name and stores it in that plugin's folder.
|
||||
You can speficy a 'shelf_file' in your .senpy file. By default the ShelfMixin creates a file based on the plugin name and stores it in that plugin's folder.
|
||||
|
||||
Shelves may get corrupted if the plugin exists unexpectedly.
|
||||
A corrupt shelf prevents the plugin from loading.
|
||||
If you do not care about the pickle, you can force your plugin to remove the corrupted file and load anyway, set the 'force_shelf' to True in your .senpy file.
|
||||
I want to implement my service as a plugin, How i can do it?
|
||||
????????????????????????????????????????????????????????????
|
||||
|
||||
How can I turn an external service into a plugin?
|
||||
?????????????????????????????????????????????????
|
||||
|
||||
This example ilustrate how to implement a plugin that accesses the Sentiment140 service.
|
||||
This example ilustrate how to implement the Sentiment140 service as a plugin in senpy
|
||||
|
||||
.. code:: python
|
||||
|
||||
@@ -318,30 +190,26 @@ This example ilustrate how to implement a plugin that accesses the Sentiment140
|
||||
yield entry
|
||||
|
||||
|
||||
Can my plugin require additional parameters from the user?
|
||||
??????????????????????????????????????????????????????????
|
||||
Where can I define extra parameters to be introduced in the request to my plugin?
|
||||
?????????????????????????????????????????????????????????????????????????????????
|
||||
|
||||
You can add extra parameters in the definition file under the attribute ``extra_params``.
|
||||
It takes a dictionary, where the keys are the name of the argument/parameter, and the value has the following fields:
|
||||
You can add these parameters in the definition file under the attribute "extra_params" : "{param_name}". The name of the parameter has new attributes-value pairs. The basic attributes are:
|
||||
|
||||
* aliases: the different names which can be used in the request to use the parameter.
|
||||
* required: if set to true, users need to provide this parameter unless a default is set.
|
||||
* options: the different acceptable values of the parameter (i.e. an enum). If set, the value provided must match one of the options.
|
||||
* default: the default value of the parameter, if none is provided in the request.
|
||||
* required: this option is a boolean and indicates if the parameters is binding in operation plugin.
|
||||
* options: the different values of the paremeter.
|
||||
* default: the default value of the parameter, this is useful in case the paremeter is required and you want to have a default value.
|
||||
|
||||
.. code:: python
|
||||
|
||||
extra_params
|
||||
language:
|
||||
aliases:
|
||||
- language
|
||||
- lang
|
||||
- l
|
||||
required: true,
|
||||
options:
|
||||
- es
|
||||
- en
|
||||
default: es
|
||||
"extra_params": {
|
||||
"language": {
|
||||
"aliases": ["language", "l"],
|
||||
"required": true,
|
||||
"options": ["es","en"],
|
||||
"default": "es"
|
||||
}
|
||||
}
|
||||
|
||||
This example shows how to introduce a parameter associated with language.
|
||||
The extraction of this paremeter is used in the analyse method of the Plugin interface.
|
||||
@@ -373,6 +241,7 @@ Additionally, with the ``--pdb`` option you will be dropped into a pdb post mort
|
||||
|
||||
senpy --pdb
|
||||
|
||||
|
||||
Where can I find more code examples?
|
||||
????????????????????????????????????
|
||||
|
||||
|
||||
@@ -1,2 +1 @@
|
||||
sphinxcontrib-httpdomain>=1.4
|
||||
nbsphinx
|
||||
|
||||
74
docs/schema.rst
Normal file
74
docs/schema.rst
Normal file
@@ -0,0 +1,74 @@
|
||||
Schema Examples
|
||||
===============
|
||||
All the examples in this page use the :download:`the main schema <_static/schemas/definitions.json>`.
|
||||
|
||||
Simple NIF annotation
|
||||
---------------------
|
||||
Description
|
||||
...........
|
||||
This example covers the basic example in the NIF documentation: `<http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core/nif-core.html>`_.
|
||||
|
||||
Representation
|
||||
..............
|
||||
.. literalinclude:: examples/example-basic.json
|
||||
:language: json-ld
|
||||
|
||||
Sentiment Analysis
|
||||
---------------------
|
||||
Description
|
||||
...........
|
||||
|
||||
Representation
|
||||
..............
|
||||
|
||||
.. literalinclude:: examples/example-sentiment.json
|
||||
:emphasize-lines: 5-10,25-33
|
||||
:language: json-ld
|
||||
|
||||
Suggestion Mining
|
||||
-----------------
|
||||
Description
|
||||
...........
|
||||
|
||||
Representation
|
||||
..............
|
||||
|
||||
.. literalinclude:: examples/example-suggestion.json
|
||||
:emphasize-lines: 5-8,22-27
|
||||
:language: json-ld
|
||||
|
||||
Emotion Analysis
|
||||
----------------
|
||||
Description
|
||||
...........
|
||||
|
||||
Representation
|
||||
..............
|
||||
|
||||
.. literalinclude:: examples/example-emotion.json
|
||||
:language: json-ld
|
||||
:emphasize-lines: 5-8,25-37
|
||||
|
||||
Named Entity Recognition
|
||||
------------------------
|
||||
Description
|
||||
...........
|
||||
|
||||
Representation
|
||||
..............
|
||||
|
||||
.. literalinclude:: examples/example-ner.json
|
||||
:emphasize-lines: 5-8,19-34
|
||||
:language: json-ld
|
||||
|
||||
Complete example
|
||||
----------------
|
||||
Description
|
||||
...........
|
||||
This example covers all of the above cases, integrating all the annotations in the same document.
|
||||
|
||||
Representation
|
||||
..............
|
||||
|
||||
.. literalinclude:: examples/example-complete.json
|
||||
:language: json-ld
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 122 KiB After Width: | Height: | Size: 65 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 48 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 49 KiB After Width: | Height: | Size: 79 KiB |
@@ -1,38 +1,20 @@
|
||||
What is Senpy?
|
||||
--------------
|
||||
|
||||
Web services can get really complex: data validation, user interaction, formatting, logging., etc.
|
||||
The figure below summarizes the typical features in an analysis service.
|
||||
Senpy implements all the common blocks, so developers can focus on what really matters: great analysis algorithms that solve real problems.
|
||||
Senpy is an open source reference implementation of a linked data model for sentiment and emotion analysis services based on the vocabularies NIF, Marl and Onyx.
|
||||
|
||||
.. image:: senpy-framework.png
|
||||
:width: 60%
|
||||
:align: center
|
||||
The overall goal of the reference implementation Senpy is easing the adoption of the proposed linked data model for sentiment and emotion analysis services, so that services from different providers become interoperable. With this aim, the design of the reference implementation has focused on its extensibility and reusability.
|
||||
|
||||
A modular approach allows organizations to replace individual components with custom ones developed in-house. Furthermore, organizations can benefit from reusing prepackages modules that provide advanced functionalities, such as algorithms for sentiment and emotion analysis, linked data publication or emotion and sentiment mapping between different providers.
|
||||
|
||||
Senpy for end users
|
||||
===================
|
||||
Specifications
|
||||
==============
|
||||
|
||||
All services built using senpy share a common interface.
|
||||
This allows users to use them (almost) interchangeably.
|
||||
Senpy comes with a :ref:`built-in client`.
|
||||
The model used in Senpy is based on the following specifications:
|
||||
|
||||
|
||||
Senpy for service developers
|
||||
============================
|
||||
|
||||
Senpy is a framework that turns your sentiment or emotion analysis algorithm into a full blown semantic service.
|
||||
Senpy takes care of:
|
||||
|
||||
* Interfacing with the user: parameter validation, error handling.
|
||||
* Formatting: JSON-LD, Turtle/n-triples input and output, or simple text input
|
||||
* Linked Data: senpy results are semantically annotated, using a series of well established vocabularies, and sane default URIs.
|
||||
* User interface: a web UI where users can explore your service and test different settings
|
||||
* A client to interact with the service. Currently only available in Python.
|
||||
|
||||
Sharing your sentiment analysis with the world has never been easier!
|
||||
|
||||
Check out the :doc:`plugins` if you have developed an analysis algorithm (e.g. sentiment analysis) and you want to publish it as a service.
|
||||
* Marl, a vocabulary designed to annotate and describe subjetive opinions expressed on the web or in information systems.
|
||||
* Onyx, which is built one the same principles as Marl to annotate and describe emotions, and provides interoperability with Emotion Markup Language.
|
||||
* NIF 2.0, which defines a semantic format and APO for improving interoperability among natural language processing services
|
||||
|
||||
Architecture
|
||||
============
|
||||
@@ -47,5 +29,7 @@ Senpy proposes a modular and dynamic architecture that allows:
|
||||
The framework consists of two main modules: Senpy core, which is the building block of the service, and Senpy plugins, which consist of the analysis algorithm. The next figure depicts a simplified version of the processes involved in an analysis with the Senpy framework.
|
||||
|
||||
.. image:: senpy-architecture.png
|
||||
:width: 100%
|
||||
:height: 400px
|
||||
:width: 800px
|
||||
:scale: 100 %
|
||||
:align: center
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
Server
|
||||
======
|
||||
|
||||
The senpy server is launched via the `senpy` command:
|
||||
|
||||
.. code:: text
|
||||
|
||||
usage: senpy [-h] [--level logging_level] [--debug] [--default-plugins]
|
||||
[--host HOST] [--port PORT] [--plugins-folder PLUGINS_FOLDER]
|
||||
[--only-install]
|
||||
|
||||
Run a Senpy server
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
--level logging_level, -l logging_level
|
||||
Logging level
|
||||
--debug, -d Run the application in debug mode
|
||||
--default-plugins Load the default plugins
|
||||
--host HOST Use 0.0.0.0 to accept requests from any host.
|
||||
--port PORT, -p PORT Port to listen on.
|
||||
--plugins-folder PLUGINS_FOLDER, -f PLUGINS_FOLDER
|
||||
Where to look for plugins.
|
||||
--only-install, -i Do not run a server, only install plugin dependencies
|
||||
|
||||
|
||||
When launched, the server will recursively look for plugins in the specified plugins folder (the current working directory by default).
|
||||
For every plugin found, it will download its dependencies, and try to activate it.
|
||||
The default server includes a playground and an endpoint with all plugins found.
|
||||
|
||||
Let's run senpy with the default plugins:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
senpy -f . --default-plugins
|
||||
|
||||
Now go to `http://localhost:5000 <http://localhost:5000>`_, you should be greeted by the senpy playground:
|
||||
|
||||
.. image:: senpy-playground.png
|
||||
:width: 100%
|
||||
:alt: Playground
|
||||
|
||||
The playground is a user-friendly way to test your plugins, but you can always use the service directly: `http://localhost:5000/api?input=hello <http://localhost:5000/api?input=hello>`_.
|
||||
|
||||
|
||||
By default, senpy will listen only on the `127.0.0.1` address.
|
||||
That means you can only access the API from your (or localhost).
|
||||
You can listen on a different address using the `--host` flag (e.g., 0.0.0.0).
|
||||
The default port is 5000.
|
||||
You can change it with the `--port` flag.
|
||||
|
||||
For instance, to accept connections on port 6000 on any interface:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
senpy --host 0.0.0.0 --port 6000
|
||||
|
||||
For more options, see the `--help` page.
|
||||
@@ -1,15 +1,80 @@
|
||||
Usage
|
||||
-----
|
||||
|
||||
First of all, you need to install the package.
|
||||
See :doc:`installation` for instructions.
|
||||
Once installed, the `senpy` command should be available.
|
||||
The easiest and recommended way is to just use the command-line tool to load your plugins and launch the server.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
.. code:: bash
|
||||
|
||||
server
|
||||
SenpyClientUse
|
||||
commandline
|
||||
senpy
|
||||
|
||||
Or, alternatively:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
python -m senpy
|
||||
|
||||
|
||||
This will create a server with any modules found in the current path.
|
||||
|
||||
Useful command-line options
|
||||
===========================
|
||||
|
||||
In case you want to load modules, which are located in different folders under the root folder, use the next option.
|
||||
|
||||
.. code:: bash
|
||||
|
||||
python -m senpy -f .
|
||||
|
||||
The default port used by senpy is 5000, but you can change it using the option `--port`.
|
||||
|
||||
.. code:: bash
|
||||
|
||||
python -m senpy --port 8080
|
||||
|
||||
Also, the host can be changed where senpy is deployed. The default value is `127.0.0.1`.
|
||||
|
||||
.. code:: bash
|
||||
|
||||
python -m senpy --host 0.0.0.0
|
||||
|
||||
For more options, see the `--help` page.
|
||||
|
||||
Alternatively, you can use the modules included in senpy to build your own application.
|
||||
|
||||
Senpy server
|
||||
============
|
||||
|
||||
Once the server is launched, there is a basic endpoint in the server, which provides a playground to use the plugins that have been loaded.
|
||||
|
||||
In case you want to know the different endpoints of the server, there is more information available in the NIF API section_.
|
||||
|
||||
CLI
|
||||
===
|
||||
|
||||
This video shows how to use senpy through command-line tool.
|
||||
|
||||
https://asciinema.org/a/9uwef1ghkjk062cw2t4mhzpyk
|
||||
|
||||
Request example in python
|
||||
=========================
|
||||
|
||||
This example shows how to make a request to the default plugin:
|
||||
|
||||
.. code:: python
|
||||
|
||||
from senpy.client import Client
|
||||
|
||||
c = Client('http://127.0.0.1:5000/api/')
|
||||
r = c.analyse('hello world')
|
||||
|
||||
for entry in r.entries:
|
||||
print('{} -> {}'.format(entry.text, entry.emotions))
|
||||
|
||||
|
||||
|
||||
.. _section: http://senpy.readthedocs.org/en/latest/api.html
|
||||
|
||||
|
||||
Conversion
|
||||
==========
|
||||
See :doc:`conversion`
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
Vocabularies and model
|
||||
======================
|
||||
|
||||
The model used in Senpy is based on the following vocabularies:
|
||||
|
||||
* Marl, a vocabulary designed to annotate and describe subjetive opinions expressed on the web or in information systems.
|
||||
* Onyx, which is built one the same principles as Marl to annotate and describe emotions, and provides interoperability with Emotion Markup Language.
|
||||
* NIF 2.0, which defines a semantic format and APO for improving interoperability among natural language processing services
|
||||
@@ -1,7 +0,0 @@
|
||||
Deploy senpy to a kubernetes cluster.
|
||||
|
||||
Usage:
|
||||
|
||||
```
|
||||
kubectl apply -f . -n senpy
|
||||
```
|
||||
@@ -1,26 +0,0 @@
|
||||
---
|
||||
apiVersion: extensions/v1beta1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: senpy-latest
|
||||
spec:
|
||||
replicas: 1
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
role: senpy-latest
|
||||
app: test
|
||||
spec:
|
||||
containers:
|
||||
- name: senpy-latest
|
||||
image: gsiupm/senpy:latest
|
||||
imagePullPolicy: Always
|
||||
args:
|
||||
- "--default-plugins"
|
||||
resources:
|
||||
limits:
|
||||
memory: "512Mi"
|
||||
cpu: "1000m"
|
||||
ports:
|
||||
- name: web
|
||||
containerPort: 5000
|
||||
@@ -1,14 +0,0 @@
|
||||
---
|
||||
apiVersion: extensions/v1beta1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: senpy-ingress
|
||||
spec:
|
||||
rules:
|
||||
- host: latest.senpy.cluster.gsi.dit.upm.es
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
backend:
|
||||
serviceName: senpy-latest
|
||||
servicePort: 5000
|
||||
@@ -1,12 +0,0 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: senpy-latest
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- port: 5000
|
||||
protocol: TCP
|
||||
selector:
|
||||
role: senpy-latest
|
||||
@@ -1,8 +1,7 @@
|
||||
Flask>=0.10.1
|
||||
requests>=2.4.1
|
||||
tornado>=4.4.3
|
||||
gevent>=1.1rc4
|
||||
PyLD>=0.6.5
|
||||
nltk
|
||||
six
|
||||
future
|
||||
jsonschema
|
||||
|
||||
@@ -22,15 +22,35 @@ the server.
|
||||
|
||||
from flask import Flask
|
||||
from senpy.extensions import Senpy
|
||||
|
||||
from gevent.wsgi import WSGIServer
|
||||
from gevent.monkey import patch_all
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import senpy
|
||||
|
||||
patch_all(thread=False)
|
||||
|
||||
SERVER_PORT = os.environ.get("PORT", 5000)
|
||||
|
||||
|
||||
def info(type, value, tb):
|
||||
if hasattr(sys, 'ps1') or not sys.stderr.isatty():
|
||||
# we are in interactive mode or we don't have a tty-like
|
||||
# device, so we call the default hook
|
||||
sys.__excepthook__(type, value, tb)
|
||||
else:
|
||||
import traceback
|
||||
import pdb
|
||||
# we are NOT in interactive mode, print the exception...
|
||||
traceback.print_exception(type, value, tb)
|
||||
print
|
||||
# ...then start the debugger in post-mortem mode.
|
||||
# pdb.pm() # deprecated
|
||||
pdb.post_mortem(tb) # more "modern"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Run a Senpy server')
|
||||
parser.add_argument(
|
||||
@@ -74,38 +94,28 @@ def main():
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Do not run a server, only install plugin dependencies')
|
||||
parser.add_argument(
|
||||
'--threaded',
|
||||
action='store_false',
|
||||
default=True,
|
||||
help='Run a threaded server')
|
||||
parser.add_argument(
|
||||
'--version',
|
||||
'-v',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Output the senpy version and exit')
|
||||
args = parser.parse_args()
|
||||
if args.version:
|
||||
print('Senpy version {}'.format(senpy.__version__))
|
||||
exit(1)
|
||||
logging.basicConfig()
|
||||
rl = logging.getLogger()
|
||||
rl.setLevel(getattr(logging, args.level))
|
||||
app = Flask(__name__)
|
||||
app.debug = args.debug
|
||||
if args.debug:
|
||||
sys.excepthook = info
|
||||
sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins)
|
||||
sp.install_deps()
|
||||
if args.only_install:
|
||||
sp.install_deps()
|
||||
return
|
||||
sp.activate_all()
|
||||
print('Senpy version {}'.format(senpy.__version__))
|
||||
print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,
|
||||
args.port))
|
||||
app.run(args.host,
|
||||
args.port,
|
||||
threaded=args.threaded,
|
||||
debug=app.debug)
|
||||
http_server = WSGIServer((args.host, args.port), app)
|
||||
try:
|
||||
print('Senpy version {}'.format(senpy.__version__))
|
||||
print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,
|
||||
args.port))
|
||||
http_server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
print('Bye!')
|
||||
http_server.stop()
|
||||
sp.deactivate_all()
|
||||
|
||||
|
||||
|
||||
147
senpy/api.py
147
senpy/api.py
@@ -1,55 +1,38 @@
|
||||
from future.utils import iteritems
|
||||
from .models import Error, Results, Entry, from_string
|
||||
from .models import Error
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
API_PARAMS = {
|
||||
"algorithm": {
|
||||
"aliases": ["algorithms", "a", "algo"],
|
||||
"aliases": ["algorithm", "a", "algo"],
|
||||
"required": False,
|
||||
"description": ("Algorithms that will be used to process the request."
|
||||
"It may be a list of comma-separated names."),
|
||||
},
|
||||
"expanded-jsonld": {
|
||||
"@id": "expanded-jsonld",
|
||||
"aliases": ["expanded"],
|
||||
"required": True,
|
||||
"default": 0
|
||||
},
|
||||
"with_parameters": {
|
||||
"aliases": ['withparameters',
|
||||
'with-parameters'],
|
||||
"options": "boolean",
|
||||
"default": False,
|
||||
"required": True
|
||||
},
|
||||
"plugin_type": {
|
||||
"@id": "pluginType",
|
||||
"description": 'What kind of plugins to list',
|
||||
"aliases": ["pluginType"],
|
||||
"required": True,
|
||||
"default": "analysisPlugin"
|
||||
},
|
||||
"outformat": {
|
||||
"@id": "outformat",
|
||||
"aliases": ["o"],
|
||||
"aliases": ["outformat", "o"],
|
||||
"default": "json-ld",
|
||||
"required": True,
|
||||
"options": ["json-ld", "turtle"],
|
||||
},
|
||||
"help": {
|
||||
"@id": "help",
|
||||
"description": "Show additional help to know more about the possible parameters",
|
||||
"aliases": ["h"],
|
||||
"expanded-jsonld": {
|
||||
"@id": "expanded-jsonld",
|
||||
"aliases": ["expanded", "expanded-jsonld"],
|
||||
"required": True,
|
||||
"options": "boolean",
|
||||
"default": False
|
||||
"default": 0
|
||||
},
|
||||
"emotionModel": {
|
||||
"@id": "emotionModel",
|
||||
"aliases": ["emoModel"],
|
||||
"aliases": ["emotionModel", "emoModel"],
|
||||
"required": False
|
||||
},
|
||||
"plugin_type": {
|
||||
"@id": "pluginType",
|
||||
"description": 'What kind of plugins to list',
|
||||
"aliases": ["pluginType", "plugin_type"],
|
||||
"required": True,
|
||||
"default": "analysisPlugin"
|
||||
},
|
||||
"conversion": {
|
||||
"@id": "conversion",
|
||||
"description": "How to show the elements that have (not) been converted",
|
||||
@@ -61,16 +44,15 @@ API_PARAMS = {
|
||||
|
||||
WEB_PARAMS = {
|
||||
"inHeaders": {
|
||||
"aliases": ["headers"],
|
||||
"aliases": ["inHeaders", "headers"],
|
||||
"required": True,
|
||||
"default": False,
|
||||
"options": "boolean"
|
||||
"default": "0"
|
||||
},
|
||||
}
|
||||
|
||||
CLI_PARAMS = {
|
||||
"plugin_folder": {
|
||||
"aliases": ["folder"],
|
||||
"aliases": ["plugin_folder", "folder"],
|
||||
"required": True,
|
||||
"default": "."
|
||||
},
|
||||
@@ -79,71 +61,64 @@ CLI_PARAMS = {
|
||||
NIF_PARAMS = {
|
||||
"input": {
|
||||
"@id": "input",
|
||||
"aliases": ["i"],
|
||||
"aliases": ["i", "input"],
|
||||
"required": True,
|
||||
"help": "Input text"
|
||||
},
|
||||
"informat": {
|
||||
"@id": "informat",
|
||||
"aliases": ["f", "informat"],
|
||||
"required": False,
|
||||
"default": "text",
|
||||
"options": ["turtle", "text"],
|
||||
},
|
||||
"intype": {
|
||||
"@id": "intype",
|
||||
"aliases": ["t"],
|
||||
"aliases": ["intype", "t"],
|
||||
"required": False,
|
||||
"default": "direct",
|
||||
"options": ["direct", "url", "file"],
|
||||
},
|
||||
"informat": {
|
||||
"@id": "informat",
|
||||
"aliases": ["f"],
|
||||
"required": False,
|
||||
"default": "text",
|
||||
"options": ["turtle", "text", "json-ld"],
|
||||
},
|
||||
"language": {
|
||||
"@id": "language",
|
||||
"aliases": ["l"],
|
||||
"aliases": ["language", "l"],
|
||||
"required": False,
|
||||
},
|
||||
"prefix": {
|
||||
"@id": "prefix",
|
||||
"aliases": ["p"],
|
||||
"aliases": ["prefix", "p"],
|
||||
"required": True,
|
||||
"default": "",
|
||||
},
|
||||
"urischeme": {
|
||||
"@id": "urischeme",
|
||||
"aliases": ["u"],
|
||||
"aliases": ["urischeme", "u"],
|
||||
"required": False,
|
||||
"default": "RFC5147String",
|
||||
"options": "RFC5147String"
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def parse_params(indict, *specs):
|
||||
if not specs:
|
||||
specs = [NIF_PARAMS]
|
||||
logger.debug("Parsing: {}\n{}".format(indict, specs))
|
||||
def parse_params(indict, spec=NIF_PARAMS):
|
||||
logger.debug("Parsing: {}\n{}".format(indict, spec))
|
||||
outdict = indict.copy()
|
||||
wrong_params = {}
|
||||
for spec in specs:
|
||||
for param, options in iteritems(spec):
|
||||
if param[0] != "@": # Exclude json-ld properties
|
||||
for alias in options.get("aliases", []):
|
||||
# Replace each alias with the correct name of the parameter
|
||||
if alias in indict and alias is not param:
|
||||
outdict[param] = indict[alias]
|
||||
del indict[alias]
|
||||
continue
|
||||
if param not in outdict:
|
||||
if options.get("required", False) and "default" not in options:
|
||||
wrong_params[param] = spec[param]
|
||||
else:
|
||||
if "default" in options:
|
||||
outdict[param] = options["default"]
|
||||
elif "options" in spec[param]:
|
||||
if spec[param]["options"] == "boolean":
|
||||
outdict[param] = outdict[param] in [None, True, 'true', '1']
|
||||
elif outdict[param] not in spec[param]["options"]:
|
||||
wrong_params[param] = spec[param]
|
||||
for param, options in iteritems(spec):
|
||||
if param[0] != "@": # Exclude json-ld properties
|
||||
for alias in options.get("aliases", []):
|
||||
if alias in indict:
|
||||
outdict[param] = indict[alias]
|
||||
if param not in outdict:
|
||||
if options.get("required", False) and "default" not in options:
|
||||
wrong_params[param] = spec[param]
|
||||
else:
|
||||
if "default" in options:
|
||||
outdict[param] = options["default"]
|
||||
else:
|
||||
if "options" in spec[param] and \
|
||||
outdict[param] not in spec[param]["options"]:
|
||||
wrong_params[param] = spec[param]
|
||||
if wrong_params:
|
||||
logger.debug("Error parsing: %s", wrong_params)
|
||||
message = Error(
|
||||
@@ -153,30 +128,4 @@ def parse_params(indict, *specs):
|
||||
errors={param: error
|
||||
for param, error in iteritems(wrong_params)})
|
||||
raise message
|
||||
if 'algorithm' in outdict and not isinstance(outdict['algorithm'], list):
|
||||
outdict['algorithm'] = outdict['algorithm'].split(',')
|
||||
return outdict
|
||||
|
||||
|
||||
def get_extra_params(request, plugin=None):
|
||||
params = request.parameters.copy()
|
||||
if plugin:
|
||||
extra_params = parse_params(params, plugin.get('extra_params', {}))
|
||||
params.update(extra_params)
|
||||
return params
|
||||
|
||||
|
||||
def parse_call(params):
|
||||
'''Return a results object based on the parameters used in a call/request.
|
||||
'''
|
||||
params = parse_params(params, NIF_PARAMS)
|
||||
if params['informat'] == 'text':
|
||||
results = Results()
|
||||
entry = Entry(nif__isString=params['input'])
|
||||
results.entries.append(entry)
|
||||
elif params['informat'] == 'json-ld':
|
||||
results = from_string(params['input'], cls=Results)
|
||||
else:
|
||||
raise NotImplemented('Informat {} is not implemented'.format(params['informat']))
|
||||
results.parameters = params
|
||||
return results
|
||||
|
||||
@@ -19,13 +19,12 @@ Blueprints for Senpy
|
||||
"""
|
||||
from flask import (Blueprint, request, current_app, render_template, url_for,
|
||||
jsonify)
|
||||
from .models import Error, Response, Help, Plugins, read_schema
|
||||
from . import api
|
||||
from .models import Error, Response, Plugins, read_schema
|
||||
from .api import WEB_PARAMS, API_PARAMS, parse_params
|
||||
from .version import __version__
|
||||
from functools import wraps
|
||||
|
||||
import logging
|
||||
import json
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -75,33 +74,33 @@ def basic_api(f):
|
||||
@wraps(f)
|
||||
def decorated_function(*args, **kwargs):
|
||||
raw_params = get_params(request)
|
||||
headers = {'X-ORIGINAL-PARAMS': json.dumps(raw_params)}
|
||||
headers = {'X-ORIGINAL-PARAMS': raw_params}
|
||||
# Get defaults
|
||||
web_params = parse_params({}, spec=WEB_PARAMS)
|
||||
api_params = parse_params({}, spec=API_PARAMS)
|
||||
|
||||
outformat = 'json-ld'
|
||||
try:
|
||||
print('Getting request:')
|
||||
print(request)
|
||||
params = api.parse_params(raw_params, api.WEB_PARAMS, api.API_PARAMS)
|
||||
if hasattr(request, 'parameters'):
|
||||
request.parameters.update(params)
|
||||
web_params = parse_params(raw_params, spec=WEB_PARAMS)
|
||||
api_params = parse_params(raw_params, spec=API_PARAMS)
|
||||
if hasattr(request, 'params'):
|
||||
request.params.update(api_params)
|
||||
else:
|
||||
request.parameters = params
|
||||
request.params = api_params
|
||||
response = f(*args, **kwargs)
|
||||
except Error as ex:
|
||||
response = ex
|
||||
response.parameters = params
|
||||
logger.error(ex)
|
||||
if current_app.debug:
|
||||
raise
|
||||
|
||||
in_headers = params['inHeaders']
|
||||
expanded = params['expanded-jsonld']
|
||||
outformat = params['outformat']
|
||||
in_headers = web_params['inHeaders'] != "0"
|
||||
expanded = api_params['expanded-jsonld']
|
||||
outformat = api_params['outformat']
|
||||
|
||||
return response.flask(
|
||||
in_headers=in_headers,
|
||||
headers=headers,
|
||||
prefix=url_for('.api_root', _external=True),
|
||||
prefix=url_for('.api', _external=True),
|
||||
context_uri=url_for('api.context',
|
||||
entity=type(response).__name__,
|
||||
_external=True),
|
||||
@@ -113,22 +112,16 @@ def basic_api(f):
|
||||
|
||||
@api_blueprint.route('/', methods=['POST', 'GET'])
|
||||
@basic_api
|
||||
def api_root():
|
||||
if request.parameters['help']:
|
||||
dic = dict(api.API_PARAMS, **api.NIF_PARAMS)
|
||||
response = Help(parameters=dic)
|
||||
return response
|
||||
else:
|
||||
req = api.parse_call(request.parameters)
|
||||
response = current_app.senpy.analyse(req)
|
||||
return response
|
||||
def api():
|
||||
response = current_app.senpy.analyse(**request.params)
|
||||
return response
|
||||
|
||||
|
||||
@api_blueprint.route('/plugins/', methods=['POST', 'GET'])
|
||||
@basic_api
|
||||
def plugins():
|
||||
sp = current_app.senpy
|
||||
ptype = request.parameters.get('plugin_type')
|
||||
ptype = request.params.get('plugin_type')
|
||||
plugins = sp.filter_plugins(plugin_type=ptype)
|
||||
dic = Plugins(plugins=list(plugins.values()))
|
||||
return dic
|
||||
|
||||
26
senpy/cli.py
26
senpy/cli.py
@@ -1,7 +1,7 @@
|
||||
import sys
|
||||
from .models import Error
|
||||
from .api import parse_params, CLI_PARAMS
|
||||
from .extensions import Senpy
|
||||
from . import api
|
||||
|
||||
|
||||
def argv_to_dict(argv):
|
||||
@@ -13,27 +13,27 @@ def argv_to_dict(argv):
|
||||
if argv[i][0] == '-':
|
||||
key = argv[i].strip('-')
|
||||
value = argv[i + 1] if len(argv) > i + 1 else None
|
||||
if not value or value[0] == '-':
|
||||
cli_dict[key] = True
|
||||
if value and value[0] == '-':
|
||||
cli_dict[key] = ""
|
||||
else:
|
||||
cli_dict[key] = value
|
||||
return cli_dict
|
||||
|
||||
|
||||
def parse_cli(argv):
|
||||
cli_dict = argv_to_dict(argv)
|
||||
cli_params = parse_params(cli_dict, spec=CLI_PARAMS)
|
||||
return cli_params, cli_dict
|
||||
|
||||
|
||||
def main_function(argv):
|
||||
'''This is the method for unit testing
|
||||
'''
|
||||
params = api.parse_params(argv_to_dict(argv),
|
||||
api.CLI_PARAMS,
|
||||
api.API_PARAMS,
|
||||
api.NIF_PARAMS)
|
||||
plugin_folder = params['plugin_folder']
|
||||
cli_params, cli_dict = parse_cli(argv)
|
||||
plugin_folder = cli_params['plugin_folder']
|
||||
sp = Senpy(default_plugins=False, plugin_folder=plugin_folder)
|
||||
request = api.parse_call(params)
|
||||
algos = request.parameters.get('algorithm', sp.plugins.keys())
|
||||
for algo in algos:
|
||||
sp.activate_plugin(algo)
|
||||
res = sp.analyse(request)
|
||||
sp.activate_all(sync=True)
|
||||
res = sp.analyse(**cli_dict)
|
||||
return res
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import requests
|
||||
import logging
|
||||
from . import models
|
||||
from .plugins import default_plugin_type
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -13,15 +12,12 @@ class Client(object):
|
||||
def analyse(self, input, method='GET', **kwargs):
|
||||
return self.request('/', method=method, input=input, **kwargs)
|
||||
|
||||
def plugins(self, ptype=default_plugin_type):
|
||||
resp = self.request(path='/plugins', plugin_type=ptype).plugins
|
||||
return {p.name: p for p in resp}
|
||||
|
||||
def request(self, path=None, method='GET', **params):
|
||||
url = '{}{}'.format(self.endpoint, path)
|
||||
response = requests.request(method=method, url=url, params=params)
|
||||
try:
|
||||
resp = models.from_dict(response.json())
|
||||
resp.validate(resp)
|
||||
except Exception as ex:
|
||||
logger.error(('There seems to be a problem with the response:\n'
|
||||
'\tURL: {url}\n'
|
||||
|
||||
@@ -5,18 +5,24 @@ It orchestrates plugin (de)activation and analysis.
|
||||
from future import standard_library
|
||||
standard_library.install_aliases()
|
||||
|
||||
from . import plugins, api
|
||||
from . import plugins
|
||||
from .plugins import SenpyPlugin
|
||||
from .models import Error
|
||||
from .models import Error, Entry, Results
|
||||
from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
|
||||
from .api import API_PARAMS, NIF_PARAMS, parse_params
|
||||
|
||||
from threading import Thread
|
||||
from functools import partial
|
||||
|
||||
import os
|
||||
import copy
|
||||
import fnmatch
|
||||
import inspect
|
||||
import sys
|
||||
import importlib
|
||||
import logging
|
||||
import traceback
|
||||
import yaml
|
||||
import pip
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -72,85 +78,74 @@ class Senpy(object):
|
||||
else:
|
||||
logger.debug("Not a folder: %s", folder)
|
||||
|
||||
def _get_plugins(self, request):
|
||||
if not self.analysis_plugins:
|
||||
def _find_plugin(self, params):
|
||||
api_params = parse_params(params, spec=API_PARAMS)
|
||||
algo = None
|
||||
if "algorithm" in api_params and api_params["algorithm"]:
|
||||
algo = api_params["algorithm"]
|
||||
elif self.plugins:
|
||||
algo = self.default_plugin and self.default_plugin.name
|
||||
if not algo:
|
||||
raise Error(
|
||||
status=404,
|
||||
message=("No plugins found."
|
||||
" Please install one."))
|
||||
algos = request.parameters.get('algorithm', None)
|
||||
if not algos:
|
||||
if self.default_plugin:
|
||||
algos = [self.default_plugin.name, ]
|
||||
else:
|
||||
raise Error(
|
||||
status=404,
|
||||
message="No default plugin found, and None provided")
|
||||
" Please install one.").format(algo))
|
||||
if algo not in self.plugins:
|
||||
logger.debug(("The algorithm '{}' is not valid\n"
|
||||
"Valid algorithms: {}").format(algo,
|
||||
self.plugins.keys()))
|
||||
raise Error(
|
||||
status=404,
|
||||
message="The algorithm '{}' is not valid".format(algo))
|
||||
|
||||
plugins = list()
|
||||
for algo in algos:
|
||||
if algo not in self.plugins:
|
||||
logger.debug(("The algorithm '{}' is not valid\n"
|
||||
"Valid algorithms: {}").format(algo,
|
||||
self.plugins.keys()))
|
||||
raise Error(
|
||||
status=404,
|
||||
message="The algorithm '{}' is not valid".format(algo))
|
||||
plugins.append(self.plugins[algo])
|
||||
return plugins
|
||||
if not self.plugins[algo].is_activated:
|
||||
logger.debug("Plugin not activated: {}".format(algo))
|
||||
raise Error(
|
||||
status=400,
|
||||
message=("The algorithm '{}'"
|
||||
" is not activated yet").format(algo))
|
||||
return self.plugins[algo]
|
||||
|
||||
def _process_entries(self, entries, req, plugins):
|
||||
"""
|
||||
Recursively process the entries with the first plugin in the list, and pass the results
|
||||
to the rest of the plugins.
|
||||
"""
|
||||
if not plugins:
|
||||
for i in entries:
|
||||
yield i
|
||||
return
|
||||
plugin = plugins[0]
|
||||
self._activate(plugin) # Make sure the plugin is activated
|
||||
specific_params = api.get_extra_params(req, plugin)
|
||||
req.analysis.append({'plugin': plugin,
|
||||
'parameters': specific_params})
|
||||
results = plugin.analyse_entries(entries, specific_params)
|
||||
for i in self._process_entries(results, req, plugins[1:]):
|
||||
yield i
|
||||
def _get_params(self, params, plugin):
|
||||
nif_params = parse_params(params, spec=NIF_PARAMS)
|
||||
extra_params = plugin.get('extra_params', {})
|
||||
specific_params = parse_params(params, spec=extra_params)
|
||||
nif_params.update(specific_params)
|
||||
return nif_params
|
||||
|
||||
def install_deps(self):
|
||||
for plugin in self.filter_plugins(is_activated=True):
|
||||
plugins.install_deps(plugin)
|
||||
def _get_entries(self, params):
|
||||
entry = None
|
||||
if params['informat'] == 'text':
|
||||
entry = Entry(text=params['input'])
|
||||
else:
|
||||
raise NotImplemented('Only text input format implemented')
|
||||
yield entry
|
||||
|
||||
def analyse(self, request):
|
||||
"""
|
||||
Main method that analyses a request, either from CLI or HTTP.
|
||||
It takes a processed request, provided by the user, as returned
|
||||
by api.parse_call().
|
||||
"""
|
||||
logger.debug("analysing request: {}".format(request))
|
||||
def analyse(self, **api_params):
|
||||
logger.debug("analysing with params: {}".format(api_params))
|
||||
plugin = self._find_plugin(api_params)
|
||||
nif_params = self._get_params(api_params, plugin)
|
||||
resp = Results()
|
||||
if 'with_parameters' in api_params:
|
||||
resp.parameters = nif_params
|
||||
try:
|
||||
entries = request.entries
|
||||
request.entries = []
|
||||
plugins = self._get_plugins(request)
|
||||
results = request
|
||||
for i in self._process_entries(entries, results, plugins):
|
||||
results.entries.append(i)
|
||||
self.convert_emotions(results)
|
||||
if 'with_parameters' not in results.parameters:
|
||||
del results.parameters
|
||||
logger.debug("Returning analysis result: {}".format(results))
|
||||
except (Error, Exception) as ex:
|
||||
if not isinstance(ex, Error):
|
||||
msg = "Error during analysis: {} \n\t{}".format(ex,
|
||||
traceback.format_exc())
|
||||
ex = Error(message=msg, status=500)
|
||||
entries = []
|
||||
for i in self._get_entries(nif_params):
|
||||
entries += list(plugin.analyse_entry(i, nif_params))
|
||||
resp.entries = entries
|
||||
self.convert_emotions(resp, plugin, nif_params)
|
||||
resp.analysis.append(plugin.id)
|
||||
logger.debug("Returning analysis result: {}".format(resp))
|
||||
except Error as ex:
|
||||
logger.exception('Error returning analysis result')
|
||||
raise ex
|
||||
results.analysis = [i['plugin'].id for i in results.analysis]
|
||||
return results
|
||||
resp = ex
|
||||
except Exception as ex:
|
||||
logger.exception('Error returning analysis result')
|
||||
resp = Error(message=str(ex), status=500)
|
||||
return resp
|
||||
|
||||
def _conversion_candidates(self, fromModel, toModel):
|
||||
candidates = self.filter_plugins(plugin_type='emotionConversionPlugin')
|
||||
candidates = self.filter_plugins(**{'@type': 'emotionConversionPlugin'})
|
||||
for name, candidate in candidates.items():
|
||||
for pair in candidate.onyx__doesConversion:
|
||||
logging.debug(pair)
|
||||
@@ -160,34 +155,30 @@ class Senpy(object):
|
||||
# logging.debug('Found candidate: {}'.format(candidate))
|
||||
yield candidate
|
||||
|
||||
def convert_emotions(self, resp):
|
||||
def convert_emotions(self, resp, plugin, params):
|
||||
"""
|
||||
Conversion of all emotions in a response **in place**.
|
||||
Conversion of all emotions in a response.
|
||||
In addition to converting from one model to another, it has
|
||||
to include the conversion plugin to the analysis list.
|
||||
Needless to say, this is far from an elegant solution, but it works.
|
||||
@todo refactor and clean up
|
||||
"""
|
||||
plugins = [i['plugin'] for i in resp.analysis]
|
||||
params = resp.parameters
|
||||
fromModel = plugin.get('onyx:usesEmotionModel', None)
|
||||
toModel = params.get('emotionModel', None)
|
||||
output = params.get('conversion', None)
|
||||
logger.debug('Asked for model: {}'.format(toModel))
|
||||
logger.debug('Analysis plugin uses model: {}'.format(fromModel))
|
||||
|
||||
if not toModel:
|
||||
return
|
||||
|
||||
logger.debug('Asked for model: {}'.format(toModel))
|
||||
output = params.get('conversion', None)
|
||||
candidates = {}
|
||||
for plugin in plugins:
|
||||
try:
|
||||
fromModel = plugin.get('onyx:usesEmotionModel', None)
|
||||
candidates[plugin.id] = next(self._conversion_candidates(fromModel, toModel))
|
||||
logger.debug('Analysis plugin {} uses model: {}'.format(plugin.id, fromModel))
|
||||
except StopIteration:
|
||||
e = Error(('No conversion plugin found for: '
|
||||
'{} -> {}'.format(fromModel, toModel)))
|
||||
e.original_response = resp
|
||||
e.parameters = params
|
||||
raise e
|
||||
try:
|
||||
candidate = next(self._conversion_candidates(fromModel, toModel))
|
||||
except StopIteration:
|
||||
e = Error(('No conversion plugin found for: '
|
||||
'{} -> {}'.format(fromModel, toModel)))
|
||||
e.original_response = resp
|
||||
e.parameters = params
|
||||
raise e
|
||||
newentries = []
|
||||
for i in resp.entries:
|
||||
if output == "full":
|
||||
@@ -195,10 +186,6 @@ class Senpy(object):
|
||||
else:
|
||||
newemotions = []
|
||||
for j in i.emotions:
|
||||
plugname = j['prov:wasGeneratedBy']
|
||||
candidate = candidates[plugname]
|
||||
resp.analysis.append({'plugin': candidate,
|
||||
'parameters': params})
|
||||
for k in candidate.convert(j, fromModel, toModel, params):
|
||||
k.prov__wasGeneratedBy = candidate.id
|
||||
if output == 'nested':
|
||||
@@ -207,13 +194,13 @@ class Senpy(object):
|
||||
i.emotions = newemotions
|
||||
newentries.append(i)
|
||||
resp.entries = newentries
|
||||
resp.analysis.append(candidate.id)
|
||||
|
||||
@property
|
||||
def default_plugin(self):
|
||||
candidate = self._default
|
||||
if not candidate:
|
||||
candidates = self.filter_plugins(plugin_type='analysisPlugin',
|
||||
is_activated=True)
|
||||
candidates = self.filter_plugins(is_activated=True)
|
||||
if len(candidates) > 0:
|
||||
candidate = list(candidates.values())[0]
|
||||
logger.debug("Default: {}".format(candidate))
|
||||
@@ -226,42 +213,25 @@ class Senpy(object):
|
||||
else:
|
||||
self._default = self.plugins[value]
|
||||
|
||||
def activate_all(self, sync=True):
|
||||
def activate_all(self, sync=False):
|
||||
ps = []
|
||||
for plug in self.plugins.keys():
|
||||
ps.append(self.activate_plugin(plug, sync=sync))
|
||||
return ps
|
||||
|
||||
def deactivate_all(self, sync=True):
|
||||
def deactivate_all(self, sync=False):
|
||||
ps = []
|
||||
for plug in self.plugins.keys():
|
||||
ps.append(self.deactivate_plugin(plug, sync=sync))
|
||||
return ps
|
||||
|
||||
def _set_active(self, plugin, active=True, *args, **kwargs):
|
||||
def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
|
||||
''' We're using a variable in the plugin itself to activate/deactive plugins.\
|
||||
Note that plugins may activate themselves by setting this variable.
|
||||
'''
|
||||
plugin.is_activated = active
|
||||
self.plugins[plugin_name].is_activated = active
|
||||
|
||||
def _activate(self, plugin):
|
||||
success = False
|
||||
with plugin._lock:
|
||||
if plugin.is_activated:
|
||||
return
|
||||
try:
|
||||
plugin.activate()
|
||||
msg = "Plugin activated: {}".format(plugin.name)
|
||||
logger.info(msg)
|
||||
success = True
|
||||
self._set_active(plugin, success)
|
||||
except Exception as ex:
|
||||
msg = "Error activating plugin {} - {} : \n\t{}".format(
|
||||
plugin.name, ex, traceback.format_exc())
|
||||
logger.error(msg)
|
||||
raise Error(msg)
|
||||
|
||||
def activate_plugin(self, plugin_name, sync=True):
|
||||
def activate_plugin(self, plugin_name, sync=False):
|
||||
try:
|
||||
plugin = self.plugins[plugin_name]
|
||||
except KeyError:
|
||||
@@ -270,17 +240,36 @@ class Senpy(object):
|
||||
|
||||
logger.info("Activating plugin: {}".format(plugin.name))
|
||||
|
||||
if sync or 'async' in plugin and not plugin.async:
|
||||
self._activate(plugin)
|
||||
else:
|
||||
th = Thread(target=partial(self._activate, plugin))
|
||||
th.start()
|
||||
return th
|
||||
def act():
|
||||
success = False
|
||||
try:
|
||||
plugin.activate()
|
||||
msg = "Plugin activated: {}".format(plugin.name)
|
||||
logger.info(msg)
|
||||
success = True
|
||||
self._set_active_plugin(plugin_name, success)
|
||||
except Exception as ex:
|
||||
msg = "Error activating plugin {} - {} : \n\t{}".format(
|
||||
plugin.name, ex, traceback.format_exc())
|
||||
logger.error(msg)
|
||||
raise Error(msg)
|
||||
|
||||
def _deactivate(self, plugin):
|
||||
with plugin._lock:
|
||||
if not plugin.is_activated:
|
||||
return
|
||||
if sync or 'async' in plugin and not plugin.async:
|
||||
act()
|
||||
else:
|
||||
th = Thread(target=act)
|
||||
th.start()
|
||||
|
||||
def deactivate_plugin(self, plugin_name, sync=False):
|
||||
try:
|
||||
plugin = self.plugins[plugin_name]
|
||||
except KeyError:
|
||||
raise Error(
|
||||
message="Plugin not found: {}".format(plugin_name), status=404)
|
||||
|
||||
self._set_active_plugin(plugin_name, False)
|
||||
|
||||
def deact():
|
||||
try:
|
||||
plugin.deactivate()
|
||||
logger.info("Plugin deactivated: {}".format(plugin.name))
|
||||
@@ -289,21 +278,83 @@ class Senpy(object):
|
||||
"Error deactivating plugin {}: {}".format(plugin.name, ex))
|
||||
logger.error("Trace: {}".format(traceback.format_exc()))
|
||||
|
||||
def deactivate_plugin(self, plugin_name, sync=True):
|
||||
try:
|
||||
plugin = self.plugins[plugin_name]
|
||||
except KeyError:
|
||||
raise Error(
|
||||
message="Plugin not found: {}".format(plugin_name), status=404)
|
||||
|
||||
self._set_active(plugin, False)
|
||||
|
||||
if sync or 'async' in plugin and not plugin.async:
|
||||
self._deactivate(plugin)
|
||||
deact()
|
||||
else:
|
||||
th = Thread(target=partial(self._deactivate, plugin))
|
||||
th = Thread(target=deact)
|
||||
th.start()
|
||||
return th
|
||||
|
||||
@classmethod
|
||||
def validate_info(cls, info):
|
||||
return all(x in info for x in ('name', 'module', 'description', 'version'))
|
||||
|
||||
def install_deps(self):
|
||||
for i in self.plugins.values():
|
||||
self._install_deps(i)
|
||||
|
||||
@classmethod
|
||||
def _install_deps(cls, info=None):
|
||||
requirements = info.get('requirements', [])
|
||||
if requirements:
|
||||
pip_args = []
|
||||
pip_args.append('install')
|
||||
pip_args.append('--use-wheel')
|
||||
for req in requirements:
|
||||
pip_args.append(req)
|
||||
logger.info('Installing requirements: ' + str(requirements))
|
||||
pip.main(pip_args)
|
||||
|
||||
@classmethod
|
||||
def _load_module(cls, name, root):
|
||||
sys.path.append(root)
|
||||
tmp = importlib.import_module(name)
|
||||
sys.path.remove(root)
|
||||
return tmp
|
||||
|
||||
@classmethod
|
||||
def _load_plugin_from_info(cls, info, root):
|
||||
if not cls.validate_info(info):
|
||||
logger.warn('The module info is not valid.\n\t{}'.format(info))
|
||||
return None, None
|
||||
module = info["module"]
|
||||
name = info["name"]
|
||||
|
||||
cls._install_deps(info)
|
||||
tmp = cls._load_module(module, root)
|
||||
|
||||
candidate = None
|
||||
for _, obj in inspect.getmembers(tmp):
|
||||
if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
|
||||
logger.debug(("Found plugin class:"
|
||||
" {}@{}").format(obj, inspect.getmodule(obj)))
|
||||
candidate = obj
|
||||
break
|
||||
if not candidate:
|
||||
logger.debug("No valid plugin for: {}".format(module))
|
||||
return
|
||||
module = candidate(info=info)
|
||||
return name, module
|
||||
|
||||
@classmethod
|
||||
def _load_plugin(cls, root, filename):
|
||||
fpath = os.path.join(root, filename)
|
||||
logger.debug("Loading plugin: {}".format(fpath))
|
||||
with open(fpath, 'r') as f:
|
||||
info = yaml.load(f)
|
||||
logger.debug("Info: {}".format(info))
|
||||
return cls._load_plugin_from_info(info, root)
|
||||
|
||||
def _load_plugins(self):
|
||||
plugins = {}
|
||||
for search_folder in self._search_folders:
|
||||
for root, dirnames, filenames in os.walk(search_folder):
|
||||
for filename in fnmatch.filter(filenames, '*.senpy'):
|
||||
name, plugin = self._load_plugin(root, filename)
|
||||
if plugin and name:
|
||||
plugins[name] = plugin
|
||||
|
||||
self._outdated = False
|
||||
return plugins
|
||||
|
||||
def teardown(self, exception):
|
||||
pass
|
||||
@@ -312,12 +363,37 @@ class Senpy(object):
|
||||
def plugins(self):
|
||||
""" Return the plugins registered for a given application. """
|
||||
if self._outdated:
|
||||
self._plugin_list = plugins.load_plugins(self._search_folders)
|
||||
self._outdated = False
|
||||
self._plugin_list = self._load_plugins()
|
||||
return self._plugin_list
|
||||
|
||||
def filter_plugins(self, **kwargs):
|
||||
return plugins.pfilter(self.plugins, **kwargs)
|
||||
""" Filter plugins by different criteria """
|
||||
ptype = kwargs.pop('plugin_type', None)
|
||||
logger.debug('#' * 100)
|
||||
logger.debug('ptype {}'.format(ptype))
|
||||
if ptype:
|
||||
try:
|
||||
ptype = ptype[0].upper() + ptype[1:]
|
||||
pclass = getattr(plugins, ptype)
|
||||
logger.debug('Class: {}'.format(pclass))
|
||||
candidates = filter(lambda x: isinstance(x, pclass),
|
||||
self.plugins.values())
|
||||
except AttributeError:
|
||||
raise Error('{} is not a valid type'.format(ptype))
|
||||
else:
|
||||
candidates = self.plugins.values()
|
||||
|
||||
logger.debug(candidates)
|
||||
|
||||
def matches(plug):
|
||||
res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
|
||||
logger.debug(
|
||||
"matching {} with {}: {}".format(plug.name, kwargs, res))
|
||||
return res
|
||||
|
||||
if kwargs:
|
||||
candidates = filter(matches, candidates)
|
||||
return {p.name: p for p in candidates}
|
||||
|
||||
@property
|
||||
def analysis_plugins(self):
|
||||
|
||||
@@ -140,7 +140,7 @@ class SenpyMixin(object):
|
||||
vp = item[kp]
|
||||
temp[kp] = ser_or_down(vp)
|
||||
return temp
|
||||
elif isinstance(item, list) or isinstance(item, set):
|
||||
elif isinstance(item, list):
|
||||
return list(ser_or_down(i) for i in item)
|
||||
else:
|
||||
return item
|
||||
@@ -181,10 +181,10 @@ class SenpyMixin(object):
|
||||
obj = self
|
||||
if hasattr(obj, "jsonld"):
|
||||
obj = obj.jsonld()
|
||||
self._validator.validate(obj)
|
||||
jsonschema.validate(obj, self.schema)
|
||||
|
||||
def __str__(self):
|
||||
return str(self.serialize())
|
||||
return str(self.to_JSON())
|
||||
|
||||
|
||||
class BaseModel(SenpyMixin, dict):
|
||||
@@ -214,15 +214,15 @@ class BaseModel(SenpyMixin, dict):
|
||||
temp['@type'] = getattr(self, '@type')
|
||||
except AttributeError:
|
||||
logger.warn('Creating an instance of an unknown model')
|
||||
|
||||
super(BaseModel, self).__init__(temp)
|
||||
|
||||
def _get_key(self, key):
|
||||
if key is 'id':
|
||||
key = '@id'
|
||||
key = key.replace("__", ":", 1)
|
||||
return key
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
dict.__setitem__(self, key, value)
|
||||
|
||||
def __delitem__(self, key):
|
||||
dict.__delitem__(self, key)
|
||||
|
||||
@@ -236,49 +236,29 @@ class BaseModel(SenpyMixin, dict):
|
||||
self.__setitem__(self._get_key(key), value)
|
||||
|
||||
def __delattr__(self, key):
|
||||
try:
|
||||
object.__delattr__(self, key)
|
||||
except AttributeError:
|
||||
self.__delitem__(self._get_key(key))
|
||||
self.__delitem__(self._get_key(key))
|
||||
|
||||
def _plain_dict(self):
|
||||
d = {k: v for (k, v) in self.items() if k[0] != "_"}
|
||||
if 'id' in d:
|
||||
d["@id"] = d.pop('id')
|
||||
return d
|
||||
|
||||
|
||||
_subtypes = {}
|
||||
|
||||
|
||||
def register(rsubclass, rtype=None):
|
||||
_subtypes[rtype or rsubclass.__name__] = rsubclass
|
||||
|
||||
|
||||
_subtypes = {}
|
||||
|
||||
|
||||
def from_dict(indict, cls=None):
|
||||
if not cls:
|
||||
target = indict.get('@type', None)
|
||||
try:
|
||||
if target and target in _subtypes:
|
||||
cls = _subtypes[target]
|
||||
else:
|
||||
cls = BaseModel
|
||||
except Exception:
|
||||
cls = BaseModel
|
||||
outdict = dict()
|
||||
for k, v in indict.items():
|
||||
if k == '@context':
|
||||
pass
|
||||
elif isinstance(v, dict):
|
||||
v = from_dict(indict[k])
|
||||
elif isinstance(v, list):
|
||||
for ix, v2 in enumerate(v):
|
||||
if isinstance(v2, dict):
|
||||
v[ix] = from_dict(v2)
|
||||
outdict[k] = v
|
||||
return cls(**outdict)
|
||||
|
||||
|
||||
def from_string(string, **kwargs):
|
||||
return from_dict(json.loads(string), **kwargs)
|
||||
def from_dict(indict):
|
||||
target = indict.get('@type', None)
|
||||
if target and target in _subtypes:
|
||||
cls = _subtypes[target]
|
||||
else:
|
||||
cls = BaseModel
|
||||
return cls(**indict)
|
||||
|
||||
|
||||
def from_json(injson):
|
||||
@@ -286,31 +266,15 @@ def from_json(injson):
|
||||
return from_dict(indict)
|
||||
|
||||
|
||||
def from_schema(name, schema=None, schema_file=None, base_classes=None):
|
||||
def from_schema(name, schema_file=None, base_classes=None):
|
||||
base_classes = base_classes or []
|
||||
base_classes.append(BaseModel)
|
||||
schema_file = schema_file or '{}.json'.format(name)
|
||||
class_name = '{}{}'.format(name[0].upper(), name[1:])
|
||||
if '/' not in 'schema_file':
|
||||
schema_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
|
||||
'schemas',
|
||||
schema_file)
|
||||
|
||||
schema_path = 'file://' + schema_file
|
||||
|
||||
with open(schema_file) as f:
|
||||
schema = json.load(f)
|
||||
|
||||
dct = {}
|
||||
|
||||
resolver = jsonschema.RefResolver(schema_path, schema)
|
||||
dct['@type'] = name
|
||||
dct['_schema_file'] = schema_file
|
||||
dct['schema'] = schema
|
||||
dct['_validator'] = jsonschema.Draft4Validator(schema, resolver=resolver)
|
||||
|
||||
newclass = type(class_name, tuple(base_classes), dct)
|
||||
|
||||
newclass = type(class_name, tuple(base_classes), {})
|
||||
setattr(newclass, '@type', name)
|
||||
setattr(newclass, 'schema', read_schema(schema_file))
|
||||
setattr(newclass, 'class_name', class_name)
|
||||
register(newclass, name)
|
||||
return newclass
|
||||
|
||||
@@ -331,7 +295,6 @@ for i in [
|
||||
'emotionPlugin',
|
||||
'emotionSet',
|
||||
'entry',
|
||||
'help',
|
||||
'plugin',
|
||||
'plugins',
|
||||
'response',
|
||||
@@ -345,15 +308,12 @@ for i in [
|
||||
_ErrorModel = from_schema('error')
|
||||
|
||||
|
||||
class Error(SenpyMixin, Exception):
|
||||
class Error(SenpyMixin, BaseException):
|
||||
def __init__(self, message, *args, **kwargs):
|
||||
super(Error, self).__init__(self, message, message)
|
||||
self._error = _ErrorModel(message=message, *args, **kwargs)
|
||||
self.message = message
|
||||
|
||||
def validate(self, obj=None):
|
||||
self._error.validate()
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self._error[key]
|
||||
|
||||
@@ -377,8 +337,5 @@ class Error(SenpyMixin, Exception):
|
||||
def __delattr__(self, key):
|
||||
delattr(self._error, key)
|
||||
|
||||
def __str__(self):
|
||||
return str(self.to_JSON(with_context=False))
|
||||
|
||||
|
||||
register(Error, 'error')
|
||||
|
||||
@@ -1,28 +1,19 @@
|
||||
from future import standard_library
|
||||
standard_library.install_aliases()
|
||||
|
||||
import inspect
|
||||
import os.path
|
||||
import os
|
||||
import pickle
|
||||
import logging
|
||||
import tempfile
|
||||
import copy
|
||||
|
||||
import fnmatch
|
||||
import inspect
|
||||
import sys
|
||||
import subprocess
|
||||
import importlib
|
||||
import yaml
|
||||
import threading
|
||||
|
||||
from .. import models, utils
|
||||
from ..api import API_PARAMS
|
||||
from .. import models
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Plugin(models.Plugin):
|
||||
class SenpyPlugin(models.Plugin):
|
||||
def __init__(self, info=None):
|
||||
"""
|
||||
Provides a canonical name for plugins and serves as base for other
|
||||
@@ -33,9 +24,8 @@ class Plugin(models.Plugin):
|
||||
"information for the plugin."))
|
||||
logger.debug("Initialising {}".format(info))
|
||||
id = 'plugins/{}_{}'.format(info['name'], info['version'])
|
||||
super(Plugin, self).__init__(id=id, **info)
|
||||
super(SenpyPlugin, self).__init__(id=id, **info)
|
||||
self.is_activated = False
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def get_folder(self):
|
||||
return os.path.dirname(inspect.getfile(self.__class__))
|
||||
@@ -46,26 +36,8 @@ class Plugin(models.Plugin):
|
||||
def deactivate(self):
|
||||
pass
|
||||
|
||||
def test(self):
|
||||
if not hasattr(self, 'test_cases'):
|
||||
import inspect
|
||||
raise AttributeError(('Plugin {} [{}] does not have any defined '
|
||||
'test cases').format(self.id, inspect.getfile(self.__class__)))
|
||||
for case in self.test_cases:
|
||||
res = list(self.analyse_entry(models.Entry(case['entry']),
|
||||
case['params']))
|
||||
exp = case['expected']
|
||||
if not isinstance(exp, list):
|
||||
exp = [exp]
|
||||
utils.check_template(res, exp)
|
||||
for r in res:
|
||||
r.validate()
|
||||
|
||||
|
||||
SenpyPlugin = Plugin
|
||||
|
||||
|
||||
class AnalysisPlugin(Plugin):
|
||||
class AnalysisPlugin(SenpyPlugin):
|
||||
|
||||
def analyse(self, *args, **kwargs):
|
||||
raise NotImplemented(
|
||||
@@ -78,21 +50,15 @@ class AnalysisPlugin(Plugin):
|
||||
Note that this method may yield an annotated entry or a list of
|
||||
entries (e.g. in a tokenizer)
|
||||
"""
|
||||
text = entry['nif:isString']
|
||||
text = entry['text']
|
||||
params = copy.copy(parameters)
|
||||
params['input'] = text
|
||||
results = self.analyse(**params)
|
||||
for i in results.entries:
|
||||
yield i
|
||||
|
||||
def analyse_entries(self, entries, parameters):
|
||||
for entry in entries:
|
||||
logger.debug('Analysing entry with plugin {}: {}'.format(self, entry))
|
||||
for result in self.analyse_entry(entry, parameters):
|
||||
yield result
|
||||
|
||||
|
||||
class ConversionPlugin(Plugin):
|
||||
class ConversionPlugin(SenpyPlugin):
|
||||
pass
|
||||
|
||||
|
||||
@@ -120,12 +86,7 @@ class ShelfMixin(object):
|
||||
if not hasattr(self, '_sh') or self._sh is None:
|
||||
self.__dict__['_sh'] = {}
|
||||
if os.path.isfile(self.shelf_file):
|
||||
try:
|
||||
self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb'))
|
||||
except (IndexError, EOFError, pickle.UnpicklingError):
|
||||
logger.warning('{} has a corrupted shelf file!'.format(self.id))
|
||||
if not self.get('force_shelf', False):
|
||||
raise
|
||||
self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb'))
|
||||
return self._sh
|
||||
|
||||
@sh.deleter
|
||||
@@ -147,137 +108,3 @@ class ShelfMixin(object):
|
||||
if hasattr(self, '_sh') and self._sh is not None:
|
||||
with open(self.shelf_file, 'wb') as f:
|
||||
pickle.dump(self._sh, f)
|
||||
|
||||
|
||||
default_plugin_type = API_PARAMS['plugin_type']['default']
|
||||
|
||||
|
||||
def pfilter(plugins, **kwargs):
|
||||
""" Filter plugins by different criteria """
|
||||
if isinstance(plugins, models.Plugins):
|
||||
plugins = plugins.plugins
|
||||
elif isinstance(plugins, dict):
|
||||
plugins = plugins.values()
|
||||
ptype = kwargs.pop('plugin_type', default_plugin_type)
|
||||
logger.debug('#' * 100)
|
||||
logger.debug('ptype {}'.format(ptype))
|
||||
if ptype:
|
||||
try:
|
||||
ptype = ptype[0].upper() + ptype[1:]
|
||||
pclass = globals()[ptype]
|
||||
logger.debug('Class: {}'.format(pclass))
|
||||
candidates = filter(lambda x: isinstance(x, pclass),
|
||||
plugins)
|
||||
except KeyError:
|
||||
raise models.Error('{} is not a valid type'.format(ptype))
|
||||
else:
|
||||
candidates = plugins
|
||||
|
||||
logger.debug(candidates)
|
||||
|
||||
def matches(plug):
|
||||
res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
|
||||
logger.debug(
|
||||
"matching {} with {}: {}".format(plug.name, kwargs, res))
|
||||
return res
|
||||
|
||||
if kwargs:
|
||||
candidates = filter(matches, candidates)
|
||||
return {p.name: p for p in candidates}
|
||||
|
||||
|
||||
def validate_info(info):
|
||||
return all(x in info for x in ('name', 'module', 'description', 'version'))
|
||||
|
||||
|
||||
def load_module(name, root=None):
|
||||
if root:
|
||||
sys.path.append(root)
|
||||
tmp = importlib.import_module(name)
|
||||
if root:
|
||||
sys.path.remove(root)
|
||||
return tmp
|
||||
|
||||
|
||||
def log_subprocess_output(process):
|
||||
for line in iter(process.stdout.readline, b''):
|
||||
logger.info('%r', line)
|
||||
for line in iter(process.stderr.readline, b''):
|
||||
logger.error('%r', line)
|
||||
|
||||
|
||||
def install_deps(*plugins):
|
||||
for info in plugins:
|
||||
requirements = info.get('requirements', [])
|
||||
if requirements:
|
||||
pip_args = ['pip']
|
||||
pip_args.append('install')
|
||||
pip_args.append('--use-wheel')
|
||||
for req in requirements:
|
||||
pip_args.append(req)
|
||||
logger.info('Installing requirements: ' + str(requirements))
|
||||
process = subprocess.Popen(pip_args,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
log_subprocess_output(process)
|
||||
exitcode = process.wait()
|
||||
if exitcode != 0:
|
||||
raise models.Error("Dependencies not properly installed")
|
||||
|
||||
|
||||
def load_plugin_from_info(info, root=None, validator=validate_info, install=True):
|
||||
if not root and '_path' in info:
|
||||
root = os.path.dirname(info['_path'])
|
||||
if not validator(info):
|
||||
raise ValueError('Plugin info is not valid: {}'.format(info))
|
||||
module = info["module"]
|
||||
|
||||
try:
|
||||
tmp = load_module(module, root)
|
||||
except ImportError:
|
||||
if not install:
|
||||
raise
|
||||
install_deps(info)
|
||||
tmp = load_module(module, root)
|
||||
candidate = None
|
||||
for _, obj in inspect.getmembers(tmp):
|
||||
if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
|
||||
logger.debug(("Found plugin class:"
|
||||
" {}@{}").format(obj, inspect.getmodule(obj)))
|
||||
candidate = obj
|
||||
break
|
||||
if not candidate:
|
||||
logger.debug("No valid plugin for: {}".format(module))
|
||||
return
|
||||
module = candidate(info=info)
|
||||
return module
|
||||
|
||||
|
||||
def parse_plugin_info(fpath):
|
||||
logger.debug("Loading plugin: {}".format(fpath))
|
||||
with open(fpath, 'r') as f:
|
||||
info = yaml.load(f)
|
||||
info['_path'] = fpath
|
||||
name = info['name']
|
||||
return name, info
|
||||
|
||||
|
||||
def load_plugin(fpath):
|
||||
name, info = parse_plugin_info(fpath)
|
||||
logger.debug("Info: {}".format(info))
|
||||
plugin = load_plugin_from_info(info)
|
||||
return name, plugin
|
||||
|
||||
|
||||
def load_plugins(folders, loader=load_plugin):
|
||||
plugins = {}
|
||||
for search_folder in folders:
|
||||
for root, dirnames, filenames in os.walk(search_folder):
|
||||
# Do not look for plugins in hidden or special folders
|
||||
dirnames[:] = [d for d in dirnames if d[0] not in ['.', '_']]
|
||||
for filename in fnmatch.filter(filenames, '*.senpy'):
|
||||
fpath = os.path.join(root, filename)
|
||||
name, plugin = loader(fpath)
|
||||
if plugin and name:
|
||||
plugins[name] = plugin
|
||||
return plugins
|
||||
|
||||
52
senpy/plugins/conversion/centroids.py
Normal file
52
senpy/plugins/conversion/centroids.py
Normal file
@@ -0,0 +1,52 @@
|
||||
from senpy.plugins import EmotionConversionPlugin
|
||||
from senpy.models import EmotionSet, Emotion, Error
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CentroidConversion(EmotionConversionPlugin):
|
||||
|
||||
def _forward_conversion(self, original):
|
||||
"""Sum the VAD value of all categories found."""
|
||||
res = Emotion()
|
||||
for e in original.onyx__hasEmotion:
|
||||
category = e.onyx__hasEmotionCategory
|
||||
if category in self.centroids:
|
||||
for dim, value in self.centroids[category].items():
|
||||
try:
|
||||
res[dim] += value
|
||||
except Exception:
|
||||
res[dim] = value
|
||||
return res
|
||||
|
||||
def _backwards_conversion(self, original):
|
||||
"""Find the closest category"""
|
||||
dimensions = list(self.centroids.values())[0]
|
||||
|
||||
def distance(e1, e2):
|
||||
return sum((e1[k] - e2.get(self.aliases[k], 0)) for k in dimensions)
|
||||
|
||||
emotion = ''
|
||||
mindistance = 10000000000000000000000.0
|
||||
for state in self.centroids:
|
||||
d = distance(self.centroids[state], original)
|
||||
if d < mindistance:
|
||||
mindistance = d
|
||||
emotion = state
|
||||
result = Emotion(onyx__hasEmotionCategory=emotion)
|
||||
return result
|
||||
|
||||
def convert(self, emotionSet, fromModel, toModel, params):
|
||||
|
||||
cf, ct = self.centroids_direction
|
||||
logger.debug('{}\n{}\n{}\n{}'.format(emotionSet, fromModel, toModel, params))
|
||||
e = EmotionSet()
|
||||
if fromModel == cf:
|
||||
e.onyx__hasEmotion.append(self._forward_conversion(emotionSet))
|
||||
elif fromModel == ct:
|
||||
for i in emotionSet.onyx__hasEmotion:
|
||||
e.onyx__hasEmotion.append(self._backwards_conversion(i))
|
||||
else:
|
||||
raise Error('EMOTION MODEL NOT KNOWN')
|
||||
yield e
|
||||
@@ -1,153 +0,0 @@
|
||||
from senpy.plugins import EmotionConversionPlugin
|
||||
from senpy.models import EmotionSet, Emotion, Error
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CentroidConversion(EmotionConversionPlugin):
|
||||
def __init__(self, info):
|
||||
if 'centroids' not in info:
|
||||
raise Error('Centroid conversion plugins should provide '
|
||||
'the centroids in their senpy file')
|
||||
if 'onyx:doesConversion' not in info:
|
||||
if 'centroids_direction' not in info:
|
||||
raise Error('Please, provide centroids direction')
|
||||
|
||||
cf, ct = info['centroids_direction']
|
||||
info['onyx:doesConversion'] = [{
|
||||
'onyx:conversionFrom': cf,
|
||||
'onyx:conversionTo': ct
|
||||
}, {
|
||||
'onyx:conversionFrom': ct,
|
||||
'onyx:conversionTo': cf
|
||||
}]
|
||||
|
||||
if 'aliases' in info:
|
||||
aliases = info['aliases']
|
||||
ncentroids = {}
|
||||
for k1, v1 in info['centroids'].items():
|
||||
nv1 = {}
|
||||
for k2, v2 in v1.items():
|
||||
nv1[aliases.get(k2, k2)] = v2
|
||||
ncentroids[aliases.get(k1, k1)] = nv1
|
||||
info['centroids'] = ncentroids
|
||||
|
||||
super(CentroidConversion, self).__init__(info)
|
||||
|
||||
self.dimensions = set()
|
||||
for c in self.centroids.values():
|
||||
self.dimensions.update(c.keys())
|
||||
self.neutralPoints = self.get("neutralPoints", dict())
|
||||
if not self.neutralPoints:
|
||||
for i in self.dimensions:
|
||||
self.neutralPoints[i] = self.get("neutralValue", 0)
|
||||
|
||||
def _forward_conversion(self, original):
|
||||
"""Sum the VAD value of all categories found weighted by intensity.
|
||||
Intensities are scaled by onyx:maxIntensityValue if it is present, else maxIntensityValue
|
||||
is assumed to be one. Emotion entries that do not have onxy:hasEmotionIntensity specified
|
||||
are assumed to have maxIntensityValue. Emotion entries that do not have
|
||||
onyx:hasEmotionCategory specified are ignored."""
|
||||
res = Emotion()
|
||||
maxIntensity = float(original.get("onyx:maxIntensityValue", 1))
|
||||
for e in original.onyx__hasEmotion:
|
||||
category = e.get("onyx:hasEmotionCategory", None)
|
||||
if not category:
|
||||
continue
|
||||
intensity = e.get("onyx:hasEmotionIntensity", maxIntensity) / maxIntensity
|
||||
if not intensity:
|
||||
continue
|
||||
centroid = self.centroids.get(category, None)
|
||||
if centroid:
|
||||
for dim, value in centroid.items():
|
||||
neutral = self.neutralPoints[dim]
|
||||
if dim not in res:
|
||||
res[dim] = 0
|
||||
res[dim] += (value - neutral) * intensity + neutral
|
||||
return res
|
||||
|
||||
def _backwards_conversion(self, original):
|
||||
"""Find the closest category"""
|
||||
centroids = self.centroids
|
||||
neutralPoints = self.neutralPoints
|
||||
dimensions = self.dimensions
|
||||
|
||||
def distance_k(centroid, original, k):
|
||||
# k component of the distance between the value and a given centroid
|
||||
return (centroid.get(k, neutralPoints[k]) - original.get(k, neutralPoints[k]))**2
|
||||
|
||||
def distance(centroid):
|
||||
return sum(distance_k(centroid, original, k) for k in dimensions)
|
||||
|
||||
emotion = min(centroids, key=lambda x: distance(centroids[x]))
|
||||
|
||||
result = Emotion(onyx__hasEmotionCategory=emotion)
|
||||
result.onyx__algorithmConfidence = distance(centroids[emotion])
|
||||
return result
|
||||
|
||||
def convert(self, emotionSet, fromModel, toModel, params):
|
||||
|
||||
cf, ct = self.centroids_direction
|
||||
logger.debug(
|
||||
'{}\n{}\n{}\n{}'.format(emotionSet, fromModel, toModel, params))
|
||||
e = EmotionSet()
|
||||
if fromModel == cf and toModel == ct:
|
||||
e.onyx__hasEmotion.append(self._forward_conversion(emotionSet))
|
||||
elif fromModel == ct and toModel == cf:
|
||||
for i in emotionSet.onyx__hasEmotion:
|
||||
e.onyx__hasEmotion.append(self._backwards_conversion(i))
|
||||
else:
|
||||
raise Error('EMOTION MODEL NOT KNOWN')
|
||||
yield e
|
||||
|
||||
def test(self, info=None):
|
||||
if not info:
|
||||
info = {
|
||||
"name": "CentroidTest",
|
||||
"description": "Centroid test",
|
||||
"version": 0,
|
||||
"centroids": {
|
||||
"c1": {"V1": 0.5,
|
||||
"V2": 0.5},
|
||||
"c2": {"V1": -0.5,
|
||||
"V2": 0.5},
|
||||
"c3": {"V1": -0.5,
|
||||
"V2": -0.5},
|
||||
"c4": {"V1": 0.5,
|
||||
"V2": -0.5}},
|
||||
"aliases": {
|
||||
"V1": "X-dimension",
|
||||
"V2": "Y-dimension"
|
||||
},
|
||||
"centroids_direction": ["emoml:big6", "emoml:fsre-dimensions"]
|
||||
}
|
||||
|
||||
c = CentroidConversion(info)
|
||||
|
||||
es1 = EmotionSet()
|
||||
e1 = Emotion()
|
||||
e1.onyx__hasEmotionCategory = "c1"
|
||||
es1.onyx__hasEmotion.append(e1)
|
||||
res = c._forward_conversion(es1)
|
||||
assert res["X-dimension"] == 0.5
|
||||
assert res["Y-dimension"] == 0.5
|
||||
|
||||
e2 = Emotion()
|
||||
e2.onyx__hasEmotionCategory = "c2"
|
||||
es1.onyx__hasEmotion.append(e2)
|
||||
res = c._forward_conversion(es1)
|
||||
assert res["X-dimension"] == 0
|
||||
assert res["Y-dimension"] == 1
|
||||
|
||||
e = Emotion()
|
||||
e["X-dimension"] = -0.2
|
||||
e["Y-dimension"] = -0.3
|
||||
res = c._backwards_conversion(e)
|
||||
assert res["onyx:hasEmotionCategory"] == "c3"
|
||||
|
||||
e = Emotion()
|
||||
e["X-dimension"] = -0.2
|
||||
e["Y-dimension"] = 0.3
|
||||
res = c._backwards_conversion(e)
|
||||
assert res["onyx:hasEmotionCategory"] == "c2"
|
||||
@@ -1,52 +0,0 @@
|
||||
---
|
||||
name: Ekman2FSRE
|
||||
module: senpy.plugins.conversion.emotion.centroids
|
||||
description: Plugin to convert emotion sets from Ekman to VAD
|
||||
version: 0.2
|
||||
# No need to specify onyx:doesConversion because centroids.py adds it automatically from centroids_direction
|
||||
neutralValue: 5.0
|
||||
centroids:
|
||||
anger:
|
||||
A: 6.95
|
||||
D: 5.1
|
||||
V: 2.7
|
||||
S: 5.0
|
||||
disgust:
|
||||
A: 5.3
|
||||
D: 8.05
|
||||
V: 2.7
|
||||
S: 5.0
|
||||
fear:
|
||||
A: 6.5
|
||||
D: 3.6
|
||||
V: 3.2
|
||||
S: 5.0
|
||||
happiness:
|
||||
A: 7.22
|
||||
D: 6.28
|
||||
V: 8.6
|
||||
S: 5.0
|
||||
sadness:
|
||||
A: 5.21
|
||||
D: 2.82
|
||||
V: 2.21
|
||||
S: 5.0
|
||||
surprise:
|
||||
A: 5.0
|
||||
D: 5.0
|
||||
V: 5.0
|
||||
S: 10.0
|
||||
centroids_direction:
|
||||
- emoml:big6
|
||||
- emoml:fsre-dimensions
|
||||
aliases: # These are aliases for any key in the centroid, to avoid repeating a long name several times
|
||||
A: emoml:fsre-dimensions_arousal
|
||||
V: emoml:fsre-dimensions_valence
|
||||
D: emoml:fsre-dimensions_potency
|
||||
S: emoml:fsre-dimensions_unpredictability
|
||||
anger: emoml:big6anger
|
||||
disgust: emoml:big6disgust
|
||||
fear: emoml:big6fear
|
||||
happiness: emoml:big6happiness
|
||||
sadness: emoml:big6sadness
|
||||
surprise: emoml:big6surprise
|
||||
@@ -1,40 +1,38 @@
|
||||
---
|
||||
name: Ekman2PAD
|
||||
module: senpy.plugins.conversion.emotion.centroids
|
||||
name: Ekman2VAD
|
||||
module: senpy.plugins.conversion.centroids
|
||||
description: Plugin to convert emotion sets from Ekman to VAD
|
||||
version: 0.2
|
||||
# No need to specify onyx:doesConversion because centroids.py adds it automatically from centroids_direction
|
||||
neutralValue: 5.0
|
||||
version: 0.1
|
||||
onyx:doesConversion:
|
||||
- onyx:conversionFrom: emoml:big6
|
||||
onyx:conversionTo: emoml:fsre-dimensions
|
||||
- onyx:conversionFrom: emoml:fsre-dimensions
|
||||
onyx:conversionTo: emoml:big6
|
||||
centroids:
|
||||
anger:
|
||||
emoml:big6anger:
|
||||
A: 6.95
|
||||
D: 5.1
|
||||
V: 2.7
|
||||
disgust:
|
||||
emoml:big6disgust:
|
||||
A: 5.3
|
||||
D: 8.05
|
||||
V: 2.7
|
||||
fear:
|
||||
emoml:big6fear:
|
||||
A: 6.5
|
||||
D: 3.6
|
||||
V: 3.2
|
||||
happiness:
|
||||
emoml:big6happiness:
|
||||
A: 7.22
|
||||
D: 6.28
|
||||
V: 8.6
|
||||
sadness:
|
||||
emoml:big6sadness:
|
||||
A: 5.21
|
||||
D: 2.82
|
||||
V: 2.21
|
||||
centroids_direction:
|
||||
- emoml:big6
|
||||
- emoml:pad
|
||||
aliases: # These are aliases for any key in the centroid, to avoid repeating a long name several times
|
||||
A: emoml:pad-dimensions:arousal
|
||||
V: emoml:pad-dimensions:pleasure
|
||||
D: emoml:pad-dimensions:dominance
|
||||
anger: emoml:big6anger
|
||||
disgust: emoml:big6disgust
|
||||
fear: emoml:big6fear
|
||||
happiness: emoml:big6happiness
|
||||
sadness: emoml:big6sadness
|
||||
- emoml:fsre-dimensions
|
||||
aliases:
|
||||
A: emoml:arousal
|
||||
V: emoml:valence
|
||||
D: emoml:dominance
|
||||
@@ -1,7 +1,7 @@
|
||||
import random
|
||||
|
||||
from senpy.plugins import EmotionPlugin
|
||||
from senpy.models import EmotionSet, Emotion, Entry
|
||||
from senpy.models import EmotionSet, Emotion
|
||||
|
||||
|
||||
class RmoRandPlugin(EmotionPlugin):
|
||||
@@ -16,11 +16,3 @@ class RmoRandPlugin(EmotionPlugin):
|
||||
emotionSet.prov__wasGeneratedBy = self.id
|
||||
entry.emotions.append(emotionSet)
|
||||
yield entry
|
||||
|
||||
def test(self):
|
||||
params = dict()
|
||||
results = list()
|
||||
for i in range(100):
|
||||
res = next(self.analyse_entry(Entry(nif__isString="Hello"), params))
|
||||
res.validate()
|
||||
results.append(res.emotions[0]['onyx:hasEmotion'][0]['onyx:hasEmotionCategory'])
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import random
|
||||
|
||||
from senpy.plugins import SentimentPlugin
|
||||
from senpy.models import Sentiment, Entry
|
||||
from senpy.models import Sentiment
|
||||
|
||||
|
||||
class RandPlugin(SentimentPlugin):
|
||||
@@ -22,13 +22,3 @@ class RandPlugin(SentimentPlugin):
|
||||
entry.sentiments.append(sentiment)
|
||||
entry.language = lang
|
||||
yield entry
|
||||
|
||||
def test(self):
|
||||
params = dict()
|
||||
results = list()
|
||||
for i in range(100):
|
||||
res = next(self.analyse_entry(Entry(nif__isString="Hello"), params))
|
||||
res.validate()
|
||||
results.append(res.sentiments[0]['marl:hasPolarity'])
|
||||
assert 'marl:Positive' in results
|
||||
assert 'marl:Negative' in results
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
from senpy.plugins import AnalysisPlugin
|
||||
from senpy.models import Entry
|
||||
from nltk.tokenize.punkt import PunktSentenceTokenizer
|
||||
from nltk.tokenize.simple import LineTokenizer
|
||||
import nltk
|
||||
|
||||
|
||||
class SplitPlugin(AnalysisPlugin):
|
||||
|
||||
def activate(self):
|
||||
nltk.download('punkt')
|
||||
|
||||
def analyse_entry(self, entry, params):
|
||||
chunker_type = params.get("delimiter", "sentence")
|
||||
original_text = entry.get('nif:isString', None)
|
||||
if chunker_type == "sentence":
|
||||
tokenizer = PunktSentenceTokenizer()
|
||||
if chunker_type == "paragraph":
|
||||
tokenizer = LineTokenizer()
|
||||
chars = tokenizer.span_tokenize(original_text)
|
||||
for i, chunk in enumerate(tokenizer.tokenize(original_text)):
|
||||
e = Entry()
|
||||
e['nif:isString'] = chunk
|
||||
if entry.id:
|
||||
e.id = entry.id + "#char={},{}".format(chars[i][0], chars[i][1])
|
||||
yield e
|
||||
|
||||
test_cases = [
|
||||
{
|
||||
'entry': {
|
||||
'nif:isString': 'Hello. World.'
|
||||
},
|
||||
'params': {
|
||||
'delimiter': 'sentence',
|
||||
},
|
||||
'expected': [
|
||||
{
|
||||
'nif:isString': 'Hello.'
|
||||
},
|
||||
{
|
||||
'nif:isString': 'World.'
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
'entry': {
|
||||
"id": ":test",
|
||||
'nif:isString': 'Hello. World.'
|
||||
},
|
||||
'params': {
|
||||
'delimiter': 'sentence',
|
||||
},
|
||||
'expected': [
|
||||
{
|
||||
"@id": ":test#char=0,6",
|
||||
'nif:isString': 'Hello.'
|
||||
},
|
||||
{
|
||||
"@id": ":test#char=7,13",
|
||||
'nif:isString': 'World.'
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,19 +0,0 @@
|
||||
---
|
||||
name: split
|
||||
module: senpy.plugins.misc.split
|
||||
description: A sample plugin that chunks input text
|
||||
author: "@militarpancho"
|
||||
version: '0.2'
|
||||
url: "https://github.com/gsi-upm/senpy"
|
||||
requirements:
|
||||
- nltk
|
||||
extra_params:
|
||||
delimiter:
|
||||
aliases:
|
||||
- type
|
||||
- t
|
||||
required: false
|
||||
default: sentence
|
||||
options:
|
||||
- sentence
|
||||
- paragraph
|
||||
@@ -12,7 +12,7 @@ class Sentiment140Plugin(SentimentPlugin):
|
||||
json.dumps({
|
||||
"language": lang,
|
||||
"data": [{
|
||||
"text": entry['nif:isString']
|
||||
"text": entry.text
|
||||
}]
|
||||
}))
|
||||
p = params.get("prefix", None)
|
||||
@@ -34,20 +34,3 @@ class Sentiment140Plugin(SentimentPlugin):
|
||||
entry.sentiments.append(sentiment)
|
||||
entry.language = lang
|
||||
yield entry
|
||||
|
||||
test_cases = [
|
||||
{
|
||||
'entry': {
|
||||
'nif:isString': 'I love Titanic'
|
||||
},
|
||||
'params': {},
|
||||
'expected': {
|
||||
"nif:isString": "I love Titanic",
|
||||
'sentiments': [
|
||||
{
|
||||
'marl:hasPolarity': 'marl:Positive',
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -20,16 +20,10 @@
|
||||
"@id": "me:hasSuggestions",
|
||||
"@container": "@set"
|
||||
},
|
||||
"onyx:hasEmotion": {
|
||||
"@container": "@set"
|
||||
},
|
||||
"emotions": {
|
||||
"@id": "onyx:hasEmotionSet",
|
||||
"@container": "@set"
|
||||
},
|
||||
"onyx:hasEmotion": {
|
||||
"@container": "@set"
|
||||
},
|
||||
"sentiments": {
|
||||
"@id": "marl:hasOpinion",
|
||||
"@container": "@set"
|
||||
@@ -43,12 +37,6 @@
|
||||
"@type": "@id",
|
||||
"@container": "@set"
|
||||
},
|
||||
"options": {
|
||||
"@container": "@set"
|
||||
},
|
||||
"plugins": {
|
||||
"@container": "@set"
|
||||
},
|
||||
"prov:wasGeneratedBy": {
|
||||
"@type": "@id"
|
||||
},
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
"nif:beginIndex": {"type": "integer"},
|
||||
"nif:endIndex": {"type": "integer"},
|
||||
"nif:anchorOf": {
|
||||
"description": "Piece of context that contains the Emotion",
|
||||
"description": "Piece of context that contains the Sentiment",
|
||||
"type": "string"
|
||||
},
|
||||
"onyx:hasDimension": {
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"type": "string"
|
||||
},
|
||||
"nif:isString": {
|
||||
"description": "String contained in this Context. Alternative: nif:isString",
|
||||
"description": "String contained in this Context",
|
||||
"type": "string"
|
||||
},
|
||||
"sentiments": {
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"allOf": [
|
||||
{"$ref": "response.json"},
|
||||
{
|
||||
"title": "Help",
|
||||
"description": "Help containing accepted parameters",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"parameters": {
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"required": "parameters"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -3,14 +3,14 @@
|
||||
"allOf": [
|
||||
{"$ref": "response.json"},
|
||||
{
|
||||
"required": ["plugins"],
|
||||
"properties": {
|
||||
"plugins": {
|
||||
"type": "array",
|
||||
"default": [],
|
||||
"items": {
|
||||
"$ref": "plugin.json"
|
||||
}
|
||||
},
|
||||
"@type": {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,12 +2,7 @@
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"@type": {"type": "string"},
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"default": {}
|
||||
|
||||
}
|
||||
"@type": {"type": "string"}
|
||||
},
|
||||
"required": ["@type"]
|
||||
|
||||
|
||||
@@ -18,16 +18,10 @@
|
||||
"type": "string"
|
||||
},
|
||||
"analysis": {
|
||||
"default": [],
|
||||
"type": "array",
|
||||
"default": [],
|
||||
"items": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "analysis.json"
|
||||
},{
|
||||
"type": "string"
|
||||
}
|
||||
]
|
||||
"$ref": "analysis.json"
|
||||
}
|
||||
},
|
||||
"entries": {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
var ONYX = "http://www.gsi.dit.upm.es/ontologies/onyx/ns#";
|
||||
var RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type";
|
||||
var plugins_params={};
|
||||
var default_params = JSON.parse($.ajax({type: "GET", url: "/api?help=True" , async: false}).responseText);
|
||||
|
||||
function replaceURLWithHTMLLinks(text) {
|
||||
console.log('Text: ' + text);
|
||||
var exp = /(\b(https?|ftp|file):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])/ig;
|
||||
@@ -28,68 +28,54 @@ function hashchanged(){
|
||||
$(document).ready(function() {
|
||||
var response = JSON.parse($.ajax({type: "GET", url: "/api/plugins/" , async: false}).responseText);
|
||||
var defaultPlugin= JSON.parse($.ajax({type: "GET", url: "/api/plugins/default" , async: false}).responseText);
|
||||
html="";
|
||||
html="";
|
||||
var availablePlugins = document.getElementById('availablePlugins');
|
||||
plugins = response.plugins;
|
||||
gplugins = {};
|
||||
for (r in plugins){
|
||||
ptype = plugins[r]['@type'];
|
||||
if(gplugins[ptype] == undefined){
|
||||
gplugins[ptype] = [r]
|
||||
}else{
|
||||
gplugins[ptype].push(r)
|
||||
}
|
||||
}
|
||||
for (g in gplugins){
|
||||
html += "<optgroup label=\""+g+"\">"
|
||||
for (r in gplugins[g]){
|
||||
plugin = plugins[r]
|
||||
if (plugin["name"]){
|
||||
if (plugin["name"] == defaultPlugin["name"]){
|
||||
if (plugin["is_activated"]){
|
||||
html+= "<option value=\""+plugin["name"]+"\" selected=\"selected\">"+plugin["name"]+"</option>"
|
||||
}else{
|
||||
html+= "<option value=\""+plugin["name"]+"\" selected=\"selected\" disabled=\"disabled\">"+plugin["name"]+"</option>"
|
||||
}
|
||||
}
|
||||
else{
|
||||
if (plugin["is_activated"]){
|
||||
html+= "<option value=\""+plugin["name"]+"\">"+plugin["name"]+"</option>"
|
||||
}
|
||||
else{
|
||||
html+= "<option value=\""+plugin["name"]+"\" disabled=\"disabled\">"+plugin["name"]+"</option>"
|
||||
}
|
||||
}
|
||||
}
|
||||
for (r in plugins){
|
||||
plugin = plugins[r]
|
||||
if (plugin["name"]){
|
||||
if (plugin["name"] == defaultPlugin["name"]){
|
||||
if (plugin["is_activated"]){
|
||||
html+= "<option value=\""+plugin["name"]+"\" selected=\"selected\">"+plugin["name"]+"</option>"
|
||||
}else{
|
||||
html+= "<option value=\""+plugin["name"]+"\" selected=\"selected\" disabled=\"disabled\">"+plugin["name"]+"</option>"
|
||||
}
|
||||
}
|
||||
else{
|
||||
if (plugin["is_activated"]){
|
||||
html+= "<option value=\""+plugin["name"]+"\">"+plugin["name"]+"</option>"
|
||||
}
|
||||
else{
|
||||
html+= "<option value=\""+plugin["name"]+"\" disabled=\"disabled\">"+plugin["name"]+"</option>"
|
||||
}
|
||||
}
|
||||
}
|
||||
if (plugin["extra_params"]){
|
||||
plugins_params[plugin["name"]]={};
|
||||
for (param in plugin["extra_params"]){
|
||||
if (typeof plugin["extra_params"][param] !="string"){
|
||||
var params = new Array();
|
||||
var alias = plugin["extra_params"][param]["aliases"][0];
|
||||
params[alias]=new Array();
|
||||
for (option in plugin["extra_params"][param]["options"]){
|
||||
params[alias].push(plugin["extra_params"][param]["options"][option])
|
||||
}
|
||||
plugins_params[plugin["name"]][alias] = (params[alias])
|
||||
}
|
||||
}
|
||||
}
|
||||
var pluginList = document.createElement('li');
|
||||
|
||||
if (plugin["extra_params"]){
|
||||
plugins_params[plugin["name"]]={};
|
||||
for (param in plugin["extra_params"]){
|
||||
if (typeof plugin["extra_params"][param] !="string"){
|
||||
var params = new Array();
|
||||
var alias = plugin["extra_params"][param]["aliases"][0];
|
||||
params[alias]=new Array();
|
||||
for (option in plugin["extra_params"][param]["options"]){
|
||||
params[alias].push(plugin["extra_params"][param]["options"][option])
|
||||
}
|
||||
plugins_params[plugin["name"]][alias] = (params[alias])
|
||||
}
|
||||
}
|
||||
}
|
||||
var pluginList = document.createElement('li');
|
||||
|
||||
newHtml = ""
|
||||
if(plugin.url) {
|
||||
newHtml= "<a href="+plugin.url+">" + plugin.name + "</a>";
|
||||
}else {
|
||||
newHtml= plugin["name"];
|
||||
}
|
||||
newHtml += ": " + replaceURLWithHTMLLinks(plugin.description);
|
||||
pluginList.innerHTML = newHtml;
|
||||
availablePlugins.appendChild(pluginList)
|
||||
}
|
||||
html += "</optgroup>"
|
||||
}
|
||||
newHtml = ""
|
||||
if(plugin.url) {
|
||||
newHtml= "<a href="+plugin.url+">" + plugin.name + "</a>";
|
||||
}else {
|
||||
newHtml= plugin["name"];
|
||||
}
|
||||
newHtml += ": " + replaceURLWithHTMLLinks(plugin.description);
|
||||
pluginList.innerHTML = newHtml;
|
||||
availablePlugins.appendChild(pluginList)
|
||||
}
|
||||
document.getElementById('plugins').innerHTML = html;
|
||||
change_params();
|
||||
|
||||
@@ -102,23 +88,8 @@ $(document).ready(function() {
|
||||
|
||||
function change_params(){
|
||||
var plugin = document.getElementById("plugins").options[document.getElementById("plugins").selectedIndex].value;
|
||||
html=""
|
||||
for (param in default_params){
|
||||
if ((default_params[param]['options']) && (['help','conversion'].indexOf(param) < 0)){
|
||||
html+= "<label> "+param+"</label>"
|
||||
html+= "<select id=\""+param+"\" name=\""+param+"\">"
|
||||
for (option in default_params[param]['options']){
|
||||
if (default_params[param]['options'][option] == default_params[param]['default']){
|
||||
html+="<option value \""+default_params[param]['options'][option]+"\" selected >"+default_params[param]['options'][option]+"</option>"
|
||||
}
|
||||
else{
|
||||
html+="<option value \""+default_params[param]['options'][option]+"\">"+default_params[param]['options'][option]+"</option>"
|
||||
|
||||
}
|
||||
}
|
||||
html+="</select><br>"
|
||||
}
|
||||
}
|
||||
html=""
|
||||
for (param in plugins_params[plugin]){
|
||||
if (param || plugins_params[plugin][param].length > 1){
|
||||
html+= "<label> Parameter "+param+"</label>"
|
||||
@@ -149,32 +120,15 @@ function load_JSON(){
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (param in default_params){
|
||||
if ((param != null) && (default_params[param]['options']) && (['help','conversion'].indexOf(param) < 0)){
|
||||
var param_value = encodeURIComponent(document.getElementById(param).options[document.getElementById(param).selectedIndex].text);
|
||||
if (param_value){
|
||||
url+="&"+param+"="+param_value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var response = $.ajax({type: "GET", url: url , async: false}).responseText;
|
||||
rawcontainer.innerHTML = replaceURLWithHTMLLinks(response)
|
||||
|
||||
document.getElementById("input_request").innerHTML = "<a href='"+url+"'>"+url+"</a>"
|
||||
document.getElementById("results-div").style.display = 'block';
|
||||
try {
|
||||
response = JSON.parse(response);
|
||||
var response = JSON.parse($.ajax({type: "GET", url: url , async: false}).responseText);
|
||||
var options = {
|
||||
mode: 'view'
|
||||
};
|
||||
var editor = new JSONEditor(container, options, response);
|
||||
editor.expandAll();
|
||||
}
|
||||
catch(err){
|
||||
console.log("Error decoding JSON (got turtle?)");
|
||||
}
|
||||
rawcontainer.innerHTML = replaceURLWithHTMLLinks(JSON.stringify(response, undefined, 2))
|
||||
document.getElementById("input_request").innerHTML = "<a href='"+url+"'>"+url+"</a>"
|
||||
document.getElementById("results-div").style.display = 'block';
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@
|
||||
This website is the senpy Playground, which allows you to test the instance of senpy in this server. It provides a user-friendly interface to the functions exposed by the senpy API.
|
||||
</p>
|
||||
<p>
|
||||
Once you get comfortable with the parameters and results, you are encouraged to issue your own requests to the API endpoint. You can find examples of API URL's when you try out a plugin with the "Analyse!" button on the "Test it" tab.
|
||||
Once you get comfortable with the parameters and results, you are encouraged to issue your own requests to the API endpoint, which should be <a href="/api">here</a>.
|
||||
</p>
|
||||
<p>
|
||||
These are some of the things you can do with the API:
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
from . import models
|
||||
|
||||
|
||||
def check_template(indict, template):
|
||||
if isinstance(template, dict) and isinstance(indict, dict):
|
||||
for k, v in template.items():
|
||||
if k not in indict:
|
||||
return '{} not in {}'.format(k, indict)
|
||||
check_template(indict[k], v)
|
||||
elif isinstance(template, list) and isinstance(indict, list):
|
||||
if len(indict) != len(template):
|
||||
raise models.Error('Different size for {} and {}'.format(indict, template))
|
||||
for e in template:
|
||||
found = False
|
||||
for i in indict:
|
||||
try:
|
||||
check_template(i, e)
|
||||
found = True
|
||||
except models.Error as ex:
|
||||
continue
|
||||
if not found:
|
||||
raise models.Error('{} not found in {}'.format(e, indict))
|
||||
else:
|
||||
if indict != template:
|
||||
raise models.Error('{} and {} are different'.format(indict, template))
|
||||
@@ -11,7 +11,7 @@ def read_version(versionfile=DEFAULT_FILE):
|
||||
try:
|
||||
with open(versionfile) as f:
|
||||
return f.read().strip()
|
||||
except IOError: # pragma: no cover
|
||||
except IOError:
|
||||
logger.error('Running an unknown version of senpy. Be careful!.')
|
||||
return '0.0'
|
||||
|
||||
|
||||
@@ -12,6 +12,3 @@ max-line-length = 100
|
||||
universal=1
|
||||
[tool:pytest]
|
||||
addopts = --cov=senpy --cov-report term-missing
|
||||
|
||||
[coverage:report]
|
||||
omit = senpy/__main__.py
|
||||
@@ -1,26 +0,0 @@
|
||||
from senpy.plugins import AnalysisPlugin
|
||||
|
||||
import multiprocessing
|
||||
|
||||
|
||||
def _train(process_number):
|
||||
return process_number
|
||||
|
||||
|
||||
class AsyncPlugin(AnalysisPlugin):
|
||||
def _do_async(self, num_processes):
|
||||
pool = multiprocessing.Pool(processes=num_processes)
|
||||
values = pool.map(_train, range(num_processes))
|
||||
|
||||
return values
|
||||
|
||||
def activate(self):
|
||||
self.value = self._do_async(4)
|
||||
|
||||
def analyse_entry(self, entry, params):
|
||||
values = self._do_async(2)
|
||||
entry.async_values = values
|
||||
yield entry
|
||||
|
||||
def test(self):
|
||||
pass
|
||||
@@ -1,8 +0,0 @@
|
||||
---
|
||||
name: Async
|
||||
module: asyncplugin
|
||||
description: I am async
|
||||
author: "@balkian"
|
||||
version: '0.1'
|
||||
async: true
|
||||
extra_params: {}
|
||||
@@ -3,9 +3,5 @@ from senpy.plugins import SentimentPlugin
|
||||
|
||||
class DummyPlugin(SentimentPlugin):
|
||||
def analyse_entry(self, entry, params):
|
||||
entry['nif:iString'] = entry['nif:isString'][::-1]
|
||||
entry.reversed = entry.get('reversed', 0) + 1
|
||||
entry.text = entry.text[::-1]
|
||||
yield entry
|
||||
|
||||
def test(self):
|
||||
pass
|
||||
|
||||
@@ -1,14 +1,11 @@
|
||||
from senpy.plugins import AnalysisPlugin
|
||||
from senpy.plugins import SenpyPlugin
|
||||
from time import sleep
|
||||
|
||||
|
||||
class SleepPlugin(AnalysisPlugin):
|
||||
class SleepPlugin(SenpyPlugin):
|
||||
def activate(self, *args, **kwargs):
|
||||
sleep(self.timeout)
|
||||
|
||||
def analyse_entry(self, entry, params):
|
||||
sleep(float(params.get("timeout", self.timeout)))
|
||||
yield entry
|
||||
|
||||
def test(self):
|
||||
pass
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"description": "I am dummy",
|
||||
"author": "@balkian",
|
||||
"version": "0.1",
|
||||
"timeout": 0.05,
|
||||
"timeout": 0.5,
|
||||
"extra_params": {
|
||||
"timeout": {
|
||||
"@id": "timeout_sleep",
|
||||
|
||||
@@ -11,24 +11,24 @@ class APITest(TestCase):
|
||||
|
||||
def test_api_params(self):
|
||||
"""The API should not define any required parameters without a default"""
|
||||
parse_params({}, API_PARAMS)
|
||||
parse_params({}, spec=API_PARAMS)
|
||||
|
||||
def test_web_params(self):
|
||||
"""The WEB should not define any required parameters without a default"""
|
||||
parse_params({}, WEB_PARAMS)
|
||||
parse_params({}, spec=WEB_PARAMS)
|
||||
|
||||
def test_basic(self):
|
||||
a = {}
|
||||
try:
|
||||
parse_params(a, NIF_PARAMS)
|
||||
parse_params(a, spec=NIF_PARAMS)
|
||||
raise AssertionError()
|
||||
except Error:
|
||||
pass
|
||||
a = {'input': 'hello'}
|
||||
p = parse_params(a, NIF_PARAMS)
|
||||
p = parse_params(a, spec=NIF_PARAMS)
|
||||
assert 'input' in p
|
||||
b = {'i': 'hello'}
|
||||
p = parse_params(b, NIF_PARAMS)
|
||||
p = parse_params(b, spec=NIF_PARAMS)
|
||||
assert 'input' in p
|
||||
|
||||
def test_plugin(self):
|
||||
@@ -40,18 +40,18 @@ class APITest(TestCase):
|
||||
}
|
||||
}
|
||||
try:
|
||||
parse_params(query, plug_params)
|
||||
parse_params(query, spec=plug_params)
|
||||
raise AssertionError()
|
||||
except Error:
|
||||
pass
|
||||
query['hello'] = 'world'
|
||||
p = parse_params(query, plug_params)
|
||||
p = parse_params(query, spec=plug_params)
|
||||
assert 'hello' in p
|
||||
assert p['hello'] == 'world'
|
||||
del query['hello']
|
||||
|
||||
query['hiya'] = 'dlrow'
|
||||
p = parse_params(query, plug_params)
|
||||
p = parse_params(query, spec=plug_params)
|
||||
assert 'hello' in p
|
||||
assert 'hiya' in p
|
||||
assert p['hello'] == 'dlrow'
|
||||
@@ -63,17 +63,6 @@ class APITest(TestCase):
|
||||
'default': 1
|
||||
}
|
||||
}
|
||||
p = parse_params({}, spec)
|
||||
p = parse_params({}, spec=spec)
|
||||
assert 'hello' in p
|
||||
assert p['hello'] == 1
|
||||
|
||||
def test_call(self):
|
||||
call = {
|
||||
'input': "Aloha my friend",
|
||||
'algo': "Dummy"
|
||||
}
|
||||
p = parse_params(call, API_PARAMS, NIF_PARAMS)
|
||||
assert 'algorithm' in p
|
||||
assert "Dummy" in p['algorithm']
|
||||
assert 'input' in p
|
||||
assert p['input'] == 'Aloha my friend'
|
||||
|
||||
@@ -17,19 +17,16 @@ def parse_resp(resp):
|
||||
|
||||
|
||||
class BlueprintsTest(TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Set up only once, and re-use in every individual test"""
|
||||
cls.app = Flask("test_extensions")
|
||||
cls.app.debug = False
|
||||
cls.client = cls.app.test_client()
|
||||
cls.senpy = Senpy()
|
||||
cls.senpy.init_app(cls.app)
|
||||
cls.dir = os.path.join(os.path.dirname(__file__), "..")
|
||||
cls.senpy.add_folder(cls.dir)
|
||||
cls.senpy.activate_plugin("Dummy", sync=True)
|
||||
cls.senpy.activate_plugin("DummyRequired", sync=True)
|
||||
cls.senpy.default_plugin = 'Dummy'
|
||||
def setUp(self):
|
||||
self.app = Flask("test_extensions")
|
||||
self.client = self.app.test_client()
|
||||
self.senpy = Senpy()
|
||||
self.senpy.init_app(self.app)
|
||||
self.dir = os.path.join(os.path.dirname(__file__), "..")
|
||||
self.senpy.add_folder(self.dir)
|
||||
self.senpy.activate_plugin("Dummy", sync=True)
|
||||
self.senpy.activate_plugin("DummyRequired", sync=True)
|
||||
self.senpy.default_plugin = 'Dummy'
|
||||
|
||||
def assertCode(self, resp, code):
|
||||
self.assertEqual(resp.status_code, code)
|
||||
@@ -38,7 +35,6 @@ class BlueprintsTest(TestCase):
|
||||
"""
|
||||
Calling with no arguments should ask the user for more arguments
|
||||
"""
|
||||
self.app.debug = False
|
||||
resp = self.client.get("/api/")
|
||||
self.assertCode(resp, 400)
|
||||
js = parse_resp(resp)
|
||||
@@ -55,7 +51,7 @@ class BlueprintsTest(TestCase):
|
||||
The dummy plugin returns an empty response,\
|
||||
it should contain the context
|
||||
"""
|
||||
resp = self.client.get("/api/?i=My aloha mohame&with_parameters=True")
|
||||
resp = self.client.get("/api/?i=My aloha mohame")
|
||||
self.assertCode(resp, 200)
|
||||
js = parse_resp(resp)
|
||||
logging.debug("Got response: %s", js)
|
||||
@@ -66,7 +62,7 @@ class BlueprintsTest(TestCase):
|
||||
"""
|
||||
Extra params that have a default should
|
||||
"""
|
||||
resp = self.client.get("/api/?i=My aloha mohame&algo=Dummy&with_parameters=true")
|
||||
resp = self.client.get("/api/?i=My aloha mohame&algo=Dummy")
|
||||
self.assertCode(resp, 200)
|
||||
js = parse_resp(resp)
|
||||
logging.debug("Got response: %s", js)
|
||||
@@ -78,7 +74,6 @@ class BlueprintsTest(TestCase):
|
||||
Extra params that have a required argument that does not
|
||||
have a default should raise an error.
|
||||
"""
|
||||
self.app.debug = False
|
||||
resp = self.client.get("/api/?i=My aloha mohame&algo=DummyRequired")
|
||||
self.assertCode(resp, 400)
|
||||
js = parse_resp(resp)
|
||||
@@ -90,7 +85,6 @@ class BlueprintsTest(TestCase):
|
||||
The dummy plugin returns an empty response,\
|
||||
it should contain the context
|
||||
"""
|
||||
self.app.debug = False
|
||||
resp = self.client.get("/api/?i=My aloha mohame&algo=DOESNOTEXIST")
|
||||
self.assertCode(resp, 404)
|
||||
js = parse_resp(resp)
|
||||
@@ -157,10 +151,3 @@ class BlueprintsTest(TestCase):
|
||||
self.assertCode(resp, 200)
|
||||
js = parse_resp(resp)
|
||||
assert "$schema" in js
|
||||
|
||||
def test_help(self):
|
||||
resp = self.client.get("/api/?help=true")
|
||||
self.assertCode(resp, 200)
|
||||
js = parse_resp(resp)
|
||||
assert "parameters" in js
|
||||
assert "help" in js["parameters"]
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
import logging
|
||||
from functools import partial
|
||||
|
||||
try:
|
||||
from unittest.mock import patch
|
||||
except ImportError:
|
||||
from mock import patch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from unittest import TestCase
|
||||
@@ -12,7 +17,11 @@ class CLITest(TestCase):
|
||||
def test_basic(self):
|
||||
self.assertRaises(Error, partial(main_function, []))
|
||||
|
||||
res = main_function(['--input', 'test', '--algo', 'rand', '--with-parameters'])
|
||||
assert res.parameters['input'] == 'test'
|
||||
assert 'rand' in res.parameters['algorithm']
|
||||
assert res.parameters['input'] == 'test'
|
||||
with patch('senpy.extensions.Senpy.analyse') as patched:
|
||||
main_function(['--input', 'test'])
|
||||
|
||||
patched.assert_called_with(input='test')
|
||||
with patch('senpy.extensions.Senpy.analyse') as patched:
|
||||
main_function(['--input', 'test', '--algo', 'rand'])
|
||||
|
||||
patched.assert_called_with(input='test', algo='rand')
|
||||
|
||||
@@ -4,21 +4,18 @@ try:
|
||||
except ImportError:
|
||||
from mock import patch
|
||||
|
||||
import json
|
||||
|
||||
from senpy.client import Client
|
||||
from senpy.models import Results, Plugins, Error
|
||||
from senpy.plugins import AnalysisPlugin, default_plugin_type
|
||||
from senpy.models import Results, Error
|
||||
|
||||
|
||||
class Call(dict):
|
||||
def __init__(self, obj):
|
||||
self.obj = obj.serialize()
|
||||
self.obj = obj.jsonld()
|
||||
self.status_code = 200
|
||||
self.content = self.json()
|
||||
|
||||
def json(self):
|
||||
return json.loads(self.obj)
|
||||
return self.obj
|
||||
|
||||
|
||||
class ModelsTest(TestCase):
|
||||
@@ -47,19 +44,3 @@ class ModelsTest(TestCase):
|
||||
method='GET',
|
||||
params={'input': 'hello',
|
||||
'algorithm': 'NONEXISTENT'})
|
||||
|
||||
def test_plugins(self):
|
||||
endpoint = 'http://dummy/'
|
||||
client = Client(endpoint)
|
||||
plugins = Plugins()
|
||||
p1 = AnalysisPlugin({'name': 'AnalysisP1', 'version': 0, 'description': 'No'})
|
||||
plugins.plugins = [p1, ]
|
||||
success = Call(plugins)
|
||||
with patch('requests.request', return_value=success) as patched:
|
||||
response = client.plugins()
|
||||
assert isinstance(response, dict)
|
||||
assert len(response) == 1
|
||||
assert "AnalysisP1" in response
|
||||
patched.assert_called_with(
|
||||
url=endpoint + '/plugins', method='GET',
|
||||
params={'plugin_type': default_plugin_type})
|
||||
|
||||
@@ -10,22 +10,15 @@ except ImportError:
|
||||
|
||||
from functools import partial
|
||||
from senpy.extensions import Senpy
|
||||
from senpy import plugins
|
||||
from senpy.models import Error, Results, Entry, EmotionSet, Emotion, Plugin
|
||||
from senpy import api
|
||||
from senpy.models import Error, Results, Entry, EmotionSet, Emotion
|
||||
from flask import Flask
|
||||
from unittest import TestCase
|
||||
|
||||
|
||||
def analyse(instance, **kwargs):
|
||||
request = api.parse_call(kwargs)
|
||||
return instance.analyse(request)
|
||||
|
||||
|
||||
class ExtensionsTest(TestCase):
|
||||
def setUp(self):
|
||||
self.app = Flask('test_extensions')
|
||||
self.dir = os.path.dirname(__file__)
|
||||
self.dir = os.path.join(os.path.dirname(__file__))
|
||||
self.senpy = Senpy(plugin_folder=self.dir,
|
||||
app=self.app,
|
||||
default_plugins=False)
|
||||
@@ -45,8 +38,8 @@ class ExtensionsTest(TestCase):
|
||||
print(self.senpy.plugins)
|
||||
assert "Dummy" in self.senpy.plugins
|
||||
|
||||
def test_installing(self):
|
||||
""" Installing a plugin """
|
||||
def test_enabling(self):
|
||||
""" Enabling a plugin """
|
||||
info = {
|
||||
'name': 'TestPip',
|
||||
'module': 'dummy',
|
||||
@@ -55,31 +48,19 @@ class ExtensionsTest(TestCase):
|
||||
'version': 0
|
||||
}
|
||||
root = os.path.join(self.dir, 'plugins', 'dummy_plugin')
|
||||
module = plugins.load_plugin_from_info(info, root=root)
|
||||
plugins.install_deps(info)
|
||||
assert module.name == 'TestPip'
|
||||
name, module = self.senpy._load_plugin_from_info(info, root=root)
|
||||
assert name == 'TestPip'
|
||||
assert module
|
||||
import noop
|
||||
dir(noop)
|
||||
self.senpy.install_deps()
|
||||
|
||||
def test_enabling(self):
|
||||
def test_installing(self):
|
||||
""" Enabling a plugin """
|
||||
self.senpy.activate_all(sync=True)
|
||||
assert len(self.senpy.plugins) >= 3
|
||||
assert self.senpy.plugins["Sleep"].is_activated
|
||||
|
||||
def test_installing_nonexistent(self):
|
||||
""" Fail if the dependencies cannot be met """
|
||||
info = {
|
||||
'name': 'TestPipFail',
|
||||
'module': 'dummy',
|
||||
'description': None,
|
||||
'requirements': ['IAmMakingThisPackageNameUpToFail'],
|
||||
'version': 0
|
||||
}
|
||||
with self.assertRaises(Error):
|
||||
plugins.install_deps(info)
|
||||
|
||||
def test_disabling(self):
|
||||
""" Disabling a plugin """
|
||||
self.senpy.deactivate_all(sync=True)
|
||||
@@ -97,65 +78,37 @@ class ExtensionsTest(TestCase):
|
||||
def test_noplugin(self):
|
||||
""" Don't analyse if there isn't any plugin installed """
|
||||
self.senpy.deactivate_all(sync=True)
|
||||
self.assertRaises(Error, partial(analyse, self.senpy, input="tupni"))
|
||||
self.assertRaises(Error, partial(self.senpy.analyse, input="tupni"))
|
||||
self.assertRaises(Error,
|
||||
partial(
|
||||
self.senpy.analyse,
|
||||
input="tupni",
|
||||
algorithm='Dummy'))
|
||||
|
||||
def test_analyse(self):
|
||||
""" Using a plugin """
|
||||
# I was using mock until plugin started inheriting
|
||||
# Leaf (defaultdict with __setattr__ and __getattr__.
|
||||
r1 = analyse(self.senpy, algorithm="Dummy", input="tupni", output="tuptuo")
|
||||
r2 = analyse(self.senpy, input="tupni", output="tuptuo")
|
||||
r1 = self.senpy.analyse(
|
||||
algorithm="Dummy", input="tupni", output="tuptuo")
|
||||
r2 = self.senpy.analyse(input="tupni", output="tuptuo")
|
||||
assert r1.analysis[0] == "plugins/Dummy_0.1"
|
||||
assert r2.analysis[0] == "plugins/Dummy_0.1"
|
||||
assert r1.entries[0]['nif:iString'] == 'input'
|
||||
|
||||
def test_analyse_jsonld(self):
|
||||
""" Using a plugin with JSON-LD input"""
|
||||
js_input = '''{
|
||||
"@id": "prueba",
|
||||
"@type": "results",
|
||||
"entries": [
|
||||
{"@id": "entry1",
|
||||
"nif:isString": "tupni",
|
||||
"@type": "entry"
|
||||
}
|
||||
]
|
||||
}'''
|
||||
r1 = analyse(self.senpy,
|
||||
algorithm="Dummy",
|
||||
input=js_input,
|
||||
informat="json-ld",
|
||||
output="tuptuo")
|
||||
r2 = analyse(self.senpy,
|
||||
input="tupni",
|
||||
output="tuptuo")
|
||||
assert r1.analysis[0] == "plugins/Dummy_0.1"
|
||||
assert r2.analysis[0] == "plugins/Dummy_0.1"
|
||||
assert r1.entries[0]['nif:iString'] == 'input'
|
||||
assert r1.entries[0].text == 'input'
|
||||
|
||||
def test_analyse_error(self):
|
||||
mm = mock.MagicMock()
|
||||
mm.id = 'magic_mock'
|
||||
mm.is_activated = True
|
||||
mm.analyse_entries.side_effect = Error('error in analysis', status=500)
|
||||
mm.analyse_entry.side_effect = Error('error on analysis', status=900)
|
||||
self.senpy.plugins['MOCK'] = mm
|
||||
try:
|
||||
analyse(self.senpy, input='nothing', algorithm='MOCK')
|
||||
assert False
|
||||
except Error as ex:
|
||||
assert 'error in analysis' in ex['message']
|
||||
assert ex['status'] == 500
|
||||
|
||||
resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
|
||||
assert resp['message'] == 'error on analysis'
|
||||
assert resp['status'] == 900
|
||||
mm.analyse.side_effect = Exception('generic exception on analysis')
|
||||
mm.analyse_entries.side_effect = Exception(
|
||||
mm.analyse_entry.side_effect = Exception(
|
||||
'generic exception on analysis')
|
||||
|
||||
try:
|
||||
analyse(self.senpy, input='nothing', algorithm='MOCK')
|
||||
assert False
|
||||
except Error as ex:
|
||||
assert 'generic exception on analysis' in ex['message']
|
||||
assert ex['status'] == 500
|
||||
resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
|
||||
assert resp['message'] == 'generic exception on analysis'
|
||||
assert resp['status'] == 500
|
||||
|
||||
def test_filtering(self):
|
||||
""" Filtering plugins """
|
||||
@@ -171,40 +124,40 @@ class ExtensionsTest(TestCase):
|
||||
assert len(senpy.plugins) > 1
|
||||
|
||||
def test_convert_emotions(self):
|
||||
self.senpy.activate_all(sync=True)
|
||||
plugin = Plugin({
|
||||
self.senpy.activate_all()
|
||||
plugin = {
|
||||
'id': 'imaginary',
|
||||
'onyx:usesEmotionModel': 'emoml:fsre-dimensions'
|
||||
})
|
||||
}
|
||||
eSet1 = EmotionSet()
|
||||
eSet1.prov__wasGeneratedBy = plugin['@id']
|
||||
eSet1['onyx:hasEmotion'].append(Emotion({
|
||||
'emoml:arousal': 1,
|
||||
'emoml:potency': 0,
|
||||
'emoml:valence': 0
|
||||
}))
|
||||
response = Results({
|
||||
'analysis': [{'plugin': plugin}],
|
||||
'entries': [Entry({
|
||||
'nif:iString': 'much ado about nothing',
|
||||
'text': 'much ado about nothing',
|
||||
'emotions': [eSet1]
|
||||
})]
|
||||
})
|
||||
params = {'emotionModel': 'emoml:big6',
|
||||
'conversion': 'full'}
|
||||
r1 = deepcopy(response)
|
||||
r1.parameters = params
|
||||
self.senpy.convert_emotions(r1)
|
||||
self.senpy.convert_emotions(r1,
|
||||
plugin,
|
||||
params)
|
||||
assert len(r1.entries[0].emotions) == 2
|
||||
params['conversion'] = 'nested'
|
||||
r2 = deepcopy(response)
|
||||
r2.parameters = params
|
||||
self.senpy.convert_emotions(r2)
|
||||
self.senpy.convert_emotions(r2,
|
||||
plugin,
|
||||
params)
|
||||
assert len(r2.entries[0].emotions) == 1
|
||||
assert r2.entries[0].emotions[0]['prov:wasDerivedFrom'] == eSet1
|
||||
params['conversion'] = 'filtered'
|
||||
r3 = deepcopy(response)
|
||||
r3.parameters = params
|
||||
self.senpy.convert_emotions(r3)
|
||||
self.senpy.convert_emotions(r3,
|
||||
plugin,
|
||||
params)
|
||||
assert len(r3.entries[0].emotions) == 1
|
||||
r3.jsonld()
|
||||
|
||||
@@ -11,12 +11,8 @@ from senpy.models import (Emotion,
|
||||
Entry,
|
||||
Error,
|
||||
Results,
|
||||
Sentiment,
|
||||
Plugins,
|
||||
Plugin,
|
||||
from_string,
|
||||
from_dict)
|
||||
from senpy import plugins
|
||||
Sentiment)
|
||||
from senpy.plugins import SenpyPlugin
|
||||
from pprint import pprint
|
||||
|
||||
|
||||
@@ -57,8 +53,8 @@ class ModelsTest(TestCase):
|
||||
assert (received["entries"][0]["nif:isString"] != "Not testing")
|
||||
|
||||
def test_id(self):
|
||||
""" Adding the id after creation should overwrite the automatic ID
|
||||
"""
|
||||
''' Adding the id after creation should overwrite the automatic ID
|
||||
'''
|
||||
r = Entry()
|
||||
j = r.jsonld()
|
||||
assert '@id' in j
|
||||
@@ -98,32 +94,20 @@ class ModelsTest(TestCase):
|
||||
r.validate()
|
||||
|
||||
def test_plugins(self):
|
||||
self.assertRaises(Error, plugins.Plugin)
|
||||
p = plugins.Plugin({"name": "dummy",
|
||||
"version": 0,
|
||||
"extra_params": {
|
||||
"none": {
|
||||
"options": ["es", ],
|
||||
"required": False,
|
||||
"default": "0"
|
||||
}
|
||||
}})
|
||||
self.assertRaises(Error, SenpyPlugin)
|
||||
p = SenpyPlugin({"name": "dummy", "version": 0})
|
||||
c = p.jsonld()
|
||||
assert '@type' in c
|
||||
assert c['@type'] == 'plugin'
|
||||
assert 'info' not in c
|
||||
assert 'repo' not in c
|
||||
assert 'extra_params' in c
|
||||
logging.debug('Framed:')
|
||||
assert "info" not in c
|
||||
assert "repo" not in c
|
||||
assert "extra_params" in c
|
||||
logging.debug("Framed:")
|
||||
logging.debug(c)
|
||||
p.validate()
|
||||
assert 'es' in c['extra_params']['none']['options']
|
||||
assert isinstance(c['extra_params']['none']['options'], list)
|
||||
|
||||
def test_str(self):
|
||||
"""The string representation shouldn't include private variables"""
|
||||
r = Results()
|
||||
p = plugins.Plugin({"name": "STR test", "version": 0})
|
||||
p = SenpyPlugin({"name": "STR test", "version": 0})
|
||||
p._testing = 0
|
||||
s = str(p)
|
||||
assert "_testing" not in s
|
||||
@@ -159,40 +143,3 @@ class ModelsTest(TestCase):
|
||||
print(t)
|
||||
g = rdflib.Graph().parse(data=t, format='turtle')
|
||||
assert len(g) == len(triples)
|
||||
|
||||
def test_plugin_list(self):
|
||||
"""The plugin list should be of type \"plugins\""""
|
||||
plugs = Plugins()
|
||||
c = plugs.jsonld()
|
||||
assert '@type' in c
|
||||
assert c['@type'] == 'plugins'
|
||||
|
||||
def test_single_plugin(self):
|
||||
"""A response with a single plugin should still return a list"""
|
||||
plugs = Plugins()
|
||||
p = Plugin({'id': str(1),
|
||||
'version': 0,
|
||||
'description': 'dummy'})
|
||||
plugs.plugins.append(p)
|
||||
assert isinstance(plugs.plugins, list)
|
||||
js = plugs.jsonld()
|
||||
assert isinstance(js['plugins'], list)
|
||||
|
||||
def test_from_string(self):
|
||||
results = {
|
||||
'@type': 'results',
|
||||
'@id': 'prueba',
|
||||
'entries': [{
|
||||
'@id': 'entry1',
|
||||
'@type': 'entry',
|
||||
'nif:isString': 'TEST'
|
||||
}]
|
||||
}
|
||||
recovered = from_dict(results)
|
||||
assert isinstance(recovered, Results)
|
||||
assert isinstance(recovered.entries[0], Entry)
|
||||
|
||||
string = json.dumps(results)
|
||||
recovered = from_string(string)
|
||||
assert isinstance(recovered, Results)
|
||||
assert isinstance(recovered.entries[0], Entry)
|
||||
|
||||
@@ -6,12 +6,11 @@ import shutil
|
||||
import tempfile
|
||||
|
||||
from unittest import TestCase
|
||||
from senpy.models import Results, Entry, EmotionSet, Emotion
|
||||
from senpy import plugins
|
||||
from senpy.plugins.conversion.emotion.centroids import CentroidConversion
|
||||
from senpy.models import Results, Entry
|
||||
from senpy.plugins import SentimentPlugin, ShelfMixin
|
||||
|
||||
|
||||
class ShelfDummyPlugin(plugins.SentimentPlugin, plugins.ShelfMixin):
|
||||
class ShelfDummyPlugin(SentimentPlugin, ShelfMixin):
|
||||
def activate(self, *args, **kwargs):
|
||||
if 'counter' not in self.sh:
|
||||
self.sh['counter'] = 0
|
||||
@@ -33,6 +32,7 @@ class PluginsTest(TestCase):
|
||||
def tearDown(self):
|
||||
if os.path.exists(self.shelf_dir):
|
||||
shutil.rmtree(self.shelf_dir)
|
||||
|
||||
if os.path.isfile(self.shelf_file):
|
||||
os.remove(self.shelf_file)
|
||||
|
||||
@@ -50,29 +50,26 @@ class PluginsTest(TestCase):
|
||||
|
||||
def test_shelf(self):
|
||||
''' A shelf is created and the value is stored '''
|
||||
newfile = self.shelf_file + "new"
|
||||
a = ShelfDummyPlugin(info={
|
||||
'name': 'shelve',
|
||||
'version': 'test',
|
||||
'shelf_file': newfile
|
||||
'shelf_file': self.shelf_file
|
||||
})
|
||||
assert a.sh == {}
|
||||
a.activate()
|
||||
assert a.sh == {'counter': 0}
|
||||
assert a.shelf_file == newfile
|
||||
assert a.shelf_file == self.shelf_file
|
||||
|
||||
a.sh['a'] = 'fromA'
|
||||
assert a.sh['a'] == 'fromA'
|
||||
|
||||
a.save()
|
||||
|
||||
sh = pickle.load(open(newfile, 'rb'))
|
||||
sh = pickle.load(open(self.shelf_file, 'rb'))
|
||||
|
||||
assert sh['a'] == 'fromA'
|
||||
|
||||
def test_dummy_shelf(self):
|
||||
with open(self.shelf_file, 'wb') as f:
|
||||
pickle.dump({'counter': 99}, f)
|
||||
a = ShelfDummyPlugin(info={
|
||||
'name': 'DummyShelf',
|
||||
'shelf_file': self.shelf_file,
|
||||
@@ -82,47 +79,11 @@ class PluginsTest(TestCase):
|
||||
|
||||
assert a.shelf_file == self.shelf_file
|
||||
res1 = a.analyse(input=1)
|
||||
assert res1.entries[0].nif__isString == 100
|
||||
a.deactivate()
|
||||
del a
|
||||
assert res1.entries[0].nif__isString == 1
|
||||
res2 = a.analyse(input=1)
|
||||
assert res2.entries[0].nif__isString == 2
|
||||
|
||||
with open(self.shelf_file, 'rb') as f:
|
||||
sh = pickle.load(f)
|
||||
assert sh['counter'] == 100
|
||||
|
||||
def test_corrupt_shelf(self):
|
||||
''' Reusing the values of a previous shelf '''
|
||||
|
||||
emptyfile = os.path.join(self.shelf_dir, "emptyfile")
|
||||
invalidfile = os.path.join(self.shelf_dir, "invalid_file")
|
||||
with open(emptyfile, 'w+b'), open(invalidfile, 'w+b') as inf:
|
||||
inf.write(b'ohno')
|
||||
|
||||
files = {emptyfile: ['empty file', (EOFError, IndexError)],
|
||||
invalidfile: ['invalid file', (pickle.UnpicklingError, IndexError)]}
|
||||
|
||||
for fn in files:
|
||||
with open(fn, 'rb') as f:
|
||||
msg, error = files[fn]
|
||||
a = ShelfDummyPlugin(info={
|
||||
'name': 'shelve',
|
||||
'version': 'test',
|
||||
'shelf_file': f.name
|
||||
})
|
||||
assert os.path.isfile(a.shelf_file)
|
||||
print('Shelf file: %s' % a.shelf_file)
|
||||
with self.assertRaises(error):
|
||||
a.sh['a'] = 'fromA'
|
||||
a.save()
|
||||
del a._sh
|
||||
assert os.path.isfile(a.shelf_file)
|
||||
a.force_shelf = True
|
||||
a.sh['a'] = 'fromA'
|
||||
a.save()
|
||||
b = pickle.load(f)
|
||||
assert b['a'] == 'fromA'
|
||||
|
||||
def test_reuse_shelf(self):
|
||||
def test_two(self):
|
||||
''' Reusing the values of a previous shelf '''
|
||||
a = ShelfDummyPlugin(info={
|
||||
'name': 'shelve',
|
||||
@@ -159,77 +120,3 @@ class PluginsTest(TestCase):
|
||||
}
|
||||
})
|
||||
assert 'example' in a.extra_params
|
||||
|
||||
def test_conversion_centroids(self):
|
||||
info = {
|
||||
"name": "CentroidTest",
|
||||
"description": "Centroid test",
|
||||
"version": 0,
|
||||
"centroids": {
|
||||
"c1": {"V1": 0.5,
|
||||
"V2": 0.5},
|
||||
"c2": {"V1": -0.5,
|
||||
"V2": 0.5},
|
||||
"c3": {"V1": -0.5,
|
||||
"V2": -0.5},
|
||||
"c4": {"V1": 0.5,
|
||||
"V2": -0.5}},
|
||||
"aliases": {
|
||||
"V1": "X-dimension",
|
||||
"V2": "Y-dimension"
|
||||
},
|
||||
"centroids_direction": ["emoml:big6", "emoml:fsre-dimensions"]
|
||||
}
|
||||
c = CentroidConversion(info)
|
||||
print(c.serialize())
|
||||
|
||||
es1 = EmotionSet()
|
||||
e1 = Emotion()
|
||||
e1.onyx__hasEmotionCategory = "c1"
|
||||
es1.onyx__hasEmotion.append(e1)
|
||||
res = c._forward_conversion(es1)
|
||||
assert res["X-dimension"] == 0.5
|
||||
assert res["Y-dimension"] == 0.5
|
||||
print(res)
|
||||
|
||||
e2 = Emotion()
|
||||
e2.onyx__hasEmotionCategory = "c2"
|
||||
es1.onyx__hasEmotion.append(e2)
|
||||
res = c._forward_conversion(es1)
|
||||
assert res["X-dimension"] == 0
|
||||
assert res["Y-dimension"] == 1
|
||||
print(res)
|
||||
|
||||
e = Emotion()
|
||||
e["X-dimension"] = -0.2
|
||||
e["Y-dimension"] = -0.3
|
||||
res = c._backwards_conversion(e)
|
||||
assert res["onyx:hasEmotionCategory"] == "c3"
|
||||
print(res)
|
||||
|
||||
e = Emotion()
|
||||
e["X-dimension"] = -0.2
|
||||
e["Y-dimension"] = 0.3
|
||||
res = c._backwards_conversion(e)
|
||||
assert res["onyx:hasEmotionCategory"] == "c2"
|
||||
|
||||
|
||||
def make_mini_test(plugin_info):
|
||||
def mini_test(self):
|
||||
plugin = plugins.load_plugin_from_info(plugin_info, install=True)
|
||||
plugin.test()
|
||||
return mini_test
|
||||
|
||||
|
||||
def _add_tests():
|
||||
root = os.path.dirname(__file__)
|
||||
plugs = plugins.load_plugins(os.path.join(root, ".."), loader=plugins.parse_plugin_info)
|
||||
for k, v in plugs.items():
|
||||
pass
|
||||
t_method = make_mini_test(v)
|
||||
t_method.__name__ = 'test_plugin_{}'.format(k)
|
||||
setattr(PluginsTest, t_method.__name__, t_method)
|
||||
del t_method
|
||||
|
||||
|
||||
_add_tests()
|
||||
|
||||
@@ -34,8 +34,7 @@ def do_create_(jsfile, success):
|
||||
except (AssertionError, ValidationError, KeyError) as ex:
|
||||
if success:
|
||||
raise
|
||||
return
|
||||
assert success
|
||||
|
||||
return do_expected
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user