From 1eec6ecbad039b946c0d7b690335f2bb4ea8f320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Tue, 12 Jun 2018 10:01:43 +0200 Subject: [PATCH] Squashed 'sentiment-meaningCloud/' content from commit 2a5d212 git-subtree-dir: sentiment-meaningCloud git-subtree-split: 2a5d212833fac38efe69b9d90588c1f0a27ff390 --- .gitignore | 5 + .gitlab-ci.yml | 67 ++++++++++++++ .makefiles/README.md | 27 ++++++ .makefiles/base.mk | 36 ++++++++ .makefiles/docker.mk | 29 ++++++ .makefiles/git.mk | 28 ++++++ .makefiles/k8s.mk | 51 +++++++++++ .makefiles/makefiles.mk | 17 ++++ .makefiles/precommit.mk | 5 + .makefiles/python.mk | 100 ++++++++++++++++++++ Dockerfile | 1 + Dockerfile-3.5 | 4 + Dockerfile.template | 4 + Makefile | 9 ++ README.md | 34 +++++++ docker-compose.yml | 28 ++++++ k8s/README.md | 7 ++ k8s/senpy-deployment.yaml | 27 ++++++ k8s/senpy-ingress.yaml | 14 +++ k8s/senpy-svc.yaml | 12 +++ mocked_request.py | 77 ++++++++++++++++ sentiment_meaningcloud.py | 171 +++++++++++++++++++++++++++++++++++ sentiment_meaningcloud.senpy | 11 +++ 23 files changed, 764 insertions(+) create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100644 .makefiles/README.md create mode 100644 .makefiles/base.mk create mode 100644 .makefiles/docker.mk create mode 100644 .makefiles/git.mk create mode 100644 .makefiles/k8s.mk create mode 100644 .makefiles/makefiles.mk create mode 100644 .makefiles/precommit.mk create mode 100644 .makefiles/python.mk create mode 120000 Dockerfile create mode 100644 Dockerfile-3.5 create mode 100644 Dockerfile.template create mode 100644 Makefile create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100644 k8s/README.md create mode 100644 k8s/senpy-deployment.yaml create mode 100644 k8s/senpy-ingress.yaml create mode 100644 k8s/senpy-svc.yaml create mode 100644 mocked_request.py create mode 100644 sentiment_meaningcloud.py create mode 100644 sentiment_meaningcloud.senpy diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b3992d9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.* +.env +__pycache__ +.pyc +VERSION diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..4806968 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,67 @@ +# Uncomment if you want to use docker-in-docker +# image: gsiupm/dockermake:latest +# services: +# - docker:dind +# When using dind, it's wise to use the overlayfs driver for +# improved performance. + +stages: + - test + - push + - deploy + - clean + +before_script: + - make -e login + +.test: &test_definition + stage: test + script: + - make -e test-$PYTHON_VERSION + +test-3.5: + <<: *test_definition + variables: + PYTHON_VERSION: "3.5" + +.image: &image_definition + stage: push + script: + - make -e push-$PYTHON_VERSION + only: + - tags + - triggers + +push-3.5: + <<: *image_definition + variables: + PYTHON_VERSION: "3.5" + +push-latest: + <<: *image_definition + variables: + PYTHON_VERSION: latest + only: + - tags + - triggers + +deploy: + stage: deploy + environment: production + script: + - make -e deploy + only: + - tags + - triggers + +clean : + stage: clean + script: + - make -e clean + when: manual + +cleanup_py: + stage: clean + when: always # this is important; run even if preceding stages failed. + script: + - docker logout diff --git a/.makefiles/README.md b/.makefiles/README.md new file mode 100644 index 0000000..2ab487e --- /dev/null +++ b/.makefiles/README.md @@ -0,0 +1,27 @@ +These makefiles are recipes for several common tasks in different types of projects. +To add them to your project, simply do: + +``` +git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git +git subtree add --prefix=.makefiles/ makefiles master +touch Makefile +echo "include .makefiles/base.mk" >> Makefile +``` + +Now you can take advantage of the recipes. +For instance, to add useful targets for a python project, just add this to your Makefile: + +``` +include .makefiles/python.mk +``` + +You may need to set special variables like the name of your project or the python versions you're targetting. +Take a look at each specific `.mk` file for more information, and the `Makefile` in the [senpy](https://lab.cluster.gsi.dit.upm.es/senpy/senpy) project for a real use case. + +If you update the makefiles from your repository, make sure to push the changes for review in upstream (this repository): + +``` +make makefiles-push +``` + +It will automatically commit all unstaged changes in the .makefiles folder. diff --git a/.makefiles/base.mk b/.makefiles/base.mk new file mode 100644 index 0000000..a4d94d7 --- /dev/null +++ b/.makefiles/base.mk @@ -0,0 +1,36 @@ +export +NAME ?= $(shell basename $(CURDIR)) +VERSION ?= $(shell git describe --tags --dirty 2>/dev/null) + +ifeq ($(VERSION),) + VERSION:="unknown" +endif + +# Get the location of this makefile. +MK_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) + +-include .env +-include ../.env + +help: ## Show this help. + @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/\(.*:\)[^#]*##\s*\(.*\)/\1\t\2/' | column -t -s " " + +config: ## Load config from the environment. You should run it once in every session before other tasks. Run: eval $(make config) + @awk '{ print "export " $$0}' ../.env + @awk '{ print "export " $$0}' .env + @echo "# Please, run: " + @echo "# eval \$$(make config)" +# If you need to run a command on the key/value pairs, use this: +# @awk '{ split($$0, a, "="); "echo " a[2] " | base64 -w 0" |& getline b64; print "export " a[1] "=" a[2]; print "export " a[1] "_BASE64=" b64}' .env + +ci: ## Run a task using gitlab-runner. Only use to debug problems in the CI pipeline + gitlab-runner exec shell --builds-dir '.builds' --env CI_PROJECT_NAME=$(NAME) ${action} + +include $(MK_DIR)/makefiles.mk +include $(MK_DIR)/docker.mk +include $(MK_DIR)/git.mk + +info:: ## List all variables + env + +.PHONY:: config help ci diff --git a/.makefiles/docker.mk b/.makefiles/docker.mk new file mode 100644 index 0000000..f74d93f --- /dev/null +++ b/.makefiles/docker.mk @@ -0,0 +1,29 @@ +IMAGENAME?=$(NAME) +IMAGEWTAG?=$(IMAGENAME):$(VERSION) + +docker-login: ## Log in to the registry. It will only be used in the server, or when running a CI task locally (if CI_BUILD_TOKEN is set). +ifeq ($(CI_BUILD_TOKEN),) + @echo "Not logging in to the docker registry" "$(CI_REGISTRY)" +else + @docker login -u gitlab-ci-token -p $(CI_BUILD_TOKEN) $(CI_REGISTRY) +endif +ifeq ($(HUB_USER),) + @echo "Not logging in to global the docker registry" +else + @docker login -u $(HUB_USER) -p $(HUB_PASSWORD) +endif + +docker-clean: ## Remove docker credentials +ifeq ($(HUB_USER),) +else + @docker logout +endif + +login:: docker-login + +clean:: docker-clean + +docker-info: + @echo IMAGEWTAG=${IMAGEWTAG} + +.PHONY:: docker-login docker-clean login clean diff --git a/.makefiles/git.mk b/.makefiles/git.mk new file mode 100644 index 0000000..836eb14 --- /dev/null +++ b/.makefiles/git.mk @@ -0,0 +1,28 @@ +commit: + git commit -a + +tag: + git tag ${VERSION} + +git-push:: + git push --tags -u origin HEAD + +git-pull: + git pull --all + +push-github: ## Push the code to github. You need to set up GITHUB_DEPLOY_KEY +ifeq ($(GITHUB_DEPLOY_KEY),) +else + $(eval KEY_FILE := "$(shell mktemp)") + @echo "$(GITHUB_DEPLOY_KEY)" > $(KEY_FILE) + @git remote rm github-deploy || true + git remote add github-deploy $(GITHUB_REPO) + -@GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git fetch github-deploy $(CI_COMMIT_REF_NAME) + @GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git push github-deploy HEAD:$(CI_COMMIT_REF_NAME) + rm $(KEY_FILE) +endif + +push:: git-push +pull:: git-pull + +.PHONY:: commit tag push git-push git-pull push-github diff --git a/.makefiles/k8s.mk b/.makefiles/k8s.mk new file mode 100644 index 0000000..a493b4a --- /dev/null +++ b/.makefiles/k8s.mk @@ -0,0 +1,51 @@ +# Deployment with Kubernetes + +# KUBE_CA_PEM_FILE is the path of a certificate file. It automatically set by GitLab +# if you enable Kubernetes integration in a project. +# +# As of this writing, Kubernetes integration can not be set on a group level, so it has to +# be manually set in every project. +# Alternatively, we use a custom KUBE_CA_BUNDLE environment variable, which can be set at +# the group level. In this case, the variable contains the whole content of the certificate, +# which we dump to a temporary file +# +# Check if the KUBE_CA_PEM_FILE exists. Otherwise, create it from KUBE_CA_BUNDLE +KUBE_CA_TEMP=false +ifndef KUBE_CA_PEM_FILE +KUBE_CA_PEM_FILE:=$$PWD/.ca.crt +CREATED:=$(shell echo -e "$(KUBE_CA_BUNDLE)" > $(KUBE_CA_PEM_FILE)) +endif +KUBE_TOKEN?="" +KUBE_NAMESPACE?=$(NAME) +KUBECTL=docker run --rm -v $(KUBE_CA_PEM_FILE):/tmp/ca.pem -i lachlanevenson/k8s-kubectl --server="$(KUBE_URL)" --token="$(KUBE_TOKEN)" --certificate-authority="/tmp/ca.pem" -n $(KUBE_NAMESPACE) +CI_COMMIT_REF_NAME?=master + +info:: ## Print variables. Useful for debugging. + @echo "#KUBERNETES" + @echo KUBE_URL=$(KUBE_URL) + @echo KUBE_CA_PEM_FILE=$(KUBE_CA_PEM_FILE) + @echo KUBE_CA_BUNDLE=$$KUBE_CA_BUNDLE + @echo KUBE_TOKEN=$(KUBE_TOKEN) + @echo KUBE_NAMESPACE=$(KUBE_NAMESPACE) + @echo KUBECTL=$(KUBECTL) + + @echo "#CI" + @echo CI_PROJECT_NAME=$(CI_PROJECT_NAME) + @echo CI_REGISTRY=$(CI_REGISTRY) + @echo CI_REGISTRY_USER=$(CI_REGISTRY_USER) + @echo CI_COMMIT_REF_NAME=$(CI_COMMIT_REF_NAME) + @echo "CREATED=$(CREATED)" + +# +# Deployment and advanced features +# + + +deploy: ## Deploy to kubernetes using the credentials in KUBE_CA_PEM_FILE (or KUBE_CA_BUNDLE ) and TOKEN + @ls k8s/*.yaml k8s/*.yml k8s/*.tmpl 2>/dev/null || true + @cat k8s/*.yaml k8s/*.yml k8s/*.tmpl 2>/dev/null | envsubst | $(KUBECTL) apply -f - + +deploy-check: ## Get the deployed configuration. + @$(KUBECTL) get deploy,pods,svc,ingress + +.PHONY:: info deploy deploy-check diff --git a/.makefiles/makefiles.mk b/.makefiles/makefiles.mk new file mode 100644 index 0000000..03dcc17 --- /dev/null +++ b/.makefiles/makefiles.mk @@ -0,0 +1,17 @@ +makefiles-remote: + @git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git 2>/dev/null || true + +makefiles-commit: makefiles-remote + git add -f .makefiles + git commit -em "Updated makefiles from ${NAME}" + +makefiles-push: + git subtree push --prefix=.makefiles/ makefiles $(NAME) + +makefiles-pull: makefiles-remote + git subtree pull --prefix=.makefiles/ makefiles master --squash + +pull:: makefiles-pull +push:: makefiles-push + +.PHONY:: makefiles-remote makefiles-commit makefiles-push makefiles-pull pull push diff --git a/.makefiles/precommit.mk b/.makefiles/precommit.mk new file mode 100644 index 0000000..82fe75f --- /dev/null +++ b/.makefiles/precommit.mk @@ -0,0 +1,5 @@ +init: ## Init pre-commit hooks (i.e. enforcing format checking before allowing a commit) + pip install --user pre-commit + pre-commit install + +.PHONY:: init diff --git a/.makefiles/python.mk b/.makefiles/python.mk new file mode 100644 index 0000000..2ad9559 --- /dev/null +++ b/.makefiles/python.mk @@ -0,0 +1,100 @@ +PYVERSIONS ?= 3.5 +PYMAIN ?= $(firstword $(PYVERSIONS)) +TARNAME ?= $(NAME)-$(VERSION).tar.gz +VERSIONFILE ?= $(NAME)/VERSION + +DEVPORT ?= 6000 + + +.FORCE: + +version: .FORCE + @echo $(VERSION) > $(VERSIONFILE) + @echo $(VERSION) + +yapf: ## Format python code + yapf -i -r $(NAME) + yapf -i -r tests + +dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS)) ## Generate dockerfiles for each python version + @unlink Dockerfile >/dev/null + ln -s Dockerfile-$(PYMAIN) Dockerfile + +Dockerfile-%: Dockerfile.template ## Generate a specific dockerfile (e.g. Dockerfile-2.7) + sed "s/{{PYVERSION}}/$*/" Dockerfile.template > Dockerfile-$* + +quick_build: $(addprefix build-, $(PYMAIN)) + +build: $(addprefix build-, $(PYVERSIONS)) ## Build all images / python versions + +build-%: version Dockerfile-% ## Build a specific version (e.g. build-2.7) + docker build -t '$(IMAGEWTAG)-python$*' --cache-from $(IMAGENAME):python$* -f Dockerfile-$* .; + +dev-%: ## Launch a specific development environment using docker (e.g. dev-2.7) + @docker start $(NAME)-dev$* || (\ + $(MAKE) build-$*; \ + docker run -d -w /usr/src/app/ -p $(DEVPORT):5000 -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti --name $(NAME)-dev$* '$(IMAGEWTAG)-python$*'; \ + )\ + + docker exec -ti $(NAME)-dev$* bash + +dev: dev-$(PYMAIN) ## Launch a development environment using docker, using the default python version + +quick_test: test-$(PYMAIN) + +test-%: ## Run setup.py from in an isolated container, built from the base image. (e.g. test-2.7) +# This speeds tests up because the image has most (if not all) of the dependencies already. + docker rm $(NAME)-test-$* || true + docker create -ti --name $(NAME)-test-$* --entrypoint="" -w /usr/src/app/ $(IMAGENAME):python$* python setup.py test + docker cp . $(NAME)-test-$*:/usr/src/app + docker start -a $(NAME)-test-$* + +test: $(addprefix test-,$(PYVERSIONS)) ## Run the tests with the main python version + +run-%: build-% + docker run --rm -p $(DEVPORT):5000 -ti '$(IMAGEWTAG)-python$(PYMAIN)' --default-plugins + +run: run-$(PYMAIN) + +# Pypy - Upload a package + +dist/$(TARNAME): version + python setup.py sdist; + +sdist: dist/$(TARNAME) ## Generate the distribution file (wheel) + +pip_test-%: sdist ## Test the distribution file using pip install and a specific python version (e.g. pip_test-2.7) + docker run --rm -v $$PWD/dist:/dist/ python:$* pip install /dist/$(TARNAME); + +pip_test: $(addprefix pip_test-,$(PYVERSIONS)) ## Test pip installation with the main python version + +pip_upload: pip_test ## Upload package to pip + python setup.py sdist upload ; + +# Pushing to docker + +push-latest: $(addprefix push-latest-,$(PYVERSIONS)) ## Push the "latest" tag to dockerhub + docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)' + docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGENAME)' + docker push '$(IMAGENAME):latest' + docker push '$(IMAGEWTAG)' + +push-latest-%: build-% ## Push the latest image for a specific python version + docker tag $(IMAGENAME):$(VERSION)-python$* $(IMAGENAME):python$* + docker push $(IMAGENAME):$(VERSION)-python$* + docker push $(IMAGENAME):python$* + +push-%: build-% ## Push the image of the current version (tagged). e.g. push-2.7 + docker push $(IMAGENAME):$(VERSION)-python$* + +push:: $(addprefix push-,$(PYVERSIONS)) ## Push an image with the current version for every python version + docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)' + docker push $(IMAGENAME):$(VERSION) + +clean:: ## Clean older docker images and containers related to this project and dev environments + @docker stop $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true + @docker rm $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true + @docker ps -a | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true + @docker images | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true + +.PHONY:: yapf dockerfiles Dockerfile-% quick_build build build-% dev-% quick-dev test quick_test push-latest push-latest-% push-% push version .FORCE diff --git a/Dockerfile b/Dockerfile new file mode 120000 index 0000000..89e531a --- /dev/null +++ b/Dockerfile @@ -0,0 +1 @@ +Dockerfile-3.5 \ No newline at end of file diff --git a/Dockerfile-3.5 b/Dockerfile-3.5 new file mode 100644 index 0000000..100885d --- /dev/null +++ b/Dockerfile-3.5 @@ -0,0 +1,4 @@ +FROM gsiupm/senpy:0.10.4-python3.5 + + +MAINTAINER manuel.garcia-amado.sancho@alumnos.upm.es diff --git a/Dockerfile.template b/Dockerfile.template new file mode 100644 index 0000000..453ae95 --- /dev/null +++ b/Dockerfile.template @@ -0,0 +1,4 @@ +FROM gsiupm/senpy:0.10.4-python{{PYVERSION}} + + +MAINTAINER manuel.garcia-amado.sancho@alumnos.upm.es diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..444c94b --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +NAME:=meaningcloud +VERSIONFILE:=VERSION +IMAGENAME:=registry.cluster.gsi.dit.upm.es/senpy/sentiment-meaningcloud +PYVERSIONS:= 3.5 +DEVPORT:=5000 + +include .makefiles/base.mk +include .makefiles/k8s.mk +include .makefiles/python.mk diff --git a/README.md b/README.md new file mode 100644 index 0000000..5bbb7a0 --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ +# Senpy Plugin MeaningCloud + +MeaningCloud plugin uses API from Meaning Cloud to perform sentiment analysis. + +For more information about Meaning Cloud and its services, please visit: https://www.meaningcloud.com/developer/apis + +## Usage + +To use this plugin, you need to obtain an API key from meaningCloud signing up here: https://www.meaningcloud.com/developer/login + +When you had obtained the meaningCloud API Key, you have to provide it to the plugin, using the param **apiKey**. + +To use this plugin, you should use a GET Requests with the following possible params: +Params: +- Language: English (en) and Spanish (es). (default: en) +- API Key: the API key from Meaning Cloud. Aliases: ["apiKey","meaningCloud-key"]. (required) +- Input: text to analyse.(required) +- Model: model provided to Meaning Cloud API (for general domain). (default: general) + +## Example of Usage + +Example request: +``` +http://senpy.cluster.gsi.dit.upm.es/api/?algo=meaningCloud&language=en&apiKey=&input=I%20love%20Madrid +``` + + +Example respond: This plugin follows the standard for the senpy plugin response. For more information, please visit [senpy documentation](http://senpy.readthedocs.io). Specifically, NIF API section. + +This plugin supports **python2.7** and **python3**. + +![alt GSI Logo][logoGSI] + +[logoGSI]: http://www.gsi.dit.upm.es/images/stories/logos/gsi.png "GSI Logo" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8e1c6d1 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,28 @@ +version: '3' +services: + dev: + image: gsiupm/senpy:latest + entrypoint: ["/bin/bash"] + working_dir: "/senpy-plugins" + tty: true + ports: + - "127.0.0.1:5005:5000" + volumes: + - ".:/senpy-plugins" + test: + image: gsiupm/senpy:latest + entrypoint: ["py.test"] + working_dir: "/usr/src/app/" + volumes: + - ".:/senpy-plugins/" + command: + [] + meaningcloud: + image: "${IMAGENAME-gsiupm/meaningcloud}:${VERSION-dev}" + build: + context: . + dockerfile: Dockerfile-3.5 + ports: + - 5001:5000 + volumes: + - "./data:/data" diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 0000000..69ca6a1 --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,7 @@ +Deploy senpy to a kubernetes cluster. + +Usage: + +``` +kubectl apply -f . -n senpy +``` diff --git a/k8s/senpy-deployment.yaml b/k8s/senpy-deployment.yaml new file mode 100644 index 0000000..c954f37 --- /dev/null +++ b/k8s/senpy-deployment.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: ${NAME} +spec: + replicas: 1 + template: + metadata: + labels: + role: senpy-plugin + app: ${NAME} + spec: + containers: + - name: senpy-latest + image: ${CI_REGISTRY_IMAGE}:${VERSION} + imagePullPolicy: Always + args: + - "-f" + - "/senpy-plugins" + resources: + limits: + memory: "512Mi" + cpu: "1000m" + ports: + - name: web + containerPort: 5000 diff --git a/k8s/senpy-ingress.yaml b/k8s/senpy-ingress.yaml new file mode 100644 index 0000000..1cb02a6 --- /dev/null +++ b/k8s/senpy-ingress.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: ${NAME} +spec: + rules: + - host: ${NAME}.senpy.cluster.gsi.dit.upm.es + http: + paths: + - path: / + backend: + serviceName: ${NAME} + servicePort: 5000 diff --git a/k8s/senpy-svc.yaml b/k8s/senpy-svc.yaml new file mode 100644 index 0000000..cde518c --- /dev/null +++ b/k8s/senpy-svc.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: ${NAME} +spec: + type: ClusterIP + ports: + - port: 5000 + protocol: TCP + selector: + app: ${NAME} diff --git a/mocked_request.py b/mocked_request.py new file mode 100644 index 0000000..113ef2e --- /dev/null +++ b/mocked_request.py @@ -0,0 +1,77 @@ +def mocked_requests_post(*args, **kwargs): + class MockResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status_code = status_code + + def json(self): + return self.json_data + + print("Mocking request") + if args[0] == 'http://api.meaningcloud.com/sentiment-2.1': + return MockResponse({ + 'model': 'general_en', + 'sentence_list': [{ + 'text': + 'Hello World', + 'endp': + '10', + 'inip': + '0', + 'segment_list': [{ + 'text': + 'Hello World', + 'segment_type': + 'secondary', + 'confidence': + '100', + 'inip': + '0', + 'agreement': + 'AGREEMENT', + 'endp': + '10', + 'polarity_term_list': [], + 'score_tag': + 'NONE' + }], + 'score_tag': + 'NONE', + }], + 'score_tag': + 'NONE' + }, 200) + elif args[0] == 'http://api.meaningcloud.com/topics-2.0': + + return MockResponse({ + 'entity_list': [{ + 'form': + 'Obama', + 'id': + '__1265958475430276310', + 'variant_list': [{ + 'endp': '16', + 'form': 'Obama', + 'inip': '12' + }], + 'sementity': { + 'fiction': 'nonfiction', + 'confidence': 'uncertain', + 'class': 'instance', + 'type': 'Top>Person' + } + }], + 'concept_list': [{ + 'form': + 'world', + 'id': + '5c053cd39d', + 'relevance': + '100', + 'semtheme_list': [{ + 'id': 'ODTHEME_ASTRONOMY', + 'type': 'Top>NaturalSciences>Astronomy' + }] + }], + }, 200) + return MockResponse(None, 404) diff --git a/sentiment_meaningcloud.py b/sentiment_meaningcloud.py new file mode 100644 index 0000000..7f27fc9 --- /dev/null +++ b/sentiment_meaningcloud.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- + +import time +import logging +import requests +import json +import string +import os +from os import path +import time +from senpy.plugins import SentimentPlugin +from senpy.models import Results, Entry, Sentiment, Error + +import mocked_request + +try: + from unittest import mock +except ImportError: + import mock + +logger = logging.getLogger(__name__) + + +class MeaningCloudPlugin(SentimentPlugin): + version = "0.1" + + extra_params = { + "language": { + "aliases": ["language", "l"], + "required": True, + "options": ["en","es","ca","it","pt","fr","auto"], + "default": "auto" + }, + "apikey":{ + "aliases": ["apiKey", "meaningcloud-key", "meaningcloud-apikey"], + "required": True + } + } + + """MeaningCloud plugin uses API from Meaning Cloud to perform sentiment analysis.""" + def _polarity(self, value): + + if 'NONE' in value: + polarity = 'marl:Neutral' + polarityValue = 0 + elif 'N' in value: + polarity = 'marl:Negative' + polarityValue = -1 + elif 'P' in value: + polarity = 'marl:Positive' + polarityValue = 1 + return polarity, polarityValue + + def analyse_entry(self, entry, params): + + txt = entry['nif:isString'] + api = 'http://api.meaningcloud.com/' + lang = params.get("language") + model = "general" + key = params["apikey"] + parameters = { + 'key': key, + 'model': model, + 'lang': lang, + 'of': 'json', + 'txt': txt, + 'tt': 'a' + } + try: + r = requests.post( + api + "sentiment-2.1", params=parameters, timeout=3) + parameters['lang'] = r.json()['model'].split('_')[1] + lang = parameters['lang'] + r2 = requests.post( + api + "topics-2.0", params=parameters, timeout=3) + except requests.exceptions.Timeout: + raise Error("Meaning Cloud API does not response") + + api_response = r.json() + api_response_topics = r2.json() + if not api_response.get('score_tag'): + raise Error(r.json()) + entry['language_detected'] = lang + logger.info(api_response) + agg_polarity, agg_polarityValue = self._polarity( + api_response.get('score_tag', None)) + agg_opinion = Sentiment( + id="Opinion0", + marl__hasPolarity=agg_polarity, + marl__polarityValue=agg_polarityValue, + marl__opinionCount=len(api_response['sentence_list'])) + entry.sentiments.append(agg_opinion) + logger.info(api_response['sentence_list']) + count = 1 + + for sentence in api_response['sentence_list']: + for nopinion in sentence['segment_list']: + logger.info(nopinion) + polarity, polarityValue = self._polarity( + nopinion.get('score_tag', None)) + opinion = Sentiment( + id="Opinion{}".format(count), + marl__hasPolarity=polarity, + marl__polarityValue=polarityValue, + marl__aggregatesOpinion=agg_opinion.get('id'), + nif__anchorOf=nopinion.get('text', None), + nif__beginIndex=nopinion.get('inip', None), + nif__endIndex=nopinion.get('endp', None)) + count += 1 + entry.sentiments.append(opinion) + + mapper = {'es': 'es.', 'en': '', 'ca': 'es.', 'it':'it.', 'fr':'fr.', 'pt':'pt.'} + + for sent_entity in api_response_topics['entity_list']: + + resource = "_".join(sent_entity.get('form', None).split()) + entity = Sentiment( + id="Entity{}".format(sent_entity.get('id')), + marl__describesObject="http://{}dbpedia.org/resource/{}".format( + mapper[lang], resource), + nif__anchorOf=sent_entity.get('form', None), + nif__beginIndex=sent_entity['variant_list'][0].get('inip', None), + nif__endIndex=sent_entity['variant_list'][0].get('endp', None)) + entity[ + '@type'] = "ODENTITY_{}".format( + sent_entity['sementity'].get('type', None).split(">")[-1]) + entry.entities.append(entity) + + for topic in api_response_topics['concept_list']: + if 'semtheme_list' in topic: + for theme in topic['semtheme_list']: + concept = Sentiment( + id="Topic{}".format(topic.get('id')), + prov__wasDerivedFrom="http://dbpedia.org/resource/{}". + format(theme['type'].split('>')[-1])) + concept[ + '@type'] = "ODTHEME_{}".format( + theme['type'].split(">")[-1]) + entry.topics.append(concept) + yield entry + + @mock.patch('requests.post', side_effect=mocked_request.mocked_requests_post) + def test(self, *args, **kwargs): + results = list() + params = {'algo': 'sentiment-meaningCloud', + 'intype': 'direct', + 'expanded-jsonld': 0, + 'informat': 'text', + 'prefix': '', + 'plugin_type': 'analysisPlugin', + 'urischeme': 'RFC5147String', + 'outformat': 'json-ld', + 'i': 'Hello World', + 'input': 'Hello World', + 'conversion': 'full', + 'language': 'en', + 'apikey': '00000', + 'algorithm': 'sentiment-meaningCloud'} + for i in range(100): + res = next(self.analyse_entry(Entry(nif__isString="Hello World Obama"), params)) + results.append(res.sentiments[0]['marl:hasPolarity']) + results.append(res.topics[0]['prov:wasDerivedFrom']) + results.append(res.entities[0]['prov:wasDerivedFrom']) + + assert 'marl:Neutral' in results + assert 'http://dbpedia.org/resource/Astronomy' in results + assert 'http://dbpedia.org/resource/Obama' in results + +if __name__ == '__main__': + from senpy import easy_test + easy_test() diff --git a/sentiment_meaningcloud.senpy b/sentiment_meaningcloud.senpy new file mode 100644 index 0000000..eaa417b --- /dev/null +++ b/sentiment_meaningcloud.senpy @@ -0,0 +1,11 @@ +{ + "name": "sentiment-meaningcloud", + "module": "sentiment_meaningcloud", + "description": "Sentiment analysis with meaningCloud service. To use this plugin, you need to obtain an API key from meaningCloud signing up here: https://www.meaningcloud.com/developer/login. When you had obtained the meaningCloud API Key, you have to provide it to the plugin, using param apiKey. Example request: http://senpy.cluster.gsi.dit.upm.es/api/?algo=meaningCloud&language=en&apiKey=&input=I%20love%20Madrid.", + "author": "GSI UPM", + "version": "1.0", + "requirements": {}, + "maxPolarityValue": "1", + "minPolarityValue": "-1" + +}