mirror of
https://github.com/gsi-upm/senpy
synced 2024-11-13 04:02:29 +00:00
Squashed 'sentiment-meaningCloud/' content from commit 2a5d212
git-subtree-dir: sentiment-meaningCloud git-subtree-split: 2a5d212833fac38efe69b9d90588c1f0a27ff390
This commit is contained in:
commit
1eec6ecbad
5
.gitignore
vendored
Normal file
5
.gitignore
vendored
Normal file
@ -0,0 +1,5 @@
|
||||
.*
|
||||
.env
|
||||
__pycache__
|
||||
.pyc
|
||||
VERSION
|
67
.gitlab-ci.yml
Normal file
67
.gitlab-ci.yml
Normal file
@ -0,0 +1,67 @@
|
||||
# Uncomment if you want to use docker-in-docker
|
||||
# image: gsiupm/dockermake:latest
|
||||
# services:
|
||||
# - docker:dind
|
||||
# When using dind, it's wise to use the overlayfs driver for
|
||||
# improved performance.
|
||||
|
||||
stages:
|
||||
- test
|
||||
- push
|
||||
- deploy
|
||||
- clean
|
||||
|
||||
before_script:
|
||||
- make -e login
|
||||
|
||||
.test: &test_definition
|
||||
stage: test
|
||||
script:
|
||||
- make -e test-$PYTHON_VERSION
|
||||
|
||||
test-3.5:
|
||||
<<: *test_definition
|
||||
variables:
|
||||
PYTHON_VERSION: "3.5"
|
||||
|
||||
.image: &image_definition
|
||||
stage: push
|
||||
script:
|
||||
- make -e push-$PYTHON_VERSION
|
||||
only:
|
||||
- tags
|
||||
- triggers
|
||||
|
||||
push-3.5:
|
||||
<<: *image_definition
|
||||
variables:
|
||||
PYTHON_VERSION: "3.5"
|
||||
|
||||
push-latest:
|
||||
<<: *image_definition
|
||||
variables:
|
||||
PYTHON_VERSION: latest
|
||||
only:
|
||||
- tags
|
||||
- triggers
|
||||
|
||||
deploy:
|
||||
stage: deploy
|
||||
environment: production
|
||||
script:
|
||||
- make -e deploy
|
||||
only:
|
||||
- tags
|
||||
- triggers
|
||||
|
||||
clean :
|
||||
stage: clean
|
||||
script:
|
||||
- make -e clean
|
||||
when: manual
|
||||
|
||||
cleanup_py:
|
||||
stage: clean
|
||||
when: always # this is important; run even if preceding stages failed.
|
||||
script:
|
||||
- docker logout
|
27
.makefiles/README.md
Normal file
27
.makefiles/README.md
Normal file
@ -0,0 +1,27 @@
|
||||
These makefiles are recipes for several common tasks in different types of projects.
|
||||
To add them to your project, simply do:
|
||||
|
||||
```
|
||||
git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git
|
||||
git subtree add --prefix=.makefiles/ makefiles master
|
||||
touch Makefile
|
||||
echo "include .makefiles/base.mk" >> Makefile
|
||||
```
|
||||
|
||||
Now you can take advantage of the recipes.
|
||||
For instance, to add useful targets for a python project, just add this to your Makefile:
|
||||
|
||||
```
|
||||
include .makefiles/python.mk
|
||||
```
|
||||
|
||||
You may need to set special variables like the name of your project or the python versions you're targetting.
|
||||
Take a look at each specific `.mk` file for more information, and the `Makefile` in the [senpy](https://lab.cluster.gsi.dit.upm.es/senpy/senpy) project for a real use case.
|
||||
|
||||
If you update the makefiles from your repository, make sure to push the changes for review in upstream (this repository):
|
||||
|
||||
```
|
||||
make makefiles-push
|
||||
```
|
||||
|
||||
It will automatically commit all unstaged changes in the .makefiles folder.
|
36
.makefiles/base.mk
Normal file
36
.makefiles/base.mk
Normal file
@ -0,0 +1,36 @@
|
||||
export
|
||||
NAME ?= $(shell basename $(CURDIR))
|
||||
VERSION ?= $(shell git describe --tags --dirty 2>/dev/null)
|
||||
|
||||
ifeq ($(VERSION),)
|
||||
VERSION:="unknown"
|
||||
endif
|
||||
|
||||
# Get the location of this makefile.
|
||||
MK_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
|
||||
|
||||
-include .env
|
||||
-include ../.env
|
||||
|
||||
help: ## Show this help.
|
||||
@fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/\(.*:\)[^#]*##\s*\(.*\)/\1\t\2/' | column -t -s " "
|
||||
|
||||
config: ## Load config from the environment. You should run it once in every session before other tasks. Run: eval $(make config)
|
||||
@awk '{ print "export " $$0}' ../.env
|
||||
@awk '{ print "export " $$0}' .env
|
||||
@echo "# Please, run: "
|
||||
@echo "# eval \$$(make config)"
|
||||
# If you need to run a command on the key/value pairs, use this:
|
||||
# @awk '{ split($$0, a, "="); "echo " a[2] " | base64 -w 0" |& getline b64; print "export " a[1] "=" a[2]; print "export " a[1] "_BASE64=" b64}' .env
|
||||
|
||||
ci: ## Run a task using gitlab-runner. Only use to debug problems in the CI pipeline
|
||||
gitlab-runner exec shell --builds-dir '.builds' --env CI_PROJECT_NAME=$(NAME) ${action}
|
||||
|
||||
include $(MK_DIR)/makefiles.mk
|
||||
include $(MK_DIR)/docker.mk
|
||||
include $(MK_DIR)/git.mk
|
||||
|
||||
info:: ## List all variables
|
||||
env
|
||||
|
||||
.PHONY:: config help ci
|
29
.makefiles/docker.mk
Normal file
29
.makefiles/docker.mk
Normal file
@ -0,0 +1,29 @@
|
||||
IMAGENAME?=$(NAME)
|
||||
IMAGEWTAG?=$(IMAGENAME):$(VERSION)
|
||||
|
||||
docker-login: ## Log in to the registry. It will only be used in the server, or when running a CI task locally (if CI_BUILD_TOKEN is set).
|
||||
ifeq ($(CI_BUILD_TOKEN),)
|
||||
@echo "Not logging in to the docker registry" "$(CI_REGISTRY)"
|
||||
else
|
||||
@docker login -u gitlab-ci-token -p $(CI_BUILD_TOKEN) $(CI_REGISTRY)
|
||||
endif
|
||||
ifeq ($(HUB_USER),)
|
||||
@echo "Not logging in to global the docker registry"
|
||||
else
|
||||
@docker login -u $(HUB_USER) -p $(HUB_PASSWORD)
|
||||
endif
|
||||
|
||||
docker-clean: ## Remove docker credentials
|
||||
ifeq ($(HUB_USER),)
|
||||
else
|
||||
@docker logout
|
||||
endif
|
||||
|
||||
login:: docker-login
|
||||
|
||||
clean:: docker-clean
|
||||
|
||||
docker-info:
|
||||
@echo IMAGEWTAG=${IMAGEWTAG}
|
||||
|
||||
.PHONY:: docker-login docker-clean login clean
|
28
.makefiles/git.mk
Normal file
28
.makefiles/git.mk
Normal file
@ -0,0 +1,28 @@
|
||||
commit:
|
||||
git commit -a
|
||||
|
||||
tag:
|
||||
git tag ${VERSION}
|
||||
|
||||
git-push::
|
||||
git push --tags -u origin HEAD
|
||||
|
||||
git-pull:
|
||||
git pull --all
|
||||
|
||||
push-github: ## Push the code to github. You need to set up GITHUB_DEPLOY_KEY
|
||||
ifeq ($(GITHUB_DEPLOY_KEY),)
|
||||
else
|
||||
$(eval KEY_FILE := "$(shell mktemp)")
|
||||
@echo "$(GITHUB_DEPLOY_KEY)" > $(KEY_FILE)
|
||||
@git remote rm github-deploy || true
|
||||
git remote add github-deploy $(GITHUB_REPO)
|
||||
-@GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git fetch github-deploy $(CI_COMMIT_REF_NAME)
|
||||
@GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git push github-deploy HEAD:$(CI_COMMIT_REF_NAME)
|
||||
rm $(KEY_FILE)
|
||||
endif
|
||||
|
||||
push:: git-push
|
||||
pull:: git-pull
|
||||
|
||||
.PHONY:: commit tag push git-push git-pull push-github
|
51
.makefiles/k8s.mk
Normal file
51
.makefiles/k8s.mk
Normal file
@ -0,0 +1,51 @@
|
||||
# Deployment with Kubernetes
|
||||
|
||||
# KUBE_CA_PEM_FILE is the path of a certificate file. It automatically set by GitLab
|
||||
# if you enable Kubernetes integration in a project.
|
||||
#
|
||||
# As of this writing, Kubernetes integration can not be set on a group level, so it has to
|
||||
# be manually set in every project.
|
||||
# Alternatively, we use a custom KUBE_CA_BUNDLE environment variable, which can be set at
|
||||
# the group level. In this case, the variable contains the whole content of the certificate,
|
||||
# which we dump to a temporary file
|
||||
#
|
||||
# Check if the KUBE_CA_PEM_FILE exists. Otherwise, create it from KUBE_CA_BUNDLE
|
||||
KUBE_CA_TEMP=false
|
||||
ifndef KUBE_CA_PEM_FILE
|
||||
KUBE_CA_PEM_FILE:=$$PWD/.ca.crt
|
||||
CREATED:=$(shell echo -e "$(KUBE_CA_BUNDLE)" > $(KUBE_CA_PEM_FILE))
|
||||
endif
|
||||
KUBE_TOKEN?=""
|
||||
KUBE_NAMESPACE?=$(NAME)
|
||||
KUBECTL=docker run --rm -v $(KUBE_CA_PEM_FILE):/tmp/ca.pem -i lachlanevenson/k8s-kubectl --server="$(KUBE_URL)" --token="$(KUBE_TOKEN)" --certificate-authority="/tmp/ca.pem" -n $(KUBE_NAMESPACE)
|
||||
CI_COMMIT_REF_NAME?=master
|
||||
|
||||
info:: ## Print variables. Useful for debugging.
|
||||
@echo "#KUBERNETES"
|
||||
@echo KUBE_URL=$(KUBE_URL)
|
||||
@echo KUBE_CA_PEM_FILE=$(KUBE_CA_PEM_FILE)
|
||||
@echo KUBE_CA_BUNDLE=$$KUBE_CA_BUNDLE
|
||||
@echo KUBE_TOKEN=$(KUBE_TOKEN)
|
||||
@echo KUBE_NAMESPACE=$(KUBE_NAMESPACE)
|
||||
@echo KUBECTL=$(KUBECTL)
|
||||
|
||||
@echo "#CI"
|
||||
@echo CI_PROJECT_NAME=$(CI_PROJECT_NAME)
|
||||
@echo CI_REGISTRY=$(CI_REGISTRY)
|
||||
@echo CI_REGISTRY_USER=$(CI_REGISTRY_USER)
|
||||
@echo CI_COMMIT_REF_NAME=$(CI_COMMIT_REF_NAME)
|
||||
@echo "CREATED=$(CREATED)"
|
||||
|
||||
#
|
||||
# Deployment and advanced features
|
||||
#
|
||||
|
||||
|
||||
deploy: ## Deploy to kubernetes using the credentials in KUBE_CA_PEM_FILE (or KUBE_CA_BUNDLE ) and TOKEN
|
||||
@ls k8s/*.yaml k8s/*.yml k8s/*.tmpl 2>/dev/null || true
|
||||
@cat k8s/*.yaml k8s/*.yml k8s/*.tmpl 2>/dev/null | envsubst | $(KUBECTL) apply -f -
|
||||
|
||||
deploy-check: ## Get the deployed configuration.
|
||||
@$(KUBECTL) get deploy,pods,svc,ingress
|
||||
|
||||
.PHONY:: info deploy deploy-check
|
17
.makefiles/makefiles.mk
Normal file
17
.makefiles/makefiles.mk
Normal file
@ -0,0 +1,17 @@
|
||||
makefiles-remote:
|
||||
@git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git 2>/dev/null || true
|
||||
|
||||
makefiles-commit: makefiles-remote
|
||||
git add -f .makefiles
|
||||
git commit -em "Updated makefiles from ${NAME}"
|
||||
|
||||
makefiles-push:
|
||||
git subtree push --prefix=.makefiles/ makefiles $(NAME)
|
||||
|
||||
makefiles-pull: makefiles-remote
|
||||
git subtree pull --prefix=.makefiles/ makefiles master --squash
|
||||
|
||||
pull:: makefiles-pull
|
||||
push:: makefiles-push
|
||||
|
||||
.PHONY:: makefiles-remote makefiles-commit makefiles-push makefiles-pull pull push
|
5
.makefiles/precommit.mk
Normal file
5
.makefiles/precommit.mk
Normal file
@ -0,0 +1,5 @@
|
||||
init: ## Init pre-commit hooks (i.e. enforcing format checking before allowing a commit)
|
||||
pip install --user pre-commit
|
||||
pre-commit install
|
||||
|
||||
.PHONY:: init
|
100
.makefiles/python.mk
Normal file
100
.makefiles/python.mk
Normal file
@ -0,0 +1,100 @@
|
||||
PYVERSIONS ?= 3.5
|
||||
PYMAIN ?= $(firstword $(PYVERSIONS))
|
||||
TARNAME ?= $(NAME)-$(VERSION).tar.gz
|
||||
VERSIONFILE ?= $(NAME)/VERSION
|
||||
|
||||
DEVPORT ?= 6000
|
||||
|
||||
|
||||
.FORCE:
|
||||
|
||||
version: .FORCE
|
||||
@echo $(VERSION) > $(VERSIONFILE)
|
||||
@echo $(VERSION)
|
||||
|
||||
yapf: ## Format python code
|
||||
yapf -i -r $(NAME)
|
||||
yapf -i -r tests
|
||||
|
||||
dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS)) ## Generate dockerfiles for each python version
|
||||
@unlink Dockerfile >/dev/null
|
||||
ln -s Dockerfile-$(PYMAIN) Dockerfile
|
||||
|
||||
Dockerfile-%: Dockerfile.template ## Generate a specific dockerfile (e.g. Dockerfile-2.7)
|
||||
sed "s/{{PYVERSION}}/$*/" Dockerfile.template > Dockerfile-$*
|
||||
|
||||
quick_build: $(addprefix build-, $(PYMAIN))
|
||||
|
||||
build: $(addprefix build-, $(PYVERSIONS)) ## Build all images / python versions
|
||||
|
||||
build-%: version Dockerfile-% ## Build a specific version (e.g. build-2.7)
|
||||
docker build -t '$(IMAGEWTAG)-python$*' --cache-from $(IMAGENAME):python$* -f Dockerfile-$* .;
|
||||
|
||||
dev-%: ## Launch a specific development environment using docker (e.g. dev-2.7)
|
||||
@docker start $(NAME)-dev$* || (\
|
||||
$(MAKE) build-$*; \
|
||||
docker run -d -w /usr/src/app/ -p $(DEVPORT):5000 -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti --name $(NAME)-dev$* '$(IMAGEWTAG)-python$*'; \
|
||||
)\
|
||||
|
||||
docker exec -ti $(NAME)-dev$* bash
|
||||
|
||||
dev: dev-$(PYMAIN) ## Launch a development environment using docker, using the default python version
|
||||
|
||||
quick_test: test-$(PYMAIN)
|
||||
|
||||
test-%: ## Run setup.py from in an isolated container, built from the base image. (e.g. test-2.7)
|
||||
# This speeds tests up because the image has most (if not all) of the dependencies already.
|
||||
docker rm $(NAME)-test-$* || true
|
||||
docker create -ti --name $(NAME)-test-$* --entrypoint="" -w /usr/src/app/ $(IMAGENAME):python$* python setup.py test
|
||||
docker cp . $(NAME)-test-$*:/usr/src/app
|
||||
docker start -a $(NAME)-test-$*
|
||||
|
||||
test: $(addprefix test-,$(PYVERSIONS)) ## Run the tests with the main python version
|
||||
|
||||
run-%: build-%
|
||||
docker run --rm -p $(DEVPORT):5000 -ti '$(IMAGEWTAG)-python$(PYMAIN)' --default-plugins
|
||||
|
||||
run: run-$(PYMAIN)
|
||||
|
||||
# Pypy - Upload a package
|
||||
|
||||
dist/$(TARNAME): version
|
||||
python setup.py sdist;
|
||||
|
||||
sdist: dist/$(TARNAME) ## Generate the distribution file (wheel)
|
||||
|
||||
pip_test-%: sdist ## Test the distribution file using pip install and a specific python version (e.g. pip_test-2.7)
|
||||
docker run --rm -v $$PWD/dist:/dist/ python:$* pip install /dist/$(TARNAME);
|
||||
|
||||
pip_test: $(addprefix pip_test-,$(PYVERSIONS)) ## Test pip installation with the main python version
|
||||
|
||||
pip_upload: pip_test ## Upload package to pip
|
||||
python setup.py sdist upload ;
|
||||
|
||||
# Pushing to docker
|
||||
|
||||
push-latest: $(addprefix push-latest-,$(PYVERSIONS)) ## Push the "latest" tag to dockerhub
|
||||
docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
|
||||
docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGENAME)'
|
||||
docker push '$(IMAGENAME):latest'
|
||||
docker push '$(IMAGEWTAG)'
|
||||
|
||||
push-latest-%: build-% ## Push the latest image for a specific python version
|
||||
docker tag $(IMAGENAME):$(VERSION)-python$* $(IMAGENAME):python$*
|
||||
docker push $(IMAGENAME):$(VERSION)-python$*
|
||||
docker push $(IMAGENAME):python$*
|
||||
|
||||
push-%: build-% ## Push the image of the current version (tagged). e.g. push-2.7
|
||||
docker push $(IMAGENAME):$(VERSION)-python$*
|
||||
|
||||
push:: $(addprefix push-,$(PYVERSIONS)) ## Push an image with the current version for every python version
|
||||
docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
|
||||
docker push $(IMAGENAME):$(VERSION)
|
||||
|
||||
clean:: ## Clean older docker images and containers related to this project and dev environments
|
||||
@docker stop $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true
|
||||
@docker rm $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true
|
||||
@docker ps -a | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true
|
||||
@docker images | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true
|
||||
|
||||
.PHONY:: yapf dockerfiles Dockerfile-% quick_build build build-% dev-% quick-dev test quick_test push-latest push-latest-% push-% push version .FORCE
|
1
Dockerfile
Symbolic link
1
Dockerfile
Symbolic link
@ -0,0 +1 @@
|
||||
Dockerfile-3.5
|
4
Dockerfile-3.5
Normal file
4
Dockerfile-3.5
Normal file
@ -0,0 +1,4 @@
|
||||
FROM gsiupm/senpy:0.10.4-python3.5
|
||||
|
||||
|
||||
MAINTAINER manuel.garcia-amado.sancho@alumnos.upm.es
|
4
Dockerfile.template
Normal file
4
Dockerfile.template
Normal file
@ -0,0 +1,4 @@
|
||||
FROM gsiupm/senpy:0.10.4-python{{PYVERSION}}
|
||||
|
||||
|
||||
MAINTAINER manuel.garcia-amado.sancho@alumnos.upm.es
|
9
Makefile
Normal file
9
Makefile
Normal file
@ -0,0 +1,9 @@
|
||||
NAME:=meaningcloud
|
||||
VERSIONFILE:=VERSION
|
||||
IMAGENAME:=registry.cluster.gsi.dit.upm.es/senpy/sentiment-meaningcloud
|
||||
PYVERSIONS:= 3.5
|
||||
DEVPORT:=5000
|
||||
|
||||
include .makefiles/base.mk
|
||||
include .makefiles/k8s.mk
|
||||
include .makefiles/python.mk
|
34
README.md
Normal file
34
README.md
Normal file
@ -0,0 +1,34 @@
|
||||
# Senpy Plugin MeaningCloud
|
||||
|
||||
MeaningCloud plugin uses API from Meaning Cloud to perform sentiment analysis.
|
||||
|
||||
For more information about Meaning Cloud and its services, please visit: https://www.meaningcloud.com/developer/apis
|
||||
|
||||
## Usage
|
||||
|
||||
To use this plugin, you need to obtain an API key from meaningCloud signing up here: https://www.meaningcloud.com/developer/login
|
||||
|
||||
When you had obtained the meaningCloud API Key, you have to provide it to the plugin, using the param **apiKey**.
|
||||
|
||||
To use this plugin, you should use a GET Requests with the following possible params:
|
||||
Params:
|
||||
- Language: English (en) and Spanish (es). (default: en)
|
||||
- API Key: the API key from Meaning Cloud. Aliases: ["apiKey","meaningCloud-key"]. (required)
|
||||
- Input: text to analyse.(required)
|
||||
- Model: model provided to Meaning Cloud API (for general domain). (default: general)
|
||||
|
||||
## Example of Usage
|
||||
|
||||
Example request:
|
||||
```
|
||||
http://senpy.cluster.gsi.dit.upm.es/api/?algo=meaningCloud&language=en&apiKey=<put here your API key>&input=I%20love%20Madrid
|
||||
```
|
||||
|
||||
|
||||
Example respond: This plugin follows the standard for the senpy plugin response. For more information, please visit [senpy documentation](http://senpy.readthedocs.io). Specifically, NIF API section.
|
||||
|
||||
This plugin supports **python2.7** and **python3**.
|
||||
|
||||
![alt GSI Logo][logoGSI]
|
||||
|
||||
[logoGSI]: http://www.gsi.dit.upm.es/images/stories/logos/gsi.png "GSI Logo"
|
28
docker-compose.yml
Normal file
28
docker-compose.yml
Normal file
@ -0,0 +1,28 @@
|
||||
version: '3'
|
||||
services:
|
||||
dev:
|
||||
image: gsiupm/senpy:latest
|
||||
entrypoint: ["/bin/bash"]
|
||||
working_dir: "/senpy-plugins"
|
||||
tty: true
|
||||
ports:
|
||||
- "127.0.0.1:5005:5000"
|
||||
volumes:
|
||||
- ".:/senpy-plugins"
|
||||
test:
|
||||
image: gsiupm/senpy:latest
|
||||
entrypoint: ["py.test"]
|
||||
working_dir: "/usr/src/app/"
|
||||
volumes:
|
||||
- ".:/senpy-plugins/"
|
||||
command:
|
||||
[]
|
||||
meaningcloud:
|
||||
image: "${IMAGENAME-gsiupm/meaningcloud}:${VERSION-dev}"
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile-3.5
|
||||
ports:
|
||||
- 5001:5000
|
||||
volumes:
|
||||
- "./data:/data"
|
7
k8s/README.md
Normal file
7
k8s/README.md
Normal file
@ -0,0 +1,7 @@
|
||||
Deploy senpy to a kubernetes cluster.
|
||||
|
||||
Usage:
|
||||
|
||||
```
|
||||
kubectl apply -f . -n senpy
|
||||
```
|
27
k8s/senpy-deployment.yaml
Normal file
27
k8s/senpy-deployment.yaml
Normal file
@ -0,0 +1,27 @@
|
||||
---
|
||||
apiVersion: extensions/v1beta1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: ${NAME}
|
||||
spec:
|
||||
replicas: 1
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
role: senpy-plugin
|
||||
app: ${NAME}
|
||||
spec:
|
||||
containers:
|
||||
- name: senpy-latest
|
||||
image: ${CI_REGISTRY_IMAGE}:${VERSION}
|
||||
imagePullPolicy: Always
|
||||
args:
|
||||
- "-f"
|
||||
- "/senpy-plugins"
|
||||
resources:
|
||||
limits:
|
||||
memory: "512Mi"
|
||||
cpu: "1000m"
|
||||
ports:
|
||||
- name: web
|
||||
containerPort: 5000
|
14
k8s/senpy-ingress.yaml
Normal file
14
k8s/senpy-ingress.yaml
Normal file
@ -0,0 +1,14 @@
|
||||
---
|
||||
apiVersion: extensions/v1beta1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: ${NAME}
|
||||
spec:
|
||||
rules:
|
||||
- host: ${NAME}.senpy.cluster.gsi.dit.upm.es
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
backend:
|
||||
serviceName: ${NAME}
|
||||
servicePort: 5000
|
12
k8s/senpy-svc.yaml
Normal file
12
k8s/senpy-svc.yaml
Normal file
@ -0,0 +1,12 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ${NAME}
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- port: 5000
|
||||
protocol: TCP
|
||||
selector:
|
||||
app: ${NAME}
|
77
mocked_request.py
Normal file
77
mocked_request.py
Normal file
@ -0,0 +1,77 @@
|
||||
def mocked_requests_post(*args, **kwargs):
|
||||
class MockResponse:
|
||||
def __init__(self, json_data, status_code):
|
||||
self.json_data = json_data
|
||||
self.status_code = status_code
|
||||
|
||||
def json(self):
|
||||
return self.json_data
|
||||
|
||||
print("Mocking request")
|
||||
if args[0] == 'http://api.meaningcloud.com/sentiment-2.1':
|
||||
return MockResponse({
|
||||
'model': 'general_en',
|
||||
'sentence_list': [{
|
||||
'text':
|
||||
'Hello World',
|
||||
'endp':
|
||||
'10',
|
||||
'inip':
|
||||
'0',
|
||||
'segment_list': [{
|
||||
'text':
|
||||
'Hello World',
|
||||
'segment_type':
|
||||
'secondary',
|
||||
'confidence':
|
||||
'100',
|
||||
'inip':
|
||||
'0',
|
||||
'agreement':
|
||||
'AGREEMENT',
|
||||
'endp':
|
||||
'10',
|
||||
'polarity_term_list': [],
|
||||
'score_tag':
|
||||
'NONE'
|
||||
}],
|
||||
'score_tag':
|
||||
'NONE',
|
||||
}],
|
||||
'score_tag':
|
||||
'NONE'
|
||||
}, 200)
|
||||
elif args[0] == 'http://api.meaningcloud.com/topics-2.0':
|
||||
|
||||
return MockResponse({
|
||||
'entity_list': [{
|
||||
'form':
|
||||
'Obama',
|
||||
'id':
|
||||
'__1265958475430276310',
|
||||
'variant_list': [{
|
||||
'endp': '16',
|
||||
'form': 'Obama',
|
||||
'inip': '12'
|
||||
}],
|
||||
'sementity': {
|
||||
'fiction': 'nonfiction',
|
||||
'confidence': 'uncertain',
|
||||
'class': 'instance',
|
||||
'type': 'Top>Person'
|
||||
}
|
||||
}],
|
||||
'concept_list': [{
|
||||
'form':
|
||||
'world',
|
||||
'id':
|
||||
'5c053cd39d',
|
||||
'relevance':
|
||||
'100',
|
||||
'semtheme_list': [{
|
||||
'id': 'ODTHEME_ASTRONOMY',
|
||||
'type': 'Top>NaturalSciences>Astronomy'
|
||||
}]
|
||||
}],
|
||||
}, 200)
|
||||
return MockResponse(None, 404)
|
171
sentiment_meaningcloud.py
Normal file
171
sentiment_meaningcloud.py
Normal file
@ -0,0 +1,171 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import time
|
||||
import logging
|
||||
import requests
|
||||
import json
|
||||
import string
|
||||
import os
|
||||
from os import path
|
||||
import time
|
||||
from senpy.plugins import SentimentPlugin
|
||||
from senpy.models import Results, Entry, Sentiment, Error
|
||||
|
||||
import mocked_request
|
||||
|
||||
try:
|
||||
from unittest import mock
|
||||
except ImportError:
|
||||
import mock
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MeaningCloudPlugin(SentimentPlugin):
|
||||
version = "0.1"
|
||||
|
||||
extra_params = {
|
||||
"language": {
|
||||
"aliases": ["language", "l"],
|
||||
"required": True,
|
||||
"options": ["en","es","ca","it","pt","fr","auto"],
|
||||
"default": "auto"
|
||||
},
|
||||
"apikey":{
|
||||
"aliases": ["apiKey", "meaningcloud-key", "meaningcloud-apikey"],
|
||||
"required": True
|
||||
}
|
||||
}
|
||||
|
||||
"""MeaningCloud plugin uses API from Meaning Cloud to perform sentiment analysis."""
|
||||
def _polarity(self, value):
|
||||
|
||||
if 'NONE' in value:
|
||||
polarity = 'marl:Neutral'
|
||||
polarityValue = 0
|
||||
elif 'N' in value:
|
||||
polarity = 'marl:Negative'
|
||||
polarityValue = -1
|
||||
elif 'P' in value:
|
||||
polarity = 'marl:Positive'
|
||||
polarityValue = 1
|
||||
return polarity, polarityValue
|
||||
|
||||
def analyse_entry(self, entry, params):
|
||||
|
||||
txt = entry['nif:isString']
|
||||
api = 'http://api.meaningcloud.com/'
|
||||
lang = params.get("language")
|
||||
model = "general"
|
||||
key = params["apikey"]
|
||||
parameters = {
|
||||
'key': key,
|
||||
'model': model,
|
||||
'lang': lang,
|
||||
'of': 'json',
|
||||
'txt': txt,
|
||||
'tt': 'a'
|
||||
}
|
||||
try:
|
||||
r = requests.post(
|
||||
api + "sentiment-2.1", params=parameters, timeout=3)
|
||||
parameters['lang'] = r.json()['model'].split('_')[1]
|
||||
lang = parameters['lang']
|
||||
r2 = requests.post(
|
||||
api + "topics-2.0", params=parameters, timeout=3)
|
||||
except requests.exceptions.Timeout:
|
||||
raise Error("Meaning Cloud API does not response")
|
||||
|
||||
api_response = r.json()
|
||||
api_response_topics = r2.json()
|
||||
if not api_response.get('score_tag'):
|
||||
raise Error(r.json())
|
||||
entry['language_detected'] = lang
|
||||
logger.info(api_response)
|
||||
agg_polarity, agg_polarityValue = self._polarity(
|
||||
api_response.get('score_tag', None))
|
||||
agg_opinion = Sentiment(
|
||||
id="Opinion0",
|
||||
marl__hasPolarity=agg_polarity,
|
||||
marl__polarityValue=agg_polarityValue,
|
||||
marl__opinionCount=len(api_response['sentence_list']))
|
||||
entry.sentiments.append(agg_opinion)
|
||||
logger.info(api_response['sentence_list'])
|
||||
count = 1
|
||||
|
||||
for sentence in api_response['sentence_list']:
|
||||
for nopinion in sentence['segment_list']:
|
||||
logger.info(nopinion)
|
||||
polarity, polarityValue = self._polarity(
|
||||
nopinion.get('score_tag', None))
|
||||
opinion = Sentiment(
|
||||
id="Opinion{}".format(count),
|
||||
marl__hasPolarity=polarity,
|
||||
marl__polarityValue=polarityValue,
|
||||
marl__aggregatesOpinion=agg_opinion.get('id'),
|
||||
nif__anchorOf=nopinion.get('text', None),
|
||||
nif__beginIndex=nopinion.get('inip', None),
|
||||
nif__endIndex=nopinion.get('endp', None))
|
||||
count += 1
|
||||
entry.sentiments.append(opinion)
|
||||
|
||||
mapper = {'es': 'es.', 'en': '', 'ca': 'es.', 'it':'it.', 'fr':'fr.', 'pt':'pt.'}
|
||||
|
||||
for sent_entity in api_response_topics['entity_list']:
|
||||
|
||||
resource = "_".join(sent_entity.get('form', None).split())
|
||||
entity = Sentiment(
|
||||
id="Entity{}".format(sent_entity.get('id')),
|
||||
marl__describesObject="http://{}dbpedia.org/resource/{}".format(
|
||||
mapper[lang], resource),
|
||||
nif__anchorOf=sent_entity.get('form', None),
|
||||
nif__beginIndex=sent_entity['variant_list'][0].get('inip', None),
|
||||
nif__endIndex=sent_entity['variant_list'][0].get('endp', None))
|
||||
entity[
|
||||
'@type'] = "ODENTITY_{}".format(
|
||||
sent_entity['sementity'].get('type', None).split(">")[-1])
|
||||
entry.entities.append(entity)
|
||||
|
||||
for topic in api_response_topics['concept_list']:
|
||||
if 'semtheme_list' in topic:
|
||||
for theme in topic['semtheme_list']:
|
||||
concept = Sentiment(
|
||||
id="Topic{}".format(topic.get('id')),
|
||||
prov__wasDerivedFrom="http://dbpedia.org/resource/{}".
|
||||
format(theme['type'].split('>')[-1]))
|
||||
concept[
|
||||
'@type'] = "ODTHEME_{}".format(
|
||||
theme['type'].split(">")[-1])
|
||||
entry.topics.append(concept)
|
||||
yield entry
|
||||
|
||||
@mock.patch('requests.post', side_effect=mocked_request.mocked_requests_post)
|
||||
def test(self, *args, **kwargs):
|
||||
results = list()
|
||||
params = {'algo': 'sentiment-meaningCloud',
|
||||
'intype': 'direct',
|
||||
'expanded-jsonld': 0,
|
||||
'informat': 'text',
|
||||
'prefix': '',
|
||||
'plugin_type': 'analysisPlugin',
|
||||
'urischeme': 'RFC5147String',
|
||||
'outformat': 'json-ld',
|
||||
'i': 'Hello World',
|
||||
'input': 'Hello World',
|
||||
'conversion': 'full',
|
||||
'language': 'en',
|
||||
'apikey': '00000',
|
||||
'algorithm': 'sentiment-meaningCloud'}
|
||||
for i in range(100):
|
||||
res = next(self.analyse_entry(Entry(nif__isString="Hello World Obama"), params))
|
||||
results.append(res.sentiments[0]['marl:hasPolarity'])
|
||||
results.append(res.topics[0]['prov:wasDerivedFrom'])
|
||||
results.append(res.entities[0]['prov:wasDerivedFrom'])
|
||||
|
||||
assert 'marl:Neutral' in results
|
||||
assert 'http://dbpedia.org/resource/Astronomy' in results
|
||||
assert 'http://dbpedia.org/resource/Obama' in results
|
||||
|
||||
if __name__ == '__main__':
|
||||
from senpy import easy_test
|
||||
easy_test()
|
11
sentiment_meaningcloud.senpy
Normal file
11
sentiment_meaningcloud.senpy
Normal file
@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "sentiment-meaningcloud",
|
||||
"module": "sentiment_meaningcloud",
|
||||
"description": "Sentiment analysis with meaningCloud service. To use this plugin, you need to obtain an API key from meaningCloud signing up here: https://www.meaningcloud.com/developer/login. When you had obtained the meaningCloud API Key, you have to provide it to the plugin, using param apiKey. Example request: http://senpy.cluster.gsi.dit.upm.es/api/?algo=meaningCloud&language=en&apiKey=<put here your API key>&input=I%20love%20Madrid.",
|
||||
"author": "GSI UPM",
|
||||
"version": "1.0",
|
||||
"requirements": {},
|
||||
"maxPolarityValue": "1",
|
||||
"minPolarityValue": "-1"
|
||||
|
||||
}
|
Loading…
Reference in New Issue
Block a user