balkian/senpy
1
0
Fork 0
mirror of https://github.com/gsi-upm/senpy synced 2024-11-21 15:52:28 +00:00
Code Issues Releases Wiki Activity

Deployment changes

Browse Source 
* Docker all the things!
* Make all the things!
* Fixed version.sh
This commit is contained in:
J. Fernando Sánchez 2017-02-27 12:14:39 +01:00
parent 9f6a6f5ecd
commit ba2e18125c
16 changed files with 145 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -6,3 +6,5 @@ build
README.html
__pycache__
VERSION
Dockerfile-*
Dockerfile

50
.gitlab-ci.yml
View File

@ -1,4 +1,4 @@
image: docker:latest
image: gsiupm/dockermake:latest
# When using dind, it's wise to use the overlayfs driver for
@ -6,56 +6,40 @@ image: docker:latest
variables:
DOCKER_DRIVER: overlay
DOCKERFILE: Dockerfile
before_script:
- sh version.sh > senpy/VERSION
VERSION: $CI_BUILD_REF
stages:
- test
- images
- release
- clean
.test: &test_definition
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
cache:
paths:
- .eggs/
- "$CI_PROJECT_DIR/pip-cache"
- .venv
key: "$CI_PROJECT_NAME"
stage: test
script:
- pip install --use-wheel -U pip setuptools virtualenv
- virtualenv .venv/$PYTHON_VERSION
- source .venv/$PYTHON_VERSION/bin/activate
- pip install --use-wheel -r requirements.txt
- pip install --use-wheel -r test-requirements.txt
- py.test --cov=senpy --cov-report term-missing
- python
- make -e test-$PYTHON_VERSION
test-3.5:
<<: *test_definition
image: "python:3.5"
test-3.4:
<<: *test_definition
image: "python:3.4"
variables:
PYTHON_VERSION: "3.5"
test-2.7:
<<: *test_definition
image: "python:2.7"
variables:
PYTHON_VERSION: "2.7"
.image: &image_definition
stage: images
variables:
PYTHON_VERSION: "3.5"
VERSION: $CI_BUILD_TAG
IMAGENAME: $CI_REGISTRY_IMAGE
before_script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
script:
- docker build -f Dockerfile-3.5 . -t $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG-$PYTHON_VERSION
- docker push $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG-$PYTHON_VERSION
stage: images
- make -e push-$PYTHON_VERSION
only:
- tags
- triggers
@ -75,10 +59,12 @@ image-latest:
before_script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
script:
- docker build -f Dockerfile . -t $CI_REGISTRY_IMAGE
- docker tag $CI_REGISTRY_IMAGE $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG
- docker push $CI_REGISTRY_IMAGE
- docker push $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG
- make -e push-latest
only:
- master
- triggers
clean :
stage: clean
script:
- make -e clean

1
Dockerfile
View File

@ -1 +0,0 @@
Dockerfile-3.5

21
Dockerfile-2.7
View File

@ -1,21 +0,0 @@
from python:2.7
RUN mkdir /cache/
ENV PIP_CACHE_DIR=/cache/
WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/
RUN pip install --use-wheel -r requirements.txt
ADD . /usr/src/app/
RUN pip install .
VOLUME /data/
RUN mkdir /senpy-plugins/
WORKDIR /senpy-plugins/
ONBUILD ADD . /senpy-plugins/
ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

21
Dockerfile-3.4
View File

@ -1,21 +0,0 @@
from python:3.4
RUN mkdir /cache/
ENV PIP_CACHE_DIR=/cache/
WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/
RUN pip install --use-wheel -r requirements.txt
ADD . /usr/src/app/
RUN pip install .
VOLUME /data/
RUN mkdir /senpy-plugins/
WORKDIR /senpy-plugins/
ONBUILD ADD . /senpy-plugins/
ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

21
Dockerfile-3.5
View File

@ -1,21 +0,0 @@
FROM python:3.5
RUN mkdir /cache/
ENV PIP_CACHE_DIR=/cache/
WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/
RUN pip install --use-wheel -r requirements.txt
ADD . /usr/src/app/
RUN pip install .
VOLUME /data/
RUN mkdir /senpy-plugins/
WORKDIR /senpy-plugins/
ONBUILD ADD . /senpy-plugins/
ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

33
Dockerfile.deps
View File

@ -1,33 +0,0 @@
from python:2.7
RUN apt-get update
RUN apt-get -y install git
RUN mkdir -p /senpy-plugins
RUN apt-get -y install python-numpy
RUN apt-get -y install python-scipy
RUN apt-get -y install python-sklearn
RUN apt-get -y install python-gevent
RUN apt-get -y install libopenblas-dev
RUN apt-get -y install gfortran
RUN apt-get -y install libxml2-dev libxslt1-dev python-dev
#RUN pip install --upgrade pip
ADD id_rsa /root/.ssh/id_rsa
RUN chmod 700 /root/.ssh/id_rsa
RUN echo "Host github.com\n\tStrictHostKeyChecking no\n" >> /root/.ssh/config
RUN git clone https://github.com/gsi-upm/senpy /usr/src/app/
RUN git clone git@github.com:gsi-upm/senpy-plugins-enterprise /senpy-plugins/enterprise
RUN git clone https://github.com/gsi-upm/senpy-plugins-community /senpy-plugins/community
RUN pip install /usr/src/app
RUN pip install --no-use-wheel -r /senpy-plugins/enterprise/requirements.txt
RUN python -m nltk.downloader stopwords
RUN python -m nltk.downloader punkt
RUN python -m nltk.downloader maxent_treebank_pos_tagger
RUN python -m nltk.downloader wordnet
WORKDIR /senpy-plugins
ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]

23
Dockerfile.template
View File

@ -1,21 +1,22 @@
from python:{{PYVERSION}}
RUN mkdir /cache/
ENV PIP_CACHE_DIR=/cache/
WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/
RUN pip install --use-wheel -r requirements.txt
ADD . /usr/src/app/
RUN pip install .
MAINTAINER J. Fernando Sánchez <jf.sanchez@upm.es>
RUN mkdir /cache/ /senpy-plugins /data/
VOLUME /data/
RUN mkdir /senpy-plugins/
ENV PIP_CACHE_DIR=/cache/
WORKDIR /senpy-plugins/
ONBUILD ADD . /senpy-plugins/
ONBUILD COPY . /senpy-plugins/
ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
ONBUILD WORKDIR /senpy-plugins/
WORKDIR /usr/src/app
COPY test-requirements.txt requirements.txt /usr/src/app/
RUN pip install --use-wheel -r test-requirements.txt -r requirements.txt
COPY . /usr/src/app/
RUN pip install --no-deps --no-index .
ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

60
Makefile
View File

@ -1,25 +1,25 @@
PYVERSIONS=3.5 3.4 2.7
PYVERSIONS=3.5 2.7
PYMAIN=$(firstword $(PYVERSIONS))
NAME=senpy
REPO=gsiupm
VERSION=$(shell ./version.sh)
VERSION=$(shell git describe --tags --dirty 2>/dev/null)
TARNAME=$(NAME)-$(VERSION).tar.gz
IMAGENAME=$(REPO)/$(NAME):$(VERSION)
TEST_COMMAND=gitlab-runner exec docker --cache-dir=/tmp/gitlabrunner --docker-volumes /tmp/gitlabrunner:/tmp/gitlabrunner --env CI_PROJECT_NAME=$(NAME)
action="test-${PYMAIN}"
all: build run
FORCE:
.FORCE:
version: FORCE
version: .FORCE
@echo $(VERSION) > $(NAME)/VERSION
@echo $(NAME) $(VERSION)
@echo $(VERSION)
yapf:
yapf -i -r senpy
yapf -i -r tests
dev:
init:
pip install --user pre-commit
pre-commit install
@ -34,26 +34,27 @@ quick_build: $(addprefix build-, $(PYMAIN))
build: $(addprefix build-, $(PYVERSIONS))
build-%: Dockerfile-%
build-%: version Dockerfile-%
docker build -t '$(IMAGENAME)-python$*' -f Dockerfile-$* .;
quick_test: $(addprefix test-,$(PYMAIN))
test: $(addprefix test-,$(PYVERSIONS))
debug-%:
@docker start $(NAME)-debug || (\
dev-%:
@docker start $(NAME)-dev || (\
$(MAKE) build-$*; \
docker run -d -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -p 5000:5000 -ti --name $(NAME)-debug '$(IMAGENAME)-python$*'; \
docker exec -ti $(NAME)-debug pip install -r test-requirements.txt; \
docker run -d -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -p 5000:5000 -ti --name $(NAME)-dev '$(IMAGENAME)-python$*'; \
)\
docker attach $(NAME)-debug
docker exec -ti $(NAME)-dev bash
debug: debug-$(PYMAIN)
dev: dev-$(PYMAIN)
test-%:
$(TEST_COMMAND) test-$*
test-all: $(addprefix test-,$(PYVERSIONS))
test-%: build-%
docker run --rm --entrypoint /usr/local/bin/python -w /usr/src/app $(IMAGENAME)-python$* setup.py test
test: test-$(PYMAIN)
dist/$(TARNAME):
docker run --rm -ti -v $$PWD:/usr/src/app/ -w /usr/src/app/ python:$(PYMAIN) python setup.py sdist;
@ -77,11 +78,16 @@ upload: test $(addprefix upload-,$(PYVERSIONS))
clean:
@docker ps -a | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true
@docker images | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true
@docker rmi $(NAME)-debug 2>/dev/null || true
@docker rmi $(NAME)-dev 2>/dev/null || true
git_commit:
git commit -a
git_tag:
git tag ${VERSION}
upload_git:
git commit -a
git tag ${VERSION}
git push --tags origin master
pip_upload:
@ -94,4 +100,14 @@ run-%: build-%
run: run-$(PYMAIN)
.PHONY: test test-% build-% build test pip_test run yapf dev
push-latest: build-$(PYMAIN)
docker tag $(IMAGENAME)-python$(PYMAIN) $(IMAGENAME)
docker push $(IMAGENAME)
push-%: build-%
docker push $(IMAGENAME)-python$*
ci:
gitlab-runner exec docker --docker-volumes /var/run/docker.sock:/var/run/docker.sock --env CI_PROJECT_NAME=$(NAME) ${action}
.PHONY: test test-% test-all build-% build test pip_test run yapf dev ci version .FORCE

4
README.rst
View File

@ -38,9 +38,9 @@ If you want to install senpy globally, use sudo instead of the ``--user`` flag.
Docker Image
************
Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy --host 0.0.0.0 --default-plugins``.
Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy --default-plugins``.
To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --host 0.0.0.0 --default-plugins -f /plugins``
To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --default-plugins -f /plugins``
Usage
-----

2
senpy/extensions.py
View File

@ -284,7 +284,7 @@ class Senpy(object):
@classmethod
def validate_info(cls, info):
return all(x in info for x in ('name', 'module', 'version'))
return all(x in info for x in ('name', 'module', 'description', 'version'))
def install_deps(self):
for i in self.plugins.values():

2
senpy/plugins/conversion/emotion/ekman2vad.senpy
View File

@ -1,7 +1,7 @@
---
name: Ekman2VAD
module: ekman2vad
description: Plugin to convert from Ekman to VAD
description: Plugin to convert emotion sets from Ekman to VAD
version: 0.1
onyx:doesConversion:
- onyx:conversionFrom: emoml:big6

2
test-requirements.txt
View File

@ -1,3 +1,3 @@
pytest
mock
pytest-cov
pytest

68
tests/test_api.py Normal file
View File

@ -0,0 +1,68 @@
import logging
logger = logging.getLogger(__name__)
from unittest import TestCase
from senpy.api import parse_params, API_PARAMS, NIF_PARAMS, WEB_PARAMS
from senpy.models import Error
class APITest(TestCase):
def test_api_params(self):
"""The API should not define any required parameters without a default"""
parse_params({}, spec=API_PARAMS)
def test_web_params(self):
"""The WEB should not define any required parameters without a default"""
parse_params({}, spec=WEB_PARAMS)
def test_basic(self):
a = {}
try:
parse_params(a, spec=NIF_PARAMS)
raise AssertionError()
except Error:
pass
a = {'input': 'hello'}
p = parse_params(a, spec=NIF_PARAMS)
assert 'input' in p
b = {'i': 'hello'}
p = parse_params(b, spec=NIF_PARAMS)
assert 'input' in p
def test_plugin(self):
query = {}
plug_params = {
'hello': {
'aliases': ['hello', 'hiya'],
'required': True
}
}
try:
parse_params(query, spec=plug_params)
raise AssertionError()
except Error:
pass
query['hello'] = 'world'
p = parse_params(query, spec=plug_params)
assert 'hello' in p
assert p['hello'] == 'world'
del query['hello']
query['hiya'] = 'dlrow'
p = parse_params(query, spec=plug_params)
assert 'hello' in p
assert 'hiya' in p
assert p['hello'] == 'dlrow'
def test_default(self):
spec = {
'hello': {
'required': True,
'default': 1
}
}
p = parse_params({}, spec=spec)
assert 'hello' in p
assert p['hello'] == 1

1
tests/test_extensions.py
View File

@ -42,6 +42,7 @@ class ExtensionsTest(TestCase):
info = {
'name': 'TestPip',
'module': 'dummy',
'description': None,
'requirements': ['noop'],
'version': 0
}

2
version.sh
View File

@ -1,2 +0,0 @@
#!/bin/sh
VERSION=$(git describe --long --tags --dirty)