diff --git a/.gitignore b/.gitignore index 4617294..9075048 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ build README.html __pycache__ VERSION +Dockerfile-* +Dockerfile \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7bfe306..75ed0e4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: docker:latest +image: gsiupm/dockermake:latest # When using dind, it's wise to use the overlayfs driver for @@ -6,56 +6,40 @@ image: docker:latest variables: DOCKER_DRIVER: overlay DOCKERFILE: Dockerfile - -before_script: - - sh version.sh > senpy/VERSION + VERSION: $CI_BUILD_REF stages: - test - images - release + - clean .test: &test_definition - variables: - PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache" - cache: - paths: - - .eggs/ - - "$CI_PROJECT_DIR/pip-cache" - - .venv - key: "$CI_PROJECT_NAME" stage: test script: - - pip install --use-wheel -U pip setuptools virtualenv - - virtualenv .venv/$PYTHON_VERSION - - source .venv/$PYTHON_VERSION/bin/activate - - pip install --use-wheel -r requirements.txt - - pip install --use-wheel -r test-requirements.txt - - py.test --cov=senpy --cov-report term-missing - - python + - make -e test-$PYTHON_VERSION test-3.5: <<: *test_definition - image: "python:3.5" - -test-3.4: - <<: *test_definition - image: "python:3.4" + variables: + PYTHON_VERSION: "3.5" test-2.7: <<: *test_definition - image: "python:2.7" + variables: + PYTHON_VERSION: "2.7" .image: &image_definition + stage: images variables: PYTHON_VERSION: "3.5" + VERSION: $CI_BUILD_TAG + IMAGENAME: $CI_REGISTRY_IMAGE before_script: - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY script: - - docker build -f Dockerfile-3.5 . -t $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG-$PYTHON_VERSION - - docker push $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG-$PYTHON_VERSION - stage: images + - make -e push-$PYTHON_VERSION only: - tags - triggers @@ -75,10 +59,12 @@ image-latest: before_script: - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY script: - - docker build -f Dockerfile . -t $CI_REGISTRY_IMAGE - - docker tag $CI_REGISTRY_IMAGE $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG - - docker push $CI_REGISTRY_IMAGE - - docker push $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG + - make -e push-latest only: - master - - triggers \ No newline at end of file + - triggers + +clean : + stage: clean + script: + - make -e clean \ No newline at end of file diff --git a/Dockerfile b/Dockerfile deleted file mode 120000 index 89e531a..0000000 --- a/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -Dockerfile-3.5 \ No newline at end of file diff --git a/Dockerfile-2.7 b/Dockerfile-2.7 deleted file mode 100644 index 5ec4a22..0000000 --- a/Dockerfile-2.7 +++ /dev/null @@ -1,21 +0,0 @@ -from python:2.7 - -RUN mkdir /cache/ -ENV PIP_CACHE_DIR=/cache/ - -WORKDIR /usr/src/app -ADD requirements.txt /usr/src/app/ -RUN pip install --use-wheel -r requirements.txt -ADD . /usr/src/app/ -RUN pip install . - - -VOLUME /data/ - -RUN mkdir /senpy-plugins/ - -WORKDIR /senpy-plugins/ -ONBUILD ADD . /senpy-plugins/ -ONBUILD RUN python -m senpy --only-install -f /senpy-plugins - -ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"] \ No newline at end of file diff --git a/Dockerfile-3.4 b/Dockerfile-3.4 deleted file mode 100644 index 118230d..0000000 --- a/Dockerfile-3.4 +++ /dev/null @@ -1,21 +0,0 @@ -from python:3.4 - -RUN mkdir /cache/ -ENV PIP_CACHE_DIR=/cache/ - -WORKDIR /usr/src/app -ADD requirements.txt /usr/src/app/ -RUN pip install --use-wheel -r requirements.txt -ADD . /usr/src/app/ -RUN pip install . - - -VOLUME /data/ - -RUN mkdir /senpy-plugins/ - -WORKDIR /senpy-plugins/ -ONBUILD ADD . /senpy-plugins/ -ONBUILD RUN python -m senpy --only-install -f /senpy-plugins - -ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"] \ No newline at end of file diff --git a/Dockerfile-3.5 b/Dockerfile-3.5 deleted file mode 100644 index 314cc0d..0000000 --- a/Dockerfile-3.5 +++ /dev/null @@ -1,21 +0,0 @@ -FROM python:3.5 - -RUN mkdir /cache/ -ENV PIP_CACHE_DIR=/cache/ - -WORKDIR /usr/src/app -ADD requirements.txt /usr/src/app/ -RUN pip install --use-wheel -r requirements.txt -ADD . /usr/src/app/ -RUN pip install . - - -VOLUME /data/ - -RUN mkdir /senpy-plugins/ - -WORKDIR /senpy-plugins/ -ONBUILD ADD . /senpy-plugins/ -ONBUILD RUN python -m senpy --only-install -f /senpy-plugins - -ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"] \ No newline at end of file diff --git a/Dockerfile.deps b/Dockerfile.deps deleted file mode 100644 index ad5cfba..0000000 --- a/Dockerfile.deps +++ /dev/null @@ -1,33 +0,0 @@ -from python:2.7 - -RUN apt-get update -RUN apt-get -y install git -RUN mkdir -p /senpy-plugins - -RUN apt-get -y install python-numpy -RUN apt-get -y install python-scipy -RUN apt-get -y install python-sklearn -RUN apt-get -y install python-gevent -RUN apt-get -y install libopenblas-dev -RUN apt-get -y install gfortran -RUN apt-get -y install libxml2-dev libxslt1-dev python-dev - -#RUN pip install --upgrade pip - -ADD id_rsa /root/.ssh/id_rsa -RUN chmod 700 /root/.ssh/id_rsa -RUN echo "Host github.com\n\tStrictHostKeyChecking no\n" >> /root/.ssh/config - -RUN git clone https://github.com/gsi-upm/senpy /usr/src/app/ -RUN git clone git@github.com:gsi-upm/senpy-plugins-enterprise /senpy-plugins/enterprise -RUN git clone https://github.com/gsi-upm/senpy-plugins-community /senpy-plugins/community - -RUN pip install /usr/src/app -RUN pip install --no-use-wheel -r /senpy-plugins/enterprise/requirements.txt -RUN python -m nltk.downloader stopwords -RUN python -m nltk.downloader punkt -RUN python -m nltk.downloader maxent_treebank_pos_tagger -RUN python -m nltk.downloader wordnet - -WORKDIR /senpy-plugins -ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"] diff --git a/Dockerfile.template b/Dockerfile.template index 79d1fc1..878d10f 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -1,21 +1,22 @@ from python:{{PYVERSION}} -RUN mkdir /cache/ -ENV PIP_CACHE_DIR=/cache/ - -WORKDIR /usr/src/app -ADD requirements.txt /usr/src/app/ -RUN pip install --use-wheel -r requirements.txt -ADD . /usr/src/app/ -RUN pip install . +MAINTAINER J. Fernando Sánchez +RUN mkdir /cache/ /senpy-plugins /data/ VOLUME /data/ -RUN mkdir /senpy-plugins/ +ENV PIP_CACHE_DIR=/cache/ -WORKDIR /senpy-plugins/ -ONBUILD ADD . /senpy-plugins/ +ONBUILD COPY . /senpy-plugins/ ONBUILD RUN python -m senpy --only-install -f /senpy-plugins +ONBUILD WORKDIR /senpy-plugins/ + + +WORKDIR /usr/src/app +COPY test-requirements.txt requirements.txt /usr/src/app/ +RUN pip install --use-wheel -r test-requirements.txt -r requirements.txt +COPY . /usr/src/app/ +RUN pip install --no-deps --no-index . ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"] \ No newline at end of file diff --git a/Makefile b/Makefile index 7b6076b..0702c94 100644 --- a/Makefile +++ b/Makefile @@ -1,25 +1,25 @@ -PYVERSIONS=3.5 3.4 2.7 +PYVERSIONS=3.5 2.7 PYMAIN=$(firstword $(PYVERSIONS)) NAME=senpy REPO=gsiupm -VERSION=$(shell ./version.sh) +VERSION=$(shell git describe --tags --dirty 2>/dev/null) TARNAME=$(NAME)-$(VERSION).tar.gz IMAGENAME=$(REPO)/$(NAME):$(VERSION) -TEST_COMMAND=gitlab-runner exec docker --cache-dir=/tmp/gitlabrunner --docker-volumes /tmp/gitlabrunner:/tmp/gitlabrunner --env CI_PROJECT_NAME=$(NAME) +action="test-${PYMAIN}" all: build run -FORCE: +.FORCE: -version: FORCE +version: .FORCE @echo $(VERSION) > $(NAME)/VERSION - @echo $(NAME) $(VERSION) + @echo $(VERSION) yapf: yapf -i -r senpy yapf -i -r tests -dev: +init: pip install --user pre-commit pre-commit install @@ -34,26 +34,27 @@ quick_build: $(addprefix build-, $(PYMAIN)) build: $(addprefix build-, $(PYVERSIONS)) -build-%: Dockerfile-% +build-%: version Dockerfile-% docker build -t '$(IMAGENAME)-python$*' -f Dockerfile-$* .; quick_test: $(addprefix test-,$(PYMAIN)) -test: $(addprefix test-,$(PYVERSIONS)) - -debug-%: - @docker start $(NAME)-debug || (\ +dev-%: + @docker start $(NAME)-dev || (\ $(MAKE) build-$*; \ - docker run -d -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -p 5000:5000 -ti --name $(NAME)-debug '$(IMAGENAME)-python$*'; \ - docker exec -ti $(NAME)-debug pip install -r test-requirements.txt; \ + docker run -d -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -p 5000:5000 -ti --name $(NAME)-dev '$(IMAGENAME)-python$*'; \ )\ - docker attach $(NAME)-debug + docker exec -ti $(NAME)-dev bash + +dev: dev-$(PYMAIN) + +test-all: $(addprefix test-,$(PYVERSIONS)) -debug: debug-$(PYMAIN) +test-%: build-% + docker run --rm --entrypoint /usr/local/bin/python -w /usr/src/app $(IMAGENAME)-python$* setup.py test -test-%: - $(TEST_COMMAND) test-$* +test: test-$(PYMAIN) dist/$(TARNAME): docker run --rm -ti -v $$PWD:/usr/src/app/ -w /usr/src/app/ python:$(PYMAIN) python setup.py sdist; @@ -77,11 +78,16 @@ upload: test $(addprefix upload-,$(PYVERSIONS)) clean: @docker ps -a | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true @docker images | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true - @docker rmi $(NAME)-debug 2>/dev/null || true + @docker rmi $(NAME)-dev 2>/dev/null || true -upload_git: + +git_commit: git commit -a + +git_tag: git tag ${VERSION} + +upload_git: git push --tags origin master pip_upload: @@ -94,4 +100,14 @@ run-%: build-% run: run-$(PYMAIN) -.PHONY: test test-% build-% build test pip_test run yapf dev +push-latest: build-$(PYMAIN) + docker tag $(IMAGENAME)-python$(PYMAIN) $(IMAGENAME) + docker push $(IMAGENAME) + +push-%: build-% + docker push $(IMAGENAME)-python$* + +ci: + gitlab-runner exec docker --docker-volumes /var/run/docker.sock:/var/run/docker.sock --env CI_PROJECT_NAME=$(NAME) ${action} + +.PHONY: test test-% test-all build-% build test pip_test run yapf dev ci version .FORCE diff --git a/README.rst b/README.rst index bdb674a..8b8633f 100644 --- a/README.rst +++ b/README.rst @@ -38,9 +38,9 @@ If you want to install senpy globally, use sudo instead of the ``--user`` flag. Docker Image ************ -Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy --host 0.0.0.0 --default-plugins``. +Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy --default-plugins``. -To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v :/plugins gsiupm/senpy --host 0.0.0.0 --default-plugins -f /plugins`` +To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v :/plugins gsiupm/senpy --default-plugins -f /plugins`` Usage ----- diff --git a/senpy/extensions.py b/senpy/extensions.py index 6231b80..3b6693b 100644 --- a/senpy/extensions.py +++ b/senpy/extensions.py @@ -284,7 +284,7 @@ class Senpy(object): @classmethod def validate_info(cls, info): - return all(x in info for x in ('name', 'module', 'version')) + return all(x in info for x in ('name', 'module', 'description', 'version')) def install_deps(self): for i in self.plugins.values(): diff --git a/senpy/plugins/conversion/emotion/ekman2vad.senpy b/senpy/plugins/conversion/emotion/ekman2vad.senpy index d50b13d..64c7854 100644 --- a/senpy/plugins/conversion/emotion/ekman2vad.senpy +++ b/senpy/plugins/conversion/emotion/ekman2vad.senpy @@ -1,7 +1,7 @@ --- name: Ekman2VAD module: ekman2vad -description: Plugin to convert from Ekman to VAD +description: Plugin to convert emotion sets from Ekman to VAD version: 0.1 onyx:doesConversion: - onyx:conversionFrom: emoml:big6 diff --git a/test-requirements.txt b/test-requirements.txt index aa973f5..5ef1112 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,3 @@ -pytest mock pytest-cov +pytest diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..7c30070 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,68 @@ +import logging + +logger = logging.getLogger(__name__) + +from unittest import TestCase +from senpy.api import parse_params, API_PARAMS, NIF_PARAMS, WEB_PARAMS +from senpy.models import Error + + +class APITest(TestCase): + + def test_api_params(self): + """The API should not define any required parameters without a default""" + parse_params({}, spec=API_PARAMS) + + def test_web_params(self): + """The WEB should not define any required parameters without a default""" + parse_params({}, spec=WEB_PARAMS) + + def test_basic(self): + a = {} + try: + parse_params(a, spec=NIF_PARAMS) + raise AssertionError() + except Error: + pass + a = {'input': 'hello'} + p = parse_params(a, spec=NIF_PARAMS) + assert 'input' in p + b = {'i': 'hello'} + p = parse_params(b, spec=NIF_PARAMS) + assert 'input' in p + + def test_plugin(self): + query = {} + plug_params = { + 'hello': { + 'aliases': ['hello', 'hiya'], + 'required': True + } + } + try: + parse_params(query, spec=plug_params) + raise AssertionError() + except Error: + pass + query['hello'] = 'world' + p = parse_params(query, spec=plug_params) + assert 'hello' in p + assert p['hello'] == 'world' + del query['hello'] + + query['hiya'] = 'dlrow' + p = parse_params(query, spec=plug_params) + assert 'hello' in p + assert 'hiya' in p + assert p['hello'] == 'dlrow' + + def test_default(self): + spec = { + 'hello': { + 'required': True, + 'default': 1 + } + } + p = parse_params({}, spec=spec) + assert 'hello' in p + assert p['hello'] == 1 diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 7c89aca..609c29e 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -42,6 +42,7 @@ class ExtensionsTest(TestCase): info = { 'name': 'TestPip', 'module': 'dummy', + 'description': None, 'requirements': ['noop'], 'version': 0 } diff --git a/version.sh b/version.sh deleted file mode 100755 index c8d1f03..0000000 --- a/version.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -VERSION=$(git describe --long --tags --dirty)