Deployment changes

* Docker all the things!
* Make all the things!
* Fixed version.sh

.gitignore

@@ -6,3 +6,5 @@ build
 README.html
 __pycache__
 VERSION
+Dockerfile-*
+Dockerfile

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-image: docker:latest
+image: gsiupm/dockermake:latest
 
 
 # When using dind, it's wise to use the overlayfs driver for
@@ -6,56 +6,40 @@ image: docker:latest
 variables:
   DOCKER_DRIVER: overlay
   DOCKERFILE: Dockerfile
-
+  VERSION: $CI_BUILD_REF
-before_script:
-  - sh version.sh > senpy/VERSION
 
 stages:
   - test
   - images
   - release
+  - clean
 
 .test: &test_definition
-  variables:
-    PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
-  cache:
-    paths:
-      - .eggs/
-      - "$CI_PROJECT_DIR/pip-cache"
-      - .venv
-    key: "$CI_PROJECT_NAME"
   stage: test
   script:
-    - pip install --use-wheel -U pip setuptools virtualenv
+    - make -e test-$PYTHON_VERSION
-    - virtualenv .venv/$PYTHON_VERSION
-    - source .venv/$PYTHON_VERSION/bin/activate
-    - pip install --use-wheel -r requirements.txt
-    - pip install --use-wheel -r test-requirements.txt
-    - py.test --cov=senpy --cov-report term-missing
-    - python 
     
 test-3.5:
   <<: *test_definition
-  image: "python:3.5"
+  variables:
-
+    PYTHON_VERSION: "3.5"
-test-3.4:
-  <<: *test_definition
-  image: "python:3.4"
 
 test-2.7:
   <<: *test_definition
-  image: "python:2.7"
+  variables:
+    PYTHON_VERSION: "2.7"
 
 
 .image: &image_definition
+  stage: images
   variables:
     PYTHON_VERSION: "3.5"
+    VERSION: $CI_BUILD_TAG
+    IMAGENAME: $CI_REGISTRY_IMAGE
   before_script:
     - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
   script:
-    - docker build -f Dockerfile-3.5 . -t $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG-$PYTHON_VERSION
+    - make -e push-$PYTHON_VERSION
-    - docker push $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG-$PYTHON_VERSION
-  stage: images
   only:
     - tags
     - triggers
@@ -75,10 +59,12 @@ image-latest:
   before_script:
     - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
   script:
-    - docker build -f Dockerfile . -t $CI_REGISTRY_IMAGE
+    - make -e push-latest
-    - docker tag $CI_REGISTRY_IMAGE $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG
-    - docker push $CI_REGISTRY_IMAGE
-    - docker push $CI_REGISTRY_IMAGE:$CI_BUILD_REF_SLUG
   only:
     - master
-    - triggers
+    - triggers
+
+clean :
+  stage: clean
+  script:
+    - make -e clean

Dockerfile

@@ -1 +0,0 @@
-Dockerfile-3.5

Dockerfile-2.7

@@ -1,21 +0,0 @@
-from python:2.7
-
-RUN mkdir /cache/
-ENV PIP_CACHE_DIR=/cache/
-
-WORKDIR /usr/src/app
-ADD requirements.txt /usr/src/app/
-RUN pip install --use-wheel -r requirements.txt
-ADD . /usr/src/app/
-RUN pip install .
-
-
-VOLUME /data/
-
-RUN mkdir /senpy-plugins/
-
-WORKDIR /senpy-plugins/
-ONBUILD ADD . /senpy-plugins/
-ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
-
-ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

Dockerfile-3.4

@@ -1,21 +0,0 @@
-from python:3.4
-
-RUN mkdir /cache/
-ENV PIP_CACHE_DIR=/cache/
-
-WORKDIR /usr/src/app
-ADD requirements.txt /usr/src/app/
-RUN pip install --use-wheel -r requirements.txt
-ADD . /usr/src/app/
-RUN pip install .
-
-
-VOLUME /data/
-
-RUN mkdir /senpy-plugins/
-
-WORKDIR /senpy-plugins/
-ONBUILD ADD . /senpy-plugins/
-ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
-
-ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

Dockerfile-3.5

@@ -1,21 +0,0 @@
-FROM python:3.5
-
-RUN mkdir /cache/
-ENV PIP_CACHE_DIR=/cache/
-
-WORKDIR /usr/src/app
-ADD requirements.txt /usr/src/app/
-RUN pip install --use-wheel -r requirements.txt
-ADD . /usr/src/app/
-RUN pip install .
-
-
-VOLUME /data/
-
-RUN mkdir /senpy-plugins/
-
-WORKDIR /senpy-plugins/
-ONBUILD ADD . /senpy-plugins/
-ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
-
-ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

Dockerfile.deps

@@ -1,33 +0,0 @@
-from python:2.7
-
-RUN apt-get update
-RUN apt-get -y install git
-RUN mkdir -p /senpy-plugins
-
-RUN apt-get -y install python-numpy
-RUN apt-get -y install python-scipy
-RUN apt-get -y install python-sklearn
-RUN apt-get -y install python-gevent
-RUN apt-get -y install libopenblas-dev
-RUN apt-get -y install gfortran
-RUN apt-get -y install libxml2-dev libxslt1-dev python-dev
-
-#RUN pip install --upgrade pip
-
-ADD id_rsa /root/.ssh/id_rsa
-RUN chmod 700 /root/.ssh/id_rsa
-RUN echo "Host github.com\n\tStrictHostKeyChecking no\n" >> /root/.ssh/config
-
-RUN git clone https://github.com/gsi-upm/senpy /usr/src/app/
-RUN git clone git@github.com:gsi-upm/senpy-plugins-enterprise /senpy-plugins/enterprise
-RUN git clone https://github.com/gsi-upm/senpy-plugins-community /senpy-plugins/community
-
-RUN pip install /usr/src/app
-RUN pip install --no-use-wheel -r /senpy-plugins/enterprise/requirements.txt
-RUN python -m nltk.downloader stopwords
-RUN python -m nltk.downloader punkt
-RUN python -m nltk.downloader maxent_treebank_pos_tagger
-RUN python -m nltk.downloader wordnet
-
-WORKDIR /senpy-plugins
-ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]

Dockerfile.template

@@ -1,21 +1,22 @@
 from python:{{PYVERSION}}
 
-RUN mkdir /cache/
+MAINTAINER J. Fernando Sánchez <jf.sanchez@upm.es>
-ENV PIP_CACHE_DIR=/cache/
-
-WORKDIR /usr/src/app
-ADD requirements.txt /usr/src/app/
-RUN pip install --use-wheel -r requirements.txt
-ADD . /usr/src/app/
-RUN pip install .
 
+RUN mkdir /cache/ /senpy-plugins /data/
 
 VOLUME /data/
 
-RUN mkdir /senpy-plugins/
+ENV PIP_CACHE_DIR=/cache/
 
-WORKDIR /senpy-plugins/
+ONBUILD COPY . /senpy-plugins/
-ONBUILD ADD . /senpy-plugins/
 ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
+ONBUILD WORKDIR /senpy-plugins/
+
+
+WORKDIR /usr/src/app
+COPY test-requirements.txt requirements.txt /usr/src/app/
+RUN pip install --use-wheel -r test-requirements.txt -r requirements.txt
+COPY . /usr/src/app/
+RUN pip install --no-deps --no-index .
 
 ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

Makefile

@@ -1,25 +1,25 @@
-PYVERSIONS=3.5 3.4 2.7
+PYVERSIONS=3.5 2.7
 PYMAIN=$(firstword $(PYVERSIONS))
 NAME=senpy
 REPO=gsiupm
-VERSION=$(shell ./version.sh)
+VERSION=$(shell git describe --tags --dirty 2>/dev/null)
 TARNAME=$(NAME)-$(VERSION).tar.gz 
 IMAGENAME=$(REPO)/$(NAME):$(VERSION)
-TEST_COMMAND=gitlab-runner exec docker --cache-dir=/tmp/gitlabrunner --docker-volumes /tmp/gitlabrunner:/tmp/gitlabrunner --env CI_PROJECT_NAME=$(NAME)
+action="test-${PYMAIN}"
 
 all: build run
 
-FORCE:
+.FORCE:
 
-version: FORCE
+version: .FORCE
 	@echo $(VERSION) > $(NAME)/VERSION
-	@echo $(NAME) $(VERSION)
+	@echo $(VERSION)
 
 yapf:
 	yapf -i -r senpy
 	yapf -i -r tests
 
-dev:
+init:
 	pip install --user pre-commit
 	pre-commit install
 
@@ -34,26 +34,27 @@ quick_build: $(addprefix build-, $(PYMAIN))
 
 build: $(addprefix build-, $(PYVERSIONS))
 
-build-%: Dockerfile-%
+build-%: version Dockerfile-%
 	docker build -t '$(IMAGENAME)-python$*' -f Dockerfile-$* .;
 
 quick_test: $(addprefix test-,$(PYMAIN))
 
-test: $(addprefix test-,$(PYVERSIONS))
+dev-%:
-
+	@docker start $(NAME)-dev || (\
-debug-%:
-	@docker start $(NAME)-debug || (\
 		$(MAKE) build-$*; \
-		docker run -d -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -p 5000:5000 -ti --name $(NAME)-debug '$(IMAGENAME)-python$*'; \
+		docker run -d -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -p 5000:5000 -ti --name $(NAME)-dev '$(IMAGENAME)-python$*'; \
-		docker exec -ti $(NAME)-debug pip install -r test-requirements.txt; \
 	)\
 
-	docker attach $(NAME)-debug
+	docker exec -ti $(NAME)-dev bash
+
+dev: dev-$(PYMAIN)
+
+test-all: $(addprefix test-,$(PYVERSIONS))
 
-debug: debug-$(PYMAIN)
+test-%: build-%
+	docker run --rm --entrypoint /usr/local/bin/python -w /usr/src/app $(IMAGENAME)-python$*  setup.py test
 
-test-%:
+test: test-$(PYMAIN)
-	$(TEST_COMMAND) test-$*
 
 dist/$(TARNAME):
 	docker run --rm -ti -v $$PWD:/usr/src/app/ -w /usr/src/app/ python:$(PYMAIN) python setup.py sdist;
@@ -77,11 +78,16 @@ upload: test $(addprefix upload-,$(PYVERSIONS))
 clean:
 	@docker ps -a | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true
 	@docker images | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true
-	@docker rmi $(NAME)-debug 2>/dev/null || true
+	@docker rmi $(NAME)-dev 2>/dev/null || true
 
-upload_git:
+
+git_commit:
 	git commit -a
+
+git_tag:
 	git tag ${VERSION}
+
+upload_git:
 	git push --tags origin master
 
 pip_upload:
@@ -94,4 +100,14 @@ run-%: build-%
 
 run: run-$(PYMAIN)
 
-.PHONY: test test-% build-% build test pip_test run yapf dev
+push-latest: build-$(PYMAIN)
+	docker tag $(IMAGENAME)-python$(PYMAIN) $(IMAGENAME)
+	docker push $(IMAGENAME)
+
+push-%: build-%
+	docker push $(IMAGENAME)-python$*
+
+ci:
+	gitlab-runner exec docker --docker-volumes /var/run/docker.sock:/var/run/docker.sock --env CI_PROJECT_NAME=$(NAME) ${action}
+
+.PHONY: test test-% test-all build-% build test pip_test run yapf dev ci version .FORCE

README.rst

@@ -38,9 +38,9 @@ If you want to install senpy globally, use sudo instead of the ``--user`` flag.
 
 Docker Image
 ************
-Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy --host 0.0.0.0 --default-plugins``.
+Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy --default-plugins``.
 
-To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --host 0.0.0.0 --default-plugins -f /plugins``
+To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --default-plugins -f /plugins``
 
 Usage
 -----

senpy/extensions.py

@@ -284,7 +284,7 @@ class Senpy(object):
 
     @classmethod
     def validate_info(cls, info):
-        return all(x in info for x in ('name', 'module', 'version'))
+        return all(x in info for x in ('name', 'module', 'description', 'version'))
 
     def install_deps(self):
         for i in self.plugins.values():

senpy/plugins/conversion/emotion/ekman2vad.senpy

@@ -1,7 +1,7 @@
 ---
 name: Ekman2VAD
 module: ekman2vad
-description: Plugin to convert from Ekman to VAD
+description: Plugin to convert emotion sets from Ekman to VAD
 version: 0.1
 onyx:doesConversion:
   - onyx:conversionFrom: emoml:big6

test-requirements.txt

@@ -1,3 +1,3 @@
-pytest
 mock
 pytest-cov
+pytest

tests/test_api.py

@@ -0,0 +1,68 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
+from unittest import TestCase
+from senpy.api import parse_params, API_PARAMS, NIF_PARAMS, WEB_PARAMS
+from senpy.models import Error
+
+
+class APITest(TestCase):
+
+    def test_api_params(self):
+        """The API should not define any required parameters without a default"""
+        parse_params({}, spec=API_PARAMS)
+
+    def test_web_params(self):
+        """The WEB should not define any required parameters without a default"""
+        parse_params({}, spec=WEB_PARAMS)
+
+    def test_basic(self):
+        a = {}
+        try:
+            parse_params(a, spec=NIF_PARAMS)
+            raise AssertionError()
+        except Error:
+            pass
+        a = {'input': 'hello'}
+        p = parse_params(a, spec=NIF_PARAMS)
+        assert 'input' in p
+        b = {'i': 'hello'}
+        p = parse_params(b, spec=NIF_PARAMS)
+        assert 'input' in p
+
+    def test_plugin(self):
+        query = {}
+        plug_params = {
+            'hello': {
+                'aliases': ['hello', 'hiya'],
+                'required': True
+            }
+        }
+        try:
+            parse_params(query, spec=plug_params)
+            raise AssertionError()
+        except Error:
+            pass
+        query['hello'] = 'world'
+        p = parse_params(query, spec=plug_params)
+        assert 'hello' in p
+        assert p['hello'] == 'world'
+        del query['hello']
+
+        query['hiya'] = 'dlrow'
+        p = parse_params(query, spec=plug_params)
+        assert 'hello' in p
+        assert 'hiya' in p
+        assert p['hello'] == 'dlrow'
+
+    def test_default(self):
+        spec = {
+            'hello': {
+                'required': True,
+                'default': 1
+            }
+        }
+        p = parse_params({}, spec=spec)
+        assert 'hello' in p
+        assert p['hello'] == 1

tests/test_extensions.py

@@ -42,6 +42,7 @@ class ExtensionsTest(TestCase):
         info = {
             'name': 'TestPip',
             'module': 'dummy',
+            'description': None,
             'requirements': ['noop'],
             'version': 0
         }

version.sh

@@ -1,2 +0,0 @@
-#!/bin/sh
-VERSION=$(git describe --long --tags --dirty)