Push to dockerhub instead of private registry

Closes gsi-upm/senpy#34

.gitlab-ci.yml

@@ -1,18 +1,22 @@
-image: gsiupm/dockermake:latest
-
-
+# Uncomment if you want to use docker-in-docker
+# image: gsiupm/dockermake:latest
+# services:
+# - docker:dind
 # When using dind, it's wise to use the overlayfs driver for
 # improved performance.
+
 variables:
-  DOCKER_DRIVER: overlay
-  DOCKERFILE: Dockerfile
   IMAGENAME: $CI_REGISTRY_IMAGE
 
 stages:
   - test
   - push
+  - deploy
   - clean
 
+before_script:
+  - docker login -u $HUB_USER -p $HUB_PASSWORD
+
 .test: &test_definition
   stage: test
   script:
@@ -28,11 +32,8 @@ test-2.7:
   variables:
     PYTHON_VERSION: "2.7"
 
-
 .image: &image_definition
   stage: push
-  before_script:
-    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
   script:
     - make -e push-$PYTHON_VERSION
   only:
@@ -57,9 +58,44 @@ push-latest:
     - master
     - triggers
 
-clean :
+push-github:
+  stage: deploy
+  script:
+    - make -e push-github
+  only:
+    - master
+    - triggers
+
+deploy_pypi:
+  stage: deploy
+  script:   # Configure the PyPI credentials, then push the package, and cleanup the creds.
+    - echo "[server-login]" >> ~/.pypirc
+    - echo "username=" ${PYPI_USER} >> ~/.pypirc
+    - echo "password=" ${PYPI_PASSWORD} >> ~/.pypirc
+    - make pip_upload
+    - echo "" > ~/.pypirc && rm ~/.pypirc  # If the above fails, this won't run.
+  only:
+    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
+  except:
+    - branches
+
+deploy:
+  stage: deploy
+  environment: test
+  script:
+    - make -e deploy
+  only:
+    - master
+
+clean_docker :
   stage: clean
   script:
     - make -e clean
-  only:
-    - master
+  when: manual
+
+cleanup_pypirc:
+   stage: clean
+   when: always   # this is important; run even if preceding stages failed.
+   script:
+    - rm -vf ~/.pypirc  # we don't want to leave these around, but GitLab may clean up anyway.
+    - docker logout

Dockerfile.template

@@ -17,6 +17,6 @@ WORKDIR /usr/src/app
 COPY test-requirements.txt requirements.txt /usr/src/app/
 RUN pip install --use-wheel -r test-requirements.txt -r requirements.txt
 COPY . /usr/src/app/
-RUN pip install --no-deps --no-index .
+RUN pip install --no-index --no-deps --editable .
 
 ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

Makefile

@@ -1,14 +1,30 @@
+NAME=senpy
+VERSION=$(shell git describe --tags --dirty 2>/dev/null)
+GITHUB_REPO=git@github.com:gsi-upm/senpy.git
+
+IMAGENAME=gsiupm/senpy
+IMAGEWTAG=$(IMAGENAME):$(VERSION)
+
 PYVERSIONS=3.5 2.7
 PYMAIN=$(firstword $(PYVERSIONS))
-NAME=senpy
-REPO=gsiupm
-VERSION=$(shell git describe --tags --dirty 2>/dev/null)
-TARNAME=$(NAME)-$(VERSION).tar.gz 
-IMAGENAME=$(REPO)/$(NAME)
-IMAGEWTAG=$(IMAGENAME):$(VERSION)
+
 DEVPORT=5000
+
+TARNAME=$(NAME)-$(VERSION).tar.gz 
 action="test-${PYMAIN}"
 
+KUBE_CA_PEM_FILE=""
+KUBE_URL=""
+KUBE_TOKEN=""
+KUBE_NAMESPACE=$(NAME)
+KUBECTL=docker run --rm -v $(KUBE_CA_PEM_FILE):/tmp/ca.pem -v $$PWD:/tmp/cwd/ -i lachlanevenson/k8s-kubectl --server="$(KUBE_URL)" --token="$(KUBE_TOKEN)" --certificate-authority="/tmp/ca.pem" -n $(KUBE_NAMESPACE)
+CI_REGISTRY=docker.io
+CI_REGISTRY_USER=gitlab
+CI_BUILD_TOKEN=""
+CI_COMMIT_REF_NAME=master
+
+
+
 all: build run
 
 .FORCE:
@@ -18,7 +34,7 @@ version: .FORCE
 	@echo $(VERSION)
 
 yapf:
-	yapf -i -r senpy
+	yapf -i -r $(NAME)
 	yapf -i -r tests
 
 init:
@@ -37,7 +53,7 @@ quick_build: $(addprefix build-, $(PYMAIN))
 build: $(addprefix build-, $(PYVERSIONS))
 
 build-%: version Dockerfile-%
-	docker build -t '$(IMAGEWTAG)-python$*' -f Dockerfile-$* .;
+	docker build -t '$(IMAGEWTAG)-python$*' --cache-from $(IMAGENAME):python$* -f Dockerfile-$* .;
 
 quick_test: $(addprefix test-,$(PYMAIN))
 
@@ -53,8 +69,8 @@ dev: dev-$(PYMAIN)
 
 test-all: $(addprefix test-,$(PYVERSIONS))
 
-test-%: build-%
-	docker run --rm --entrypoint /usr/local/bin/python -w /usr/src/app $(IMAGEWTAG)-python$*  setup.py test
+test-%:
+	docker run --rm --entrypoint /usr/local/bin/python -w /usr/src/app $(IMAGENAME):python$*  setup.py test
 
 test: test-$(PYMAIN)
 
@@ -70,9 +86,12 @@ pip_test-%: sdist
 
 pip_test: $(addprefix pip_test-,$(PYVERSIONS))
 
+pip_upload: pip_test 
+	python setup.py sdist upload ;
+
 clean:
-	@docker ps -a | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true
-	@docker images | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true
+	@docker ps -a | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true
+	@docker images | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true
 	@docker stop $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true
 	@docker rm $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true
 
@@ -85,20 +104,22 @@ git_tag:
 git_push:
 	git push --tags origin master
 
-pip_upload: pip_test 
-	python setup.py sdist upload ;
-
 run-%: build-%
 	docker run --rm -p $(DEVPORT):5000 -ti '$(IMAGEWTAG)-python$(PYMAIN)' --default-plugins
 
 run: run-$(PYMAIN)
 
-push-latest: build-$(PYMAIN)
+push-latest: $(addprefix push-latest-,$(PYVERSIONS))
 	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
 	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGENAME)'
 	docker push '$(IMAGENAME):latest'
 	docker push '$(IMAGEWTAG)'
 
+push-latest-%: build-%
+	docker tag $(IMAGENAME):$(VERSION)-python$* $(IMAGENAME):python$*
+	docker push $(IMAGENAME):$(VERSION)-python$*
+	docker push $(IMAGENAME):python$*
+
 push-%: build-%
 	docker push $(IMAGENAME):$(VERSION)-python$*
 
@@ -106,7 +127,21 @@ push: $(addprefix push-,$(PYVERSIONS))
 	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
 	docker push  $(IMAGENAME):$(VERSION)
 
+push-github:
+	$(eval KEY_FILE := $(shell mktemp))
+	@echo "$$GITHUB_DEPLOY_KEY" > $(KEY_FILE)
+	@git remote rm github-deploy || true
+	git remote add github-deploy $(GITHUB_REPO)
+	@GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git push github-deploy $(CI_COMMIT_REF_NAME)
+	rm $(KEY_FILE)
+
 ci:
 	gitlab-runner exec docker --docker-volumes /var/run/docker.sock:/var/run/docker.sock --env CI_PROJECT_NAME=$(NAME) ${action}
 
-.PHONY: test test-% test-all build-% build test pip_test run yapf push-main push-% dev ci version .FORCE
+deploy:
+	@$(KUBECTL) delete secret $(CI_REGISTRY) || true
+	@$(KUBECTL) create secret docker-registry $(CI_REGISTRY) --docker-server=$(CI_REGISTRY) --docker-username=$(CI_REGISTRY_USER) --docker-email=$(CI_REGISTRY_USER) --docker-password=$(CI_BUILD_TOKEN)
+	@$(KUBECTL) apply -f /tmp/cwd/k8s/
+
+
+.PHONY: test test-% test-all build-% build test pip_test run yapf push-main push-% dev ci version .FORCE deploy

docs/plugins.rst

@@ -183,7 +183,11 @@ Training a classifier can be time time consuming. To avoid running the training
               def deactivate(self):
                   self.close()
 
-You can speficy a 'shelf_file' in your .senpy file. By default the ShelfMixin creates a file based on the plugin name and stores it in that plugin's folder.
+You can specify a 'shelf_file' in your .senpy file. By default the ShelfMixin creates a file based on the plugin name and stores it in that plugin's folder.
+
+Shelves may get corrupted if the plugin exists unexpectedly.
+A corrupt shelf prevents the plugin from loading.
+If you do not care about the pickle, you can force your plugin to remove the corrupted file and load anyway, set the  'force_shelf' to True in your .senpy file.
 
 I want to implement my service as a plugin, How i can do it?
 ????????????????????????????????????????????????????????????

k8s/README.md

@@ -0,0 +1,7 @@
+Deploy senpy to a kubernetes cluster.
+
+Usage:
+
+```
+kubectl apply -f . -n senpy
+```

k8s/senpy-deployment.yaml

@@ -0,0 +1,25 @@
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: senpy-latest
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        role: senpy-latest
+        app: test
+    spec:
+      containers:
+      - name: senpy-latest
+        image: registry.cluster.gsi.dit.upm.es/senpy/senpy:latest
+        args:
+          - "--default-plugins"
+        resources:
+          limits:
+            memory: "512Mi"
+            cpu: "1000m"
+        ports:
+          - name: web
+            containerPort: 5000

k8s/senpy-ingress.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: senpy-ingress
+spec:
+  rules:
+  - host: latest.senpy.cluster.gsi.dit.upm.es
+    http:
+      paths:
+      - path: /
+        backend:
+          serviceName: senpy-latest
+          servicePort: 5000

k8s/senpy-svc.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: senpy-latest
+spec:
+  type: ClusterIP
+  ports:
+    - port: 5000
+      protocol: TCP
+  selector:
+    role: senpy-latest

senpy/__main__.py

@@ -78,7 +78,16 @@ def main():
         action='store_true',
         default=False,
         help='Do not run a server, only install plugin dependencies')
+    parser.add_argument(
+        '--version',
+        '-v',
+        action='store_true',
+        default=False,
+        help='Output the senpy version and exit')
     args = parser.parse_args()
+    if args.version:
+        print('Senpy version {}'.format(senpy.__version__))
+        exit(1)
     logging.basicConfig()
     rl = logging.getLogger()
     rl.setLevel(getattr(logging, args.level))

senpy/extensions.py

@@ -178,7 +178,7 @@ class Senpy(object):
         except (Error, Exception) as ex:
             if not isinstance(ex, Error):
                 ex = Error(message=str(ex), status=500)
-            logger.error('Error returning analysis result')
+            logger.exception('Error returning analysis result')
             raise ex
         return resp
 
@@ -195,7 +195,7 @@ class Senpy(object):
 
     def convert_emotions(self, resp, plugins, params):
         """
-        Conversion of all emotions in a response.
+        Conversion of all emotions in a response **in place**.
         In addition to converting from one model to another, it has
         to include the conversion plugin to the analysis list.
         Needless to say, this is far from an elegant solution, but it works.
@@ -220,7 +220,6 @@ class Senpy(object):
                 e.parameters = params
                 raise e
         newentries = []
-        resp.analysis = set(resp.analysis)
         for i in resp.entries:
             if output == "full":
                 newemotions = copy.deepcopy(i.emotions)
@@ -229,7 +228,7 @@ class Senpy(object):
             for j in i.emotions:
                 plugname = j['prov:wasGeneratedBy']
                 candidate = candidates[plugname]
-                resp.analysis.add(candidate.id)
+                resp.analysis.append(candidate.id)
                 for k in candidate.convert(j, fromModel, toModel, params):
                     k.prov__wasGeneratedBy = candidate.id
                     if output == 'nested':
@@ -238,6 +237,7 @@ class Senpy(object):
             i.emotions = newemotions
             newentries.append(i)
         resp.entries = newentries
+        resp.analysis = list(set(resp.analysis))
 
     @property
     def default_plugin(self):

senpy/models.py

@@ -237,7 +237,10 @@ class BaseModel(SenpyMixin, dict):
         self.__setitem__(self._get_key(key), value)
 
     def __delattr__(self, key):
-        self.__delitem__(self._get_key(key))
+        try:
+            object.__delattr__(self, key)
+        except AttributeError:
+            self.__delitem__(self._get_key(key))
 
     def _plain_dict(self):
         d = {k: v for (k, v) in self.items() if k[0] != "_"}

senpy/plugins/__init__.py

@@ -96,7 +96,12 @@ class ShelfMixin(object):
         if not hasattr(self, '_sh') or self._sh is None:
             self.__dict__['_sh'] = {}
             if os.path.isfile(self.shelf_file):
-                self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb'))
+                try:
+                    self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb'))
+                except (IndexError, EOFError, pickle.UnpicklingError):
+                    logger.warning('{} has a corrupted shelf file!'.format(self.id))
+                    if not self.get('force_shelf', False):
+                        raise
         return self._sh
 
     @sh.deleter

tests/test_extensions.py

@@ -205,6 +205,7 @@ class ExtensionsTest(TestCase):
                                     [plugin, ],
                                     params)
         assert len(r3.entries[0].emotions) == 1
+        r3.jsonld()
 
     # def test_async_plugin(self):
     #     """ We should accept multiprocessing plugins with async=False"""

tests/test_plugins.py

@@ -83,7 +83,39 @@ class PluginsTest(TestCase):
         res2 = a.analyse(input=1)
         assert res2.entries[0].nif__isString == 2
 
-    def test_two(self):
+    def test_corrupt_shelf(self):
+        ''' Reusing the values of a previous shelf '''
+
+        emptyfile = os.path.join(self.shelf_dir, "emptyfile")
+        invalidfile = os.path.join(self.shelf_dir, "invalid_file")
+        with open(emptyfile, 'w+b'), open(invalidfile, 'w+b') as inf:
+            inf.write(b'ohno')
+
+        files = {emptyfile: ['empty file', (EOFError, IndexError)],
+                 invalidfile: ['invalid file', (pickle.UnpicklingError, IndexError)]}
+
+        for fn in files:
+            with open(fn, 'rb') as f:
+                msg, error = files[fn]
+                a = ShelfDummyPlugin(info={
+                    'name': 'shelve',
+                    'version': 'test',
+                    'shelf_file': f.name
+                })
+                assert os.path.isfile(a.shelf_file)
+                print('Shelf file: %s' % a.shelf_file)
+                with self.assertRaises(error):
+                    a.sh['a'] = 'fromA'
+                    a.save()
+                del a._sh
+                assert os.path.isfile(a.shelf_file)
+                a.force_shelf = True
+                a.sh['a'] = 'fromA'
+                a.save()
+                b = pickle.load(f)
+                assert b['a'] == 'fromA'
+
+    def test_reuse_shelf(self):
         ''' Reusing the values of a previous shelf '''
         a = ShelfDummyPlugin(info={
             'name': 'shelve',