From 6f489acdfc965401c988156baaf313f6c137da47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Wed, 27 Sep 2017 21:08:21 +0200 Subject: [PATCH 01/26] First version of makefiles --- base.mk | 24 ++++++++++++++ docker.mk | 21 ++++++++++++ git.mk | 19 +++++++++++ k8s.mk | 50 ++++++++++++++++++++++++++++ precommit.mk | 5 +++ python.mk | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 211 insertions(+) create mode 100644 base.mk create mode 100644 docker.mk create mode 100644 git.mk create mode 100644 k8s.mk create mode 100644 precommit.mk create mode 100644 python.mk diff --git a/base.mk b/base.mk new file mode 100644 index 0000000..55b8a34 --- /dev/null +++ b/base.mk @@ -0,0 +1,24 @@ +VERSION ?= $(shell git describe --tags --dirty 2>/dev/null) + +.FORCE: + +version: .FORCE + @echo $(VERSION) > $(NAME)/VERSION + @echo $(VERSION) + +help: ## Show this help. + @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/\(.*:\)[^#]*##\s*\(.*\)/\1\t\2/' | column -t -s " " + +config: ## Load config from the environment. You should run it once in every session before other tasks. Run: eval $(make config) + @echo ". ../.env || true;" + @awk '{ print "export " $$0}' .env + @echo "# Please, run: " + @echo "# eval \$$(make config)" +# If you need to run a command on the key/value pairs, use this: +# @awk '{ split($$0, a, "="); "echo " a[2] " | base64 -w 0" |& getline b64; print "export " a[1] "=" a[2]; print "export " a[1] "_BASE64=" b64}' .env + +ci: ## Run a task using gitlab-runner. Only use to debug problems in the CI pipeline + gitlab-runner exec shell --builds-dir '.builds' --env CI_PROJECT_NAME=$(NAME) ${action} + + +.PHONY:: config help ci version .FORCE diff --git a/docker.mk b/docker.mk new file mode 100644 index 0000000..aab2a8a --- /dev/null +++ b/docker.mk @@ -0,0 +1,21 @@ +IMAGEWTAG ?= $(IMAGENAME):$(VERSION) + +login: ## Log in to the registry. It will only be used in the server, or when running a CI task locally (if CI_BUILD_TOKEN is set). +ifeq ($(CI_BUILD_TOKEN),) + @echo "Not logging in to the docker registry" "$(CI_REGISTRY)" +else + @docker login -u gitlab-ci-token -p $(CI_BUILD_TOKEN) $(CI_REGISTRY) +endif +ifeq ($(HUB_USER),) + @echo "Not logging in to global the docker registry" +else + @docker login -u $(HUB_USER) -p $(HUB_PASSWORD) +endif + +clean:: ## Remove docker credentials +ifeq ($(HUB_USER),) +else + @docker logout +endif + +.PHONY:: login clean diff --git a/git.mk b/git.mk new file mode 100644 index 0000000..c4f6493 --- /dev/null +++ b/git.mk @@ -0,0 +1,19 @@ +git_commit: + git commit -a + +git_tag: + git tag ${VERSION} + +git_push: + git push --tags origin master + +push-github: ## Push the code to github. You need to set up HUB_USER and HUB_PASSWORD + $(eval KEY_FILE := $(shell mktemp)) + @echo "$$GITHUB_DEPLOY_KEY" > $(KEY_FILE) + @git remote rm github-deploy || true + git remote add github-deploy $(GITHUB_REPO) + @GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git fetch github-deploy $(CI_COMMIT_REF_NAME) || true + @GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git push github-deploy $(CI_COMMIT_REF_NAME) + rm $(KEY_FILE) + +.PHONY:: git_commit git_tag git_push push-github diff --git a/k8s.mk b/k8s.mk new file mode 100644 index 0000000..812abe8 --- /dev/null +++ b/k8s.mk @@ -0,0 +1,50 @@ +# Deployment with Kubernetes + +# KUBE_CA_PEM_FILE is the path of a certificate file. It automatically set by GitLab +# if you enable Kubernetes integration in a project. +# +# As of this writing, Kubernetes integration can not be set on a group level, so it has to +# be manually set in every project. +# Alternatively, we use a custom KUBE_CA_BUNDLE environment variable, which can be set at +# the group level. In this case, the variable contains the whole content of the certificate, +# which we dump to a temporary file +# +# Check if the KUBE_CA_PEM_FILE exists. Otherwise, create it from KUBE_CA_BUNDLE +KUBE_CA_TEMP=false +ifeq ($(wildcard $(KUBE_CA_PEM_FILE)),) + KUBE_CA_PEM_FILE:="$$PWD/.ca.crt" + CREATED:=$(shell echo -e "$$KUBE_CA_BUNDLE" > $(KUBE_CA_PEM_FILE)) +endif +KUBE_URL?="" +KUBE_TOKEN?="" +KUBE_NAMESPACE?=$(NAME) +KUBECTL=docker run --rm -v $(KUBE_CA_PEM_FILE):/tmp/ca.pem -i lachlanevenson/k8s-kubectl --server="$(KUBE_URL)" --token="$(KUBE_TOKEN)" --certificate-authority="/tmp/ca.pem" -n $(KUBE_NAMESPACE) +CI_COMMIT_REF_NAME?=master + +info: ## Print variables. Useful for debugging. + @echo "#KUBERNETES" + @echo KUBE_URL=$(KUBE_URL) + @echo KUBE_CA_PEM_FILE=$(KUBE_CA_PEM_FILE) + @echo KUBE_CA_BUNDLE=$$KUBE_CA_BUNDLE + @echo KUBE_TOKEN=$(KUBE_TOKEN) + @echo KUBE_NAMESPACE=$(KUBE_NAMESPACE) + @echo KUBECTL=$(KUBECTL) + + @echo "#CI" + @echo CI_PROJECT_NAME=$(CI_PROJECT_NAME) + @echo CI_REGISTRY=$(CI_REGISTRY) + @echo CI_REGISTRY_USER=$(CI_REGISTRY_USER) + @echo CI_COMMIT_REF_NAME=$(CI_COMMIT_REF_NAME) + +# +# Deployment and advanced features +# + + +deploy: ## Deploy to kubernetes using the credentials in KUBE_CA_PEM_FILE (or KUBE_CA_BUNDLE ) and TOKEN + @cat k8s/{*.yaml,*.yml,*.tmpl} | envsubst | $(KUBECTL) apply -f - + +deploy-check: ## Get the deployed configuration. + @$(KUBECTL) get deploy,pods,svc,ingress + +.PHONY:: info deploy deploy-check diff --git a/precommit.mk b/precommit.mk new file mode 100644 index 0000000..82fe75f --- /dev/null +++ b/precommit.mk @@ -0,0 +1,5 @@ +init: ## Init pre-commit hooks (i.e. enforcing format checking before allowing a commit) + pip install --user pre-commit + pre-commit install + +.PHONY:: init diff --git a/python.mk b/python.mk new file mode 100644 index 0000000..a69d70b --- /dev/null +++ b/python.mk @@ -0,0 +1,92 @@ +PYVERSIONS ?= 2.7 +PYMAIN ?= $(firstword $(PYVERSIONS)) +TARNAME ?= $(NAME)-$(VERSION).tar.gz + +DEVPORT ?= 6000 + +yapf: ## Format python code + yapf -i -r $(NAME) + yapf -i -r tests + +dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS)) ## Generate dockerfiles for each python version + @unlink Dockerfile >/dev/null + ln -s Dockerfile-$(PYMAIN) Dockerfile + +Dockerfile-%: Dockerfile.template ## Generate a specific dockerfile (e.g. Dockerfile-2.7) + sed "s/{{PYVERSION}}/$*/" Dockerfile.template > Dockerfile-$* + +quick_build: $(addprefix build-, $(PYMAIN)) + +build: $(addprefix build-, $(PYVERSIONS)) ## Build all images / python versions + +build-%: version Dockerfile-% ## Build a specific version (e.g. build-2.7) + docker build -t '$(IMAGEWTAG)-python$*' --cache-from $(IMAGENAME):python$* -f Dockerfile-$* .; + +dev-%: ## Launch a specific development environment using docker (e.g. dev-2.7) + @docker start $(NAME)-dev$* || (\ + $(MAKE) build-$*; \ + docker run -d -w /usr/src/app/ -p $(DEVPORT):5000 -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti --name $(NAME)-dev$* '$(IMAGEWTAG)-python$*'; \ + )\ + + docker exec -ti $(NAME)-dev$* bash + +dev: dev-$(PYMAIN) ## Launch a development environment using docker, using the default python version + +quick_test: test-$(PYMAIN) + +test-%: ## Run setup.py from in an isolated container, built from the base image. (e.g. test-2.7) +# This speeds tests up because the image has most (if not all) of the dependencies already. + docker rm $(NAME)-test-$* || true + docker create -ti --name $(NAME)-test-$* --entrypoint="" -w /usr/src/app/ $(IMAGENAME):python$* python setup.py test + docker cp . $(NAME)-test-$*:/usr/src/app + docker start -a $(NAME)-test-$* + +test: $(addprefix test-,$(PYVERSIONS)) ## Run the tests with the main python version + +run-%: build-% + docker run --rm -p $(DEVPORT):5000 -ti '$(IMAGEWTAG)-python$(PYMAIN)' --default-plugins + +run: run-$(PYMAIN) + +# Pypy - Upload a package + +dist/$(TARNAME): version + python setup.py sdist; + +sdist: dist/$(TARNAME) ## Generate the distribution file (wheel) + +pip_test-%: sdist ## Test the distribution file using pip install and a specific python version (e.g. pip_test-2.7) + docker run --rm -v $$PWD/dist:/dist/ python:$* pip install /dist/$(TARNAME); + +pip_test: $(addprefix pip_test-,$(PYVERSIONS)) ## Test pip installation with the main python version + +pip_upload: pip_test ## Upload package to pip + python setup.py sdist upload ; + +# Pushing to docker + +push-latest: $(addprefix push-latest-,$(PYVERSIONS)) ## Push the "latest" tag to dockerhub + docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)' + docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGENAME)' + docker push '$(IMAGENAME):latest' + docker push '$(IMAGEWTAG)' + +push-latest-%: build-% ## Push the latest image for a specific python version + docker tag $(IMAGENAME):$(VERSION)-python$* $(IMAGENAME):python$* + docker push $(IMAGENAME):$(VERSION)-python$* + docker push $(IMAGENAME):python$* + +push-%: build-% ## Push the image of the current version (tagged). e.g. push-2.7 + docker push $(IMAGENAME):$(VERSION)-python$* + +push: $(addprefix push-,$(PYVERSIONS)) ## Push an image with the current version for every python version + docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)' + docker push $(IMAGENAME):$(VERSION) + +clean:: ## Clean older docker images and containers related to this project and dev environments + @docker stop $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true + @docker rm $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true + @docker ps -a | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true + @docker images | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true + +.PHONY:: yapf dockerfiles Dockerfile-% quick_build build build-% dev-% quick-dev test quick_test push-latest push-latest-% push-% push From 473efd8dd7102c18d4fe6fa0c58fe848f2753aac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Wed, 27 Sep 2017 21:13:49 +0200 Subject: [PATCH 02/26] Updated makefiles from senpy --- makefiles.mk | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 makefiles.mk diff --git a/makefiles.mk b/makefiles.mk new file mode 100644 index 0000000..2b80a94 --- /dev/null +++ b/makefiles.mk @@ -0,0 +1,14 @@ +makefiles-remote: + git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git || true + +makefiles-commit: + git add -f .makefiles + git commit -m "Updated makefiles from ${NAME}" + +makefiles-push: makefiles-remote + git subtree push --prefix=.makefiles/ makefiles $(NAME) + +makefiles-pull: makefiles-remote + git subtree pull --prefix=.makefiles/ makefiles master --squash + +.PHONY:: makefiles-remote makefiles-commit makefiles-push makefiles-pull From cda9f5c4caefca7d4bc88b6d6b602db58b3380b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Wed, 27 Sep 2017 21:32:08 +0200 Subject: [PATCH 03/26] Updated makefiles from senpy --- README.md | 27 +++++++++++++++++++++++++++ makefiles.mk | 14 ++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 README.md create mode 100644 makefiles.mk diff --git a/README.md b/README.md new file mode 100644 index 0000000..e025233 --- /dev/null +++ b/README.md @@ -0,0 +1,27 @@ +These makefiles are recipes for several common tasks in different types of projects. +To add them to your project, simply do: + +``` +git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git +git subtree add --prefix=.makefiles/ makefiles master +touch Makefile +echo "include .makefiles/makefiles.mk" >> Makefile +``` + +Now you can take advantage of the recipes. +For instance, to add useful targets for a python project, just add this to your Makefile: + +``` +include .makefiles/python.mk +``` + +You may need to set special variables like the name of your project or the python versions you're targetting. +Take a look at each specific `.mk` file for more information, and the `Makefile` in the [senpy](https://lab.cluster.gsi.dit.upm.es/senpy/senpy) project for a real use case. + +If you update the makefiles from your repository, make sure to push the changes for review in upstream (this repository): + +``` +make makefiles-push +``` + +It will automatically commit all unstaged changes in the .makefiles folder. diff --git a/makefiles.mk b/makefiles.mk new file mode 100644 index 0000000..0589458 --- /dev/null +++ b/makefiles.mk @@ -0,0 +1,14 @@ +makefiles-remote: + git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git || true + +makefiles-commit: makefiles-remote + git add -f .makefiles + git commit -em "Updated makefiles from ${NAME}" + +makefiles-push: makefiles-commit + git subtree push --prefix=.makefiles/ makefiles $(NAME) + +makefiles-pull: makefiles-remote + git subtree pull --prefix=.makefiles/ makefiles master --squash + +.PHONY:: makefiles-remote makefiles-commit makefiles-push makefiles-pull From 7444aa7ec82dec8bac284d268ef27303ce48082a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Wed, 27 Sep 2017 21:43:54 +0200 Subject: [PATCH 04/26] Updated makefiles from senpy --- makefiles.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefiles.mk b/makefiles.mk index 0589458..4bee03b 100644 --- a/makefiles.mk +++ b/makefiles.mk @@ -5,7 +5,7 @@ makefiles-commit: makefiles-remote git add -f .makefiles git commit -em "Updated makefiles from ${NAME}" -makefiles-push: makefiles-commit +makefiles-push: git subtree push --prefix=.makefiles/ makefiles $(NAME) makefiles-pull: makefiles-remote From 407fd718a3138a450141e5d28e9efa68d090c323 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Thu, 28 Sep 2017 10:22:46 +0200 Subject: [PATCH 05/26] Fixed wildcard in k8s recipe --- k8s.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s.mk b/k8s.mk index 812abe8..faf1e0c 100644 --- a/k8s.mk +++ b/k8s.mk @@ -42,7 +42,7 @@ info: ## Print variables. Useful for debugging. deploy: ## Deploy to kubernetes using the credentials in KUBE_CA_PEM_FILE (or KUBE_CA_BUNDLE ) and TOKEN - @cat k8s/{*.yaml,*.yml,*.tmpl} | envsubst | $(KUBECTL) apply -f - + @cat k8s/*.{yaml,yml,tmpl} | envsubst | $(KUBECTL) apply -f - deploy-check: ## Get the deployed configuration. @$(KUBECTL) get deploy,pods,svc,ingress From 8ad2fae774675e26d4ca45c744ed6015488fa9e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Thu, 28 Sep 2017 10:33:26 +0200 Subject: [PATCH 06/26] More fixes to k8s recipe --- k8s.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/k8s.mk b/k8s.mk index faf1e0c..bc1ce2d 100644 --- a/k8s.mk +++ b/k8s.mk @@ -42,7 +42,8 @@ info: ## Print variables. Useful for debugging. deploy: ## Deploy to kubernetes using the credentials in KUBE_CA_PEM_FILE (or KUBE_CA_BUNDLE ) and TOKEN - @cat k8s/*.{yaml,yml,tmpl} | envsubst | $(KUBECTL) apply -f - + @ls k8s/ + @cat k8s/*.{yaml,yml,tmpl} 2>/dev/null | envsubst | $(KUBECTL) apply -f - deploy-check: ## Get the deployed configuration. @$(KUBECTL) get deploy,pods,svc,ingress From 921d7f23ce1cba604a4f38da5b2551b8b620293e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Thu, 28 Sep 2017 10:39:29 +0200 Subject: [PATCH 07/26] Remove curly braces from makefile In gitlab, make is using /bin/sh, which does not accept brace expansion --- k8s.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s.mk b/k8s.mk index bc1ce2d..96892b2 100644 --- a/k8s.mk +++ b/k8s.mk @@ -43,7 +43,7 @@ info: ## Print variables. Useful for debugging. deploy: ## Deploy to kubernetes using the credentials in KUBE_CA_PEM_FILE (or KUBE_CA_BUNDLE ) and TOKEN @ls k8s/ - @cat k8s/*.{yaml,yml,tmpl} 2>/dev/null | envsubst | $(KUBECTL) apply -f - + @cat k8s/*.yaml k8s/*.yml k8s/*.tmpl 2>/dev/null | envsubst | $(KUBECTL) apply -f - deploy-check: ## Get the deployed configuration. @$(KUBECTL) get deploy,pods,svc,ingress From 963211caf2c870ecad8a0e5449e9952edd2e934a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Fri, 29 Sep 2017 15:00:37 +0200 Subject: [PATCH 08/26] Updated makefiles from gsictl --- README.md | 2 +- base.mk | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e025233..2ab487e 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ To add them to your project, simply do: git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git git subtree add --prefix=.makefiles/ makefiles master touch Makefile -echo "include .makefiles/makefiles.mk" >> Makefile +echo "include .makefiles/base.mk" >> Makefile ``` Now you can take advantage of the recipes. diff --git a/base.mk b/base.mk index 55b8a34..aaeae57 100644 --- a/base.mk +++ b/base.mk @@ -1,5 +1,8 @@ +NAME ?= $(shell basename $(CURDIR)) VERSION ?= $(shell git describe --tags --dirty 2>/dev/null) +# Get the location of this makefile. +MK_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) .FORCE: version: .FORCE @@ -20,5 +23,6 @@ config: ## Load config from the environment. You should run it once in every se ci: ## Run a task using gitlab-runner. Only use to debug problems in the CI pipeline gitlab-runner exec shell --builds-dir '.builds' --env CI_PROJECT_NAME=$(NAME) ${action} +include $(MK_DIR)/makefiles.mk .PHONY:: config help ci version .FORCE From 53db670715c4fad5ca7de8bfa071a75982e506ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Fri, 29 Sep 2017 15:11:08 +0200 Subject: [PATCH 09/26] Updated makefiles from gsictl --- base.mk | 1 + docker.mk | 10 +++++++--- git.mk | 11 +++++++---- makefiles.mk | 4 +++- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/base.mk b/base.mk index aaeae57..1390d44 100644 --- a/base.mk +++ b/base.mk @@ -24,5 +24,6 @@ ci: ## Run a task using gitlab-runner. Only use to debug problems in the CI pip gitlab-runner exec shell --builds-dir '.builds' --env CI_PROJECT_NAME=$(NAME) ${action} include $(MK_DIR)/makefiles.mk +include $(MK_DIR)/docker.mk .PHONY:: config help ci version .FORCE diff --git a/docker.mk b/docker.mk index aab2a8a..61d9f0a 100644 --- a/docker.mk +++ b/docker.mk @@ -1,6 +1,6 @@ IMAGEWTAG ?= $(IMAGENAME):$(VERSION) -login: ## Log in to the registry. It will only be used in the server, or when running a CI task locally (if CI_BUILD_TOKEN is set). +docker-login: ## Log in to the registry. It will only be used in the server, or when running a CI task locally (if CI_BUILD_TOKEN is set). ifeq ($(CI_BUILD_TOKEN),) @echo "Not logging in to the docker registry" "$(CI_REGISTRY)" else @@ -12,10 +12,14 @@ else @docker login -u $(HUB_USER) -p $(HUB_PASSWORD) endif -clean:: ## Remove docker credentials +login:: docker-login + +clean:: docker-clean ## Remove docker credentials + +docker-clean: ifeq ($(HUB_USER),) else @docker logout endif -.PHONY:: login clean +.PHONY:: docker-login docker-clean login clean diff --git a/git.mk b/git.mk index c4f6493..991b929 100644 --- a/git.mk +++ b/git.mk @@ -1,12 +1,15 @@ -git_commit: +commit: git commit -a -git_tag: +tag: git tag ${VERSION} -git_push: +push:: git push --tags origin master +pull:: + git pull --all + push-github: ## Push the code to github. You need to set up HUB_USER and HUB_PASSWORD $(eval KEY_FILE := $(shell mktemp)) @echo "$$GITHUB_DEPLOY_KEY" > $(KEY_FILE) @@ -16,4 +19,4 @@ push-github: ## Push the code to github. You need to set up HUB_USER and HUB_PAS @GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git push github-deploy $(CI_COMMIT_REF_NAME) rm $(KEY_FILE) -.PHONY:: git_commit git_tag git_push push-github +.PHONY:: commit tag push push-github diff --git a/makefiles.mk b/makefiles.mk index 4bee03b..158c332 100644 --- a/makefiles.mk +++ b/makefiles.mk @@ -1,5 +1,5 @@ makefiles-remote: - git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git || true + @git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git 2>/dev/null || true makefiles-commit: makefiles-remote git add -f .makefiles @@ -11,4 +11,6 @@ makefiles-push: makefiles-pull: makefiles-remote git subtree pull --prefix=.makefiles/ makefiles master --squash +update:: makefiles-pull + .PHONY:: makefiles-remote makefiles-commit makefiles-push makefiles-pull From fc94b454482181bbf2dffc1ddb91a048cd9e2949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Fri, 29 Sep 2017 15:14:29 +0200 Subject: [PATCH 10/26] Updated makefiles from gsictl --- makefiles.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefiles.mk b/makefiles.mk index 158c332..d97127b 100644 --- a/makefiles.mk +++ b/makefiles.mk @@ -11,6 +11,6 @@ makefiles-push: makefiles-pull: makefiles-remote git subtree pull --prefix=.makefiles/ makefiles master --squash -update:: makefiles-pull +pull:: makefiles-pull .PHONY:: makefiles-remote makefiles-commit makefiles-push makefiles-pull From 99d4bc70bcd9dc22b3cdffed552ea8b3c90813dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Fri, 29 Sep 2017 15:32:11 +0200 Subject: [PATCH 11/26] Updated makefiles from gsictl --- docker.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker.mk b/docker.mk index 61d9f0a..16973a7 100644 --- a/docker.mk +++ b/docker.mk @@ -14,9 +14,9 @@ endif login:: docker-login -clean:: docker-clean ## Remove docker credentials +clean:: docker-clean -docker-clean: +docker-clean: ## Remove docker credentials ifeq ($(HUB_USER),) else @docker logout From f9a75f4e215398064f334342f5f586d42f75a4b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Fri, 29 Sep 2017 15:32:51 +0200 Subject: [PATCH 12/26] Updated makefiles from gsictl --- docker.mk | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker.mk b/docker.mk index 16973a7..4845c2f 100644 --- a/docker.mk +++ b/docker.mk @@ -12,14 +12,14 @@ else @docker login -u $(HUB_USER) -p $(HUB_PASSWORD) endif -login:: docker-login - -clean:: docker-clean - docker-clean: ## Remove docker credentials ifeq ($(HUB_USER),) else @docker logout endif +login:: docker-login + +clean:: docker-clean + .PHONY:: docker-login docker-clean login clean From 3d23370a5957e074775d94837bdb41e2c568082a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Fri, 29 Sep 2017 15:33:58 +0200 Subject: [PATCH 13/26] Updated makefiles from gsictl --- base.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/base.mk b/base.mk index 1390d44..d0c392b 100644 --- a/base.mk +++ b/base.mk @@ -25,5 +25,6 @@ ci: ## Run a task using gitlab-runner. Only use to debug problems in the CI pip include $(MK_DIR)/makefiles.mk include $(MK_DIR)/docker.mk +include $(MK_DIR)/git.mk .PHONY:: config help ci version .FORCE From 14bcfd511fc42c5734bc3697e72867b72a6d53e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Fri, 29 Sep 2017 15:38:59 +0200 Subject: [PATCH 14/26] Updated makefiles from gsictl --- git.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git.mk b/git.mk index 991b929..6a2413f 100644 --- a/git.mk +++ b/git.mk @@ -5,7 +5,7 @@ tag: git tag ${VERSION} push:: - git push --tags origin master + git push -u --tags origin HEAD pull:: git pull --all From a3a9414073ab454ce9d27444e80cd24534037c43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Fri, 29 Sep 2017 15:39:26 +0200 Subject: [PATCH 15/26] Updated makefiles from gsictl --- git.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git.mk b/git.mk index 6a2413f..178cd98 100644 --- a/git.mk +++ b/git.mk @@ -5,7 +5,7 @@ tag: git tag ${VERSION} push:: - git push -u --tags origin HEAD + git push --tags -u origin HEAD pull:: git pull --all From cbef9630b4e8bcc3f34d52bf4d42c7e3d638f310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Fri, 29 Sep 2017 15:51:56 +0200 Subject: [PATCH 16/26] Updated makefiles from gsictl --- git.mk | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/git.mk b/git.mk index 178cd98..dae0e82 100644 --- a/git.mk +++ b/git.mk @@ -4,19 +4,25 @@ commit: tag: git tag ${VERSION} -push:: +git-push:: git push --tags -u origin HEAD -pull:: +git-pull: git pull --all -push-github: ## Push the code to github. You need to set up HUB_USER and HUB_PASSWORD +push-github: ## Push the code to github. You need to set up GITHUB_DEPLOY_KEY +ifeq ($(GITHUB_DEPLOY_KEY),) +else $(eval KEY_FILE := $(shell mktemp)) - @echo "$$GITHUB_DEPLOY_KEY" > $(KEY_FILE) + @echo "$(GITHUB_DEPLOY_KEY)" > $(KEY_FILE) @git remote rm github-deploy || true git remote add github-deploy $(GITHUB_REPO) @GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git fetch github-deploy $(CI_COMMIT_REF_NAME) || true @GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git push github-deploy $(CI_COMMIT_REF_NAME) rm $(KEY_FILE) +endif -.PHONY:: commit tag push push-github +push:: git-push +pull:: git-pull + +.PHONY:: commit tag push git-push git-pull push-github From 0dc93fc16be7e659fef1c9cbed838659f89a7538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Fri, 29 Sep 2017 15:53:14 +0200 Subject: [PATCH 17/26] Updated makefiles from gsictl --- makefiles.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/makefiles.mk b/makefiles.mk index d97127b..03dcc17 100644 --- a/makefiles.mk +++ b/makefiles.mk @@ -12,5 +12,6 @@ makefiles-pull: makefiles-remote git subtree pull --prefix=.makefiles/ makefiles master --squash pull:: makefiles-pull +push:: makefiles-push -.PHONY:: makefiles-remote makefiles-commit makefiles-push makefiles-pull +.PHONY:: makefiles-remote makefiles-commit makefiles-push makefiles-pull pull push From 3dc27f12f7c408ccbda43fc0bb49162bffbe1fad Mon Sep 17 00:00:00 2001 From: militarpancho Date: Mon, 2 Oct 2017 13:12:17 +0200 Subject: [PATCH 18/26] Updated makefiles from sentiment-meaningCloud --- python.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python.mk b/python.mk index a69d70b..33d012e 100644 --- a/python.mk +++ b/python.mk @@ -79,7 +79,7 @@ push-latest-%: build-% ## Push the latest image for a specific python version push-%: build-% ## Push the image of the current version (tagged). e.g. push-2.7 docker push $(IMAGENAME):$(VERSION)-python$* -push: $(addprefix push-,$(PYVERSIONS)) ## Push an image with the current version for every python version +push:: $(addprefix push-,$(PYVERSIONS)) ## Push an image with the current version for every python version docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)' docker push $(IMAGENAME):$(VERSION) From f04cbeeddbcbefaf16671fbe5587b156d8fe3902 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Tue, 3 Oct 2017 13:41:51 +0200 Subject: [PATCH 19/26] Testing new k8s mk --- base.mk | 10 +++++++++- k8s.mk | 13 ++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/base.mk b/base.mk index d0c392b..f170cfa 100644 --- a/base.mk +++ b/base.mk @@ -1,8 +1,13 @@ +export NAME ?= $(shell basename $(CURDIR)) VERSION ?= $(shell git describe --tags --dirty 2>/dev/null) # Get the location of this makefile. MK_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) + +include .env +include ../.env + .FORCE: version: .FORCE @@ -13,7 +18,7 @@ help: ## Show this help. @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/\(.*:\)[^#]*##\s*\(.*\)/\1\t\2/' | column -t -s " " config: ## Load config from the environment. You should run it once in every session before other tasks. Run: eval $(make config) - @echo ". ../.env || true;" + @awk '{ print "export " $$0}' ../.env @awk '{ print "export " $$0}' .env @echo "# Please, run: " @echo "# eval \$$(make config)" @@ -27,4 +32,7 @@ include $(MK_DIR)/makefiles.mk include $(MK_DIR)/docker.mk include $(MK_DIR)/git.mk +info:: ## List all variables + env + .PHONY:: config help ci version .FORCE diff --git a/k8s.mk b/k8s.mk index 96892b2..e380a8c 100644 --- a/k8s.mk +++ b/k8s.mk @@ -11,17 +11,16 @@ # # Check if the KUBE_CA_PEM_FILE exists. Otherwise, create it from KUBE_CA_BUNDLE KUBE_CA_TEMP=false -ifeq ($(wildcard $(KUBE_CA_PEM_FILE)),) - KUBE_CA_PEM_FILE:="$$PWD/.ca.crt" - CREATED:=$(shell echo -e "$$KUBE_CA_BUNDLE" > $(KUBE_CA_PEM_FILE)) +ifndef KUBE_CA_PEM_FILE +KUBE_CA_PEM_FILE:=$$PWD/.ca.crt +CREATED:=$(shell echo -e "$(KUBE_CA_BUNDLE)" > $(KUBE_CA_PEM_FILE)) endif -KUBE_URL?="" KUBE_TOKEN?="" KUBE_NAMESPACE?=$(NAME) KUBECTL=docker run --rm -v $(KUBE_CA_PEM_FILE):/tmp/ca.pem -i lachlanevenson/k8s-kubectl --server="$(KUBE_URL)" --token="$(KUBE_TOKEN)" --certificate-authority="/tmp/ca.pem" -n $(KUBE_NAMESPACE) CI_COMMIT_REF_NAME?=master -info: ## Print variables. Useful for debugging. +info:: ## Print variables. Useful for debugging. @echo "#KUBERNETES" @echo KUBE_URL=$(KUBE_URL) @echo KUBE_CA_PEM_FILE=$(KUBE_CA_PEM_FILE) @@ -35,6 +34,7 @@ info: ## Print variables. Useful for debugging. @echo CI_REGISTRY=$(CI_REGISTRY) @echo CI_REGISTRY_USER=$(CI_REGISTRY_USER) @echo CI_COMMIT_REF_NAME=$(CI_COMMIT_REF_NAME) + @echo "CREATED=$(CREATED)" # # Deployment and advanced features @@ -42,8 +42,7 @@ info: ## Print variables. Useful for debugging. deploy: ## Deploy to kubernetes using the credentials in KUBE_CA_PEM_FILE (or KUBE_CA_BUNDLE ) and TOKEN - @ls k8s/ - @cat k8s/*.yaml k8s/*.yml k8s/*.tmpl 2>/dev/null | envsubst | $(KUBECTL) apply -f - + cat k8s/*.yaml | envsubst | $(KUBECTL) apply -f - deploy-check: ## Get the deployed configuration. @$(KUBECTL) get deploy,pods,svc,ingress From 1543f5550e4ed3e7b990f1a6d22f95db9cad7263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Tue, 3 Oct 2017 13:46:09 +0200 Subject: [PATCH 20/26] Updated makefiles from senpy --- base.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base.mk b/base.mk index f170cfa..cc27cfa 100644 --- a/base.mk +++ b/base.mk @@ -5,8 +5,8 @@ VERSION ?= $(shell git describe --tags --dirty 2>/dev/null) # Get the location of this makefile. MK_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) -include .env -include ../.env +-include .env +-include ../.env .FORCE: From 8fe7616bae24e4e01e1ae331a47fe82401bc262d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Tue, 3 Oct 2017 15:08:16 +0200 Subject: [PATCH 21/26] Updated makefiles from senpy --- k8s.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/k8s.mk b/k8s.mk index e380a8c..a493b4a 100644 --- a/k8s.mk +++ b/k8s.mk @@ -42,7 +42,8 @@ info:: ## Print variables. Useful for debugging. deploy: ## Deploy to kubernetes using the credentials in KUBE_CA_PEM_FILE (or KUBE_CA_BUNDLE ) and TOKEN - cat k8s/*.yaml | envsubst | $(KUBECTL) apply -f - + @ls k8s/*.yaml k8s/*.yml k8s/*.tmpl 2>/dev/null || true + @cat k8s/*.yaml k8s/*.yml k8s/*.tmpl 2>/dev/null | envsubst | $(KUBECTL) apply -f - deploy-check: ## Get the deployed configuration. @$(KUBECTL) get deploy,pods,svc,ingress From c23f7986b461af2860d0ebee3d058ae4ffe094c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Tue, 3 Oct 2017 15:40:56 +0200 Subject: [PATCH 22/26] Trying to fix push to github --- git.mk | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git.mk b/git.mk index dae0e82..836eb14 100644 --- a/git.mk +++ b/git.mk @@ -13,12 +13,12 @@ git-pull: push-github: ## Push the code to github. You need to set up GITHUB_DEPLOY_KEY ifeq ($(GITHUB_DEPLOY_KEY),) else - $(eval KEY_FILE := $(shell mktemp)) + $(eval KEY_FILE := "$(shell mktemp)") @echo "$(GITHUB_DEPLOY_KEY)" > $(KEY_FILE) @git remote rm github-deploy || true git remote add github-deploy $(GITHUB_REPO) - @GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git fetch github-deploy $(CI_COMMIT_REF_NAME) || true - @GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git push github-deploy $(CI_COMMIT_REF_NAME) + -@GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git fetch github-deploy $(CI_COMMIT_REF_NAME) + @GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git push github-deploy HEAD:$(CI_COMMIT_REF_NAME) rm $(KEY_FILE) endif From d150321741e958947c1aa1afd90bf0675ad3746e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Tue, 3 Oct 2017 18:24:30 +0200 Subject: [PATCH 23/26] Updated makefiles from meaningcloud * Fixed some python+docker variables * Improved defaults for docker image names --- base.mk | 12 +++++------- docker.mk | 6 +++++- python.mk | 10 +++++++++- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/base.mk b/base.mk index cc27cfa..f67f682 100644 --- a/base.mk +++ b/base.mk @@ -2,18 +2,16 @@ export NAME ?= $(shell basename $(CURDIR)) VERSION ?= $(shell git describe --tags --dirty 2>/dev/null) +ifeq ($(wildcard $VERSION),) + VERSION:=unknown +endif + # Get the location of this makefile. MK_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) -include .env -include ../.env -.FORCE: - -version: .FORCE - @echo $(VERSION) > $(NAME)/VERSION - @echo $(VERSION) - help: ## Show this help. @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/\(.*:\)[^#]*##\s*\(.*\)/\1\t\2/' | column -t -s " " @@ -35,4 +33,4 @@ include $(MK_DIR)/git.mk info:: ## List all variables env -.PHONY:: config help ci version .FORCE +.PHONY:: config help ci diff --git a/docker.mk b/docker.mk index 4845c2f..f74d93f 100644 --- a/docker.mk +++ b/docker.mk @@ -1,4 +1,5 @@ -IMAGEWTAG ?= $(IMAGENAME):$(VERSION) +IMAGENAME?=$(NAME) +IMAGEWTAG?=$(IMAGENAME):$(VERSION) docker-login: ## Log in to the registry. It will only be used in the server, or when running a CI task locally (if CI_BUILD_TOKEN is set). ifeq ($(CI_BUILD_TOKEN),) @@ -22,4 +23,7 @@ login:: docker-login clean:: docker-clean +docker-info: + @echo IMAGEWTAG=${IMAGEWTAG} + .PHONY:: docker-login docker-clean login clean diff --git a/python.mk b/python.mk index 33d012e..f14cc7a 100644 --- a/python.mk +++ b/python.mk @@ -1,9 +1,17 @@ PYVERSIONS ?= 2.7 PYMAIN ?= $(firstword $(PYVERSIONS)) TARNAME ?= $(NAME)-$(VERSION).tar.gz +VERSIONFILE ?= $(NAME)/VERSION DEVPORT ?= 6000 + +.FORCE: + +version: .FORCE + @echo $(VERSION) > $(VERSIONFILE) + @echo $(VERSION) + yapf: ## Format python code yapf -i -r $(NAME) yapf -i -r tests @@ -89,4 +97,4 @@ clean:: ## Clean older docker images and containers related to this project and @docker ps -a | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true @docker images | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true -.PHONY:: yapf dockerfiles Dockerfile-% quick_build build build-% dev-% quick-dev test quick_test push-latest push-latest-% push-% push +.PHONY:: yapf dockerfiles Dockerfile-% quick_build build build-% dev-% quick-dev test quick_test push-latest push-latest-% push-% push version .FORCE From f0c211c00a79c703df2f906cb749f72d97efba25 Mon Sep 17 00:00:00 2001 From: militarpancho Date: Wed, 4 Oct 2017 15:37:05 +0200 Subject: [PATCH 24/26] PYVERSION changed --- python.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python.mk b/python.mk index f14cc7a..2ad9559 100644 --- a/python.mk +++ b/python.mk @@ -1,4 +1,4 @@ -PYVERSIONS ?= 2.7 +PYVERSIONS ?= 3.5 PYMAIN ?= $(firstword $(PYVERSIONS)) TARNAME ?= $(NAME)-$(VERSION).tar.gz VERSIONFILE ?= $(NAME)/VERSION From 42224e343cc662b8f70149da37957c0d1374a531 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Thu, 5 Oct 2017 11:19:02 +0200 Subject: [PATCH 25/26] Updated makefiles from meaningcloud Version was "unknown" due to a bug --- base.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base.mk b/base.mk index f67f682..a4d94d7 100644 --- a/base.mk +++ b/base.mk @@ -2,8 +2,8 @@ export NAME ?= $(shell basename $(CURDIR)) VERSION ?= $(shell git describe --tags --dirty 2>/dev/null) -ifeq ($(wildcard $VERSION),) - VERSION:=unknown +ifeq ($(VERSION),) + VERSION:="unknown" endif # Get the location of this makefile. From 919c4a07a26517e0354fc3343ee7d52891dda105 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Thu, 5 Oct 2017 13:25:33 +0000 Subject: [PATCH 26/26] Update base.mk --- base.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base.mk b/base.mk index a4d94d7..4371ebe 100644 --- a/base.mk +++ b/base.mk @@ -3,7 +3,7 @@ NAME ?= $(shell basename $(CURDIR)) VERSION ?= $(shell git describe --tags --dirty 2>/dev/null) ifeq ($(VERSION),) - VERSION:="unknown" + VERSION:=unknown endif # Get the location of this makefile.