commit 6f489acdfc965401c988156baaf313f6c137da47 Author: J. Fernando Sánchez Date: Wed Sep 27 21:08:21 2017 +0200 First version of makefiles diff --git a/base.mk b/base.mk new file mode 100644 index 0000000..55b8a34 --- /dev/null +++ b/base.mk @@ -0,0 +1,24 @@ +VERSION ?= $(shell git describe --tags --dirty 2>/dev/null) + +.FORCE: + +version: .FORCE + @echo $(VERSION) > $(NAME)/VERSION + @echo $(VERSION) + +help: ## Show this help. + @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/\(.*:\)[^#]*##\s*\(.*\)/\1\t\2/' | column -t -s " " + +config: ## Load config from the environment. You should run it once in every session before other tasks. Run: eval $(make config) + @echo ". ../.env || true;" + @awk '{ print "export " $$0}' .env + @echo "# Please, run: " + @echo "# eval \$$(make config)" +# If you need to run a command on the key/value pairs, use this: +# @awk '{ split($$0, a, "="); "echo " a[2] " | base64 -w 0" |& getline b64; print "export " a[1] "=" a[2]; print "export " a[1] "_BASE64=" b64}' .env + +ci: ## Run a task using gitlab-runner. Only use to debug problems in the CI pipeline + gitlab-runner exec shell --builds-dir '.builds' --env CI_PROJECT_NAME=$(NAME) ${action} + + +.PHONY:: config help ci version .FORCE diff --git a/docker.mk b/docker.mk new file mode 100644 index 0000000..aab2a8a --- /dev/null +++ b/docker.mk @@ -0,0 +1,21 @@ +IMAGEWTAG ?= $(IMAGENAME):$(VERSION) + +login: ## Log in to the registry. It will only be used in the server, or when running a CI task locally (if CI_BUILD_TOKEN is set). +ifeq ($(CI_BUILD_TOKEN),) + @echo "Not logging in to the docker registry" "$(CI_REGISTRY)" +else + @docker login -u gitlab-ci-token -p $(CI_BUILD_TOKEN) $(CI_REGISTRY) +endif +ifeq ($(HUB_USER),) + @echo "Not logging in to global the docker registry" +else + @docker login -u $(HUB_USER) -p $(HUB_PASSWORD) +endif + +clean:: ## Remove docker credentials +ifeq ($(HUB_USER),) +else + @docker logout +endif + +.PHONY:: login clean diff --git a/git.mk b/git.mk new file mode 100644 index 0000000..c4f6493 --- /dev/null +++ b/git.mk @@ -0,0 +1,19 @@ +git_commit: + git commit -a + +git_tag: + git tag ${VERSION} + +git_push: + git push --tags origin master + +push-github: ## Push the code to github. You need to set up HUB_USER and HUB_PASSWORD + $(eval KEY_FILE := $(shell mktemp)) + @echo "$$GITHUB_DEPLOY_KEY" > $(KEY_FILE) + @git remote rm github-deploy || true + git remote add github-deploy $(GITHUB_REPO) + @GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git fetch github-deploy $(CI_COMMIT_REF_NAME) || true + @GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git push github-deploy $(CI_COMMIT_REF_NAME) + rm $(KEY_FILE) + +.PHONY:: git_commit git_tag git_push push-github diff --git a/k8s.mk b/k8s.mk new file mode 100644 index 0000000..812abe8 --- /dev/null +++ b/k8s.mk @@ -0,0 +1,50 @@ +# Deployment with Kubernetes + +# KUBE_CA_PEM_FILE is the path of a certificate file. It automatically set by GitLab +# if you enable Kubernetes integration in a project. +# +# As of this writing, Kubernetes integration can not be set on a group level, so it has to +# be manually set in every project. +# Alternatively, we use a custom KUBE_CA_BUNDLE environment variable, which can be set at +# the group level. In this case, the variable contains the whole content of the certificate, +# which we dump to a temporary file +# +# Check if the KUBE_CA_PEM_FILE exists. Otherwise, create it from KUBE_CA_BUNDLE +KUBE_CA_TEMP=false +ifeq ($(wildcard $(KUBE_CA_PEM_FILE)),) + KUBE_CA_PEM_FILE:="$$PWD/.ca.crt" + CREATED:=$(shell echo -e "$$KUBE_CA_BUNDLE" > $(KUBE_CA_PEM_FILE)) +endif +KUBE_URL?="" +KUBE_TOKEN?="" +KUBE_NAMESPACE?=$(NAME) +KUBECTL=docker run --rm -v $(KUBE_CA_PEM_FILE):/tmp/ca.pem -i lachlanevenson/k8s-kubectl --server="$(KUBE_URL)" --token="$(KUBE_TOKEN)" --certificate-authority="/tmp/ca.pem" -n $(KUBE_NAMESPACE) +CI_COMMIT_REF_NAME?=master + +info: ## Print variables. Useful for debugging. + @echo "#KUBERNETES" + @echo KUBE_URL=$(KUBE_URL) + @echo KUBE_CA_PEM_FILE=$(KUBE_CA_PEM_FILE) + @echo KUBE_CA_BUNDLE=$$KUBE_CA_BUNDLE + @echo KUBE_TOKEN=$(KUBE_TOKEN) + @echo KUBE_NAMESPACE=$(KUBE_NAMESPACE) + @echo KUBECTL=$(KUBECTL) + + @echo "#CI" + @echo CI_PROJECT_NAME=$(CI_PROJECT_NAME) + @echo CI_REGISTRY=$(CI_REGISTRY) + @echo CI_REGISTRY_USER=$(CI_REGISTRY_USER) + @echo CI_COMMIT_REF_NAME=$(CI_COMMIT_REF_NAME) + +# +# Deployment and advanced features +# + + +deploy: ## Deploy to kubernetes using the credentials in KUBE_CA_PEM_FILE (or KUBE_CA_BUNDLE ) and TOKEN + @cat k8s/{*.yaml,*.yml,*.tmpl} | envsubst | $(KUBECTL) apply -f - + +deploy-check: ## Get the deployed configuration. + @$(KUBECTL) get deploy,pods,svc,ingress + +.PHONY:: info deploy deploy-check diff --git a/precommit.mk b/precommit.mk new file mode 100644 index 0000000..82fe75f --- /dev/null +++ b/precommit.mk @@ -0,0 +1,5 @@ +init: ## Init pre-commit hooks (i.e. enforcing format checking before allowing a commit) + pip install --user pre-commit + pre-commit install + +.PHONY:: init diff --git a/python.mk b/python.mk new file mode 100644 index 0000000..a69d70b --- /dev/null +++ b/python.mk @@ -0,0 +1,92 @@ +PYVERSIONS ?= 2.7 +PYMAIN ?= $(firstword $(PYVERSIONS)) +TARNAME ?= $(NAME)-$(VERSION).tar.gz + +DEVPORT ?= 6000 + +yapf: ## Format python code + yapf -i -r $(NAME) + yapf -i -r tests + +dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS)) ## Generate dockerfiles for each python version + @unlink Dockerfile >/dev/null + ln -s Dockerfile-$(PYMAIN) Dockerfile + +Dockerfile-%: Dockerfile.template ## Generate a specific dockerfile (e.g. Dockerfile-2.7) + sed "s/{{PYVERSION}}/$*/" Dockerfile.template > Dockerfile-$* + +quick_build: $(addprefix build-, $(PYMAIN)) + +build: $(addprefix build-, $(PYVERSIONS)) ## Build all images / python versions + +build-%: version Dockerfile-% ## Build a specific version (e.g. build-2.7) + docker build -t '$(IMAGEWTAG)-python$*' --cache-from $(IMAGENAME):python$* -f Dockerfile-$* .; + +dev-%: ## Launch a specific development environment using docker (e.g. dev-2.7) + @docker start $(NAME)-dev$* || (\ + $(MAKE) build-$*; \ + docker run -d -w /usr/src/app/ -p $(DEVPORT):5000 -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti --name $(NAME)-dev$* '$(IMAGEWTAG)-python$*'; \ + )\ + + docker exec -ti $(NAME)-dev$* bash + +dev: dev-$(PYMAIN) ## Launch a development environment using docker, using the default python version + +quick_test: test-$(PYMAIN) + +test-%: ## Run setup.py from in an isolated container, built from the base image. (e.g. test-2.7) +# This speeds tests up because the image has most (if not all) of the dependencies already. + docker rm $(NAME)-test-$* || true + docker create -ti --name $(NAME)-test-$* --entrypoint="" -w /usr/src/app/ $(IMAGENAME):python$* python setup.py test + docker cp . $(NAME)-test-$*:/usr/src/app + docker start -a $(NAME)-test-$* + +test: $(addprefix test-,$(PYVERSIONS)) ## Run the tests with the main python version + +run-%: build-% + docker run --rm -p $(DEVPORT):5000 -ti '$(IMAGEWTAG)-python$(PYMAIN)' --default-plugins + +run: run-$(PYMAIN) + +# Pypy - Upload a package + +dist/$(TARNAME): version + python setup.py sdist; + +sdist: dist/$(TARNAME) ## Generate the distribution file (wheel) + +pip_test-%: sdist ## Test the distribution file using pip install and a specific python version (e.g. pip_test-2.7) + docker run --rm -v $$PWD/dist:/dist/ python:$* pip install /dist/$(TARNAME); + +pip_test: $(addprefix pip_test-,$(PYVERSIONS)) ## Test pip installation with the main python version + +pip_upload: pip_test ## Upload package to pip + python setup.py sdist upload ; + +# Pushing to docker + +push-latest: $(addprefix push-latest-,$(PYVERSIONS)) ## Push the "latest" tag to dockerhub + docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)' + docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGENAME)' + docker push '$(IMAGENAME):latest' + docker push '$(IMAGEWTAG)' + +push-latest-%: build-% ## Push the latest image for a specific python version + docker tag $(IMAGENAME):$(VERSION)-python$* $(IMAGENAME):python$* + docker push $(IMAGENAME):$(VERSION)-python$* + docker push $(IMAGENAME):python$* + +push-%: build-% ## Push the image of the current version (tagged). e.g. push-2.7 + docker push $(IMAGENAME):$(VERSION)-python$* + +push: $(addprefix push-,$(PYVERSIONS)) ## Push an image with the current version for every python version + docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)' + docker push $(IMAGENAME):$(VERSION) + +clean:: ## Clean older docker images and containers related to this project and dev environments + @docker stop $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true + @docker rm $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true + @docker ps -a | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true + @docker images | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true + +.PHONY:: yapf dockerfiles Dockerfile-% quick_build build build-% dev-% quick-dev test quick_test push-latest push-latest-% push-% push