diff --git a/.makefiles/README.md b/.makefiles/README.md deleted file mode 100644 index e025233..0000000 --- a/.makefiles/README.md +++ /dev/null @@ -1,27 +0,0 @@ -These makefiles are recipes for several common tasks in different types of projects. -To add them to your project, simply do: - -``` -git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git -git subtree add --prefix=.makefiles/ makefiles master -touch Makefile -echo "include .makefiles/makefiles.mk" >> Makefile -``` - -Now you can take advantage of the recipes. -For instance, to add useful targets for a python project, just add this to your Makefile: - -``` -include .makefiles/python.mk -``` - -You may need to set special variables like the name of your project or the python versions you're targetting. -Take a look at each specific `.mk` file for more information, and the `Makefile` in the [senpy](https://lab.cluster.gsi.dit.upm.es/senpy/senpy) project for a real use case. - -If you update the makefiles from your repository, make sure to push the changes for review in upstream (this repository): - -``` -make makefiles-push -``` - -It will automatically commit all unstaged changes in the .makefiles folder. diff --git a/.makefiles/base.mk b/.makefiles/base.mk deleted file mode 100644 index 55b8a34..0000000 --- a/.makefiles/base.mk +++ /dev/null @@ -1,24 +0,0 @@ -VERSION ?= $(shell git describe --tags --dirty 2>/dev/null) - -.FORCE: - -version: .FORCE - @echo $(VERSION) > $(NAME)/VERSION - @echo $(VERSION) - -help: ## Show this help. - @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/\(.*:\)[^#]*##\s*\(.*\)/\1\t\2/' | column -t -s " " - -config: ## Load config from the environment. You should run it once in every session before other tasks. Run: eval $(make config) - @echo ". ../.env || true;" - @awk '{ print "export " $$0}' .env - @echo "# Please, run: " - @echo "# eval \$$(make config)" -# If you need to run a command on the key/value pairs, use this: -# @awk '{ split($$0, a, "="); "echo " a[2] " | base64 -w 0" |& getline b64; print "export " a[1] "=" a[2]; print "export " a[1] "_BASE64=" b64}' .env - -ci: ## Run a task using gitlab-runner. Only use to debug problems in the CI pipeline - gitlab-runner exec shell --builds-dir '.builds' --env CI_PROJECT_NAME=$(NAME) ${action} - - -.PHONY:: config help ci version .FORCE diff --git a/.makefiles/docker.mk b/.makefiles/docker.mk deleted file mode 100644 index aab2a8a..0000000 --- a/.makefiles/docker.mk +++ /dev/null @@ -1,21 +0,0 @@ -IMAGEWTAG ?= $(IMAGENAME):$(VERSION) - -login: ## Log in to the registry. It will only be used in the server, or when running a CI task locally (if CI_BUILD_TOKEN is set). -ifeq ($(CI_BUILD_TOKEN),) - @echo "Not logging in to the docker registry" "$(CI_REGISTRY)" -else - @docker login -u gitlab-ci-token -p $(CI_BUILD_TOKEN) $(CI_REGISTRY) -endif -ifeq ($(HUB_USER),) - @echo "Not logging in to global the docker registry" -else - @docker login -u $(HUB_USER) -p $(HUB_PASSWORD) -endif - -clean:: ## Remove docker credentials -ifeq ($(HUB_USER),) -else - @docker logout -endif - -.PHONY:: login clean diff --git a/.makefiles/git.mk b/.makefiles/git.mk deleted file mode 100644 index c4f6493..0000000 --- a/.makefiles/git.mk +++ /dev/null @@ -1,19 +0,0 @@ -git_commit: - git commit -a - -git_tag: - git tag ${VERSION} - -git_push: - git push --tags origin master - -push-github: ## Push the code to github. You need to set up HUB_USER and HUB_PASSWORD - $(eval KEY_FILE := $(shell mktemp)) - @echo "$$GITHUB_DEPLOY_KEY" > $(KEY_FILE) - @git remote rm github-deploy || true - git remote add github-deploy $(GITHUB_REPO) - @GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git fetch github-deploy $(CI_COMMIT_REF_NAME) || true - @GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git push github-deploy $(CI_COMMIT_REF_NAME) - rm $(KEY_FILE) - -.PHONY:: git_commit git_tag git_push push-github diff --git a/.makefiles/k8s.mk b/.makefiles/k8s.mk deleted file mode 100644 index 96892b2..0000000 --- a/.makefiles/k8s.mk +++ /dev/null @@ -1,51 +0,0 @@ -# Deployment with Kubernetes - -# KUBE_CA_PEM_FILE is the path of a certificate file. It automatically set by GitLab -# if you enable Kubernetes integration in a project. -# -# As of this writing, Kubernetes integration can not be set on a group level, so it has to -# be manually set in every project. -# Alternatively, we use a custom KUBE_CA_BUNDLE environment variable, which can be set at -# the group level. In this case, the variable contains the whole content of the certificate, -# which we dump to a temporary file -# -# Check if the KUBE_CA_PEM_FILE exists. Otherwise, create it from KUBE_CA_BUNDLE -KUBE_CA_TEMP=false -ifeq ($(wildcard $(KUBE_CA_PEM_FILE)),) - KUBE_CA_PEM_FILE:="$$PWD/.ca.crt" - CREATED:=$(shell echo -e "$$KUBE_CA_BUNDLE" > $(KUBE_CA_PEM_FILE)) -endif -KUBE_URL?="" -KUBE_TOKEN?="" -KUBE_NAMESPACE?=$(NAME) -KUBECTL=docker run --rm -v $(KUBE_CA_PEM_FILE):/tmp/ca.pem -i lachlanevenson/k8s-kubectl --server="$(KUBE_URL)" --token="$(KUBE_TOKEN)" --certificate-authority="/tmp/ca.pem" -n $(KUBE_NAMESPACE) -CI_COMMIT_REF_NAME?=master - -info: ## Print variables. Useful for debugging. - @echo "#KUBERNETES" - @echo KUBE_URL=$(KUBE_URL) - @echo KUBE_CA_PEM_FILE=$(KUBE_CA_PEM_FILE) - @echo KUBE_CA_BUNDLE=$$KUBE_CA_BUNDLE - @echo KUBE_TOKEN=$(KUBE_TOKEN) - @echo KUBE_NAMESPACE=$(KUBE_NAMESPACE) - @echo KUBECTL=$(KUBECTL) - - @echo "#CI" - @echo CI_PROJECT_NAME=$(CI_PROJECT_NAME) - @echo CI_REGISTRY=$(CI_REGISTRY) - @echo CI_REGISTRY_USER=$(CI_REGISTRY_USER) - @echo CI_COMMIT_REF_NAME=$(CI_COMMIT_REF_NAME) - -# -# Deployment and advanced features -# - - -deploy: ## Deploy to kubernetes using the credentials in KUBE_CA_PEM_FILE (or KUBE_CA_BUNDLE ) and TOKEN - @ls k8s/ - @cat k8s/*.yaml k8s/*.yml k8s/*.tmpl 2>/dev/null | envsubst | $(KUBECTL) apply -f - - -deploy-check: ## Get the deployed configuration. - @$(KUBECTL) get deploy,pods,svc,ingress - -.PHONY:: info deploy deploy-check diff --git a/.makefiles/makefiles.mk b/.makefiles/makefiles.mk deleted file mode 100644 index 4bee03b..0000000 --- a/.makefiles/makefiles.mk +++ /dev/null @@ -1,14 +0,0 @@ -makefiles-remote: - git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git || true - -makefiles-commit: makefiles-remote - git add -f .makefiles - git commit -em "Updated makefiles from ${NAME}" - -makefiles-push: - git subtree push --prefix=.makefiles/ makefiles $(NAME) - -makefiles-pull: makefiles-remote - git subtree pull --prefix=.makefiles/ makefiles master --squash - -.PHONY:: makefiles-remote makefiles-commit makefiles-push makefiles-pull diff --git a/.makefiles/precommit.mk b/.makefiles/precommit.mk deleted file mode 100644 index 82fe75f..0000000 --- a/.makefiles/precommit.mk +++ /dev/null @@ -1,5 +0,0 @@ -init: ## Init pre-commit hooks (i.e. enforcing format checking before allowing a commit) - pip install --user pre-commit - pre-commit install - -.PHONY:: init diff --git a/.makefiles/python.mk b/.makefiles/python.mk deleted file mode 100644 index a69d70b..0000000 --- a/.makefiles/python.mk +++ /dev/null @@ -1,92 +0,0 @@ -PYVERSIONS ?= 2.7 -PYMAIN ?= $(firstword $(PYVERSIONS)) -TARNAME ?= $(NAME)-$(VERSION).tar.gz - -DEVPORT ?= 6000 - -yapf: ## Format python code - yapf -i -r $(NAME) - yapf -i -r tests - -dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS)) ## Generate dockerfiles for each python version - @unlink Dockerfile >/dev/null - ln -s Dockerfile-$(PYMAIN) Dockerfile - -Dockerfile-%: Dockerfile.template ## Generate a specific dockerfile (e.g. Dockerfile-2.7) - sed "s/{{PYVERSION}}/$*/" Dockerfile.template > Dockerfile-$* - -quick_build: $(addprefix build-, $(PYMAIN)) - -build: $(addprefix build-, $(PYVERSIONS)) ## Build all images / python versions - -build-%: version Dockerfile-% ## Build a specific version (e.g. build-2.7) - docker build -t '$(IMAGEWTAG)-python$*' --cache-from $(IMAGENAME):python$* -f Dockerfile-$* .; - -dev-%: ## Launch a specific development environment using docker (e.g. dev-2.7) - @docker start $(NAME)-dev$* || (\ - $(MAKE) build-$*; \ - docker run -d -w /usr/src/app/ -p $(DEVPORT):5000 -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti --name $(NAME)-dev$* '$(IMAGEWTAG)-python$*'; \ - )\ - - docker exec -ti $(NAME)-dev$* bash - -dev: dev-$(PYMAIN) ## Launch a development environment using docker, using the default python version - -quick_test: test-$(PYMAIN) - -test-%: ## Run setup.py from in an isolated container, built from the base image. (e.g. test-2.7) -# This speeds tests up because the image has most (if not all) of the dependencies already. - docker rm $(NAME)-test-$* || true - docker create -ti --name $(NAME)-test-$* --entrypoint="" -w /usr/src/app/ $(IMAGENAME):python$* python setup.py test - docker cp . $(NAME)-test-$*:/usr/src/app - docker start -a $(NAME)-test-$* - -test: $(addprefix test-,$(PYVERSIONS)) ## Run the tests with the main python version - -run-%: build-% - docker run --rm -p $(DEVPORT):5000 -ti '$(IMAGEWTAG)-python$(PYMAIN)' --default-plugins - -run: run-$(PYMAIN) - -# Pypy - Upload a package - -dist/$(TARNAME): version - python setup.py sdist; - -sdist: dist/$(TARNAME) ## Generate the distribution file (wheel) - -pip_test-%: sdist ## Test the distribution file using pip install and a specific python version (e.g. pip_test-2.7) - docker run --rm -v $$PWD/dist:/dist/ python:$* pip install /dist/$(TARNAME); - -pip_test: $(addprefix pip_test-,$(PYVERSIONS)) ## Test pip installation with the main python version - -pip_upload: pip_test ## Upload package to pip - python setup.py sdist upload ; - -# Pushing to docker - -push-latest: $(addprefix push-latest-,$(PYVERSIONS)) ## Push the "latest" tag to dockerhub - docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)' - docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGENAME)' - docker push '$(IMAGENAME):latest' - docker push '$(IMAGEWTAG)' - -push-latest-%: build-% ## Push the latest image for a specific python version - docker tag $(IMAGENAME):$(VERSION)-python$* $(IMAGENAME):python$* - docker push $(IMAGENAME):$(VERSION)-python$* - docker push $(IMAGENAME):python$* - -push-%: build-% ## Push the image of the current version (tagged). e.g. push-2.7 - docker push $(IMAGENAME):$(VERSION)-python$* - -push: $(addprefix push-,$(PYVERSIONS)) ## Push an image with the current version for every python version - docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)' - docker push $(IMAGENAME):$(VERSION) - -clean:: ## Clean older docker images and containers related to this project and dev environments - @docker stop $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true - @docker rm $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true - @docker ps -a | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true - @docker images | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true - -.PHONY:: yapf dockerfiles Dockerfile-% quick_build build build-% dev-% quick-dev test quick_test push-latest push-latest-% push-% push