mirror of
https://github.com/gsi-upm/senpy
synced 2024-11-14 12:42:27 +00:00
Add '.makefiles/' from commit 'a75ba6994d93ca027b6f3ba0b08b75dd60d3aa78'
git-subtree-dir: .makefiles git-subtree-mainline:c52a894017
git-subtree-split:a75ba6994d
This commit is contained in:
commit
982baa04cf
27
.makefiles/README.md
Normal file
27
.makefiles/README.md
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
These makefiles are recipes for several common tasks in different types of projects.
|
||||||
|
To add them to your project, simply do:
|
||||||
|
|
||||||
|
```
|
||||||
|
git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git
|
||||||
|
git subtree add --prefix=.makefiles/ makefiles master
|
||||||
|
touch Makefile
|
||||||
|
echo "include .makefiles/base.mk" >> Makefile
|
||||||
|
```
|
||||||
|
|
||||||
|
Now you can take advantage of the recipes.
|
||||||
|
For instance, to add useful targets for a python project, just add this to your Makefile:
|
||||||
|
|
||||||
|
```
|
||||||
|
include .makefiles/python.mk
|
||||||
|
```
|
||||||
|
|
||||||
|
You may need to set special variables like the name of your project or the python versions you're targetting.
|
||||||
|
Take a look at each specific `.mk` file for more information, and the `Makefile` in the [senpy](https://lab.cluster.gsi.dit.upm.es/senpy/senpy) project for a real use case.
|
||||||
|
|
||||||
|
If you update the makefiles from your repository, make sure to push the changes for review in upstream (this repository):
|
||||||
|
|
||||||
|
```
|
||||||
|
make makefiles-push
|
||||||
|
```
|
||||||
|
|
||||||
|
It will automatically commit all unstaged changes in the .makefiles folder.
|
36
.makefiles/base.mk
Normal file
36
.makefiles/base.mk
Normal file
@ -0,0 +1,36 @@
|
|||||||
|
export
|
||||||
|
NAME ?= $(shell basename $(CURDIR))
|
||||||
|
VERSION ?= $(shell git describe --tags --dirty 2>/dev/null)
|
||||||
|
|
||||||
|
ifeq ($(VERSION),)
|
||||||
|
VERSION:=unknown
|
||||||
|
endif
|
||||||
|
|
||||||
|
# Get the location of this makefile.
|
||||||
|
MK_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
|
||||||
|
|
||||||
|
-include .env
|
||||||
|
-include ../.env
|
||||||
|
|
||||||
|
help: ## Show this help.
|
||||||
|
@fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/\(.*:\)[^#]*##\s*\(.*\)/\1\t\2/' | column -t -s " "
|
||||||
|
|
||||||
|
config: ## Load config from the environment. You should run it once in every session before other tasks. Run: eval $(make config)
|
||||||
|
@awk '{ print "export " $$0}' ../.env
|
||||||
|
@awk '{ print "export " $$0}' .env
|
||||||
|
@echo "# Please, run: "
|
||||||
|
@echo "# eval \$$(make config)"
|
||||||
|
# If you need to run a command on the key/value pairs, use this:
|
||||||
|
# @awk '{ split($$0, a, "="); "echo " a[2] " | base64 -w 0" |& getline b64; print "export " a[1] "=" a[2]; print "export " a[1] "_BASE64=" b64}' .env
|
||||||
|
|
||||||
|
ci: ## Run a task using gitlab-runner. Only use to debug problems in the CI pipeline
|
||||||
|
gitlab-runner exec shell --builds-dir '.builds' --env CI_PROJECT_NAME=$(NAME) ${action}
|
||||||
|
|
||||||
|
include $(MK_DIR)/makefiles.mk
|
||||||
|
include $(MK_DIR)/docker.mk
|
||||||
|
include $(MK_DIR)/git.mk
|
||||||
|
|
||||||
|
info:: ## List all variables
|
||||||
|
env
|
||||||
|
|
||||||
|
.PHONY:: config help ci
|
29
.makefiles/docker.mk
Normal file
29
.makefiles/docker.mk
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
IMAGENAME?=$(NAME)
|
||||||
|
IMAGEWTAG?=$(IMAGENAME):$(VERSION)
|
||||||
|
|
||||||
|
docker-login: ## Log in to the registry. It will only be used in the server, or when running a CI task locally (if CI_BUILD_TOKEN is set).
|
||||||
|
ifeq ($(CI_BUILD_TOKEN),)
|
||||||
|
@echo "Not logging in to the docker registry" "$(CI_REGISTRY)"
|
||||||
|
else
|
||||||
|
@docker login -u gitlab-ci-token -p $(CI_BUILD_TOKEN) $(CI_REGISTRY)
|
||||||
|
endif
|
||||||
|
ifeq ($(HUB_USER),)
|
||||||
|
@echo "Not logging in to global the docker registry"
|
||||||
|
else
|
||||||
|
@docker login -u $(HUB_USER) -p $(HUB_PASSWORD)
|
||||||
|
endif
|
||||||
|
|
||||||
|
docker-clean: ## Remove docker credentials
|
||||||
|
ifeq ($(HUB_USER),)
|
||||||
|
else
|
||||||
|
@docker logout
|
||||||
|
endif
|
||||||
|
|
||||||
|
login:: docker-login
|
||||||
|
|
||||||
|
clean:: docker-clean
|
||||||
|
|
||||||
|
docker-info:
|
||||||
|
@echo IMAGEWTAG=${IMAGEWTAG}
|
||||||
|
|
||||||
|
.PHONY:: docker-login docker-clean login clean
|
28
.makefiles/git.mk
Normal file
28
.makefiles/git.mk
Normal file
@ -0,0 +1,28 @@
|
|||||||
|
commit:
|
||||||
|
git commit -a
|
||||||
|
|
||||||
|
tag:
|
||||||
|
git tag ${VERSION}
|
||||||
|
|
||||||
|
git-push::
|
||||||
|
git push --tags -u origin HEAD
|
||||||
|
|
||||||
|
git-pull:
|
||||||
|
git pull --all
|
||||||
|
|
||||||
|
push-github: ## Push the code to github. You need to set up GITHUB_DEPLOY_KEY
|
||||||
|
ifeq ($(GITHUB_DEPLOY_KEY),)
|
||||||
|
else
|
||||||
|
$(eval KEY_FILE := "$(shell mktemp)")
|
||||||
|
@echo "$(GITHUB_DEPLOY_KEY)" > $(KEY_FILE)
|
||||||
|
@git remote rm github-deploy || true
|
||||||
|
git remote add github-deploy $(GITHUB_REPO)
|
||||||
|
-@GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git fetch github-deploy $(CI_COMMIT_REF_NAME)
|
||||||
|
@GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git push github-deploy HEAD:$(CI_COMMIT_REF_NAME)
|
||||||
|
rm $(KEY_FILE)
|
||||||
|
endif
|
||||||
|
|
||||||
|
push:: git-push
|
||||||
|
pull:: git-pull
|
||||||
|
|
||||||
|
.PHONY:: commit tag push git-push git-pull push-github
|
51
.makefiles/k8s.mk
Normal file
51
.makefiles/k8s.mk
Normal file
@ -0,0 +1,51 @@
|
|||||||
|
# Deployment with Kubernetes
|
||||||
|
|
||||||
|
# KUBE_CA_PEM_FILE is the path of a certificate file. It automatically set by GitLab
|
||||||
|
# if you enable Kubernetes integration in a project.
|
||||||
|
#
|
||||||
|
# As of this writing, Kubernetes integration can not be set on a group level, so it has to
|
||||||
|
# be manually set in every project.
|
||||||
|
# Alternatively, we use a custom KUBE_CA_BUNDLE environment variable, which can be set at
|
||||||
|
# the group level. In this case, the variable contains the whole content of the certificate,
|
||||||
|
# which we dump to a temporary file
|
||||||
|
#
|
||||||
|
# Check if the KUBE_CA_PEM_FILE exists. Otherwise, create it from KUBE_CA_BUNDLE
|
||||||
|
KUBE_CA_TEMP=false
|
||||||
|
ifndef KUBE_CA_PEM_FILE
|
||||||
|
KUBE_CA_PEM_FILE:=$$PWD/.ca.crt
|
||||||
|
CREATED:=$(shell echo -e "$(KUBE_CA_BUNDLE)" > $(KUBE_CA_PEM_FILE))
|
||||||
|
endif
|
||||||
|
KUBE_TOKEN?=""
|
||||||
|
KUBE_NAMESPACE?=$(NAME)
|
||||||
|
KUBECTL=docker run --rm -v $(KUBE_CA_PEM_FILE):/tmp/ca.pem -i lachlanevenson/k8s-kubectl --server="$(KUBE_URL)" --token="$(KUBE_TOKEN)" --certificate-authority="/tmp/ca.pem" -n $(KUBE_NAMESPACE)
|
||||||
|
CI_COMMIT_REF_NAME?=master
|
||||||
|
|
||||||
|
info:: ## Print variables. Useful for debugging.
|
||||||
|
@echo "#KUBERNETES"
|
||||||
|
@echo KUBE_URL=$(KUBE_URL)
|
||||||
|
@echo KUBE_CA_PEM_FILE=$(KUBE_CA_PEM_FILE)
|
||||||
|
@echo KUBE_CA_BUNDLE=$$KUBE_CA_BUNDLE
|
||||||
|
@echo KUBE_TOKEN=$(KUBE_TOKEN)
|
||||||
|
@echo KUBE_NAMESPACE=$(KUBE_NAMESPACE)
|
||||||
|
@echo KUBECTL=$(KUBECTL)
|
||||||
|
|
||||||
|
@echo "#CI"
|
||||||
|
@echo CI_PROJECT_NAME=$(CI_PROJECT_NAME)
|
||||||
|
@echo CI_REGISTRY=$(CI_REGISTRY)
|
||||||
|
@echo CI_REGISTRY_USER=$(CI_REGISTRY_USER)
|
||||||
|
@echo CI_COMMIT_REF_NAME=$(CI_COMMIT_REF_NAME)
|
||||||
|
@echo "CREATED=$(CREATED)"
|
||||||
|
|
||||||
|
#
|
||||||
|
# Deployment and advanced features
|
||||||
|
#
|
||||||
|
|
||||||
|
|
||||||
|
deploy: ## Deploy to kubernetes using the credentials in KUBE_CA_PEM_FILE (or KUBE_CA_BUNDLE ) and TOKEN
|
||||||
|
@ls k8s/*.yaml k8s/*.yml k8s/*.tmpl 2>/dev/null || true
|
||||||
|
@cat k8s/*.yaml k8s/*.yml k8s/*.tmpl 2>/dev/null | envsubst | $(KUBECTL) apply -f -
|
||||||
|
|
||||||
|
deploy-check: ## Get the deployed configuration.
|
||||||
|
@$(KUBECTL) get deploy,pods,svc,ingress
|
||||||
|
|
||||||
|
.PHONY:: info deploy deploy-check
|
17
.makefiles/makefiles.mk
Normal file
17
.makefiles/makefiles.mk
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
makefiles-remote:
|
||||||
|
@git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git 2>/dev/null || true
|
||||||
|
|
||||||
|
makefiles-commit: makefiles-remote
|
||||||
|
git add -f .makefiles
|
||||||
|
git commit -em "Updated makefiles from ${NAME}"
|
||||||
|
|
||||||
|
makefiles-push:
|
||||||
|
git subtree push --prefix=.makefiles/ makefiles $(NAME)
|
||||||
|
|
||||||
|
makefiles-pull: makefiles-remote
|
||||||
|
git subtree pull --prefix=.makefiles/ makefiles master --squash
|
||||||
|
|
||||||
|
pull:: makefiles-pull
|
||||||
|
push:: makefiles-push
|
||||||
|
|
||||||
|
.PHONY:: makefiles-remote makefiles-commit makefiles-push makefiles-pull pull push
|
5
.makefiles/precommit.mk
Normal file
5
.makefiles/precommit.mk
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
init: ## Init pre-commit hooks (i.e. enforcing format checking before allowing a commit)
|
||||||
|
pip install --user pre-commit
|
||||||
|
pre-commit install
|
||||||
|
|
||||||
|
.PHONY:: init
|
100
.makefiles/python.mk
Normal file
100
.makefiles/python.mk
Normal file
@ -0,0 +1,100 @@
|
|||||||
|
PYVERSIONS ?= 3.5
|
||||||
|
PYMAIN ?= $(firstword $(PYVERSIONS))
|
||||||
|
TARNAME ?= $(NAME)-$(VERSION).tar.gz
|
||||||
|
VERSIONFILE ?= $(NAME)/VERSION
|
||||||
|
|
||||||
|
DEVPORT ?= 6000
|
||||||
|
|
||||||
|
|
||||||
|
.FORCE:
|
||||||
|
|
||||||
|
version: .FORCE
|
||||||
|
@echo $(VERSION) > $(VERSIONFILE)
|
||||||
|
@echo $(VERSION)
|
||||||
|
|
||||||
|
yapf: ## Format python code
|
||||||
|
yapf -i -r $(NAME)
|
||||||
|
yapf -i -r tests
|
||||||
|
|
||||||
|
dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS)) ## Generate dockerfiles for each python version
|
||||||
|
@unlink Dockerfile >/dev/null
|
||||||
|
ln -s Dockerfile-$(PYMAIN) Dockerfile
|
||||||
|
|
||||||
|
Dockerfile-%: Dockerfile.template ## Generate a specific dockerfile (e.g. Dockerfile-2.7)
|
||||||
|
sed "s/{{PYVERSION}}/$*/" Dockerfile.template > Dockerfile-$*
|
||||||
|
|
||||||
|
quick_build: $(addprefix build-, $(PYMAIN))
|
||||||
|
|
||||||
|
build: $(addprefix build-, $(PYVERSIONS)) ## Build all images / python versions
|
||||||
|
|
||||||
|
build-%: version Dockerfile-% ## Build a specific version (e.g. build-2.7)
|
||||||
|
docker build -t '$(IMAGEWTAG)-python$*' --cache-from $(IMAGENAME):python$* -f Dockerfile-$* .;
|
||||||
|
|
||||||
|
dev-%: ## Launch a specific development environment using docker (e.g. dev-2.7)
|
||||||
|
@docker start $(NAME)-dev$* || (\
|
||||||
|
$(MAKE) build-$*; \
|
||||||
|
docker run -d -w /usr/src/app/ -p $(DEVPORT):5000 -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti --name $(NAME)-dev$* '$(IMAGEWTAG)-python$*'; \
|
||||||
|
)\
|
||||||
|
|
||||||
|
docker exec -ti $(NAME)-dev$* bash
|
||||||
|
|
||||||
|
dev: dev-$(PYMAIN) ## Launch a development environment using docker, using the default python version
|
||||||
|
|
||||||
|
quick_test: test-$(PYMAIN)
|
||||||
|
|
||||||
|
test-%: ## Run setup.py from in an isolated container, built from the base image. (e.g. test-2.7)
|
||||||
|
# This speeds tests up because the image has most (if not all) of the dependencies already.
|
||||||
|
docker rm $(NAME)-test-$* || true
|
||||||
|
docker create -ti --name $(NAME)-test-$* --entrypoint="" -w /usr/src/app/ $(IMAGENAME):python$* python setup.py test
|
||||||
|
docker cp . $(NAME)-test-$*:/usr/src/app
|
||||||
|
docker start -a $(NAME)-test-$*
|
||||||
|
|
||||||
|
test: $(addprefix test-,$(PYVERSIONS)) ## Run the tests with the main python version
|
||||||
|
|
||||||
|
run-%: build-%
|
||||||
|
docker run --rm -p $(DEVPORT):5000 -ti '$(IMAGEWTAG)-python$(PYMAIN)' --default-plugins
|
||||||
|
|
||||||
|
run: run-$(PYMAIN)
|
||||||
|
|
||||||
|
# Pypy - Upload a package
|
||||||
|
|
||||||
|
dist/$(TARNAME): version
|
||||||
|
python setup.py sdist;
|
||||||
|
|
||||||
|
sdist: dist/$(TARNAME) ## Generate the distribution file (wheel)
|
||||||
|
|
||||||
|
pip_test-%: sdist ## Test the distribution file using pip install and a specific python version (e.g. pip_test-2.7)
|
||||||
|
docker run --rm -v $$PWD/dist:/dist/ python:$* pip install /dist/$(TARNAME);
|
||||||
|
|
||||||
|
pip_test: $(addprefix pip_test-,$(PYVERSIONS)) ## Test pip installation with the main python version
|
||||||
|
|
||||||
|
pip_upload: pip_test ## Upload package to pip
|
||||||
|
python setup.py sdist upload ;
|
||||||
|
|
||||||
|
# Pushing to docker
|
||||||
|
|
||||||
|
push-latest: $(addprefix push-latest-,$(PYVERSIONS)) ## Push the "latest" tag to dockerhub
|
||||||
|
docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
|
||||||
|
docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGENAME)'
|
||||||
|
docker push '$(IMAGENAME):latest'
|
||||||
|
docker push '$(IMAGEWTAG)'
|
||||||
|
|
||||||
|
push-latest-%: build-% ## Push the latest image for a specific python version
|
||||||
|
docker tag $(IMAGENAME):$(VERSION)-python$* $(IMAGENAME):python$*
|
||||||
|
docker push $(IMAGENAME):$(VERSION)-python$*
|
||||||
|
docker push $(IMAGENAME):python$*
|
||||||
|
|
||||||
|
push-%: build-% ## Push the image of the current version (tagged). e.g. push-2.7
|
||||||
|
docker push $(IMAGENAME):$(VERSION)-python$*
|
||||||
|
|
||||||
|
push:: $(addprefix push-,$(PYVERSIONS)) ## Push an image with the current version for every python version
|
||||||
|
docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
|
||||||
|
docker push $(IMAGENAME):$(VERSION)
|
||||||
|
|
||||||
|
clean:: ## Clean older docker images and containers related to this project and dev environments
|
||||||
|
@docker stop $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true
|
||||||
|
@docker rm $(addprefix $(NAME)-dev,$(PYVERSIONS)) 2>/dev/null || true
|
||||||
|
@docker ps -a | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1;}}' | xargs docker rm -v 2>/dev/null|| true
|
||||||
|
@docker images | grep $(IMAGENAME) | awk '{ split($$2, vers, "-"); if(vers[0] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true
|
||||||
|
|
||||||
|
.PHONY:: yapf dockerfiles Dockerfile-% quick_build build build-% dev-% quick-dev test quick_test push-latest push-latest-% push-% push version .FORCE
|
Loading…
Reference in New Issue
Block a user