From e5687ed83ecb7ebc9a572c0af36f6d8407a25985 Mon Sep 17 00:00:00 2001 From: Ian Fijolek Date: Thu, 20 Jun 2019 12:58:51 -0700 Subject: [PATCH] Add multi-arch build pipeline --- .drone.yml | 225 ++++++++++++++++++++++++++++++++++++++++++++------ Dockerfile | 7 +- Makefile | 40 ++++++++- get_qemu.sh | 14 ++++ manifest.tmpl | 25 ++++++ 5 files changed, 285 insertions(+), 26 deletions(-) create mode 100755 get_qemu.sh create mode 100644 manifest.tmpl diff --git a/.drone.yml b/.drone.yml index 9e34f2c..b2ca497 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,5 +1,5 @@ kind: pipeline -name: python-latest +name: test-python-latest workspace: base: /app @@ -9,6 +9,7 @@ steps: - name: test image: python:3 commands: + - python -V - make test-env test - name: notify @@ -26,7 +27,7 @@ steps: --- kind: pipeline -name: python-3.4 +name: test-python3.4 workspace: base: /app @@ -36,6 +37,7 @@ steps: - name: test image: python:3.4 commands: + - python -V - make test-env test - name: notify @@ -53,7 +55,7 @@ steps: --- kind: pipeline -name: python-3.6 +name: test-python3.6 workspace: base: /app @@ -63,6 +65,7 @@ steps: - name: test image: python:3.6 commands: + - python -V - make test-env test - name: notify @@ -80,7 +83,7 @@ steps: --- kind: pipeline -name: python-3.7 +name: test-python3.7 workspace: base: /app @@ -90,6 +93,7 @@ steps: - name: test image: python:3.7 commands: + - python -V - make test-env test - name: notify @@ -105,14 +109,64 @@ steps: when: status: [ changed, failure ] +# --- +# kind: pipeline +# name: test-pypy3.6 +# +# workspace: +# base: /app +# path: . +# +# steps: +# - name: test +# image: pypy:3.6 +# commands: +# - python -V +# # python -m env doesn't exist for pypy +# - pip install virtualenv +# - virtualenv env +# - ./env/bin/pip install tox +# - make test +# +# - name: notify +# image: drillster/drone-email +# settings: +# host: +# from_secret: SMTP_HOST +# username: +# from_secret: SMTP_USER +# password: +# from_secret: SMTP_PASS +# from: drone@iamthefij.com +# when: +# status: [ changed, failure ] + --- kind: pipeline -name: deploy +name: tests depends_on: - - python-latest - - python-3.6 - - python-3.7 + - test-python-latest + - test-python3.4 + - test-python3.6 + - test-python3.7 + # - test-pypy3.6 + +steps: [] + +--- +kind: pipeline +name: deploy-pypi + +trigger: + event: + - tag + ref: + - refs/heads/master + - refs/tags/v* + +depends_on: + - tests workspace: base: /app @@ -120,21 +174,15 @@ workspace: steps: - - name: push to docker hub - image: plugins/docker - settings: - repo: iamthefij/minitor - auto_tag: true - username: - from_secret: docker_username - password: - from_secret: docker_password - when: - branch: - - master - event: - - push - - tag + - name: push to test pypi + image: python:3 + commands: + - make build-env upload-test + environment: + TWINE_USERNAME: + from_secret: PYPI_USERNAME + TWINE_PASSWORD: + from_secret: TEST_PYPI_PASSWORD - name: push to pypi image: python:3 @@ -158,3 +206,134 @@ steps: password: from_secret: SMTP_PASS from: drone@iamthefij.com + +--- +kind: pipeline +name: deploy-docker-amd64 + +depends_on: + - tests + +trigger: + event: + - push + - tag + ref: + - refs/heads/master + - refs/tags/v* + +steps: + - name: get qemu + image: busybox + commands: + - sh ./get_qemu.sh x86_64 + + - name: build + image: plugins/docker + settings: + repo: iamthefij/minitor + auto_tag: true + auto_tag_suffix: linux-amd64 + username: + from_secret: docker_username + password: + from_secret: docker_password + +--- +kind: pipeline +name: deploy-docker-arm + +depends_on: + - tests + +trigger: + event: + - push + - tag + ref: + - refs/heads/master + - refs/tags/v* + +steps: + - name: get qemu + image: busybox + commands: + - sh ./get_qemu.sh arm + + - name: build + image: plugins/docker + settings: + repo: iamthefij/minitor + auto_tag: true + auto_tag_suffix: linux-arm + username: + from_secret: docker_username + password: + from_secret: docker_password + build_args: + - ARCH=arm + - REPO=arm32v6 + +--- +kind: pipeline +name: deploy-docker-arm64 + +depends_on: + - tests + +trigger: + event: + - push + - tag + ref: + - refs/heads/master + - refs/tags/v* + +steps: + - name: get qemu + image: busybox + commands: + - sh ./get_qemu.sh aarch64 + + - name: build + image: plugins/docker + settings: + repo: iamthefij/minitor + auto_tag: true + auto_tag_suffix: linux-arm64 + username: + from_secret: docker_username + password: + from_secret: docker_password + build_args: + - ARCH=aarch64 + - REPO=arm64v8 + +--- +kind: pipeline +name: deploy-docker-manifest + +depends_on: + - deploy-docker-amd64 + - deploy-docker-arm + - deploy-docker-arm64 + +trigger: + event: + - push + - tag + ref: + - refs/heads/master + - refs/tags/v* + +steps: + - name: publish manifest + image: plugins/manifest + settings: + spec: manifest.tmpl + auto_tag: true + ignore_missing: true + username: + from_secret: docker_username + password: + from_secret: docker_password diff --git a/Dockerfile b/Dockerfile index 4c94b19..ad28b42 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,12 @@ -FROM python:3 +ARG REPO=library +FROM ${REPO}/python:3-alpine LABEL maintainer="ian@iamthefij.com" # Minitor: https://git.iamthefij.com/iamthefij/minitor +# This should be the target qemu arch +ARG ARCH=x86_64 +COPY ./build/qemu-${ARCH}-static /usr/bin/ + COPY ./sample-config.yml /app/config.yml WORKDIR /app diff --git a/Makefile b/Makefile index 444504d..70d8177 100644 --- a/Makefile +++ b/Makefile @@ -88,6 +88,42 @@ htmlcov/index.html: .coverage open-coverage: htmlcov/index.html $(OPEN_CMD) htmlcov/index.html +# Docker targets + +# Targets to download required qemu binaries for running on an amd64 machine +build/qemu-x86_64-static: + ./get_qemu.sh x86_64 + +build/qemu-arm-static: + ./get_qemu.sh arm + +build/qemu-aarch64-static: + ./get_qemu.sh aarch64 + +# Build Docker image for host architechture (amd64) .PHONY: docker-build -docker-build: - docker build . -t $(DOCKER_TAG) +docker-build: build/qemu-x86_64-static + docker build . -t ${DOCKER_TAG}-linux-amd64 + +# Cross build for arm architechtures +.PHONY: docker-cross-build-arm +docker-cross-build-arm: build/qemu-arm-static + docker build --build-arg REPO=arm32v6 --build-arg ARCH=arm . -t ${DOCKER_TAG}-linux-arm + +.PHONY: docker-cross-build-arm +docker-cross-build-arm64: build/qemu-aarch64-static + docker build --build-arg REPO=arm64v8 --build-arg ARCH=aarch64 . -t ${DOCKER_TAG}-linux-arm64 + +# Run on host architechture +.PHONY: run +docker-run: docker-build + docker run ${DOCKER_TAG}-linux-amd64 + +# Cross run on host architechture +.PHONY: docker-cross-run-arm +docker-cross-run-arm: docker-cross-build-arm + docker run --rm ${DOCKER_TAG}-linux-arm + +.PHONY: docker-cross-run-arm64 +docker-cross-run-arm64: docker-cross-build-arm64 + docker run --rm ${DOCKER_TAG}-linux-arm64 diff --git a/get_qemu.sh b/get_qemu.sh new file mode 100755 index 0000000..6d09feb --- /dev/null +++ b/get_qemu.sh @@ -0,0 +1,14 @@ +#! /bin/bash + +HOST_ARCH=x86_64 +VERSION=v2.9.1-1 + +mkdir -p build +cd build + +# Multiple args can be passed in, but in most cases (Makefile and .drone.yml) we only use one at a time +for target_arch in $*; do + wget https://github.com/multiarch/qemu-user-static/releases/download/$VERSION/${HOST_ARCH}_qemu-${target_arch}-static.tar.gz + tar -xvf ${HOST_ARCH}_qemu-${target_arch}-static.tar.gz + rm ${HOST_ARCH}_qemu-${target_arch}-static.tar.gz +done diff --git a/manifest.tmpl b/manifest.tmpl new file mode 100644 index 0000000..f508f9a --- /dev/null +++ b/manifest.tmpl @@ -0,0 +1,25 @@ +image: iamthefij/minitor:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}} +{{#if build.tags}} +tags: +{{#each build.tags}} + - {{this}} +{{/each}} +{{/if}} +manifests: + - + image: iamthefij/minitor:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64 + platform: + architecture: amd64 + os: linux + - + image: iamthefij/minitor:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64 + platform: + architecture: arm64 + os: linux + variant: v8 + - + image: iamthefij/minitor:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm + platform: + architecture: arm + os: linux + variant: v7