Add multi-arch build pipeline

2 years ago · e5687ed83e
5 changed files with 285 additions and 26 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -1,5 +1,5 @@
 kind: pipeline
-name: python-latest
+name: test-python-latest

 workspace:
  base: /app
@ -9,6 +9,7 @@ steps:
  - name: test
    image: python:3
    commands:
+      - python -V
      - make test-env test

  - name: notify
@ -26,7 +27,7 @@ steps:

 ---
 kind: pipeline
-name: python-3.4
+name: test-python3.4

 workspace:
  base: /app
@ -36,6 +37,7 @@ steps:
  - name: test
    image: python:3.4
    commands:
+      - python -V
      - make test-env test

  - name: notify
@ -53,7 +55,7 @@ steps:

 ---
 kind: pipeline
-name: python-3.6
+name: test-python3.6

 workspace:
  base: /app
@ -63,6 +65,7 @@ steps:
  - name: test
    image: python:3.6
    commands:
+      - python -V
      - make test-env test

  - name: notify
@ -80,7 +83,7 @@ steps:

 ---
 kind: pipeline
-name: python-3.7
+name: test-python3.7

 workspace:
  base: /app
@ -90,6 +93,7 @@ steps:
  - name: test
    image: python:3.7
    commands:
+      - python -V
      - make test-env test

  - name: notify
@ -105,14 +109,64 @@ steps:
    when:
      status: [ changed, failure ]

+# ---
+# kind: pipeline
+# name: test-pypy3.6
+#
+# workspace:
+#   base: /app
+#   path: .
+#
+# steps:
+#   - name: test
+#     image: pypy:3.6
+#     commands:
+#       - python -V
+#       # python -m env doesn't exist for pypy
+#       - pip install virtualenv
+#       - virtualenv env
+#       - ./env/bin/pip install tox
+#       - make test
+#
+#   - name: notify
+#     image: drillster/drone-email
+#     settings:
+#       host:
+#         from_secret: SMTP_HOST
+#       username:
+#         from_secret: SMTP_USER
+#       password:
+#         from_secret: SMTP_PASS
+#       from: drone@iamthefij.com
+#     when:
+#       status: [ changed, failure ]
+
+---
+kind: pipeline
+name: tests
+
+depends_on:
+  - test-python-latest
+  - test-python3.4
+  - test-python3.6
+  - test-python3.7
+  # - test-pypy3.6
+
+steps: []
+
 ---
 kind: pipeline
-name: deploy
+name: deploy-pypi
+
+trigger:
+  event:
+    - tag
+  ref:
+    - refs/heads/master
+    - refs/tags/v*

 depends_on:
-  - python-latest
-  - python-3.6
-  - python-3.7
+  - tests

 workspace:
  base: /app
@ -120,21 +174,15 @@ workspace:

 steps:

-  - name: push to docker hub
-    image: plugins/docker
-    settings:
-      repo: iamthefij/minitor
-      auto_tag: true
-      username:
-        from_secret: docker_username
-      password:
-        from_secret: docker_password
-    when:
-      branch:
-        - master
-      event:
-        - push
-        - tag
+  - name: push to test pypi
+    image: python:3
+    commands:
+      - make build-env upload-test
+    environment:
+      TWINE_USERNAME:
+        from_secret: PYPI_USERNAME
+      TWINE_PASSWORD:
+        from_secret: TEST_PYPI_PASSWORD

  - name: push to pypi
    image: python:3
@ -158,3 +206,134 @@ steps:
      password:
        from_secret: SMTP_PASS
      from: drone@iamthefij.com
+
+---
+kind: pipeline
+name: deploy-docker-amd64
+
+depends_on:
+  - tests
+
+trigger:
+  event:
+    - push
+    - tag
+  ref:
+    - refs/heads/master
+    - refs/tags/v*
+
+steps:
+  - name: get qemu
+    image: busybox
+    commands:
+      - sh ./get_qemu.sh x86_64
+
+  - name: build
+    image: plugins/docker
+    settings:
+      repo: iamthefij/minitor
+      auto_tag: true
+      auto_tag_suffix: linux-amd64
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+
+---
+kind: pipeline
+name: deploy-docker-arm
+
+depends_on:
+  - tests
+
+trigger:
+  event:
+    - push
+    - tag
+  ref:
+    - refs/heads/master
+    - refs/tags/v*
+
+steps:
+  - name: get qemu
+    image: busybox
+    commands:
+      - sh ./get_qemu.sh arm
+
+  - name: build
+    image: plugins/docker
+    settings:
+      repo: iamthefij/minitor
+      auto_tag: true
+      auto_tag_suffix: linux-arm
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      build_args:
+        - ARCH=arm
+        - REPO=arm32v6
+
+---
+kind: pipeline
+name: deploy-docker-arm64
+
+depends_on:
+  - tests
+
+trigger:
+  event:
+    - push
+    - tag
+  ref:
+    - refs/heads/master
+    - refs/tags/v*
+
+steps:
+  - name: get qemu
+    image: busybox
+    commands:
+      - sh ./get_qemu.sh aarch64
+
+  - name: build
+    image: plugins/docker
+    settings:
+      repo: iamthefij/minitor
+      auto_tag: true
+      auto_tag_suffix: linux-arm64
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      build_args:
+        - ARCH=aarch64
+        - REPO=arm64v8
+
+---
+kind: pipeline
+name: deploy-docker-manifest
+
+depends_on:
+  - deploy-docker-amd64
+  - deploy-docker-arm
+  - deploy-docker-arm64
+
+trigger:
+  event:
+    - push
+    - tag
+  ref:
+    - refs/heads/master
+    - refs/tags/v*
+
+steps:
+  - name: publish manifest
+    image: plugins/manifest
+    settings:
+      spec: manifest.tmpl
+      auto_tag: true
+      ignore_missing: true
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
--- a/7
+++ b/7
@ -1,7 +1,12 @@
-FROM python:3
+ARG REPO=library
+FROM ${REPO}/python:3-alpine
 LABEL maintainer="ian@iamthefij.com"
 # Minitor: https://git.iamthefij.com/iamthefij/minitor

+# This should be the target qemu arch
+ARG ARCH=x86_64
+COPY ./build/qemu-${ARCH}-static /usr/bin/
+
 COPY ./sample-config.yml /app/config.yml
 WORKDIR /app

--- a/40
+++ b/40
@ -88,6 +88,42 @@ htmlcov/index.html: .coverage
 open-coverage: htmlcov/index.html
 	$(OPEN_CMD) htmlcov/index.html

+# Docker targets
+
+# Targets to download required qemu binaries for running on an amd64 machine
+build/qemu-x86_64-static:
+	./get_qemu.sh x86_64
+
+build/qemu-arm-static:
+	./get_qemu.sh arm
+
+build/qemu-aarch64-static:
+	./get_qemu.sh aarch64
+
+# Build Docker image for host architechture (amd64)
 .PHONY: docker-build
-docker-build:
-	docker build . -t $(DOCKER_TAG)
+docker-build: build/qemu-x86_64-static
+	docker build . -t ${DOCKER_TAG}-linux-amd64
+
+# Cross build for arm architechtures
+.PHONY: docker-cross-build-arm
+docker-cross-build-arm: build/qemu-arm-static
+	docker build --build-arg REPO=arm32v6 --build-arg ARCH=arm . -t ${DOCKER_TAG}-linux-arm
+
+.PHONY: docker-cross-build-arm
+docker-cross-build-arm64: build/qemu-aarch64-static
+	docker build --build-arg REPO=arm64v8 --build-arg ARCH=aarch64 . -t ${DOCKER_TAG}-linux-arm64
+
+# Run on host architechture
+.PHONY: run
+docker-run: docker-build
+	docker run ${DOCKER_TAG}-linux-amd64
+
+# Cross run on host architechture
+.PHONY: docker-cross-run-arm
+docker-cross-run-arm: docker-cross-build-arm
+	docker run --rm ${DOCKER_TAG}-linux-arm
+
+.PHONY: docker-cross-run-arm64
+docker-cross-run-arm64: docker-cross-build-arm64
+	docker run --rm ${DOCKER_TAG}-linux-arm64
--- a/get_qemu.sh
+++ b/get_qemu.sh
@ -0,0 +1,14 @@
+#! /bin/bash
+
+HOST_ARCH=x86_64
+VERSION=v2.9.1-1
+
+mkdir -p build
+cd build
+
+# Multiple args can be passed in, but in most cases (Makefile and .drone.yml) we only use one at a time
+for target_arch in $*; do
+  wget https://github.com/multiarch/qemu-user-static/releases/download/$VERSION/${HOST_ARCH}_qemu-${target_arch}-static.tar.gz
+  tar -xvf ${HOST_ARCH}_qemu-${target_arch}-static.tar.gz
+  rm ${HOST_ARCH}_qemu-${target_arch}-static.tar.gz
+done
--- a/manifest.tmpl
+++ b/manifest.tmpl
@ -0,0 +1,25 @@
+image: iamthefij/minitor:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
+{{#if build.tags}}
+tags:
+{{#each build.tags}}
+  - {{this}}
+{{/each}}
+{{/if}}
+manifests:
+  -
+    image: iamthefij/minitor:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64
+    platform:
+      architecture: amd64
+      os: linux
+  -
+    image: iamthefij/minitor:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64
+    platform:
+      architecture: arm64
+      os: linux
+      variant: v8
+  -
+    image: iamthefij/minitor:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm
+    platform:
+      architecture: arm
+      os: linux
+      variant: v7