From d070c89f84eadb3b8e29e867f8a8e2d8555cb510 Mon Sep 17 00:00:00 2001
From: strawberry <strawberry@puppygock.gay>
Date: Wed, 17 Apr 2024 17:59:01 -0400
Subject: [PATCH] split up CI steps
Signed-off-by: strawberry <strawberry@puppygock.gay>
---
.github/workflows/ci.yml | 403 ++++++++++++++-------------------------
1 file changed, 148 insertions(+), 255 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 71388115..1370a43e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -26,11 +26,9 @@ permissions:
contents: read
jobs:
- ci:
- name: CI and Artifacts
-
+ setup:
+ name: CI Setup
runs-on: ubuntu-latest
-
steps:
- name: Sync repository
uses: actions/checkout@v4
@@ -94,291 +92,186 @@ jobs:
./bin/nix-build-and-cache .#devShells.x86_64-linux.default.inputDerivation
+ build-and-test:
+ name: CI and Artifacts
+ needs: setup
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ target: [
+ "static-x86_64-unknown-linux-musl",
+ "static-x86_64-unknown-linux-musl-jemalloc",
+ "static-x86_64-unknown-linux-musl-hmalloc",
+ "static-aarch64-unknown-linux-musl",
+ "static-aarch64-unknown-linux-musl-jemalloc",
+ "static-aarch64-unknown-linux-musl-hmalloc",
+ ]
+ oci-target: [
+ "x86_64-unknown-linux-gnu",
+ "x86_64-unknown-linux-musl",
+ "x86_64-unknown-linux-musl-jemalloc",
+ "x86_64-unknown-linux-musl-hmalloc",
+ "aarch64-unknown-linux-musl",
+ "aarch64-unknown-linux-musl-jemalloc",
+ "aarch64-unknown-linux-musl-hmalloc",
+ ]
+
+ steps:
+ - name: Download build environment
+ uses: actions/download-artifact@v4
+ with:
+ name: build-environment
+
+
- name: Perform continuous integration
run: direnv exec . engage
- - name: Build static-x86_64-unknown-linux-musl and Create static deb-x86_64-unknown-linux-musl
+ - name: Build static artifacts
run: |
- ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl
+ ./bin/nix-build-and-cache .#${{ matrix.target }}
mkdir -p target/release
cp -v -f result/bin/conduit target/release
- direnv exec . cargo deb --no-build
+ direnv exec . cargo deb --no-build --output target/debian/${{ matrix.target }}.deb
- - name: Upload artifact static-x86_64-unknown-linux-musl
+ - name: Upload static artifacts
uses: actions/upload-artifact@v4
with:
- name: static-x86_64-unknown-linux-musl
+ name: ${{ matrix.target }}
path: result/bin/conduit
if-no-files-found: error
- - name: Upload artifact deb-x86_64-unknown-linux-musl
+ - name: Upload static deb artifacts
uses: actions/upload-artifact@v4
with:
- name: x86_64-unknown-linux-musl.deb
- path: target/debian/*.deb
+ name: ${{ matrix.target }}.deb
+ path: target/debian/${{ matrix.target }}.deb
if-no-files-found: error
- - name: Build static-x86_64-unknown-linux-musl-jemalloc and Create static deb-x86_64-unknown-linux-musl-jemalloc
+
+ - name: Build OCI images
run: |
- ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl-jemalloc
- mkdir -p target/release
- cp -v -f result/bin/conduit target/release
- direnv exec . cargo deb --no-build
+ ./bin/nix-build-and-cache .#oci-image-${{ matrix.oci-target }}
+ cp -v -f result oci-image-${{ matrix.oci-target }}.tar.gz
- - name: Upload artifact static-x86_64-unknown-linux-musl-jemalloc
+ - name: Upload OCI image artifacts
uses: actions/upload-artifact@v4
with:
- name: static-x86_64-unknown-linux-musl-jemalloc
- path: result/bin/conduit
- if-no-files-found: error
-
- - name: Upload artifact deb-x86_64-unknown-linux-musl-jemalloc
- uses: actions/upload-artifact@v4
- with:
- name: x86_64-unknown-linux-musl-jemalloc.deb
- path: target/debian/*.deb
- if-no-files-found: error
-
- - name: Build static-x86_64-unknown-linux-musl-hmalloc and Create static deb-x86_64-unknown-linux-musl-hmalloc
- run: |
- ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl-hmalloc
- mkdir -p target/release
- cp -v -f result/bin/conduit target/release
- direnv exec . cargo deb --no-build
-
- - name: Upload artifact static-x86_64-unknown-linux-musl-hmalloc
- uses: actions/upload-artifact@v4
- with:
- name: static-x86_64-unknown-linux-musl-hmalloc
- path: result/bin/conduit
- if-no-files-found: error
-
- - name: Upload artifact deb-x86_64-unknown-linux-musl-hmalloc
- uses: actions/upload-artifact@v4
- with:
- name: x86_64-unknown-linux-musl-hmalloc.deb
- path: target/debian/*.deb
- if-no-files-found: error
-
-
- - name: Build static-aarch64-unknown-linux-musl
- run: |
- ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl
-
- - name: Upload artifact static-aarch64-unknown-linux-musl
- uses: actions/upload-artifact@v4
- with:
- name: static-aarch64-unknown-linux-musl
- path: result/bin/conduit
- if-no-files-found: error
-
- - name: Build static-aarch64-unknown-linux-musl-jemalloc
- run: |
- ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl-jemalloc
-
- - name: Upload artifact static-aarch64-unknown-linux-musl-jemalloc
- uses: actions/upload-artifact@v4
- with:
- name: static-aarch64-unknown-linux-musl-jemalloc
- path: result/bin/conduit
- if-no-files-found: error
-
- - name: Build static-aarch64-unknown-linux-musl-hmalloc
- run: |
- ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl-hmalloc
-
- - name: Upload artifact static-aarch64-unknown-linux-musl-hmalloc
- uses: actions/upload-artifact@v4
- with:
- name: static-aarch64-unknown-linux-musl-hmalloc
- path: result/bin/conduit
- if-no-files-found: error
-
-
- - name: Build oci-image-x86_64-unknown-linux-gnu
- run: |
- ./bin/nix-build-and-cache .#oci-image
- cp -v -f result oci-image-amd64.tar.gz
-
- - name: Upload artifact oci-image-x86_64-unknown-linux-gnu
- uses: actions/upload-artifact@v4
- with:
- name: oci-image-x86_64-unknown-linux-gnu
- path: oci-image-amd64.tar.gz
- if-no-files-found: error
- # don't compress again
- compression-level: 0
-
- - name: Build oci-image-x86_64-unknown-linux-gnu-jemalloc
- run: |
- ./bin/nix-build-and-cache .#oci-image-jemalloc
- cp -v -f result oci-image-amd64.tar.gz
-
- - name: Upload artifact oci-image-x86_64-unknown-linux-gnu-jemalloc
- uses: actions/upload-artifact@v4
- with:
- name: oci-image-x86_64-unknown-linux-gnu-jemalloc
- path: oci-image-amd64.tar.gz
- if-no-files-found: error
- # don't compress again
- compression-level: 0
-
- - name: Build oci-image-x86_64-unknown-linux-gnu-hmalloc
- run: |
- ./bin/nix-build-and-cache .#oci-image-hmalloc
- cp -v -f result oci-image-amd64.tar.gz
-
- - name: Upload artifact oci-image-x86_64-unknown-linux-gnu-hmalloc
- uses: actions/upload-artifact@v4
- with:
- name: oci-image-x86_64-unknown-linux-gnu-hmalloc
- path: oci-image-amd64.tar.gz
+ name: oci-image-${{ matrix.oci-target }}
+ path: oci-image-${{ matrix.oci-target }}.tar.gz
if-no-files-found: error
# don't compress again
compression-level: 0
- - name: Build oci-image-aarch64-unknown-linux-musl
- run: |
- ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl
- cp -v -f result oci-image-arm64v8.tar.gz
-
- - name: Upload artifact oci-image-aarch64-unknown-linux-musl
- uses: actions/upload-artifact@v4
- with:
- name: oci-image-aarch64-unknown-linux-musl
- path: oci-image-arm64v8.tar.gz
- if-no-files-found: error
- # don't compress again
- compression-level: 0
-
- - name: Build oci-image-aarch64-unknown-linux-musl-jemalloc
- run: |
- ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl-jemalloc
- cp -v -f result oci-image-arm64v8.tar.gz
-
- - name: Upload artifact oci-image-aarch64-unknown-linux-musl-jemalloc
- uses: actions/upload-artifact@v4
- with:
- name: oci-image-aarch64-unknown-linux-musl-jemalloc
- path: oci-image-arm64v8.tar.gz
- if-no-files-found: error
- # don't compress again
- compression-level: 0
-
- - name: Build oci-image-aarch64-unknown-linux-musl-hmalloc
- run: |
- ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl-hmalloc
- cp -v -f result oci-image-arm64v8.tar.gz
-
- - name: Upload artifact oci-image-aarch64-unknown-linux-musl-hmalloc
- uses: actions/upload-artifact@v4
- with:
- name: oci-image-aarch64-unknown-linux-musl-hmalloc
- path: oci-image-arm64v8.tar.gz
- if-no-files-found: error
- # don't compress again
- compression-level: 0
- - name: Extract metadata for Dockerhub
- env:
- REGISTRY: registry.hub.docker.com
- IMAGE_NAME: ${{ github.repository }}
- id: meta-dockerhub
- uses: docker/metadata-action@v5
- with:
- images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+ publish:
+ needs: build-and-test
+ runs-on: ubuntu-latest
+ steps:
+ - name: Download build environment
+ uses: actions/download-artifact@v4
+ with:
+ name: build-environment
- - name: Extract metadata for GitHub Container Registry
- env:
- REGISTRY: ghcr.io
- IMAGE_NAME: ${{ github.repository }}
- id: meta-ghcr
- uses: docker/metadata-action@v5
- with:
- images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+ - name: Extract metadata for Dockerhub
+ env:
+ REGISTRY: registry.hub.docker.com
+ IMAGE_NAME: ${{ github.repository }}
+ id: meta-dockerhub
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+ - name: Extract metadata for GitHub Container Registry
+ env:
+ REGISTRY: ghcr.io
+ IMAGE_NAME: ${{ github.repository }}
+ id: meta-ghcr
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
- - name: Login to Dockerhub
- env:
- DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
- DOCKER_USERNAME: ${{ vars.DOCKER_USERNAME }}
- if: ${{ (github.event_name != 'pull_request') && (env.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }}
- uses: docker/login-action@v3
- with:
- # username is not really a secret
- username: ${{ vars.DOCKER_USERNAME }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
+ - name: Login to Dockerhub
+ env:
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ DOCKER_USERNAME: ${{ vars.DOCKER_USERNAME }}
+ if: ${{ (github.event_name != 'pull_request') && (env.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }}
+ uses: docker/login-action@v3
+ with:
+ # username is not really a secret
+ username: ${{ vars.DOCKER_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
- - name: Login to GitHub Container Registry
- if: github.event_name != 'pull_request'
- uses: docker/login-action@v3
- env:
- REGISTRY: ghcr.io
- with:
- registry: ${{ env.REGISTRY }}
- username: ${{ github.repository_owner }}
- password: ${{ secrets.GITHUB_TOKEN }}
+ - name: Login to GitHub Container Registry
+ if: github.event_name != 'pull_request'
+ uses: docker/login-action@v3
+ env:
+ REGISTRY: ghcr.io
+ with:
+ registry: ${{ env.REGISTRY }}
+ username: ${{ github.repository_owner }}
+ password: ${{ secrets.GITHUB_TOKEN }}
- - name: Publish to Dockerhub
- env:
- DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
- DOCKER_USERNAME: ${{ vars.DOCKER_USERNAME }}
- IMAGE_NAME: docker.io/${{ github.repository }}
- IMAGE_SUFFIX_AMD64: amd64
- IMAGE_SUFFIX_ARM64V8: arm64v8
- if: ${{ (github.event_name != 'pull_request') && (env.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }}
- run: |
- docker load -i oci-image-amd64.tar.gz
- IMAGE_ID_AMD64=$(docker images -q conduit:main)
- docker load -i oci-image-arm64v8.tar.gz
- IMAGE_ID_ARM64V8=$(docker images -q conduit:main)
+ - name: Publish to Dockerhub
+ env:
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ DOCKER_USERNAME: ${{ vars.DOCKER_USERNAME }}
+ IMAGE_NAME: docker.io/${{ github.repository }}
+ IMAGE_SUFFIX_AMD64: amd64
+ IMAGE_SUFFIX_ARM64V8: arm64v8
+ if: ${{ (github.event_name != 'pull_request') && (env.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }}
+ run: |
+ docker load -i oci-image-amd64.tar.gz
+ IMAGE_ID_AMD64=$(docker images -q conduit:main)
+ docker load -i oci-image-arm64v8.tar.gz
+ IMAGE_ID_ARM64V8=$(docker images -q conduit:main)
- # Tag and push the architecture specific images
- docker tag $IMAGE_ID_AMD64 $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64
- docker tag $IMAGE_ID_ARM64V8 $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
- docker push $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64
- docker push $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
- # Tag the multi-arch image
- docker manifest create $IMAGE_NAME:$GITHUB_SHA --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
- docker manifest push $IMAGE_NAME:$GITHUB_SHA
- # Tag and push the git ref
- docker manifest create $IMAGE_NAME:$GITHUB_REF_NAME --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
- docker manifest push $IMAGE_NAME:$GITHUB_REF_NAME
- # Tag "main" as latest (stable branch)
- if [[ "$GITHUB_REF_NAME" = "main" ]]; then
- docker manifest create $IMAGE_NAME:latest --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
- docker manifest push $IMAGE_NAME:latest
- fi
+ # Tag and push the architecture specific images
+ docker tag $IMAGE_ID_AMD64 $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64
+ docker tag $IMAGE_ID_ARM64V8 $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
+ docker push $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64
+ docker push $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
+ # Tag the multi-arch image
+ docker manifest create $IMAGE_NAME:$GITHUB_SHA --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
+ docker manifest push $IMAGE_NAME:$GITHUB_SHA
+ # Tag and push the git ref
+ docker manifest create $IMAGE_NAME:$GITHUB_REF_NAME --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
+ docker manifest push $IMAGE_NAME:$GITHUB_REF_NAME
+ # Tag "main" as latest (stable branch)
+ if [[ "$GITHUB_REF_NAME" = "main" ]]; then
+ docker manifest create $IMAGE_NAME:latest --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
+ docker manifest push $IMAGE_NAME:latest
+ fi
- - name: Publish to GitHub Container Registry
- if: github.event_name != 'pull_request'
- env:
- IMAGE_NAME: ghcr.io/${{ github.repository }}
- IMAGE_SUFFIX_AMD64: amd64
- IMAGE_SUFFIX_ARM64V8: arm64v8
- run: |
- docker load -i oci-image-amd64.tar.gz
- IMAGE_ID_AMD64=$(docker images -q conduit:main)
- docker load -i oci-image-arm64v8.tar.gz
- IMAGE_ID_ARM64V8=$(docker images -q conduit:main)
+ - name: Publish to GitHub Container Registry
+ if: github.event_name != 'pull_request'
+ env:
+ IMAGE_NAME: ghcr.io/${{ github.repository }}
+ IMAGE_SUFFIX_AMD64: amd64
+ IMAGE_SUFFIX_ARM64V8: arm64v8
+ run: |
+ docker load -i oci-image-amd64.tar.gz
+ IMAGE_ID_AMD64=$(docker images -q conduit:main)
+ docker load -i oci-image-arm64v8.tar.gz
+ IMAGE_ID_ARM64V8=$(docker images -q conduit:main)
- # Tag and push the architecture specific images
- docker tag $IMAGE_ID_AMD64 $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64
- docker tag $IMAGE_ID_ARM64V8 $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
- docker push $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64
- docker push $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
- # Tag the multi-arch image
- docker manifest create $IMAGE_NAME:$GITHUB_SHA --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
- docker manifest push $IMAGE_NAME:$GITHUB_SHA
- # Tag and push the git ref
- docker manifest create $IMAGE_NAME:$GITHUB_REF_NAME --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
- docker manifest push $IMAGE_NAME:$GITHUB_REF_NAME
- # Tag "main" as latest (stable branch)
- if [[ "$GITHUB_REF_NAME" = "main" ]]; then
- docker manifest create $IMAGE_NAME:latest --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
- docker manifest push $IMAGE_NAME:latest
- fi
+ # Tag and push the architecture specific images
+ docker tag $IMAGE_ID_AMD64 $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64
+ docker tag $IMAGE_ID_ARM64V8 $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
+ docker push $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64
+ docker push $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
+ # Tag the multi-arch image
+ docker manifest create $IMAGE_NAME:$GITHUB_SHA --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
+ docker manifest push $IMAGE_NAME:$GITHUB_SHA
+ # Tag and push the git ref
+ docker manifest create $IMAGE_NAME:$GITHUB_REF_NAME --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
+ docker manifest push $IMAGE_NAME:$GITHUB_REF_NAME
+ # Tag "main" as latest (stable branch)
+ if [[ "$GITHUB_REF_NAME" = "main" ]]; then
+ docker manifest create $IMAGE_NAME:latest --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8
+ docker manifest push $IMAGE_NAME:latest
+ fi