From d070c89f84eadb3b8e29e867f8a8e2d8555cb510 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 17 Apr 2024 17:59:01 -0400 Subject: [PATCH] split up CI steps Signed-off-by: strawberry --- .github/workflows/ci.yml | 403 ++++++++++++++------------------------- 1 file changed, 148 insertions(+), 255 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71388115..1370a43e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,11 +26,9 @@ permissions: contents: read jobs: - ci: - name: CI and Artifacts - + setup: + name: CI Setup runs-on: ubuntu-latest - steps: - name: Sync repository uses: actions/checkout@v4 @@ -94,291 +92,186 @@ jobs: ./bin/nix-build-and-cache .#devShells.x86_64-linux.default.inputDerivation + build-and-test: + name: CI and Artifacts + needs: setup + runs-on: ubuntu-latest + strategy: + matrix: + target: [ + "static-x86_64-unknown-linux-musl", + "static-x86_64-unknown-linux-musl-jemalloc", + "static-x86_64-unknown-linux-musl-hmalloc", + "static-aarch64-unknown-linux-musl", + "static-aarch64-unknown-linux-musl-jemalloc", + "static-aarch64-unknown-linux-musl-hmalloc", + ] + oci-target: [ + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "x86_64-unknown-linux-musl-jemalloc", + "x86_64-unknown-linux-musl-hmalloc", + "aarch64-unknown-linux-musl", + "aarch64-unknown-linux-musl-jemalloc", + "aarch64-unknown-linux-musl-hmalloc", + ] + + steps: + - name: Download build environment + uses: actions/download-artifact@v4 + with: + name: build-environment + + - name: Perform continuous integration run: direnv exec . engage - - name: Build static-x86_64-unknown-linux-musl and Create static deb-x86_64-unknown-linux-musl + - name: Build static artifacts run: | - ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl + ./bin/nix-build-and-cache .#${{ matrix.target }} mkdir -p target/release cp -v -f result/bin/conduit target/release - direnv exec . cargo deb --no-build + direnv exec . cargo deb --no-build --output target/debian/${{ matrix.target }}.deb - - name: Upload artifact static-x86_64-unknown-linux-musl + - name: Upload static artifacts uses: actions/upload-artifact@v4 with: - name: static-x86_64-unknown-linux-musl + name: ${{ matrix.target }} path: result/bin/conduit if-no-files-found: error - - name: Upload artifact deb-x86_64-unknown-linux-musl + - name: Upload static deb artifacts uses: actions/upload-artifact@v4 with: - name: x86_64-unknown-linux-musl.deb - path: target/debian/*.deb + name: ${{ matrix.target }}.deb + path: target/debian/${{ matrix.target }}.deb if-no-files-found: error - - name: Build static-x86_64-unknown-linux-musl-jemalloc and Create static deb-x86_64-unknown-linux-musl-jemalloc + + - name: Build OCI images run: | - ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl-jemalloc - mkdir -p target/release - cp -v -f result/bin/conduit target/release - direnv exec . cargo deb --no-build + ./bin/nix-build-and-cache .#oci-image-${{ matrix.oci-target }} + cp -v -f result oci-image-${{ matrix.oci-target }}.tar.gz - - name: Upload artifact static-x86_64-unknown-linux-musl-jemalloc + - name: Upload OCI image artifacts uses: actions/upload-artifact@v4 with: - name: static-x86_64-unknown-linux-musl-jemalloc - path: result/bin/conduit - if-no-files-found: error - - - name: Upload artifact deb-x86_64-unknown-linux-musl-jemalloc - uses: actions/upload-artifact@v4 - with: - name: x86_64-unknown-linux-musl-jemalloc.deb - path: target/debian/*.deb - if-no-files-found: error - - - name: Build static-x86_64-unknown-linux-musl-hmalloc and Create static deb-x86_64-unknown-linux-musl-hmalloc - run: | - ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl-hmalloc - mkdir -p target/release - cp -v -f result/bin/conduit target/release - direnv exec . cargo deb --no-build - - - name: Upload artifact static-x86_64-unknown-linux-musl-hmalloc - uses: actions/upload-artifact@v4 - with: - name: static-x86_64-unknown-linux-musl-hmalloc - path: result/bin/conduit - if-no-files-found: error - - - name: Upload artifact deb-x86_64-unknown-linux-musl-hmalloc - uses: actions/upload-artifact@v4 - with: - name: x86_64-unknown-linux-musl-hmalloc.deb - path: target/debian/*.deb - if-no-files-found: error - - - - name: Build static-aarch64-unknown-linux-musl - run: | - ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl - - - name: Upload artifact static-aarch64-unknown-linux-musl - uses: actions/upload-artifact@v4 - with: - name: static-aarch64-unknown-linux-musl - path: result/bin/conduit - if-no-files-found: error - - - name: Build static-aarch64-unknown-linux-musl-jemalloc - run: | - ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl-jemalloc - - - name: Upload artifact static-aarch64-unknown-linux-musl-jemalloc - uses: actions/upload-artifact@v4 - with: - name: static-aarch64-unknown-linux-musl-jemalloc - path: result/bin/conduit - if-no-files-found: error - - - name: Build static-aarch64-unknown-linux-musl-hmalloc - run: | - ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl-hmalloc - - - name: Upload artifact static-aarch64-unknown-linux-musl-hmalloc - uses: actions/upload-artifact@v4 - with: - name: static-aarch64-unknown-linux-musl-hmalloc - path: result/bin/conduit - if-no-files-found: error - - - - name: Build oci-image-x86_64-unknown-linux-gnu - run: | - ./bin/nix-build-and-cache .#oci-image - cp -v -f result oci-image-amd64.tar.gz - - - name: Upload artifact oci-image-x86_64-unknown-linux-gnu - uses: actions/upload-artifact@v4 - with: - name: oci-image-x86_64-unknown-linux-gnu - path: oci-image-amd64.tar.gz - if-no-files-found: error - # don't compress again - compression-level: 0 - - - name: Build oci-image-x86_64-unknown-linux-gnu-jemalloc - run: | - ./bin/nix-build-and-cache .#oci-image-jemalloc - cp -v -f result oci-image-amd64.tar.gz - - - name: Upload artifact oci-image-x86_64-unknown-linux-gnu-jemalloc - uses: actions/upload-artifact@v4 - with: - name: oci-image-x86_64-unknown-linux-gnu-jemalloc - path: oci-image-amd64.tar.gz - if-no-files-found: error - # don't compress again - compression-level: 0 - - - name: Build oci-image-x86_64-unknown-linux-gnu-hmalloc - run: | - ./bin/nix-build-and-cache .#oci-image-hmalloc - cp -v -f result oci-image-amd64.tar.gz - - - name: Upload artifact oci-image-x86_64-unknown-linux-gnu-hmalloc - uses: actions/upload-artifact@v4 - with: - name: oci-image-x86_64-unknown-linux-gnu-hmalloc - path: oci-image-amd64.tar.gz + name: oci-image-${{ matrix.oci-target }} + path: oci-image-${{ matrix.oci-target }}.tar.gz if-no-files-found: error # don't compress again compression-level: 0 - - name: Build oci-image-aarch64-unknown-linux-musl - run: | - ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl - cp -v -f result oci-image-arm64v8.tar.gz - - - name: Upload artifact oci-image-aarch64-unknown-linux-musl - uses: actions/upload-artifact@v4 - with: - name: oci-image-aarch64-unknown-linux-musl - path: oci-image-arm64v8.tar.gz - if-no-files-found: error - # don't compress again - compression-level: 0 - - - name: Build oci-image-aarch64-unknown-linux-musl-jemalloc - run: | - ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl-jemalloc - cp -v -f result oci-image-arm64v8.tar.gz - - - name: Upload artifact oci-image-aarch64-unknown-linux-musl-jemalloc - uses: actions/upload-artifact@v4 - with: - name: oci-image-aarch64-unknown-linux-musl-jemalloc - path: oci-image-arm64v8.tar.gz - if-no-files-found: error - # don't compress again - compression-level: 0 - - - name: Build oci-image-aarch64-unknown-linux-musl-hmalloc - run: | - ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl-hmalloc - cp -v -f result oci-image-arm64v8.tar.gz - - - name: Upload artifact oci-image-aarch64-unknown-linux-musl-hmalloc - uses: actions/upload-artifact@v4 - with: - name: oci-image-aarch64-unknown-linux-musl-hmalloc - path: oci-image-arm64v8.tar.gz - if-no-files-found: error - # don't compress again - compression-level: 0 - - name: Extract metadata for Dockerhub - env: - REGISTRY: registry.hub.docker.com - IMAGE_NAME: ${{ github.repository }} - id: meta-dockerhub - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + publish: + needs: build-and-test + runs-on: ubuntu-latest + steps: + - name: Download build environment + uses: actions/download-artifact@v4 + with: + name: build-environment - - name: Extract metadata for GitHub Container Registry - env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - id: meta-ghcr - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Extract metadata for Dockerhub + env: + REGISTRY: registry.hub.docker.com + IMAGE_NAME: ${{ github.repository }} + id: meta-dockerhub + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Extract metadata for GitHub Container Registry + env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + id: meta-ghcr + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - name: Login to Dockerhub - env: - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - DOCKER_USERNAME: ${{ vars.DOCKER_USERNAME }} - if: ${{ (github.event_name != 'pull_request') && (env.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} - uses: docker/login-action@v3 - with: - # username is not really a secret - username: ${{ vars.DOCKER_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to Dockerhub + env: + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + DOCKER_USERNAME: ${{ vars.DOCKER_USERNAME }} + if: ${{ (github.event_name != 'pull_request') && (env.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} + uses: docker/login-action@v3 + with: + # username is not really a secret + username: ${{ vars.DOCKER_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GitHub Container Registry - if: github.event_name != 'pull_request' - uses: docker/login-action@v3 - env: - REGISTRY: ghcr.io - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to GitHub Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + env: + REGISTRY: ghcr.io + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Publish to Dockerhub - env: - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - DOCKER_USERNAME: ${{ vars.DOCKER_USERNAME }} - IMAGE_NAME: docker.io/${{ github.repository }} - IMAGE_SUFFIX_AMD64: amd64 - IMAGE_SUFFIX_ARM64V8: arm64v8 - if: ${{ (github.event_name != 'pull_request') && (env.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} - run: | - docker load -i oci-image-amd64.tar.gz - IMAGE_ID_AMD64=$(docker images -q conduit:main) - docker load -i oci-image-arm64v8.tar.gz - IMAGE_ID_ARM64V8=$(docker images -q conduit:main) + - name: Publish to Dockerhub + env: + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + DOCKER_USERNAME: ${{ vars.DOCKER_USERNAME }} + IMAGE_NAME: docker.io/${{ github.repository }} + IMAGE_SUFFIX_AMD64: amd64 + IMAGE_SUFFIX_ARM64V8: arm64v8 + if: ${{ (github.event_name != 'pull_request') && (env.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} + run: | + docker load -i oci-image-amd64.tar.gz + IMAGE_ID_AMD64=$(docker images -q conduit:main) + docker load -i oci-image-arm64v8.tar.gz + IMAGE_ID_ARM64V8=$(docker images -q conduit:main) - # Tag and push the architecture specific images - docker tag $IMAGE_ID_AMD64 $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 - docker tag $IMAGE_ID_ARM64V8 $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 - docker push $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 - docker push $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 - # Tag the multi-arch image - docker manifest create $IMAGE_NAME:$GITHUB_SHA --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 - docker manifest push $IMAGE_NAME:$GITHUB_SHA - # Tag and push the git ref - docker manifest create $IMAGE_NAME:$GITHUB_REF_NAME --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 - docker manifest push $IMAGE_NAME:$GITHUB_REF_NAME - # Tag "main" as latest (stable branch) - if [[ "$GITHUB_REF_NAME" = "main" ]]; then - docker manifest create $IMAGE_NAME:latest --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 - docker manifest push $IMAGE_NAME:latest - fi + # Tag and push the architecture specific images + docker tag $IMAGE_ID_AMD64 $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 + docker tag $IMAGE_ID_ARM64V8 $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 + docker push $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 + docker push $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 + # Tag the multi-arch image + docker manifest create $IMAGE_NAME:$GITHUB_SHA --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 + docker manifest push $IMAGE_NAME:$GITHUB_SHA + # Tag and push the git ref + docker manifest create $IMAGE_NAME:$GITHUB_REF_NAME --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 + docker manifest push $IMAGE_NAME:$GITHUB_REF_NAME + # Tag "main" as latest (stable branch) + if [[ "$GITHUB_REF_NAME" = "main" ]]; then + docker manifest create $IMAGE_NAME:latest --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 + docker manifest push $IMAGE_NAME:latest + fi - - name: Publish to GitHub Container Registry - if: github.event_name != 'pull_request' - env: - IMAGE_NAME: ghcr.io/${{ github.repository }} - IMAGE_SUFFIX_AMD64: amd64 - IMAGE_SUFFIX_ARM64V8: arm64v8 - run: | - docker load -i oci-image-amd64.tar.gz - IMAGE_ID_AMD64=$(docker images -q conduit:main) - docker load -i oci-image-arm64v8.tar.gz - IMAGE_ID_ARM64V8=$(docker images -q conduit:main) + - name: Publish to GitHub Container Registry + if: github.event_name != 'pull_request' + env: + IMAGE_NAME: ghcr.io/${{ github.repository }} + IMAGE_SUFFIX_AMD64: amd64 + IMAGE_SUFFIX_ARM64V8: arm64v8 + run: | + docker load -i oci-image-amd64.tar.gz + IMAGE_ID_AMD64=$(docker images -q conduit:main) + docker load -i oci-image-arm64v8.tar.gz + IMAGE_ID_ARM64V8=$(docker images -q conduit:main) - # Tag and push the architecture specific images - docker tag $IMAGE_ID_AMD64 $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 - docker tag $IMAGE_ID_ARM64V8 $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 - docker push $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 - docker push $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 - # Tag the multi-arch image - docker manifest create $IMAGE_NAME:$GITHUB_SHA --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 - docker manifest push $IMAGE_NAME:$GITHUB_SHA - # Tag and push the git ref - docker manifest create $IMAGE_NAME:$GITHUB_REF_NAME --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 - docker manifest push $IMAGE_NAME:$GITHUB_REF_NAME - # Tag "main" as latest (stable branch) - if [[ "$GITHUB_REF_NAME" = "main" ]]; then - docker manifest create $IMAGE_NAME:latest --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 - docker manifest push $IMAGE_NAME:latest - fi + # Tag and push the architecture specific images + docker tag $IMAGE_ID_AMD64 $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 + docker tag $IMAGE_ID_ARM64V8 $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 + docker push $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 + docker push $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 + # Tag the multi-arch image + docker manifest create $IMAGE_NAME:$GITHUB_SHA --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 + docker manifest push $IMAGE_NAME:$GITHUB_SHA + # Tag and push the git ref + docker manifest create $IMAGE_NAME:$GITHUB_REF_NAME --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 + docker manifest push $IMAGE_NAME:$GITHUB_REF_NAME + # Tag "main" as latest (stable branch) + if [[ "$GITHUB_REF_NAME" = "main" ]]; then + docker manifest create $IMAGE_NAME:latest --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_AMD64 --amend $IMAGE_NAME:$GITHUB_SHA-$IMAGE_SUFFIX_ARM64V8 + docker manifest push $IMAGE_NAME:latest + fi