From a4ec0daafadfdbdf617abe62f292c7ccd24e9358 Mon Sep 17 00:00:00 2001 From: Charles Hall Date: Tue, 5 Mar 2024 21:42:17 -0500 Subject: [PATCH] make CI more efficient (github and gitlab) squashed from https://gitlab.com/famedly/conduit/-/merge_requests/596 ported the relevant parts to GitHub Actions Co-authored-by: strawberry Signed-off-by: strawberry --- .github/workflows/ci.yml | 52 +++++++++------ .gitlab-ci.yml | 139 ++++++++++++++++----------------------- bin/nix-build-and-cache | 47 ++++++------- flake.lock | 26 ++++---- flake.nix | 20 ++++-- 5 files changed, 134 insertions(+), 150 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b58c542f..c8479ee5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: with: nix_path: nixpkgs=channel:nixos-unstable - # Add the `nix-community` cachix to speed up things that leverage it + # Add `nix-community`, Crane, upstream Conduit, conduwuit, and alternative binary caches extra_nix_config: | experimental-features = nix-command flakes extra-substituters = https://nix-community.cachix.org @@ -43,6 +43,8 @@ jobs: extra-trusted-public-keys = conduit:Isq8FGyEC6FOXH6nD+BOeAA+bKp6X6UIbupSlGEPuOg= extra-substituters = https://attic.kennel.juneis.dog/conduwuit extra-trusted-public-keys = conduwuit:lYPVh7o1hLu1idH4Xt2QHaRa49WRGSAqzcfFd94aOTw= + extra-substituters = $ATTIC_ENDPOINT + extra-trusted-public-keys = $ATTIC_PUBLIC_KEY - name: Pop/push Magic Nix Cache uses: DeterminateSystems/magic-nix-cache-action@main @@ -70,14 +72,24 @@ jobs: - name: Populate `/nix/store` run: nix develop --command true - - name: Perform continuous integration - run: | - direnv allow - direnv exec . engage + - name: Allow direnv + run: direnv allow - - name: Build static-x86_64-unknown-linux-musl + - name: Cache x86_64 inputs for devShell + run: | + ./bin/nix-build-and-cache .#devShells.x86_64-linux.default.inputDerivation + + + - name: Perform continuous integration + run: direnv exec . engage + + + - name: Build static-x86_64-unknown-linux-musl and Create static deb-x86_64-unknown-linux-musl run: | ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl + mkdir -p target/release + cp -v -f result/bin/conduit target/release + direnv exec . cargo deb --no-build - name: Upload artifact static-x86_64-unknown-linux-musl uses: actions/upload-artifact@v4 @@ -86,6 +98,14 @@ jobs: path: result/bin/conduit if-no-files-found: error + - name: Upload artifact deb-x86_64-unknown-linux-musl + uses: actions/upload-artifact@v4 + with: + name: x86_64-unknown-linux-musl.deb + path: target/debian/*.deb + if-no-files-found: error + + - name: Build static-aarch64-unknown-linux-musl run: | ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl @@ -97,10 +117,11 @@ jobs: path: result/bin/conduit if-no-files-found: error + - name: Build oci-image-x86_64-unknown-linux-gnu run: | ./bin/nix-build-and-cache .#oci-image - cp -f result oci-image-amd64.tar.gz + cp -v -f result oci-image-amd64.tar.gz - name: Upload artifact oci-image-x86_64-unknown-linux-gnu uses: actions/upload-artifact@v4 @@ -111,10 +132,11 @@ jobs: # don't compress again compression-level: 0 + - name: Build oci-image-aarch64-unknown-linux-musl run: | ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl - cp -f result oci-image-arm64v8.tar.gz + cp -v -f result oci-image-arm64v8.tar.gz - name: Upload artifact oci-image-aarch64-unknown-linux-musl uses: actions/upload-artifact@v4 @@ -125,18 +147,6 @@ jobs: # don't compress again compression-level: 0 - - name: Build deb-x86_64-unknown-linux-gnu - run: | - sudo apt-get update && sudo apt-get install -y --no-install-recommends libclang-dev - cargo install cargo-deb - cargo deb - - - name: Upload artifact deb-x86_64-unknown-linux-gnu - uses: actions/upload-artifact@v4 - with: - name: deb-x86_64-unknown-linux-gnu - path: target/debian/*.deb - if-no-files-found: error - name: Extract metadata for Dockerhub env: @@ -156,6 +166,7 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Login to Dockerhub if: github.event_name != 'pull_request' uses: docker/login-action@v3 @@ -173,6 +184,7 @@ jobs: username: girlbossceo password: ${{ secrets.GITHUB_TOKEN }} + - name: Publish to Dockerhub if: github.event_name != 'pull_request' env: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 01b69584..52fbc59e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,22 +6,36 @@ stages: variables: # Makes some things print in color TERM: ansi - NIX_CONFIG: | - experimental-features = nix-command flake - extra-substituters = https://nix.computer.surgery/conduit - extra-trusted-public-keys = conduit:ZGAf6P6LhNvnoJJ3Me3PRg7tlLSrPxcQ2RiE5LIppjo= - extra-substituters = https://crane.cachix.org - extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk= - extra-substituters = https://nix-community.cachix.org - extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= - extra-substituters = https://attic.kennel.juneis.dog/conduit - extra-trusted-public-keys = conduit:Isq8FGyEC6FOXH6nD+BOeAA+bKp6X6UIbupSlGEPuOg= - extra-substituters = https://attic.kennel.juneis.dog/conduwuit - extra-trusted-public-keys = conduwuit:lYPVh7o1hLu1idH4Xt2QHaRa49WRGSAqzcfFd94aOTw= before_script: + # Enable nix-command and flakes + - if command -v nix > /dev/null; then echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf; fi + + # Add conduwuit binary cache + - if command -v nix > /dev/null; then echo "extra-substituters = https://attic.kennel.juneis.dog/conduwuit" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduwuit:lYPVh7o1hLu1idH4Xt2QHaRa49WRGSAqzcfFd94aOTw=" >> /etc/nix/nix.conf; fi + + - if command -v nix > /dev/null; then echo "extra-substituters = https://attic.kennel.juneis.dog/conduit" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduit:Isq8FGyEC6FOXH6nD+BOeAA+bKp6X6UIbupSlGEPuOg=" >> /etc/nix/nix.conf; fi + + # Add upstream Conduit binary cache + - if command -v nix > /dev/null; then echo "extra-substituters = https://nix.computer.surgery/conduit" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduit:ZGAf6P6LhNvnoJJ3Me3PRg7tlLSrPxcQ2RiE5LIppjo=" >> /etc/nix/nix.conf; fi + + # Add alternate binary cache + - if command -v nix > /dev/null && [ -n "$ATTIC_ENDPOINT" ]; then echo "extra-substituters = $ATTIC_ENDPOINT" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null && [ -n "$ATTIC_PUBLIC_KEY" ]; then echo "extra-trusted-public-keys = $ATTIC_PUBLIC_KEY" >> /etc/nix/nix.conf; fi + + # Add crane binary cache + - if command -v nix > /dev/null; then echo "extra-substituters = https://crane.cachix.org" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk=" >> /etc/nix/nix.conf; fi + + # Add nix-community binary cache + - if command -v nix > /dev/null; then echo "extra-substituters = https://nix-community.cachix.org" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" >> /etc/nix/nix.conf; fi + # Install direnv and nix-direnv - - if command -v nix > /dev/null; then nix-env -iA nixpkgs.direnv nixpkgs.nix-direnv nixpkgs.engage; fi + - if command -v nix > /dev/null; then nix-env -iA nixpkgs.direnv nixpkgs.nix-direnv; fi # Allow .envrc - if command -v nix > /dev/null; then direnv allow; fi @@ -33,6 +47,9 @@ ci: stage: ci image: nixos/nix:2.20.4 script: + # Cache the inputs required for the devShell + - ./bin/nix-build-and-cache .#devShells.x86_64-linux.default.inputDerivation + - direnv exec . engage cache: key: nix @@ -40,100 +57,50 @@ ci: - target - .gitlab-ci.d -static:x86_64-unknown-linux-musl: +artifacts: stage: artifacts image: nixos/nix:2.20.4 script: - # Push artifacts and build requirements to binary cache - ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl + - cp result/bin/conduit x86_64-unknown-linux-musl - # Make the output less difficult to find - - cp result/bin/conduit conduit - artifacts: - paths: - - conduit + - mkdir -p target/release + - cp result/bin/conduit target/release + - direnv exec . cargo deb --no-build + - mv target/debian/*.deb x86_64-unknown-linux-musl.deb -static:aarch64-unknown-linux-musl: - stage: artifacts - image: nixos/nix:2.20.4 - script: - # Push artifacts and build requirements to binary cache - - ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl - - # Make the output less difficult to find - - cp result/bin/conduit conduit - artifacts: - paths: - - conduit - -# Note that although we have an `oci-image-x86_64-unknown-linux-musl` output, -# we don't build it because it would be largely redundant to this one since it's -# all containerized anyway. -oci-image:x86_64-unknown-linux-gnu: - stage: artifacts - image: nixos/nix:2.20.4 - script: - # Push artifacts and build requirements to binary cache - # # Since the OCI image package is based on the binary package, this has the # fun side effect of uploading the normal binary too. Conduit users who are # deploying with Nix can leverage this fact by adding our binary cache to # their systems. + # + # Note that although we have an `oci-image-x86_64-unknown-linux-musl` + # output, we don't build it because it would be largely redundant to this + # one since it's all containerized anyway. - ./bin/nix-build-and-cache .#oci-image - - # Make the output less difficult to find - cp result oci-image-amd64.tar.gz - artifacts: - paths: - - oci-image-amd64.tar.gz -oci-image:aarch64-unknown-linux-musl: - stage: artifacts - needs: - # Wait for the static binary job to finish before starting so we don't have - # to build that twice for no reason - - static:aarch64-unknown-linux-musl - image: nixos/nix:2.20.4 - script: - # Push artifacts and build requirements to binary cache + - ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl + - cp result/bin/conduit aarch64-unknown-linux-musl + - ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl - - # Make the output less difficult to find - cp result oci-image-arm64v8.tar.gz artifacts: paths: + - x86_64-unknown-linux-musl + - aarch64-unknown-linux-musl + - x86_64-unknown-linux-musl.deb + - oci-image-amd64.tar.gz - oci-image-arm64v8.tar.gz -debian:x86_64-unknown-linux-gnu: - stage: artifacts - # See also `rust-toolchain.toml` - image: rust:1.75.0 - script: - - cargo install cargo-deb - - cargo deb - - # Make the output less difficult to find - - mv target/debian/*.deb conduit.deb - artifacts: - paths: - - conduit.deb - cache: - key: debian - paths: - - target - - .gitlab-ci.d - -docker-publish: +.push-oci-image: stage: publish image: docker:25.0.3 services: - docker:25.0.3-dind variables: - IMAGE_NAME: $CI_REGISTRY_IMAGE/conduwuit IMAGE_SUFFIX_AMD64: amd64 IMAGE_SUFFIX_ARM64V8: arm64v8 - before_script: - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY script: - docker load -i oci-image-amd64.tar.gz - IMAGE_ID_AMD64=$(docker images -q conduit:main) @@ -157,8 +124,14 @@ docker-publish: docker manifest push $IMAGE_NAME:latest fi dependencies: - - oci-image:x86_64-unknown-linux-gnu - - oci-image:aarch64-unknown-linux-musl + - artifacts only: - main - tags + +oci-image:push-gitlab: + extends: .push-oci-image + variables: + IMAGE_NAME: $CI_REGISTRY_IMAGE/conduwuit + before_script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY diff --git a/bin/nix-build-and-cache b/bin/nix-build-and-cache index 65b1913f..5ba5991a 100755 --- a/bin/nix-build-and-cache +++ b/bin/nix-build-and-cache @@ -6,45 +6,36 @@ set -euo pipefail INSTALLABLE="$1" # Build the installable and forward any other arguments too -nix build -L "$@" +nix build "$@" if [ ! -z "$ATTIC_TOKEN" ]; then - -nix run --inputs-from . attic -- login \ - conduit \ - https://attic.kennel.juneis.dog/conduit \ + nix run --inputs-from . attic -- \ + login \ + conduit \ + "${ATTIC_ENDPOINT:-https://attic.kennel.juneis.dog/conduit}" \ "$ATTIC_TOKEN" - push_args=( - # Attic and its build dependencies - "$(nix path-info --inputs-from . attic)" - "$(nix path-info --inputs-from . attic --derivation)" - - # The target installable and its build dependencies - "$(nix path-info "$INSTALLABLE" --derivation)" + # Push the target installable and its build dependencies + nix run --inputs-from . attic -- \ + push \ + conduit \ + "$(nix path-info "$INSTALLABLE" --derivation)" \ "$(nix path-info "$INSTALLABLE")" - ) - nix run --inputs-from . attic -- push conduit "${push_args[@]}" # push to "conduwuit" too -nix run --inputs-from . attic -- login \ - conduwuit \ - https://attic.kennel.juneis.dog/conduwuit \ + nix run --inputs-from . attic -- \ + login \ + conduwuit \ + "${ATTIC_ENDPOINT:-https://attic.kennel.juneis.dog/conduwuit}" \ "$ATTIC_TOKEN" - push_args=( - # Attic and its build dependencies - "$(nix path-info --inputs-from . attic)" - "$(nix path-info --inputs-from . attic --derivation)" - - # The target installable and its build dependencies - "$(nix path-info "$INSTALLABLE" --derivation)" + # Push the target installable and its build dependencies + nix run --inputs-from . attic -- \ + push \ + conduwuit \ + "$(nix path-info "$INSTALLABLE" --derivation)" \ "$(nix path-info "$INSTALLABLE")" - ) - - nix run --inputs-from . attic -- push conduwuit "${push_args[@]}" - else echo "\$ATTIC_TOKEN is unset, skipping uploading to the binary cache" fi diff --git a/flake.lock b/flake.lock index 2cc822f2..1c2142f2 100644 --- a/flake.lock +++ b/flake.lock @@ -60,8 +60,8 @@ }, "original": { "owner": "ipetkov", - "ref": "master", "repo": "crane", + "rev": "2c653e4478476a52c6aa3ac0495e4dea7449ea0e", "type": "github" } }, @@ -73,11 +73,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1707891749, - "narHash": "sha256-SeikNYElHgv8uVMbiA9/pU3Cce7ssIsiM8CnEiwd1Nc=", + "lastModified": 1709619709, + "narHash": "sha256-l6EPVJfwfelWST7qWQeP6t/TDK3HHv5uUB1b2vw4mOQ=", "owner": "nix-community", "repo": "fenix", - "rev": "3115aab064ef38cccd792c45429af8df43d6d277", + "rev": "c8943ea9e98d41325ff57d4ec14736d330b321b2", "type": "github" }, "original": { @@ -138,11 +138,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1705309234, - "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", + "lastModified": 1709126324, + "narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=", "owner": "numtide", "repo": "flake-utils", - "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", + "rev": "d465f4819400de7c8d874d50b982301f28a84605", "type": "github" }, "original": { @@ -200,11 +200,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1707689078, - "narHash": "sha256-UUGmRa84ZJHpGZ1WZEBEUOzaPOWG8LZ0yPg1pdDF/yM=", + "lastModified": 1709479366, + "narHash": "sha256-n6F0n8UV6lnTZbYPl1A9q1BS0p4hduAv1mGAP17CVd0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "f9d39fb9aff0efee4a3d5f4a6d7c17701d38a1d8", + "rev": "b8697e57f10292a6165a20f03d2f42920dfaf973", "type": "github" }, "original": { @@ -228,11 +228,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1707849817, - "narHash": "sha256-If6T0MDErp3/z7DBlpG4bV46IPP+7BWSlgTI88cmbw0=", + "lastModified": 1709571018, + "narHash": "sha256-ISFrxHxE0J5g7lDAscbK88hwaT5uewvWoma9TlFmRzM=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "a02a219773629686bd8ff123ca1aa995fa50d976", + "rev": "9f14343f9ee24f53f17492c5f9b653427e2ad15e", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 111afefa..cc370174 100644 --- a/flake.nix +++ b/flake.nix @@ -13,7 +13,12 @@ inputs.nixpkgs.follows = "nixpkgs"; }; crane = { - url = "github:ipetkov/crane?ref=master"; + # Pin latest crane that's not affected by the following bugs: + # + # * + # * + # * + url = "github:ipetkov/crane?rev=2c653e4478476a52c6aa3ac0495e4dea7449ea0e"; inputs.nixpkgs.follows = "nixpkgs"; }; attic.url = "github:zhaofengli/attic?ref=main"; @@ -58,7 +63,7 @@ # bindgen needs the build platform's libclang. Apparently due to # "splicing weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't # quite do the right thing here. - pkgs.buildPackages.rustPlatform.bindgenHook + pkgs.pkgsBuildHost.rustPlatform.bindgenHook ]; env = pkgs: { @@ -86,7 +91,7 @@ # these flags when using a different linker. Don't ask me why, # though, because I don't know. All I know is it breaks otherwise. # - # [0]: https://github.com/NixOS/nixpkgs/blob/612f97239e2cc474c13c9dafa0df378058c5ad8d/pkgs/build-support/rust/lib/default.nix#L36-L39 + # [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L37-L40 ( # Nixpkgs doesn't check for x86_64 here but we do, because I # observed a failure building statically for x86_64 without @@ -110,7 +115,7 @@ # even covers the case of build scripts that need native code compiled and # run on the build platform (I think). # - # [0]: https://github.com/NixOS/nixpkgs/blob/612f97239e2cc474c13c9dafa0df378058c5ad8d/pkgs/build-support/rust/lib/default.nix#L64-L78 + # [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L57-L80 // ( let inherit (pkgs.rust.lib) envVars; @@ -148,8 +153,8 @@ "CC_${cargoEnvVarTarget}" = envVars.ccForBuild; "CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild; "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForBuild; - HOST_CC = "${pkgs.buildPackages.stdenv.cc}/bin/cc"; - HOST_CXX = "${pkgs.buildPackages.stdenv.cc}/bin/c++"; + HOST_CC = "${pkgs.pkgsBuildHost.stdenv.cc}/bin/cc"; + HOST_CXX = "${pkgs.pkgsBuildHost.stdenv.cc}/bin/c++"; } )); @@ -255,6 +260,9 @@ ] ++ (with pkgsHost; [ engage + # Needed for producing Debian packages + cargo-deb + # Needed for Complement go olm