From cad16b92687dc1aef6797fc3edb4fed0bfaa7df7 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 1 May 2024 23:30:49 -0400 Subject: [PATCH] ci, nix: build and cache all packages and CI dependencies from https://or.computer.surgery/charles/matrix/-/commit/f5bd9bc45e5e5eaf76cff31f1c259ed3f39fb88a with changes for GitHub CI and misc Co-authored-by: Charles Hall Signed-off-by: strawberry --- .github/workflows/ci.yml | 14 ++-- .github/workflows/documentation.yml | 6 +- .gitlab-ci.yml | 14 ++-- bin/complement | 4 +- bin/nix-build-and-cache | 110 ++++++++++++++++++---------- 5 files changed, 93 insertions(+), 55 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 481eec67..5e711620 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,7 +53,7 @@ jobs: - name: Enable Cachix binary cache run: | - nix-env -iA cachix -f https://cachix.org/api/v1/install + nix profile install nixpkgs#cachix cachix use crane cachix use nix-community @@ -78,7 +78,7 @@ jobs: - name: Prepare build environment run: | echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" - nix-env -f "" -iA direnv -iA nix-direnv + nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv direnv allow nix develop --command true @@ -144,9 +144,9 @@ jobs: - name: Install Nix uses: DeterminateSystems/nix-installer-action@main - - name: Enable Cachix binary cache + - name: Install and enable Cachix binary cache run: | - nix-env -iA cachix -f https://cachix.org/api/v1/install + nix profile install nixpkgs#cachix cachix use crane cachix use nix-community @@ -171,13 +171,13 @@ jobs: - name: Prepare build environment run: | echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" - nix-env -f "" -iA direnv -iA nix-direnv + nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv direnv allow nix develop --command true - name: Build static ${{ matrix.target }} run: | - bin/nix-build-and-cache .#static-${{ matrix.target }} + bin/nix-build-and-cache just .#static-${{ matrix.target }} mkdir -p target/release cp -v -f result/bin/conduit target/release/ direnv exec . cargo deb --no-build --no-strip --output target/debian/${{ matrix.target }}.deb @@ -199,7 +199,7 @@ jobs: - name: Build OCI image ${{ matrix.target }} run: | - bin/nix-build-and-cache .#oci-image-${{ matrix.target }} + bin/nix-build-and-cache just .#oci-image-${{ matrix.target }} cp -v -f result oci-image-${{ matrix.target }}.tar.gz - name: Upload OCI image ${{ matrix.target }} diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index cdeeef42..fe6c9626 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -88,13 +88,13 @@ jobs: - name: Allow direnv run: direnv allow - - name: Cache x86_64 inputs for devShell + - name: Cache CI dependencies run: | - ./bin/nix-build-and-cache .#devShells.x86_64-linux.default.inputDerivation + ./bin/nix-build-and-cache ci - name: Build documentation (book) run: | - ./bin/nix-build-and-cache .#book + ./bin/nix-build-and-cache just .#book cp -r --dereference result public - name: Upload generated documentation (book) as normal artifact uses: actions/upload-artifact@v4 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f4f62ef7..4a3b1c25 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,8 +56,8 @@ ci: stage: ci image: nixos/nix:2.22.0 script: - # Cache the inputs required for the devShell - - ./bin/nix-build-and-cache .#devShells.x86_64-linux.default.inputDerivation + # Cache CI dependencies + - ./bin/nix-build-and-cache ci - direnv exec . engage cache: @@ -81,7 +81,7 @@ artifacts: stage: artifacts image: nixos/nix:2.22.0 script: - - ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl + - ./bin/nix-build-and-cache just .#static-x86_64-unknown-linux-musl - cp result/bin/conduit x86_64-unknown-linux-musl - mkdir -p target/release @@ -97,16 +97,16 @@ artifacts: # Note that although we have an `oci-image-x86_64-unknown-linux-musl` # output, we don't build it because it would be largely redundant to this # one since it's all containerized anyway. - - ./bin/nix-build-and-cache .#oci-image + - ./bin/nix-build-and-cache just .#oci-image - cp result oci-image-amd64.tar.gz - - ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl + - ./bin/nix-build-and-cache just .#static-aarch64-unknown-linux-musl - cp result/bin/conduit aarch64-unknown-linux-musl - - ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl + - ./bin/nix-build-and-cache just .#oci-image-aarch64-unknown-linux-musl - cp result oci-image-arm64v8.tar.gz - - ./bin/nix-build-and-cache .#book + - ./bin/nix-build-and-cache just .#book # We can't just copy the symlink, we need to dereference it https://gitlab.com/gitlab-org/gitlab/-/issues/19746 - cp -r --dereference result public artifacts: diff --git a/bin/complement b/bin/complement index c0a72fb1..b42905e2 100755 --- a/bin/complement +++ b/bin/complement @@ -17,7 +17,9 @@ RESULTS_FILE="$3" OCI_IMAGE="complement-conduit:dev" -pushd "$(git rev-parse --show-toplevel)" > /dev/null +toplevel="$(git rev-parse --show-toplevel)" + +pushd "$toplevel" > /dev/null # uses nix-output-monitor (nom) if available if command -v nom &> /dev/null; then nom build .#complement diff --git a/bin/nix-build-and-cache b/bin/nix-build-and-cache index 2f29bb17..140fd561 100755 --- a/bin/nix-build-and-cache +++ b/bin/nix-build-and-cache @@ -2,45 +2,81 @@ set -eo pipefail -# The first argument must be the desired installable -INSTALLABLE="$1" +toplevel="$(git rev-parse --show-toplevel)" -# Build the installable and forward any other arguments too -# uses nix-output-monitor (nom) if available -if command -v nom &> /dev/null; then - nom build "$@" -else - nix build -L "$@" -fi +# Build just the single installable and forward any other arguments too +just() { + # uses nix-output-monitor (nom) if available + if command -v nom &> /dev/null; then + nom build "$@" + else + nix build -L "$@" + fi -if [ ! -z "$ATTIC_TOKEN" ]; then - nix run --inputs-from . attic -- \ - login \ - conduit \ - "${ATTIC_ENDPOINT:-https://attic.kennel.juneis.dog/conduit}" \ - "$ATTIC_TOKEN" + if [ ! -z "$ATTIC_TOKEN" ]; then + # historical "conduit" store for compatibility purposes, same as conduwuit + nix run --inputs-from "$toplevel" attic -- \ + login \ + conduit \ + "${ATTIC_ENDPOINT:-https://attic.kennel.juneis.dog/conduit}" \ + "$ATTIC_TOKEN" - # Push the target installable and its build dependencies - nix run --inputs-from . attic -- \ - push \ - conduit \ - "$(nix path-info "$INSTALLABLE" --derivation)" \ - "$(nix path-info "$INSTALLABLE")" + readarray -t outputs < <(nix path-info "$@") + readarray -t derivations < <(nix path-info "$@" --derivation) + + # Push the target installable and its build dependencies + nix run --inputs-from "$toplevel" attic -- \ + push \ + conduit \ + "${outputs[@]}" \ + "${derivations[@]}" + + # main "conduwuit" store + nix run --inputs-from "$toplevel" attic -- \ + login \ + conduwuit \ + "${ATTIC_ENDPOINT:-https://attic.kennel.juneis.dog/conduwuit}" \ + "$ATTIC_TOKEN" + + # Push the target installable and its build dependencies + nix run --inputs-from "$toplevel" attic -- \ + push \ + conduwuit \ + "${outputs[@]}" \ + "${derivations[@]}" + else + echo "\$ATTIC_TOKEN is unset, skipping uploading to the binary cache" + fi + +} + +# Build and cache things needed for CI +ci() { + cache=( + --inputs-from "$toplevel" + + # Keep sorted + "$toplevel#devShells.x86_64-linux.default.inputDerivation" + attic#default + nixpkgs#direnv + nixpkgs#jq + nixpkgs#nix-direnv + ) + + just "${cache[@]}" +} + +# Build and cache *all* the package outputs from the flake.nix +packages() { + declare -a cache="($( + nix flake show --json 2> /dev/null | + nix run --inputs-from "$toplevel" nixpkgs#jq -- \ + -r \ + '.packages."x86_64-linux" | keys | map("'"$toplevel"'#" + .) | @sh' + ))" + + just "${cache[@]}" +} - # push to "conduwuit" too - nix run --inputs-from . attic -- \ - login \ - conduwuit \ - "${ATTIC_ENDPOINT:-https://attic.kennel.juneis.dog/conduwuit}" \ - "$ATTIC_TOKEN" - - # Push the target installable and its build dependencies - nix run --inputs-from . attic -- \ - push \ - conduwuit \ - "$(nix path-info "$INSTALLABLE" --derivation)" \ - "$(nix path-info "$INSTALLABLE")" -else - echo "\$ATTIC_TOKEN is unset, skipping uploading to the binary cache" -fi +eval "$@"