ci, nix: build and cache all packages and CI dependencies

from f5bd9bc45e
with changes for GitHub CI and misc

Co-authored-by: Charles Hall <charles@computer.surgery>
Signed-off-by: strawberry <strawberry@puppygock.gay>

.github/workflows/ci.yml

@@ -53,7 +53,7 @@ jobs:
 
             - name: Enable Cachix binary cache
               run: |
-                  nix-env -iA cachix -f https://cachix.org/api/v1/install
+                  nix profile install nixpkgs#cachix
                   cachix use crane
                   cachix use nix-community
 
@@ -78,7 +78,7 @@ jobs:
             - name: Prepare build environment
               run: |
                   echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc"
-                  nix-env -f "<nixpkgs>" -iA direnv -iA nix-direnv
+                  nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv
                   direnv allow
                   nix develop --command true
 
@@ -144,9 +144,9 @@ jobs:
             - name: Install Nix
               uses: DeterminateSystems/nix-installer-action@main
 
-            - name: Enable Cachix binary cache
+            - name: Install and enable Cachix binary cache
               run: |
-                  nix-env -iA cachix -f https://cachix.org/api/v1/install
+                  nix profile install nixpkgs#cachix
                   cachix use crane
                   cachix use nix-community
 
@@ -171,13 +171,13 @@ jobs:
             - name: Prepare build environment
               run: |
                   echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc"
-                  nix-env -f "<nixpkgs>" -iA direnv -iA nix-direnv
+                  nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv
                   direnv allow
                   nix develop --command true
 
             - name: Build static ${{ matrix.target }}
               run: |
-                  bin/nix-build-and-cache .#static-${{ matrix.target }}
+                  bin/nix-build-and-cache just .#static-${{ matrix.target }}
                   mkdir -p target/release
                   cp -v -f result/bin/conduit target/release/
                   direnv exec . cargo deb --no-build --no-strip --output target/debian/${{ matrix.target }}.deb
@@ -199,7 +199,7 @@ jobs:
 
             - name: Build OCI image ${{ matrix.target }}
               run: |
-                  bin/nix-build-and-cache .#oci-image-${{ matrix.target }}
+                  bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}
                   cp -v -f result oci-image-${{ matrix.target }}.tar.gz
 
             - name: Upload OCI image ${{ matrix.target }}

.github/workflows/documentation.yml

@@ -88,13 +88,13 @@ jobs:
       - name: Allow direnv
         run: direnv allow
 
-      - name: Cache x86_64 inputs for devShell
+      - name: Cache CI dependencies
         run: |
-          ./bin/nix-build-and-cache .#devShells.x86_64-linux.default.inputDerivation
+          ./bin/nix-build-and-cache ci
 
       - name: Build documentation (book)
         run: |
-          ./bin/nix-build-and-cache .#book
+          ./bin/nix-build-and-cache just .#book
           cp -r --dereference result public
       - name: Upload generated documentation (book) as normal artifact
         uses: actions/upload-artifact@v4

.gitlab-ci.yml

@@ -56,8 +56,8 @@ ci:
   stage: ci
   image: nixos/nix:2.22.0
   script:
-    # Cache the inputs required for the devShell
-    - ./bin/nix-build-and-cache .#devShells.x86_64-linux.default.inputDerivation
+    # Cache CI dependencies
+    - ./bin/nix-build-and-cache ci
 
     - direnv exec . engage
   cache:
@@ -81,7 +81,7 @@ artifacts:
   stage: artifacts
   image: nixos/nix:2.22.0
   script:
-    - ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl
+    - ./bin/nix-build-and-cache just .#static-x86_64-unknown-linux-musl
     - cp result/bin/conduit x86_64-unknown-linux-musl
 
     - mkdir -p target/release
@@ -97,16 +97,16 @@ artifacts:
     # Note that although we have an `oci-image-x86_64-unknown-linux-musl`
     # output, we don't build it because it would be largely redundant to this
     # one since it's all containerized anyway.
-    - ./bin/nix-build-and-cache .#oci-image
+    - ./bin/nix-build-and-cache just .#oci-image
     - cp result oci-image-amd64.tar.gz
 
-    - ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl
+    - ./bin/nix-build-and-cache just .#static-aarch64-unknown-linux-musl
     - cp result/bin/conduit aarch64-unknown-linux-musl
 
-    - ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl
+    - ./bin/nix-build-and-cache just .#oci-image-aarch64-unknown-linux-musl
     - cp result oci-image-arm64v8.tar.gz
 
-    - ./bin/nix-build-and-cache .#book
+    - ./bin/nix-build-and-cache just .#book
     # We can't just copy the symlink, we need to dereference it https://gitlab.com/gitlab-org/gitlab/-/issues/19746
     - cp -r --dereference result public
   artifacts:

bin/complement

@@ -17,7 +17,9 @@ RESULTS_FILE="$3"
 
 OCI_IMAGE="complement-conduit:dev"
 
-pushd "$(git rev-parse --show-toplevel)" > /dev/null
+toplevel="$(git rev-parse --show-toplevel)"
+
+pushd "$toplevel" > /dev/null
 # uses nix-output-monitor (nom) if available
 if command -v nom &> /dev/null; then
     nom build .#complement

bin/nix-build-and-cache

@@ -2,45 +2,81 @@
 
 set -eo pipefail
 
-# The first argument must be the desired installable
-INSTALLABLE="$1"
+toplevel="$(git rev-parse --show-toplevel)"
 
-# Build the installable and forward any other arguments too
-# uses nix-output-monitor (nom) if available
-if command -v nom &> /dev/null; then
-    nom build "$@"
-else
-    nix build -L "$@"
-fi
+# Build just the single installable and forward any other arguments too
+just() {
+    # uses nix-output-monitor (nom) if available
+    if command -v nom &> /dev/null; then
+        nom build "$@"
+    else
+        nix build -L "$@"
+    fi
 
-if [ ! -z "$ATTIC_TOKEN" ]; then
-    nix run --inputs-from . attic -- \
-        login \
-        conduit \
-        "${ATTIC_ENDPOINT:-https://attic.kennel.juneis.dog/conduit}" \
-        "$ATTIC_TOKEN"
+    if [ ! -z "$ATTIC_TOKEN" ]; then
+        # historical "conduit" store for compatibility purposes, same as conduwuit
+        nix run --inputs-from "$toplevel" attic -- \
+            login \
+            conduit \
+            "${ATTIC_ENDPOINT:-https://attic.kennel.juneis.dog/conduit}" \
+            "$ATTIC_TOKEN"
 
-    # Push the target installable and its build dependencies
-    nix run --inputs-from . attic -- \
-        push \
-        conduit \
-        "$(nix path-info "$INSTALLABLE" --derivation)" \
-        "$(nix path-info "$INSTALLABLE")"
+        readarray -t outputs < <(nix path-info "$@")
+        readarray -t derivations < <(nix path-info "$@" --derivation)
+
+        # Push the target installable and its build dependencies
+        nix run --inputs-from "$toplevel" attic -- \
+            push \
+            conduit \
+            "${outputs[@]}" \
+            "${derivations[@]}"
+
+        # main "conduwuit" store
+        nix run --inputs-from "$toplevel" attic -- \
+            login \
+            conduwuit \
+            "${ATTIC_ENDPOINT:-https://attic.kennel.juneis.dog/conduwuit}" \
+            "$ATTIC_TOKEN"
+
+        # Push the target installable and its build dependencies
+        nix run --inputs-from "$toplevel" attic -- \
+            push \
+            conduwuit \
+            "${outputs[@]}" \
+            "${derivations[@]}"
+    else
+        echo "\$ATTIC_TOKEN is unset, skipping uploading to the binary cache"
+    fi
+
+}
+
+# Build and cache things needed for CI
+ci() {
+    cache=(
+        --inputs-from "$toplevel"
+
+        # Keep sorted
+        "$toplevel#devShells.x86_64-linux.default.inputDerivation"
+        attic#default
+        nixpkgs#direnv
+        nixpkgs#jq
+        nixpkgs#nix-direnv
+    )
+
+    just "${cache[@]}"
+}
+
+# Build and cache *all* the package outputs from the flake.nix
+packages() {
+    declare -a cache="($(
+        nix flake show --json 2> /dev/null |
+            nix run --inputs-from "$toplevel" nixpkgs#jq -- \
+            -r \
+            '.packages."x86_64-linux" | keys | map("'"$toplevel"'#" + .) | @sh'
+    ))"
+
+    just "${cache[@]}"
+}
 
 
-    # push to "conduwuit" too
-    nix run --inputs-from . attic -- \
-        login \
-        conduwuit \
-        "${ATTIC_ENDPOINT:-https://attic.kennel.juneis.dog/conduwuit}" \
-        "$ATTIC_TOKEN"
-
-    # Push the target installable and its build dependencies
-    nix run --inputs-from . attic -- \
-        push \
-        conduwuit \
-        "$(nix path-info "$INSTALLABLE" --derivation)" \
-        "$(nix path-info "$INSTALLABLE")"
-else
-    echo "\$ATTIC_TOKEN is unset, skipping uploading to the binary cache"
-fi
+eval "$@"