make CI more efficient (github and gitlab)

squashed from https://gitlab.com/famedly/conduit/-/merge_requests/596

ported the relevant parts to GitHub Actions

Co-authored-by: strawberry <strawberry@puppygock.gay>
Signed-off-by: strawberry <strawberry@puppygock.gay>

.github/workflows/ci.yml

@@ -30,7 +30,7 @@ jobs:
         with:
           nix_path: nixpkgs=channel:nixos-unstable
 
-          # Add the `nix-community` cachix to speed up things that leverage it
+          # Add `nix-community`, Crane, upstream Conduit, conduwuit, and alternative binary caches
           extra_nix_config: |
             experimental-features = nix-command flakes
             extra-substituters = https://nix-community.cachix.org
@@ -43,6 +43,8 @@ jobs:
             extra-trusted-public-keys = conduit:Isq8FGyEC6FOXH6nD+BOeAA+bKp6X6UIbupSlGEPuOg=
             extra-substituters = https://attic.kennel.juneis.dog/conduwuit
             extra-trusted-public-keys = conduwuit:lYPVh7o1hLu1idH4Xt2QHaRa49WRGSAqzcfFd94aOTw=
+            extra-substituters = $ATTIC_ENDPOINT
+            extra-trusted-public-keys = $ATTIC_PUBLIC_KEY
 
       - name: Pop/push Magic Nix Cache
         uses: DeterminateSystems/magic-nix-cache-action@main
@@ -70,14 +72,24 @@ jobs:
       - name: Populate `/nix/store`
         run: nix develop --command true
 
-      - name: Perform continuous integration
+      - name: Allow direnv
-        run: |
+        run: direnv allow
-          direnv allow
-          direnv exec . engage
 
-      - name: Build static-x86_64-unknown-linux-musl
+      - name: Cache x86_64 inputs for devShell
+        run: |
+          ./bin/nix-build-and-cache .#devShells.x86_64-linux.default.inputDerivation
+
+
+      - name: Perform continuous integration
+        run: direnv exec . engage
+
+
+      - name: Build static-x86_64-unknown-linux-musl and Create static deb-x86_64-unknown-linux-musl
         run: |
           ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl
+          mkdir -p target/release
+          cp -v -f result/bin/conduit target/release
+          direnv exec . cargo deb --no-build
 
       - name: Upload artifact static-x86_64-unknown-linux-musl
         uses: actions/upload-artifact@v4
@@ -86,6 +98,14 @@ jobs:
           path: result/bin/conduit
           if-no-files-found: error
 
+      - name: Upload artifact deb-x86_64-unknown-linux-musl
+        uses: actions/upload-artifact@v4
+        with:
+          name: x86_64-unknown-linux-musl.deb
+          path: target/debian/*.deb
+          if-no-files-found: error
+
+
       - name: Build static-aarch64-unknown-linux-musl
         run: |
           ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl
@@ -97,10 +117,11 @@ jobs:
           path: result/bin/conduit
           if-no-files-found: error
 
+
       - name: Build oci-image-x86_64-unknown-linux-gnu
         run: |
           ./bin/nix-build-and-cache .#oci-image
-          cp -f result oci-image-amd64.tar.gz
+          cp -v -f result oci-image-amd64.tar.gz
 
       - name: Upload artifact oci-image-x86_64-unknown-linux-gnu
         uses: actions/upload-artifact@v4
@@ -111,10 +132,11 @@ jobs:
           # don't compress again
           compression-level: 0
 
+
       - name: Build oci-image-aarch64-unknown-linux-musl
         run: |
           ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl
-          cp -f result oci-image-arm64v8.tar.gz
+          cp -v -f result oci-image-arm64v8.tar.gz
 
       - name: Upload artifact oci-image-aarch64-unknown-linux-musl
         uses: actions/upload-artifact@v4
@@ -125,18 +147,6 @@ jobs:
           # don't compress again
           compression-level: 0
 
-      - name: Build deb-x86_64-unknown-linux-gnu
-        run: |
-          sudo apt-get update && sudo apt-get install -y --no-install-recommends libclang-dev
-          cargo install cargo-deb
-          cargo deb
-
-      - name: Upload artifact deb-x86_64-unknown-linux-gnu
-        uses: actions/upload-artifact@v4
-        with:
-          name: deb-x86_64-unknown-linux-gnu
-          path: target/debian/*.deb
-          if-no-files-found: error
 
       - name: Extract metadata for Dockerhub
         env:
@@ -156,6 +166,7 @@ jobs:
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
+
       - name: Login to Dockerhub
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v3
@@ -173,6 +184,7 @@ jobs:
           username: girlbossceo
           password: ${{ secrets.GITHUB_TOKEN }}
 
+
       - name: Publish to Dockerhub
         if: github.event_name != 'pull_request'
         env:

.gitlab-ci.yml

@@ -6,22 +6,36 @@ stages:
 variables:
   # Makes some things print in color
   TERM: ansi
-  NIX_CONFIG: |
-    experimental-features = nix-command flake
-    extra-substituters = https://nix.computer.surgery/conduit
-    extra-trusted-public-keys = conduit:ZGAf6P6LhNvnoJJ3Me3PRg7tlLSrPxcQ2RiE5LIppjo=
-    extra-substituters = https://crane.cachix.org
-    extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk=
-    extra-substituters = https://nix-community.cachix.org
-    extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=
-    extra-substituters = https://attic.kennel.juneis.dog/conduit
-    extra-trusted-public-keys = conduit:Isq8FGyEC6FOXH6nD+BOeAA+bKp6X6UIbupSlGEPuOg=
-    extra-substituters = https://attic.kennel.juneis.dog/conduwuit
-    extra-trusted-public-keys = conduwuit:lYPVh7o1hLu1idH4Xt2QHaRa49WRGSAqzcfFd94aOTw=
 
 before_script:
+  # Enable nix-command and flakes
+  - if command -v nix > /dev/null; then echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf; fi
+
+  # Add conduwuit binary cache
+  - if command -v nix > /dev/null; then echo "extra-substituters = https://attic.kennel.juneis.dog/conduwuit" >> /etc/nix/nix.conf; fi
+  - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduwuit:lYPVh7o1hLu1idH4Xt2QHaRa49WRGSAqzcfFd94aOTw=" >> /etc/nix/nix.conf; fi
+
+  - if command -v nix > /dev/null; then echo "extra-substituters = https://attic.kennel.juneis.dog/conduit" >> /etc/nix/nix.conf; fi
+  - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduit:Isq8FGyEC6FOXH6nD+BOeAA+bKp6X6UIbupSlGEPuOg=" >> /etc/nix/nix.conf; fi
+
+  # Add upstream Conduit binary cache
+  - if command -v nix > /dev/null; then echo "extra-substituters = https://nix.computer.surgery/conduit" >> /etc/nix/nix.conf; fi
+  - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduit:ZGAf6P6LhNvnoJJ3Me3PRg7tlLSrPxcQ2RiE5LIppjo=" >> /etc/nix/nix.conf; fi
+  
+  # Add alternate binary cache
+  - if command -v nix > /dev/null && [ -n "$ATTIC_ENDPOINT" ]; then echo "extra-substituters = $ATTIC_ENDPOINT" >> /etc/nix/nix.conf; fi
+  - if command -v nix > /dev/null && [ -n "$ATTIC_PUBLIC_KEY" ]; then echo "extra-trusted-public-keys = $ATTIC_PUBLIC_KEY" >> /etc/nix/nix.conf; fi
+
+  # Add crane binary cache
+  - if command -v nix > /dev/null; then echo "extra-substituters = https://crane.cachix.org" >> /etc/nix/nix.conf; fi
+  - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk=" >> /etc/nix/nix.conf; fi
+
+  # Add nix-community binary cache
+  - if command -v nix > /dev/null; then echo "extra-substituters = https://nix-community.cachix.org" >> /etc/nix/nix.conf; fi
+  - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" >> /etc/nix/nix.conf; fi
+
   # Install direnv and nix-direnv
-  - if command -v nix > /dev/null; then nix-env -iA nixpkgs.direnv nixpkgs.nix-direnv nixpkgs.engage; fi
+  - if command -v nix > /dev/null; then nix-env -iA nixpkgs.direnv nixpkgs.nix-direnv; fi
 
   # Allow .envrc
   - if command -v nix > /dev/null; then direnv allow; fi
@@ -33,6 +47,9 @@ ci:
   stage: ci
   image: nixos/nix:2.20.4
   script:
+    # Cache the inputs required for the devShell
+    - ./bin/nix-build-and-cache .#devShells.x86_64-linux.default.inputDerivation
+
     - direnv exec . engage
   cache:
     key: nix
@@ -40,100 +57,50 @@ ci:
       - target
       - .gitlab-ci.d
 
-static:x86_64-unknown-linux-musl:
+artifacts:
   stage: artifacts
   image: nixos/nix:2.20.4
   script:
-    # Push artifacts and build requirements to binary cache
     - ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl
+    - cp result/bin/conduit x86_64-unknown-linux-musl
 
-    # Make the output less difficult to find
+    - mkdir -p target/release
-    - cp result/bin/conduit conduit
+    - cp result/bin/conduit target/release
-  artifacts:
+    - direnv exec . cargo deb --no-build
-    paths:
+    - mv target/debian/*.deb x86_64-unknown-linux-musl.deb
-      - conduit
 
-static:aarch64-unknown-linux-musl:
-  stage: artifacts
-  image: nixos/nix:2.20.4
-  script:
-    # Push artifacts and build requirements to binary cache
-    - ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl
-
-    # Make the output less difficult to find
-    - cp result/bin/conduit conduit
-  artifacts:
-    paths:
-      - conduit
-
-# Note that although we have an `oci-image-x86_64-unknown-linux-musl` output,
-# we don't build it because it would be largely redundant to this one since it's
-# all containerized anyway.
-oci-image:x86_64-unknown-linux-gnu:
-  stage: artifacts
-  image: nixos/nix:2.20.4
-  script:
-    # Push artifacts and build requirements to binary cache
-    #
     # Since the OCI image package is based on the binary package, this has the
     # fun side effect of uploading the normal binary too. Conduit users who are
     # deploying with Nix can leverage this fact by adding our binary cache to
     # their systems.
+    #
+    # Note that although we have an `oci-image-x86_64-unknown-linux-musl`
+    # output, we don't build it because it would be largely redundant to this
+    # one since it's all containerized anyway.
     - ./bin/nix-build-and-cache .#oci-image
-
-    # Make the output less difficult to find
     - cp result oci-image-amd64.tar.gz
-  artifacts:
-    paths:
-      - oci-image-amd64.tar.gz
 
-oci-image:aarch64-unknown-linux-musl:
+    - ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl
-  stage: artifacts
+    - cp result/bin/conduit aarch64-unknown-linux-musl
-  needs:
+
-    # Wait for the static binary job to finish before starting so we don't have
-    # to build that twice for no reason
-    - static:aarch64-unknown-linux-musl
-  image: nixos/nix:2.20.4
-  script:
-    # Push artifacts and build requirements to binary cache
     - ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl
-
-    # Make the output less difficult to find
     - cp result oci-image-arm64v8.tar.gz
   artifacts:
     paths:
+      - x86_64-unknown-linux-musl
+      - aarch64-unknown-linux-musl
+      - x86_64-unknown-linux-musl.deb
+      - oci-image-amd64.tar.gz
       - oci-image-arm64v8.tar.gz
 
-debian:x86_64-unknown-linux-gnu:
+.push-oci-image:
-  stage: artifacts
-  # See also `rust-toolchain.toml`
-  image: rust:1.75.0
-  script:
-    - cargo install cargo-deb
-    - cargo deb
-
-    # Make the output less difficult to find
-    - mv target/debian/*.deb conduit.deb
-  artifacts:
-    paths:
-      - conduit.deb
-  cache:
-    key: debian
-    paths:
-      - target
-      - .gitlab-ci.d
-
-docker-publish:
   stage: publish
   image: docker:25.0.3
   services:
     - docker:25.0.3-dind
   variables:
-    IMAGE_NAME: $CI_REGISTRY_IMAGE/conduwuit
     IMAGE_SUFFIX_AMD64: amd64
     IMAGE_SUFFIX_ARM64V8: arm64v8
-  before_script:
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
   script:
     - docker load -i oci-image-amd64.tar.gz
     - IMAGE_ID_AMD64=$(docker images -q conduit:main)
@@ -157,8 +124,14 @@ docker-publish:
         docker manifest push $IMAGE_NAME:latest
       fi
   dependencies:
-    - oci-image:x86_64-unknown-linux-gnu
+    - artifacts
-    - oci-image:aarch64-unknown-linux-musl
   only:
     - main
     - tags
+
+oci-image:push-gitlab:
+  extends: .push-oci-image
+  variables:
+    IMAGE_NAME: $CI_REGISTRY_IMAGE/conduwuit
+  before_script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY

bin/nix-build-and-cache

@@ -6,45 +6,36 @@ set -euo pipefail
 INSTALLABLE="$1"
 
 # Build the installable and forward any other arguments too
-nix build -L "$@"
+nix build "$@"
 
 if [ ! -z "$ATTIC_TOKEN" ]; then
-
+    nix run --inputs-from . attic -- \
-nix run --inputs-from . attic -- login \
+        login \
-    conduit \
+        conduit \
-        https://attic.kennel.juneis.dog/conduit \
+        "${ATTIC_ENDPOINT:-https://attic.kennel.juneis.dog/conduit}" \
         "$ATTIC_TOKEN"
 
-    push_args=(
+    # Push the target installable and its build dependencies
-        # Attic and its build dependencies
+    nix run --inputs-from . attic -- \
-        "$(nix path-info --inputs-from . attic)"
+        push \
-        "$(nix path-info --inputs-from . attic --derivation)"
+        conduit \
-
+        "$(nix path-info "$INSTALLABLE" --derivation)" \
-        # The target installable and its build dependencies
-        "$(nix path-info "$INSTALLABLE" --derivation)"
         "$(nix path-info "$INSTALLABLE")"
-    )
 
-    nix run --inputs-from . attic -- push conduit "${push_args[@]}"
 
     # push to "conduwuit" too
-nix run --inputs-from . attic -- login \
+    nix run --inputs-from . attic -- \
-    conduwuit \
+        login \
-        https://attic.kennel.juneis.dog/conduwuit \
+        conduwuit \
+        "${ATTIC_ENDPOINT:-https://attic.kennel.juneis.dog/conduwuit}" \
         "$ATTIC_TOKEN"
 
-    push_args=(
+    # Push the target installable and its build dependencies
-        # Attic and its build dependencies
+    nix run --inputs-from . attic -- \
-        "$(nix path-info --inputs-from . attic)"
+        push \
-        "$(nix path-info --inputs-from . attic --derivation)"
+        conduwuit \
-
+        "$(nix path-info "$INSTALLABLE" --derivation)" \
-        # The target installable and its build dependencies
-        "$(nix path-info "$INSTALLABLE" --derivation)"
         "$(nix path-info "$INSTALLABLE")"
-    )
-
-    nix run --inputs-from . attic -- push conduwuit "${push_args[@]}"
-
 else
     echo "\$ATTIC_TOKEN is unset, skipping uploading to the binary cache"
 fi

flake.lock

@@ -60,8 +60,8 @@
       },
       "original": {
         "owner": "ipetkov",
-        "ref": "master",
         "repo": "crane",
+        "rev": "2c653e4478476a52c6aa3ac0495e4dea7449ea0e",
         "type": "github"
       }
     },
@@ -73,11 +73,11 @@
         "rust-analyzer-src": "rust-analyzer-src"
       },
       "locked": {
-        "lastModified": 1707891749,
+        "lastModified": 1709619709,
-        "narHash": "sha256-SeikNYElHgv8uVMbiA9/pU3Cce7ssIsiM8CnEiwd1Nc=",
+        "narHash": "sha256-l6EPVJfwfelWST7qWQeP6t/TDK3HHv5uUB1b2vw4mOQ=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "3115aab064ef38cccd792c45429af8df43d6d277",
+        "rev": "c8943ea9e98d41325ff57d4ec14736d330b321b2",
         "type": "github"
       },
       "original": {
@@ -138,11 +138,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1705309234,
+        "lastModified": 1709126324,
-        "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=",
+        "narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26",
+        "rev": "d465f4819400de7c8d874d50b982301f28a84605",
         "type": "github"
       },
       "original": {
@@ -200,11 +200,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1707689078,
+        "lastModified": 1709479366,
-        "narHash": "sha256-UUGmRa84ZJHpGZ1WZEBEUOzaPOWG8LZ0yPg1pdDF/yM=",
+        "narHash": "sha256-n6F0n8UV6lnTZbYPl1A9q1BS0p4hduAv1mGAP17CVd0=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "f9d39fb9aff0efee4a3d5f4a6d7c17701d38a1d8",
+        "rev": "b8697e57f10292a6165a20f03d2f42920dfaf973",
         "type": "github"
       },
       "original": {
@@ -228,11 +228,11 @@
     "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1707849817,
+        "lastModified": 1709571018,
-        "narHash": "sha256-If6T0MDErp3/z7DBlpG4bV46IPP+7BWSlgTI88cmbw0=",
+        "narHash": "sha256-ISFrxHxE0J5g7lDAscbK88hwaT5uewvWoma9TlFmRzM=",
         "owner": "rust-lang",
         "repo": "rust-analyzer",
-        "rev": "a02a219773629686bd8ff123ca1aa995fa50d976",
+        "rev": "9f14343f9ee24f53f17492c5f9b653427e2ad15e",
         "type": "github"
       },
       "original": {

flake.nix

@@ -13,7 +13,12 @@
       inputs.nixpkgs.follows = "nixpkgs";
     };
     crane = {
-      url = "github:ipetkov/crane?ref=master";
+      # Pin latest crane that's not affected by the following bugs:
+      #
+      # * <https://github.com/ipetkov/crane/issues/527#issuecomment-1978079140>
+      # * <https://github.com/toml-rs/toml/issues/691>
+      # * <https://github.com/toml-rs/toml/issues/267>
+      url = "github:ipetkov/crane?rev=2c653e4478476a52c6aa3ac0495e4dea7449ea0e";
       inputs.nixpkgs.follows = "nixpkgs";
     };
     attic.url = "github:zhaofengli/attic?ref=main";
@@ -58,7 +63,7 @@
         # bindgen needs the build platform's libclang. Apparently due to
         # "splicing weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't
         # quite do the right thing here.
-        pkgs.buildPackages.rustPlatform.bindgenHook
+        pkgs.pkgsBuildHost.rustPlatform.bindgenHook
       ];
 
       env = pkgs: {
@@ -86,7 +91,7 @@
               # these flags when using a different linker. Don't ask me why,
               # though, because I don't know. All I know is it breaks otherwise.
               #
-              # [0]: https://github.com/NixOS/nixpkgs/blob/612f97239e2cc474c13c9dafa0df378058c5ad8d/pkgs/build-support/rust/lib/default.nix#L36-L39
+              # [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L37-L40
               (
                 # Nixpkgs doesn't check for x86_64 here but we do, because I
                 # observed a failure building statically for x86_64 without
@@ -110,7 +115,7 @@
       # even covers the case of build scripts that need native code compiled and
       # run on the build platform (I think).
       #
-      # [0]: https://github.com/NixOS/nixpkgs/blob/612f97239e2cc474c13c9dafa0df378058c5ad8d/pkgs/build-support/rust/lib/default.nix#L64-L78
+      # [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L57-L80
       // (
         let
           inherit (pkgs.rust.lib) envVars;
@@ -148,8 +153,8 @@
           "CC_${cargoEnvVarTarget}" = envVars.ccForBuild;
           "CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild;
           "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForBuild;
-          HOST_CC = "${pkgs.buildPackages.stdenv.cc}/bin/cc";
+          HOST_CC = "${pkgs.pkgsBuildHost.stdenv.cc}/bin/cc";
-          HOST_CXX = "${pkgs.buildPackages.stdenv.cc}/bin/c++";
+          HOST_CXX = "${pkgs.pkgsBuildHost.stdenv.cc}/bin/c++";
         }
       ));
 
@@ -255,6 +260,9 @@
         ] ++ (with pkgsHost; [
           engage
 
+          # Needed for producing Debian packages
+          cargo-deb
+
           # Needed for Complement
           go
           olm