diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4f4aa695..0e56598c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,10 @@ before_script: - if command -v nix > /dev/null; then echo "extra-substituters = https://nix.computer.surgery/conduit" >> /etc/nix/nix.conf; fi - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduit:ZGAf6P6LhNvnoJJ3Me3PRg7tlLSrPxcQ2RiE5LIppjo=" >> /etc/nix/nix.conf; fi + # Add alternate binary cache + - if command -v nix > /dev/null && [ -n "$ATTIC_ENDPOINT" ]; then echo "extra-substituters = $ATTIC_ENDPOINT" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null && [ -n "$ATTIC_PUBLIC_KEY" ]; then echo "extra-trusted-public-keys = $ATTIC_PUBLIC_KEY" >> /etc/nix/nix.conf; fi + # Add crane binary cache - if command -v nix > /dev/null; then echo "extra-substituters = https://crane.cachix.org" >> /etc/nix/nix.conf; fi - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk=" >> /etc/nix/nix.conf; fi @@ -34,8 +38,11 @@ before_script: ci: stage: ci - image: nixos/nix:2.19.2 + image: nixos/nix:2.20.4 script: + # Cache the inputs required for the devShell + - ./bin/nix-build-and-cache .#devShells.x86_64-linux.default.inputDerivation + - direnv exec . engage cache: key: nix @@ -43,90 +50,42 @@ ci: - target - .gitlab-ci.d -static:x86_64-unknown-linux-musl: +artifacts: stage: artifacts - image: nixos/nix:2.19.2 + image: nixos/nix:2.20.4 script: - # Push artifacts and build requirements to binary cache - ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl + - cp result/bin/conduit x86_64-unknown-linux-musl - # Make the output less difficult to find - - cp result/bin/conduit conduit - artifacts: - paths: - - conduit + - mkdir -p target/release + - cp result/bin/conduit target/release + - direnv exec . cargo deb --no-build + - mv target/debian/*.deb x86_64-unknown-linux-musl.deb -static:aarch64-unknown-linux-musl: - stage: artifacts - image: nixos/nix:2.19.2 - script: - # Push artifacts and build requirements to binary cache - - ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl - - # Make the output less difficult to find - - cp result/bin/conduit conduit - artifacts: - paths: - - conduit - -# Note that although we have an `oci-image-x86_64-unknown-linux-musl` output, -# we don't build it because it would be largely redundant to this one since it's -# all containerized anyway. -oci-image:x86_64-unknown-linux-gnu: - stage: artifacts - image: nixos/nix:2.19.2 - script: - # Push artifacts and build requirements to binary cache - # # Since the OCI image package is based on the binary package, this has the # fun side effect of uploading the normal binary too. Conduit users who are # deploying with Nix can leverage this fact by adding our binary cache to # their systems. + # + # Note that although we have an `oci-image-x86_64-unknown-linux-musl` + # output, we don't build it because it would be largely redundant to this + # one since it's all containerized anyway. - ./bin/nix-build-and-cache .#oci-image - - # Make the output less difficult to find - cp result oci-image-amd64.tar.gz - artifacts: - paths: - - oci-image-amd64.tar.gz -oci-image:aarch64-unknown-linux-musl: - stage: artifacts - needs: - # Wait for the static binary job to finish before starting so we don't have - # to build that twice for no reason - - static:aarch64-unknown-linux-musl - image: nixos/nix:2.19.2 - script: - # Push artifacts and build requirements to binary cache + - ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl + - cp result/bin/conduit aarch64-unknown-linux-musl + - ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl - - # Make the output less difficult to find - cp result oci-image-arm64v8.tar.gz artifacts: paths: + - x86_64-unknown-linux-musl + - aarch64-unknown-linux-musl + - x86_64-unknown-linux-musl.deb + - oci-image-amd64.tar.gz - oci-image-arm64v8.tar.gz -debian:x86_64-unknown-linux-gnu: - stage: artifacts - # See also `rust-toolchain.toml` - image: rust:1.75.0 - script: - - apt-get update && apt-get install -y --no-install-recommends libclang-dev - - cargo install cargo-deb - - cargo deb - - # Make the output less difficult to find - - mv target/debian/*.deb conduit.deb - artifacts: - paths: - - conduit.deb - cache: - key: debian - paths: - - target - - .gitlab-ci.d - .push-oci-image: stage: publish image: docker:25.0.0 @@ -158,8 +117,7 @@ debian:x86_64-unknown-linux-gnu: docker manifest push $IMAGE_NAME:latest fi dependencies: - - oci-image:x86_64-unknown-linux-gnu - - oci-image:aarch64-unknown-linux-musl + - artifacts only: - next - master @@ -177,4 +135,4 @@ oci-image:push-dockerhub: variables: IMAGE_NAME: matrixconduit/matrix-conduit before_script: - - docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PASSWORD \ No newline at end of file + - docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PASSWORD diff --git a/DEPLOY.md b/DEPLOY.md index dcc41bdc..dd279688 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -34,11 +34,11 @@ If you use a system with an older glibc version (e.g. RHEL8), you might need to | Target | Type | Download | |-|-|-| -| `x86_64-unknown-linux-gnu` | Dynamically linked Debian package | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/conduit.deb?job=debian:x86_64-unknown-linux-gnu) | -| `x86_64-unknown-linux-musl` | Statically linked binary | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/conduit?job=static:x86_64-unknown-linux-musl) | -| `aarch64-unknown-linux-musl` | Statically linked binary | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/conduit?job=static:aarch64-unknown-linux-musl) | -| `x86_64-unknown-linux-musl` | OCI image | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/oci-image-amd64.tar.gz?job=oci-image:x86_64-unknown-linux-musl) | -| `aarch64-unknown-linux-musl` | OCI image | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/oci-image-arm64v8.tar.gz?job=oci-image:aarch64-unknown-linux-musl) | +| `x86_64-unknown-linux-musl` | Statically linked Debian package | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/x86_64-unknown-linux-musl.deb?job=artifacts) | +| `x86_64-unknown-linux-musl` | Statically linked binary | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/x86_64-unknown-linux-musl?job=artifacts) | +| `aarch64-unknown-linux-musl` | Statically linked binary | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/aarch64-unknown-linux-musl?job=artifacts) | +| `x86_64-unknown-linux-gnu` | OCI image | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/oci-image-amd64.tar.gz?job=artifacts) | +| `aarch64-unknown-linux-musl` | OCI image | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/oci-image-arm64v8.tar.gz?job=artifacts) | ```bash $ sudo wget -O /usr/local/bin/matrix-conduit diff --git a/bin/nix-build-and-cache b/bin/nix-build-and-cache index b37ebd85..350e1717 100755 --- a/bin/nix-build-and-cache +++ b/bin/nix-build-and-cache @@ -9,23 +9,18 @@ INSTALLABLE="$1" nix build "$@" if [ ! -z ${ATTIC_TOKEN+x} ]; then - -nix run --inputs-from . attic -- login \ - conduit \ - https://nix.computer.surgery/conduit \ + nix run --inputs-from . attic -- \ + login \ + conduit \ + "${ATTIC_ENDPOINT:-https://nix.computer.surgery/conduit}" \ "$ATTIC_TOKEN" - push_args=( - # Attic and its build dependencies - "$(nix path-info --inputs-from . attic)" - "$(nix path-info --inputs-from . attic --derivation)" - - # The target installable and its build dependencies - "$(nix path-info "$INSTALLABLE" --derivation)" + # Push the target installable and its build dependencies + nix run --inputs-from . attic -- \ + push \ + conduit \ + "$(nix path-info "$INSTALLABLE" --derivation)" \ "$(nix path-info "$INSTALLABLE")" - ) - - nix run --inputs-from . attic -- push conduit "${push_args[@]}" else echo "\$ATTIC_TOKEN is unset, skipping uploading to the binary cache" fi diff --git a/flake.lock b/flake.lock index 1c97e5c6..1c2142f2 100644 --- a/flake.lock +++ b/flake.lock @@ -9,11 +9,11 @@ "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1705617092, - "narHash": "sha256-n9PK4O4X4S1JkwpkMuYm1wHZYJzRqif8g3RuVIPD+rY=", + "lastModified": 1707922053, + "narHash": "sha256-wSZjK+rOXn+UQiP1NbdNn5/UW6UcBxjvlqr2wh++MbM=", "owner": "zhaofengli", "repo": "attic", - "rev": "fbe252a5c21febbe920c025560cbd63b20e24f3b", + "rev": "6eabc3f02fae3683bffab483e614bebfcd476b21", "type": "github" }, "original": { @@ -51,17 +51,17 @@ ] }, "locked": { - "lastModified": 1706473964, - "narHash": "sha256-Fq6xleee/TsX6NbtoRuI96bBuDHMU57PrcK9z1QEKbk=", + "lastModified": 1707685877, + "narHash": "sha256-XoXRS+5whotelr1rHiZle5t5hDg9kpguS5yk8c8qzOc=", "owner": "ipetkov", "repo": "crane", - "rev": "c798790eabec3e3da48190ae3698ac227aab770c", + "rev": "2c653e4478476a52c6aa3ac0495e4dea7449ea0e", "type": "github" }, "original": { "owner": "ipetkov", - "ref": "master", "repo": "crane", + "rev": "2c653e4478476a52c6aa3ac0495e4dea7449ea0e", "type": "github" } }, @@ -73,11 +73,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1705559032, - "narHash": "sha256-Cb+Jd1+Gz4Wi+8elPnUIHnqQmE1qjDRZ+PsJaPaAffY=", + "lastModified": 1709619709, + "narHash": "sha256-l6EPVJfwfelWST7qWQeP6t/TDK3HHv5uUB1b2vw4mOQ=", "owner": "nix-community", "repo": "fenix", - "rev": "e132ea0eb0c799a2109a91688e499d7bf4962801", + "rev": "c8943ea9e98d41325ff57d4ec14736d330b321b2", "type": "github" }, "original": { @@ -138,11 +138,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1705309234, - "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", + "lastModified": 1709126324, + "narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=", "owner": "numtide", "repo": "flake-utils", - "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", + "rev": "d465f4819400de7c8d874d50b982301f28a84605", "type": "github" }, "original": { @@ -200,11 +200,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1705496572, - "narHash": "sha256-rPIe9G5EBLXdBdn9ilGc0nq082lzQd0xGGe092R/5QE=", + "lastModified": 1709479366, + "narHash": "sha256-n6F0n8UV6lnTZbYPl1A9q1BS0p4hduAv1mGAP17CVd0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "842d9d80cfd4560648c785f8a4e6f3b096790e19", + "rev": "b8697e57f10292a6165a20f03d2f42920dfaf973", "type": "github" }, "original": { @@ -228,11 +228,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1705523001, - "narHash": "sha256-TWq5vJ6m+9HGSDMsQAmz1TMegMi79R3TTyKjnPWsQp8=", + "lastModified": 1709571018, + "narHash": "sha256-ISFrxHxE0J5g7lDAscbK88hwaT5uewvWoma9TlFmRzM=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "9d9b34354d2f13e33568c9c55b226dd014a146a0", + "rev": "9f14343f9ee24f53f17492c5f9b653427e2ad15e", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 6ca4c72d..cc95aecb 100644 --- a/flake.nix +++ b/flake.nix @@ -13,7 +13,12 @@ inputs.nixpkgs.follows = "nixpkgs"; }; crane = { - url = "github:ipetkov/crane?ref=master"; + # Pin latest crane that's not affected by the following bugs: + # + # * + # * + # * + url = "github:ipetkov/crane?rev=2c653e4478476a52c6aa3ac0495e4dea7449ea0e"; inputs.nixpkgs.follows = "nixpkgs"; }; attic.url = "github:zhaofengli/attic?ref=main"; @@ -50,7 +55,7 @@ # bindgen needs the build platform's libclang. Apparently due to # "splicing weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't # quite do the right thing here. - pkgs.buildPackages.rustPlatform.bindgenHook + pkgs.pkgsBuildHost.rustPlatform.bindgenHook ]; env = pkgs: { @@ -78,7 +83,7 @@ # these flags when using a different linker. Don't ask me why, # though, because I don't know. All I know is it breaks otherwise. # - # [0]: https://github.com/NixOS/nixpkgs/blob/612f97239e2cc474c13c9dafa0df378058c5ad8d/pkgs/build-support/rust/lib/default.nix#L36-L39 + # [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L37-L40 ( # Nixpkgs doesn't check for x86_64 here but we do, because I # observed a failure building statically for x86_64 without @@ -102,7 +107,7 @@ # even covers the case of build scripts that need native code compiled and # run on the build platform (I think). # - # [0]: https://github.com/NixOS/nixpkgs/blob/612f97239e2cc474c13c9dafa0df378058c5ad8d/pkgs/build-support/rust/lib/default.nix#L64-L78 + # [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L57-L80 // ( let inherit (pkgs.rust.lib) envVars; @@ -140,8 +145,8 @@ "CC_${cargoEnvVarTarget}" = envVars.ccForBuild; "CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild; "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForBuild; - HOST_CC = "${pkgs.buildPackages.stdenv.cc}/bin/cc"; - HOST_CXX = "${pkgs.buildPackages.stdenv.cc}/bin/c++"; + HOST_CC = "${pkgs.pkgsBuildHost.stdenv.cc}/bin/cc"; + HOST_CXX = "${pkgs.pkgsBuildHost.stdenv.cc}/bin/c++"; } )); @@ -247,6 +252,9 @@ ] ++ (with pkgsHost; [ engage + # Needed for producing Debian packages + cargo-deb + # Needed for Complement go olm