TudbuT/conduit
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'ci-efficiency' into 'next'

Browse source 
make CI more efficient

See merge request famedly/conduit!596
This commit is contained in:
Charles Hall 2024-03-06 00:17:39 +00:00
commit 20bb214d7e
5 changed files with 75 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

96
.gitlab-ci.yml
View file

@ -15,6 +15,10 @@ before_script:
- if command -v nix > /dev/null; then echo "extra-substituters = https://nix.computer.surgery/conduit" >> /etc/nix/nix.conf; fi
- if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduit:ZGAf6P6LhNvnoJJ3Me3PRg7tlLSrPxcQ2RiE5LIppjo=" >> /etc/nix/nix.conf; fi
# Add alternate binary cache
- if command -v nix > /dev/null && [ -n "$ATTIC_ENDPOINT" ]; then echo "extra-substituters = $ATTIC_ENDPOINT" >> /etc/nix/nix.conf; fi
- if command -v nix > /dev/null && [ -n "$ATTIC_PUBLIC_KEY" ]; then echo "extra-trusted-public-keys = $ATTIC_PUBLIC_KEY" >> /etc/nix/nix.conf; fi
# Add crane binary cache
- if command -v nix > /dev/null; then echo "extra-substituters = https://crane.cachix.org" >> /etc/nix/nix.conf; fi
- if command -v nix > /dev/null; then echo "extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk=" >> /etc/nix/nix.conf; fi
@ -34,8 +38,11 @@ before_script:
ci:
stage: ci
image: nixos/nix:2.19.2
image: nixos/nix:2.20.4
script:
# Cache the inputs required for the devShell
- ./bin/nix-build-and-cache .#devShells.x86_64-linux.default.inputDerivation
- direnv exec . engage
cache:
key: nix
@ -43,90 +50,42 @@ ci:
- target
- .gitlab-ci.d
static:x86_64-unknown-linux-musl:
artifacts:
stage: artifacts
image: nixos/nix:2.19.2
image: nixos/nix:2.20.4
script:
# Push artifacts and build requirements to binary cache
- ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl
- cp result/bin/conduit x86_64-unknown-linux-musl
# Make the output less difficult to find
- cp result/bin/conduit conduit
artifacts:
paths:
- conduit
- mkdir -p target/release
- cp result/bin/conduit target/release
- direnv exec . cargo deb --no-build
- mv target/debian/*.deb x86_64-unknown-linux-musl.deb
static:aarch64-unknown-linux-musl:
stage: artifacts
image: nixos/nix:2.19.2
script:
# Push artifacts and build requirements to binary cache
- ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl
# Make the output less difficult to find
- cp result/bin/conduit conduit
artifacts:
paths:
- conduit
# Note that although we have an `oci-image-x86_64-unknown-linux-musl` output,
# we don't build it because it would be largely redundant to this one since it's
# all containerized anyway.
oci-image:x86_64-unknown-linux-gnu:
stage: artifacts
image: nixos/nix:2.19.2
script:
# Push artifacts and build requirements to binary cache
#
# Since the OCI image package is based on the binary package, this has the
# fun side effect of uploading the normal binary too. Conduit users who are
# deploying with Nix can leverage this fact by adding our binary cache to
# their systems.
#
# Note that although we have an `oci-image-x86_64-unknown-linux-musl`
# output, we don't build it because it would be largely redundant to this
# one since it's all containerized anyway.
- ./bin/nix-build-and-cache .#oci-image
# Make the output less difficult to find
- cp result oci-image-amd64.tar.gz
artifacts:
paths:
- oci-image-amd64.tar.gz
oci-image:aarch64-unknown-linux-musl:
stage: artifacts
needs:
# Wait for the static binary job to finish before starting so we don't have
# to build that twice for no reason
- static:aarch64-unknown-linux-musl
image: nixos/nix:2.19.2
script:
# Push artifacts and build requirements to binary cache
- ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl
- cp result/bin/conduit aarch64-unknown-linux-musl
- ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl
# Make the output less difficult to find
- cp result oci-image-arm64v8.tar.gz
artifacts:
paths:
- x86_64-unknown-linux-musl
- aarch64-unknown-linux-musl
- x86_64-unknown-linux-musl.deb
- oci-image-amd64.tar.gz
- oci-image-arm64v8.tar.gz
debian:x86_64-unknown-linux-gnu:
stage: artifacts
# See also `rust-toolchain.toml`
image: rust:1.75.0
script:
- apt-get update && apt-get install -y --no-install-recommends libclang-dev
- cargo install cargo-deb
- cargo deb
# Make the output less difficult to find
- mv target/debian/*.deb conduit.deb
artifacts:
paths:
- conduit.deb
cache:
key: debian
paths:
- target
- .gitlab-ci.d
.push-oci-image:
stage: publish
image: docker:25.0.0
@ -158,8 +117,7 @@ debian:x86_64-unknown-linux-gnu:
docker manifest push $IMAGE_NAME:latest
fi
dependencies:
- oci-image:x86_64-unknown-linux-gnu
- oci-image:aarch64-unknown-linux-musl
- artifacts
only:
- next
- master

10
DEPLOY.md
View file

@ -34,11 +34,11 @@ If you use a system with an older glibc version (e.g. RHEL8), you might need to
| Target | Type | Download |
|-|-|-|
| `x86_64-unknown-linux-gnu` | Dynamically linked Debian package | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/conduit.deb?job=debian:x86_64-unknown-linux-gnu) |
| `x86_64-unknown-linux-musl` | Statically linked binary | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/conduit?job=static:x86_64-unknown-linux-musl) |
| `aarch64-unknown-linux-musl` | Statically linked binary | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/conduit?job=static:aarch64-unknown-linux-musl) |
| `x86_64-unknown-linux-musl` | OCI image | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/oci-image-amd64.tar.gz?job=oci-image:x86_64-unknown-linux-musl) |
| `aarch64-unknown-linux-musl` | OCI image | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/oci-image-arm64v8.tar.gz?job=oci-image:aarch64-unknown-linux-musl) |
| `x86_64-unknown-linux-musl` | Statically linked Debian package | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/x86_64-unknown-linux-musl.deb?job=artifacts) |
| `x86_64-unknown-linux-musl` | Statically linked binary | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/x86_64-unknown-linux-musl?job=artifacts) |
| `aarch64-unknown-linux-musl` | Statically linked binary | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/aarch64-unknown-linux-musl?job=artifacts) |
| `x86_64-unknown-linux-gnu` | OCI image | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/oci-image-amd64.tar.gz?job=artifacts) |
| `aarch64-unknown-linux-musl` | OCI image | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/oci-image-arm64v8.tar.gz?job=artifacts) |
```bash
$ sudo wget -O /usr/local/bin/matrix-conduit <url>

21
bin/nix-build-and-cache
View file

@ -9,23 +9,18 @@ INSTALLABLE="$1"
nix build "$@"
if [ ! -z ${ATTIC_TOKEN+x} ]; then
nix run --inputs-from . attic -- login \
nix run --inputs-from . attic -- \
login \
conduit \
https://nix.computer.surgery/conduit \
"${ATTIC_ENDPOINT:-https://nix.computer.surgery/conduit}" \
"$ATTIC_TOKEN"
push_args=(
# Attic and its build dependencies
"$(nix path-info --inputs-from . attic)"
"$(nix path-info --inputs-from . attic --derivation)"
# The target installable and its build dependencies
"$(nix path-info "$INSTALLABLE" --derivation)"
# Push the target installable and its build dependencies
nix run --inputs-from . attic -- \
push \
conduit \
"$(nix path-info "$INSTALLABLE" --derivation)" \
"$(nix path-info "$INSTALLABLE")"
)
nix run --inputs-from . attic -- push conduit "${push_args[@]}"
else
echo "\$ATTIC_TOKEN is unset, skipping uploading to the binary cache"
fi

38
flake.lock generated
View file

@ -9,11 +9,11 @@
"nixpkgs-stable": "nixpkgs-stable"
},
"locked": {
"lastModified": 1705617092,
"narHash": "sha256-n9PK4O4X4S1JkwpkMuYm1wHZYJzRqif8g3RuVIPD+rY=",
"lastModified": 1707922053,
"narHash": "sha256-wSZjK+rOXn+UQiP1NbdNn5/UW6UcBxjvlqr2wh++MbM=",
"owner": "zhaofengli",
"repo": "attic",
"rev": "fbe252a5c21febbe920c025560cbd63b20e24f3b",
"rev": "6eabc3f02fae3683bffab483e614bebfcd476b21",
"type": "github"
},
"original": {
@ -51,17 +51,17 @@
]
},
"locked": {
"lastModified": 1706473964,
"narHash": "sha256-Fq6xleee/TsX6NbtoRuI96bBuDHMU57PrcK9z1QEKbk=",
"lastModified": 1707685877,
"narHash": "sha256-XoXRS+5whotelr1rHiZle5t5hDg9kpguS5yk8c8qzOc=",
"owner": "ipetkov",
"repo": "crane",
"rev": "c798790eabec3e3da48190ae3698ac227aab770c",
"rev": "2c653e4478476a52c6aa3ac0495e4dea7449ea0e",
"type": "github"
},
"original": {
"owner": "ipetkov",
"ref": "master",
"repo": "crane",
"rev": "2c653e4478476a52c6aa3ac0495e4dea7449ea0e",
"type": "github"
}
},
@ -73,11 +73,11 @@
"rust-analyzer-src": "rust-analyzer-src"
},
"locked": {
"lastModified": 1705559032,
"narHash": "sha256-Cb+Jd1+Gz4Wi+8elPnUIHnqQmE1qjDRZ+PsJaPaAffY=",
"lastModified": 1709619709,
"narHash": "sha256-l6EPVJfwfelWST7qWQeP6t/TDK3HHv5uUB1b2vw4mOQ=",
"owner": "nix-community",
"repo": "fenix",
"rev": "e132ea0eb0c799a2109a91688e499d7bf4962801",
"rev": "c8943ea9e98d41325ff57d4ec14736d330b321b2",
"type": "github"
},
"original": {
@ -138,11 +138,11 @@
"systems": "systems"
},
"locked": {
"lastModified": 1705309234,
"narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=",
"lastModified": 1709126324,
"narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26",
"rev": "d465f4819400de7c8d874d50b982301f28a84605",
"type": "github"
},
"original": {
@ -200,11 +200,11 @@
},
"nixpkgs_2": {
"locked": {
"lastModified": 1705496572,
"narHash": "sha256-rPIe9G5EBLXdBdn9ilGc0nq082lzQd0xGGe092R/5QE=",
"lastModified": 1709479366,
"narHash": "sha256-n6F0n8UV6lnTZbYPl1A9q1BS0p4hduAv1mGAP17CVd0=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "842d9d80cfd4560648c785f8a4e6f3b096790e19",
"rev": "b8697e57f10292a6165a20f03d2f42920dfaf973",
"type": "github"
},
"original": {
@ -228,11 +228,11 @@
"rust-analyzer-src": {
"flake": false,
"locked": {
"lastModified": 1705523001,
"narHash": "sha256-TWq5vJ6m+9HGSDMsQAmz1TMegMi79R3TTyKjnPWsQp8=",
"lastModified": 1709571018,
"narHash": "sha256-ISFrxHxE0J5g7lDAscbK88hwaT5uewvWoma9TlFmRzM=",
"owner": "rust-lang",
"repo": "rust-analyzer",
"rev": "9d9b34354d2f13e33568c9c55b226dd014a146a0",
"rev": "9f14343f9ee24f53f17492c5f9b653427e2ad15e",
"type": "github"
},
"original": {

20
flake.nix
View file

@ -13,7 +13,12 @@
inputs.nixpkgs.follows = "nixpkgs";
};
crane = {
url = "github:ipetkov/crane?ref=master";
# Pin latest crane that's not affected by the following bugs:
#
# * <https://github.com/ipetkov/crane/issues/527#issuecomment-1978079140>
# * <https://github.com/toml-rs/toml/issues/691>
# * <https://github.com/toml-rs/toml/issues/267>
url = "github:ipetkov/crane?rev=2c653e4478476a52c6aa3ac0495e4dea7449ea0e";
inputs.nixpkgs.follows = "nixpkgs";
};
attic.url = "github:zhaofengli/attic?ref=main";
@ -50,7 +55,7 @@
# bindgen needs the build platform's libclang. Apparently due to
# "splicing weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't
# quite do the right thing here.
pkgs.buildPackages.rustPlatform.bindgenHook
pkgs.pkgsBuildHost.rustPlatform.bindgenHook
];
env = pkgs: {
@ -78,7 +83,7 @@
# these flags when using a different linker. Don't ask me why,
# though, because I don't know. All I know is it breaks otherwise.
#
# [0]: https://github.com/NixOS/nixpkgs/blob/612f97239e2cc474c13c9dafa0df378058c5ad8d/pkgs/build-support/rust/lib/default.nix#L36-L39
# [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L37-L40
(
# Nixpkgs doesn't check for x86_64 here but we do, because I
# observed a failure building statically for x86_64 without
@ -102,7 +107,7 @@
# even covers the case of build scripts that need native code compiled and
# run on the build platform (I think).
#
# [0]: https://github.com/NixOS/nixpkgs/blob/612f97239e2cc474c13c9dafa0df378058c5ad8d/pkgs/build-support/rust/lib/default.nix#L64-L78
# [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L57-L80
// (
let
inherit (pkgs.rust.lib) envVars;
@ -140,8 +145,8 @@
"CC_${cargoEnvVarTarget}" = envVars.ccForBuild;
"CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild;
"CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForBuild;
HOST_CC = "${pkgs.buildPackages.stdenv.cc}/bin/cc";
HOST_CXX = "${pkgs.buildPackages.stdenv.cc}/bin/c++";
HOST_CC = "${pkgs.pkgsBuildHost.stdenv.cc}/bin/cc";
HOST_CXX = "${pkgs.pkgsBuildHost.stdenv.cc}/bin/c++";
}
));
@ -247,6 +252,9 @@
] ++ (with pkgsHost; [
engage
# Needed for producing Debian packages
cargo-deb
# Needed for Complement
go
olm