TudbuT/conduit
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'ci-efficiency' into 'next'

Browse source 
make CI more efficient

See merge request famedly/conduit!596
This commit is contained in:
Charles Hall 2024-03-06 00:17:39 +00:00
commit 20bb214d7e
5 changed files with 75 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

98
.gitlab-ci.yml
View file

@ -15,6 +15,10 @@ before_script:
- if command -v nix > /dev/null; then echo "extra-substituters = https://nix.computer.surgery/conduit" >> /etc/nix/nix.conf; fi - if command -v nix > /dev/null; then echo "extra-substituters = https://nix.computer.surgery/conduit" >> /etc/nix/nix.conf; fi
- if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduit:ZGAf6P6LhNvnoJJ3Me3PRg7tlLSrPxcQ2RiE5LIppjo=" >> /etc/nix/nix.conf; fi - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduit:ZGAf6P6LhNvnoJJ3Me3PRg7tlLSrPxcQ2RiE5LIppjo=" >> /etc/nix/nix.conf; fi
# Add alternate binary cache
- if command -v nix > /dev/null && [ -n "$ATTIC_ENDPOINT" ]; then echo "extra-substituters = $ATTIC_ENDPOINT" >> /etc/nix/nix.conf; fi
- if command -v nix > /dev/null && [ -n "$ATTIC_PUBLIC_KEY" ]; then echo "extra-trusted-public-keys = $ATTIC_PUBLIC_KEY" >> /etc/nix/nix.conf; fi
# Add crane binary cache # Add crane binary cache
- if command -v nix > /dev/null; then echo "extra-substituters = https://crane.cachix.org" >> /etc/nix/nix.conf; fi - if command -v nix > /dev/null; then echo "extra-substituters = https://crane.cachix.org" >> /etc/nix/nix.conf; fi
- if command -v nix > /dev/null; then echo "extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk=" >> /etc/nix/nix.conf; fi - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk=" >> /etc/nix/nix.conf; fi
@ -34,8 +38,11 @@ before_script:
ci: ci:
stage: ci stage: ci
image: nixos/nix:2.19.2 image: nixos/nix:2.20.4
script: script:
# Cache the inputs required for the devShell
- ./bin/nix-build-and-cache .#devShells.x86_64-linux.default.inputDerivation
- direnv exec . engage - direnv exec . engage
cache: cache:
key: nix key: nix
@ -43,90 +50,42 @@ ci:
- target - target
- .gitlab-ci.d - .gitlab-ci.d
static:x86_64-unknown-linux-musl: artifacts:
stage: artifacts stage: artifacts
image: nixos/nix:2.19.2 image: nixos/nix:2.20.4
script: script:
# Push artifacts and build requirements to binary cache
- ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl - ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl
- cp result/bin/conduit x86_64-unknown-linux-musl
# Make the output less difficult to find - mkdir -p target/release
- cp result/bin/conduit conduit - cp result/bin/conduit target/release
artifacts: - direnv exec . cargo deb --no-build
paths: - mv target/debian/*.deb x86_64-unknown-linux-musl.deb
- conduit
static:aarch64-unknown-linux-musl:
stage: artifacts
image: nixos/nix:2.19.2
script:
# Push artifacts and build requirements to binary cache
- ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl
# Make the output less difficult to find
- cp result/bin/conduit conduit
artifacts:
paths:
- conduit
# Note that although we have an `oci-image-x86_64-unknown-linux-musl` output,
# we don't build it because it would be largely redundant to this one since it's
# all containerized anyway.
oci-image:x86_64-unknown-linux-gnu:
stage: artifacts
image: nixos/nix:2.19.2
script:
# Push artifacts and build requirements to binary cache
#
# Since the OCI image package is based on the binary package, this has the # Since the OCI image package is based on the binary package, this has the
# fun side effect of uploading the normal binary too. Conduit users who are # fun side effect of uploading the normal binary too. Conduit users who are
# deploying with Nix can leverage this fact by adding our binary cache to # deploying with Nix can leverage this fact by adding our binary cache to
# their systems. # their systems.
#
# Note that although we have an `oci-image-x86_64-unknown-linux-musl`
# output, we don't build it because it would be largely redundant to this
# one since it's all containerized anyway.
- ./bin/nix-build-and-cache .#oci-image - ./bin/nix-build-and-cache .#oci-image
# Make the output less difficult to find
- cp result oci-image-amd64.tar.gz - cp result oci-image-amd64.tar.gz
artifacts:
paths:
- oci-image-amd64.tar.gz
oci-image:aarch64-unknown-linux-musl: - ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl
stage: artifacts - cp result/bin/conduit aarch64-unknown-linux-musl
needs:
# Wait for the static binary job to finish before starting so we don't have
# to build that twice for no reason
- static:aarch64-unknown-linux-musl
image: nixos/nix:2.19.2
script:
# Push artifacts and build requirements to binary cache
- ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl - ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl
# Make the output less difficult to find
- cp result oci-image-arm64v8.tar.gz - cp result oci-image-arm64v8.tar.gz
artifacts: artifacts:
paths: paths:
- x86_64-unknown-linux-musl
- aarch64-unknown-linux-musl
- x86_64-unknown-linux-musl.deb
- oci-image-amd64.tar.gz
- oci-image-arm64v8.tar.gz - oci-image-arm64v8.tar.gz
debian:x86_64-unknown-linux-gnu:
stage: artifacts
# See also `rust-toolchain.toml`
image: rust:1.75.0
script:
- apt-get update && apt-get install -y --no-install-recommends libclang-dev
- cargo install cargo-deb
- cargo deb
# Make the output less difficult to find
- mv target/debian/*.deb conduit.deb
artifacts:
paths:
- conduit.deb
cache:
key: debian
paths:
- target
- .gitlab-ci.d
.push-oci-image: .push-oci-image:
stage: publish stage: publish
image: docker:25.0.0 image: docker:25.0.0
@ -158,8 +117,7 @@ debian:x86_64-unknown-linux-gnu:
docker manifest push $IMAGE_NAME:latest docker manifest push $IMAGE_NAME:latest
fi fi
dependencies: dependencies:
- oci-image:x86_64-unknown-linux-gnu - artifacts
- oci-image:aarch64-unknown-linux-musl
only: only:
- next - next
- master - master
@ -177,4 +135,4 @@ oci-image:push-dockerhub:
variables: variables:
IMAGE_NAME: matrixconduit/matrix-conduit IMAGE_NAME: matrixconduit/matrix-conduit
before_script: before_script:
- docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PASSWORD - docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PASSWORD

10
DEPLOY.md
View file

@ -34,11 +34,11 @@ If you use a system with an older glibc version (e.g. RHEL8), you might need to
| Target | Type | Download | | Target | Type | Download |
|-|-|-| |-|-|-|
| `x86_64-unknown-linux-gnu` | Dynamically linked Debian package | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/conduit.deb?job=debian:x86_64-unknown-linux-gnu) | | `x86_64-unknown-linux-musl` | Statically linked Debian package | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/x86_64-unknown-linux-musl.deb?job=artifacts) |
| `x86_64-unknown-linux-musl` | Statically linked binary | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/conduit?job=static:x86_64-unknown-linux-musl) | | `x86_64-unknown-linux-musl` | Statically linked binary | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/x86_64-unknown-linux-musl?job=artifacts) |
| `aarch64-unknown-linux-musl` | Statically linked binary | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/conduit?job=static:aarch64-unknown-linux-musl) | | `aarch64-unknown-linux-musl` | Statically linked binary | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/aarch64-unknown-linux-musl?job=artifacts) |
| `x86_64-unknown-linux-musl` | OCI image | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/oci-image-amd64.tar.gz?job=oci-image:x86_64-unknown-linux-musl) | | `x86_64-unknown-linux-gnu` | OCI image | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/oci-image-amd64.tar.gz?job=artifacts) |
| `aarch64-unknown-linux-musl` | OCI image | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/oci-image-arm64v8.tar.gz?job=oci-image:aarch64-unknown-linux-musl) | | `aarch64-unknown-linux-musl` | OCI image | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/oci-image-arm64v8.tar.gz?job=artifacts) |
```bash ```bash
$ sudo wget -O /usr/local/bin/matrix-conduit <url> $ sudo wget -O /usr/local/bin/matrix-conduit <url>

23
bin/nix-build-and-cache
View file

@ -9,23 +9,18 @@ INSTALLABLE="$1"
nix build "$@" nix build "$@"
if [ ! -z ${ATTIC_TOKEN+x} ]; then if [ ! -z ${ATTIC_TOKEN+x} ]; then
nix run --inputs-from . attic -- \
nix run --inputs-from . attic -- login \ login \
conduit \ conduit \
https://nix.computer.surgery/conduit \ "${ATTIC_ENDPOINT:-https://nix.computer.surgery/conduit}" \
"$ATTIC_TOKEN" "$ATTIC_TOKEN"
push_args=( # Push the target installable and its build dependencies
# Attic and its build dependencies nix run --inputs-from . attic -- \
"$(nix path-info --inputs-from . attic)" push \
"$(nix path-info --inputs-from . attic --derivation)" conduit \
"$(nix path-info "$INSTALLABLE" --derivation)" \
# The target installable and its build dependencies
"$(nix path-info "$INSTALLABLE" --derivation)"
"$(nix path-info "$INSTALLABLE")" "$(nix path-info "$INSTALLABLE")"
)
nix run --inputs-from . attic -- push conduit "${push_args[@]}"
else else
echo "\$ATTIC_TOKEN is unset, skipping uploading to the binary cache" echo "\$ATTIC_TOKEN is unset, skipping uploading to the binary cache"
fi fi

38
flake.lock generated
View file

@ -9,11 +9,11 @@
"nixpkgs-stable": "nixpkgs-stable" "nixpkgs-stable": "nixpkgs-stable"
}, },
"locked": { "locked": {
"lastModified": 1705617092, "lastModified": 1707922053,
"narHash": "sha256-n9PK4O4X4S1JkwpkMuYm1wHZYJzRqif8g3RuVIPD+rY=", "narHash": "sha256-wSZjK+rOXn+UQiP1NbdNn5/UW6UcBxjvlqr2wh++MbM=",
"owner": "zhaofengli", "owner": "zhaofengli",
"repo": "attic", "repo": "attic",
"rev": "fbe252a5c21febbe920c025560cbd63b20e24f3b", "rev": "6eabc3f02fae3683bffab483e614bebfcd476b21",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -51,17 +51,17 @@
] ]
}, },
"locked": { "locked": {
"lastModified": 1706473964, "lastModified": 1707685877,
"narHash": "sha256-Fq6xleee/TsX6NbtoRuI96bBuDHMU57PrcK9z1QEKbk=", "narHash": "sha256-XoXRS+5whotelr1rHiZle5t5hDg9kpguS5yk8c8qzOc=",
"owner": "ipetkov", "owner": "ipetkov",
"repo": "crane", "repo": "crane",
"rev": "c798790eabec3e3da48190ae3698ac227aab770c", "rev": "2c653e4478476a52c6aa3ac0495e4dea7449ea0e",
"type": "github" "type": "github"
}, },
"original": { "original": {
"owner": "ipetkov", "owner": "ipetkov",
"ref": "master",
"repo": "crane", "repo": "crane",
"rev": "2c653e4478476a52c6aa3ac0495e4dea7449ea0e",
"type": "github" "type": "github"
} }
}, },
@ -73,11 +73,11 @@
"rust-analyzer-src": "rust-analyzer-src" "rust-analyzer-src": "rust-analyzer-src"
}, },
"locked": { "locked": {
"lastModified": 1705559032, "lastModified": 1709619709,
"narHash": "sha256-Cb+Jd1+Gz4Wi+8elPnUIHnqQmE1qjDRZ+PsJaPaAffY=", "narHash": "sha256-l6EPVJfwfelWST7qWQeP6t/TDK3HHv5uUB1b2vw4mOQ=",
"owner": "nix-community", "owner": "nix-community",
"repo": "fenix", "repo": "fenix",
"rev": "e132ea0eb0c799a2109a91688e499d7bf4962801", "rev": "c8943ea9e98d41325ff57d4ec14736d330b321b2",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -138,11 +138,11 @@
"systems": "systems" "systems": "systems"
}, },
"locked": { "locked": {
"lastModified": 1705309234, "lastModified": 1709126324,
"narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", "narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=",
"owner": "numtide", "owner": "numtide",
"repo": "flake-utils", "repo": "flake-utils",
"rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", "rev": "d465f4819400de7c8d874d50b982301f28a84605",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -200,11 +200,11 @@
}, },
"nixpkgs_2": { "nixpkgs_2": {
"locked": { "locked": {
"lastModified": 1705496572, "lastModified": 1709479366,
"narHash": "sha256-rPIe9G5EBLXdBdn9ilGc0nq082lzQd0xGGe092R/5QE=", "narHash": "sha256-n6F0n8UV6lnTZbYPl1A9q1BS0p4hduAv1mGAP17CVd0=",
"owner": "NixOS", "owner": "NixOS",
"repo": "nixpkgs", "repo": "nixpkgs",
"rev": "842d9d80cfd4560648c785f8a4e6f3b096790e19", "rev": "b8697e57f10292a6165a20f03d2f42920dfaf973",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -228,11 +228,11 @@
"rust-analyzer-src": { "rust-analyzer-src": {
"flake": false, "flake": false,
"locked": { "locked": {
"lastModified": 1705523001, "lastModified": 1709571018,
"narHash": "sha256-TWq5vJ6m+9HGSDMsQAmz1TMegMi79R3TTyKjnPWsQp8=", "narHash": "sha256-ISFrxHxE0J5g7lDAscbK88hwaT5uewvWoma9TlFmRzM=",
"owner": "rust-lang", "owner": "rust-lang",
"repo": "rust-analyzer", "repo": "rust-analyzer",
"rev": "9d9b34354d2f13e33568c9c55b226dd014a146a0", "rev": "9f14343f9ee24f53f17492c5f9b653427e2ad15e",
"type": "github" "type": "github"
}, },
"original": { "original": {

20
flake.nix
View file

@ -13,7 +13,12 @@
inputs.nixpkgs.follows = "nixpkgs"; inputs.nixpkgs.follows = "nixpkgs";
}; };
crane = { crane = {
url = "github:ipetkov/crane?ref=master"; # Pin latest crane that's not affected by the following bugs:
#
# * <https://github.com/ipetkov/crane/issues/527#issuecomment-1978079140>
# * <https://github.com/toml-rs/toml/issues/691>
# * <https://github.com/toml-rs/toml/issues/267>
url = "github:ipetkov/crane?rev=2c653e4478476a52c6aa3ac0495e4dea7449ea0e";
inputs.nixpkgs.follows = "nixpkgs"; inputs.nixpkgs.follows = "nixpkgs";
}; };
attic.url = "github:zhaofengli/attic?ref=main"; attic.url = "github:zhaofengli/attic?ref=main";
@ -50,7 +55,7 @@
# bindgen needs the build platform's libclang. Apparently due to # bindgen needs the build platform's libclang. Apparently due to
# "splicing weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't # "splicing weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't
# quite do the right thing here. # quite do the right thing here.
pkgs.buildPackages.rustPlatform.bindgenHook pkgs.pkgsBuildHost.rustPlatform.bindgenHook
]; ];
env = pkgs: { env = pkgs: {
@ -78,7 +83,7 @@
# these flags when using a different linker. Don't ask me why, # these flags when using a different linker. Don't ask me why,
# though, because I don't know. All I know is it breaks otherwise. # though, because I don't know. All I know is it breaks otherwise.
# #
# [0]: https://github.com/NixOS/nixpkgs/blob/612f97239e2cc474c13c9dafa0df378058c5ad8d/pkgs/build-support/rust/lib/default.nix#L36-L39 # [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L37-L40
( (
# Nixpkgs doesn't check for x86_64 here but we do, because I # Nixpkgs doesn't check for x86_64 here but we do, because I
# observed a failure building statically for x86_64 without # observed a failure building statically for x86_64 without
@ -102,7 +107,7 @@
# even covers the case of build scripts that need native code compiled and # even covers the case of build scripts that need native code compiled and
# run on the build platform (I think). # run on the build platform (I think).
# #
# [0]: https://github.com/NixOS/nixpkgs/blob/612f97239e2cc474c13c9dafa0df378058c5ad8d/pkgs/build-support/rust/lib/default.nix#L64-L78 # [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L57-L80
// ( // (
let let
inherit (pkgs.rust.lib) envVars; inherit (pkgs.rust.lib) envVars;
@ -140,8 +145,8 @@
"CC_${cargoEnvVarTarget}" = envVars.ccForBuild; "CC_${cargoEnvVarTarget}" = envVars.ccForBuild;
"CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild; "CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild;
"CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForBuild; "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForBuild;
HOST_CC = "${pkgs.buildPackages.stdenv.cc}/bin/cc"; HOST_CC = "${pkgs.pkgsBuildHost.stdenv.cc}/bin/cc";
HOST_CXX = "${pkgs.buildPackages.stdenv.cc}/bin/c++"; HOST_CXX = "${pkgs.pkgsBuildHost.stdenv.cc}/bin/c++";
} }
)); ));
@ -247,6 +252,9 @@
] ++ (with pkgsHost; [ ] ++ (with pkgsHost; [
engage engage
# Needed for producing Debian packages
cargo-deb
# Needed for Complement # Needed for Complement
go go
olm olm