From 3ecf835b50064f604940af5c48c8d138c0ee7dd8 Mon Sep 17 00:00:00 2001 From: Matthias Ahouansou Date: Thu, 2 May 2024 09:53:53 +0100 Subject: [PATCH 1/8] ci: cache attic on artifacts job --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3aecebf6..7347931d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -45,13 +45,13 @@ before_script: # Set CARGO_HOME to a cacheable path - export CARGO_HOME="$(git rev-parse --show-toplevel)/.gitlab-ci.d/cargo" + # Cache attic client + - ./bin/nix-build-and-cache --inputs-from . attic + ci: stage: ci image: nixos/nix:2.20.4 script: - # Cache attic client - - ./bin/nix-build-and-cache --inputs-from . attic - # Cache the inputs required for the devShell - ./bin/nix-build-and-cache .#devShells.x86_64-linux.default.inputDerivation From c9ee4a920ead36d6986ae9aa24865ee116dde001 Mon Sep 17 00:00:00 2001 From: Charles Hall Date: Sat, 4 May 2024 10:03:49 +0100 Subject: [PATCH 2/8] always go through inputs This way we don't have to modify the destructuring of outputs' argument when adding or removing inputs. --- flake.nix | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/flake.nix b/flake.nix index 162eb3e0..984398db 100644 --- a/flake.nix +++ b/flake.nix @@ -19,24 +19,15 @@ attic.url = "github:zhaofengli/attic?ref=main"; }; - outputs = - { self - , nixpkgs - , flake-utils - , nix-filter - - , fenix - , crane - , ... - }: flake-utils.lib.eachDefaultSystem (system: + outputs = inputs: inputs.flake-utils.lib.eachDefaultSystem (system: let - pkgsHost = nixpkgs.legacyPackages.${system}; + pkgsHost = inputs.nixpkgs.legacyPackages.${system}; # Nix-accessible `Cargo.toml` cargoToml = builtins.fromTOML (builtins.readFile ./Cargo.toml); # The Rust toolchain to use - toolchain = fenix.packages.${system}.fromToolchainFile { + toolchain = inputs.fenix.packages.${system}.fromToolchainFile { file = ./rust-toolchain.toml; # See also `rust-toolchain.toml` @@ -44,7 +35,7 @@ }; builder = pkgs: - ((crane.mkLib pkgs).overrideToolchain toolchain).buildPackage; + ((inputs.crane.mkLib pkgs).overrideToolchain toolchain).buildPackage; nativeBuildInputs = pkgs: [ # bindgen needs the build platform's libclang. Apparently due to @@ -68,7 +59,8 @@ }); env = pkgs: { - CONDUIT_VERSION_EXTRA = self.shortRev or self.dirtyShortRev; + CONDUIT_VERSION_EXTRA = + inputs.self.shortRev or inputs.self.dirtyShortRev; ROCKSDB_INCLUDE_DIR = "${rocksdb' pkgs}/include"; ROCKSDB_LIB_DIR = "${rocksdb' pkgs}/lib"; } @@ -161,7 +153,7 @@ )); package = pkgs: builder pkgs { - src = nix-filter { + src = inputs.nix-filter { root = ./.; include = [ "src" @@ -202,17 +194,17 @@ { packages = { default = package pkgsHost; - oci-image = mkOciImage pkgsHost self.packages.${system}.default; + oci-image = mkOciImage pkgsHost inputs.self.packages.${system}.default; book = let - package = self.packages.${system}.default; + package = inputs.self.packages.${system}.default; in pkgsHost.stdenv.mkDerivation { pname = "${package.pname}-book"; version = package.version; - src = nix-filter { + src = inputs.nix-filter { root = ./.; include = [ "book.toml" @@ -241,7 +233,7 @@ let binaryName = "static-${crossSystem}"; pkgsCrossStatic = - (import nixpkgs { + (import inputs.nixpkgs { inherit system; crossSystem = { config = crossSystem; @@ -260,7 +252,7 @@ name = "oci-image-${crossSystem}"; value = mkOciImage pkgsCrossStatic - self.packages.${system}.${binaryName}; + inputs.self.packages.${system}.${binaryName}; } ] ) @@ -285,7 +277,7 @@ # # This needs to come before `toolchain` in this list, otherwise # `$PATH` will have stable rustfmt instead. - fenix.packages.${system}.latest.rustfmt + inputs.fenix.packages.${system}.latest.rustfmt toolchain ] ++ (with pkgsHost; [ From 3336d3f8125539f20b5dd4b39eefad6ff7790c0c Mon Sep 17 00:00:00 2001 From: Charles Hall Date: Tue, 30 Apr 2024 16:00:19 -0700 Subject: [PATCH 3/8] factor out nix code into new files via `makeScope` This makes the Nix code a lot easier to reason about. --- flake.nix | 347 +++++---------------- nix/pkgs/book/default.nix | 34 ++ nix/pkgs/default/cross-compilation-env.nix | 100 ++++++ nix/pkgs/default/default.nix | 64 ++++ nix/pkgs/oci-image/default.nix | 25 ++ nix/shell.nix | 55 ++++ 6 files changed, 357 insertions(+), 268 deletions(-) create mode 100644 nix/pkgs/book/default.nix create mode 100644 nix/pkgs/default/cross-compilation-env.nix create mode 100644 nix/pkgs/default/default.nix create mode 100644 nix/pkgs/oci-image/default.nix create mode 100644 nix/shell.nix diff --git a/flake.nix b/flake.nix index 984398db..09aa9e09 100644 --- a/flake.nix +++ b/flake.nix @@ -19,286 +19,97 @@ attic.url = "github:zhaofengli/attic?ref=main"; }; - outputs = inputs: inputs.flake-utils.lib.eachDefaultSystem (system: + outputs = inputs: let - pkgsHost = inputs.nixpkgs.legacyPackages.${system}; + # Keep sorted + mkScope = pkgs: pkgs.lib.makeScope pkgs.newScope (self: { + craneLib = + (inputs.crane.mkLib pkgs).overrideToolchain self.toolchain; - # Nix-accessible `Cargo.toml` - cargoToml = builtins.fromTOML (builtins.readFile ./Cargo.toml); + default = self.callPackage ./nix/pkgs/default {}; - # The Rust toolchain to use - toolchain = inputs.fenix.packages.${system}.fromToolchainFile { - file = ./rust-toolchain.toml; + inherit inputs; - # See also `rust-toolchain.toml` - sha256 = "sha256-SXRtAuO4IqNOQq+nLbrsDFbVk+3aVA8NNpSZsKlVH/8="; - }; + oci-image = self.callPackage ./nix/pkgs/oci-image {}; - builder = pkgs: - ((inputs.crane.mkLib pkgs).overrideToolchain toolchain).buildPackage; + book = self.callPackage ./nix/pkgs/book {}; - nativeBuildInputs = pkgs: [ - # bindgen needs the build platform's libclang. Apparently due to - # "splicing weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't - # quite do the right thing here. - pkgs.pkgsBuildHost.rustPlatform.bindgenHook - ]; + rocksdb = + let + version = "9.1.1"; + in + pkgs.rocksdb.overrideAttrs (old: { + inherit version; + src = pkgs.fetchFromGitHub { + owner = "facebook"; + repo = "rocksdb"; + rev = "v${version}"; + hash = "sha256-/Xf0bzNJPclH9IP80QNaABfhj4IAR5LycYET18VFCXc="; + }; + }); - rocksdb' = pkgs: - let - version = "9.1.1"; - in - pkgs.rocksdb.overrideAttrs (old: { - inherit version; - src = pkgs.fetchFromGitHub { - owner = "facebook"; - repo = "rocksdb"; - rev = "v${version}"; - hash = "sha256-/Xf0bzNJPclH9IP80QNaABfhj4IAR5LycYET18VFCXc="; - }; + shell = self.callPackage ./nix/shell.nix {}; + + # The Rust toolchain to use + toolchain = inputs + .fenix + .packages + .${pkgs.pkgsBuildHost.system} + .fromToolchainFile { + file = ./rust-toolchain.toml; + + # See also `rust-toolchain.toml` + sha256 = "sha256-SXRtAuO4IqNOQq+nLbrsDFbVk+3aVA8NNpSZsKlVH/8="; + }; }); + in + inputs.flake-utils.lib.eachDefaultSystem (system: + let + pkgs = inputs.nixpkgs.legacyPackages.${system}; + in + { + packages = { + default = (mkScope pkgs).default; + oci-image = (mkScope pkgs).oci-image; + book = (mkScope pkgs).book; + } + // + builtins.listToAttrs + (builtins.concatLists + (builtins.map + (crossSystem: + let + binaryName = "static-${crossSystem}"; + pkgsCrossStatic = + (import inputs.nixpkgs { + inherit system; + crossSystem = { + config = crossSystem; + }; + }).pkgsStatic; + in + [ + # An output for a statically-linked binary + { + name = binaryName; + value = (mkScope pkgsCrossStatic).default; + } - env = pkgs: { - CONDUIT_VERSION_EXTRA = - inputs.self.shortRev or inputs.self.dirtyShortRev; - ROCKSDB_INCLUDE_DIR = "${rocksdb' pkgs}/include"; - ROCKSDB_LIB_DIR = "${rocksdb' pkgs}/lib"; - } - // pkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isStatic { - ROCKSDB_STATIC = ""; - } - // { - CARGO_BUILD_RUSTFLAGS = let inherit (pkgs) lib stdenv; in - lib.concatStringsSep " " ([] - ++ lib.optionals - # This disables PIE for static builds, which isn't great in terms - # of security. Unfortunately, my hand is forced because nixpkgs' - # `libstdc++.a` is built without `-fPIE`, which precludes us from - # leaving PIE enabled. - stdenv.hostPlatform.isStatic - ["-C" "relocation-model=static"] - ++ lib.optionals - (stdenv.buildPlatform.config != stdenv.hostPlatform.config) - ["-l" "c"] - ++ lib.optionals - # This check has to match the one [here][0]. We only need to set - # these flags when using a different linker. Don't ask me why, - # though, because I don't know. All I know is it breaks otherwise. - # - # [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L37-L40 - ( - # Nixpkgs doesn't check for x86_64 here but we do, because I - # observed a failure building statically for x86_64 without - # including it here. Linkers are weird. - (stdenv.hostPlatform.isAarch64 || stdenv.hostPlatform.isx86_64) - && stdenv.hostPlatform.isStatic - && !stdenv.isDarwin - && !stdenv.cc.bintools.isLLVM + # An output for an OCI image based on that binary + { + name = "oci-image-${crossSystem}"; + value = (mkScope pkgsCrossStatic).oci-image; + } + ] ) [ - "-l" - "stdc++" - "-L" - "${stdenv.cc.cc.lib}/${stdenv.hostPlatform.config}/lib" - ] - ); - } - - # What follows is stolen from [here][0]. Its purpose is to properly - # configure compilers and linkers for various stages of the build, and - # even covers the case of build scripts that need native code compiled and - # run on the build platform (I think). - # - # [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L57-L80 - // ( - let - inherit (pkgs.rust.lib) envVars; - in - pkgs.lib.optionalAttrs - (pkgs.stdenv.targetPlatform.rust.rustcTarget - != pkgs.stdenv.hostPlatform.rust.rustcTarget) - ( - let - inherit (pkgs.stdenv.targetPlatform.rust) cargoEnvVarTarget; - in - { - "CC_${cargoEnvVarTarget}" = envVars.ccForTarget; - "CXX_${cargoEnvVarTarget}" = envVars.cxxForTarget; - "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = - envVars.linkerForTarget; - } - ) - // ( - let - inherit (pkgs.stdenv.hostPlatform.rust) cargoEnvVarTarget rustcTarget; - in - { - "CC_${cargoEnvVarTarget}" = envVars.ccForHost; - "CXX_${cargoEnvVarTarget}" = envVars.cxxForHost; - "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForHost; - CARGO_BUILD_TARGET = rustcTarget; - } - ) - // ( - let - inherit (pkgs.stdenv.buildPlatform.rust) cargoEnvVarTarget; - in - { - "CC_${cargoEnvVarTarget}" = envVars.ccForBuild; - "CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild; - "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForBuild; - HOST_CC = "${pkgs.pkgsBuildHost.stdenv.cc}/bin/cc"; - HOST_CXX = "${pkgs.pkgsBuildHost.stdenv.cc}/bin/c++"; - } - )); - - package = pkgs: builder pkgs { - src = inputs.nix-filter { - root = ./.; - include = [ - "src" - "Cargo.toml" - "Cargo.lock" - ]; - }; - - # This is redundant with CI - doCheck = false; - - env = env pkgs; - nativeBuildInputs = nativeBuildInputs pkgs; - - meta.mainProgram = cargoToml.package.name; - }; - - mkOciImage = pkgs: package: - pkgs.dockerTools.buildImage { - name = package.pname; - tag = "next"; - copyToRoot = [ - pkgs.dockerTools.caCertificates - ]; - config = { - # Use the `tini` init system so that signals (e.g. ctrl+c/SIGINT) - # are handled as expected - Entrypoint = [ - "${pkgs.lib.getExe' pkgs.tini "tini"}" - "--" - ]; - Cmd = [ - "${pkgs.lib.getExe package}" - ]; - }; - }; - in - { - packages = { - default = package pkgsHost; - oci-image = mkOciImage pkgsHost inputs.self.packages.${system}.default; - - book = - let - package = inputs.self.packages.${system}.default; - in - pkgsHost.stdenv.mkDerivation { - pname = "${package.pname}-book"; - version = package.version; - - src = inputs.nix-filter { - root = ./.; - include = [ - "book.toml" - "conduit-example.toml" - "README.md" - "debian/README.md" - "docs" - ]; - }; - - nativeBuildInputs = (with pkgsHost; [ - mdbook - ]); - - buildPhase = '' - mdbook build - mv public $out - ''; - }; - } - // - builtins.listToAttrs - (builtins.concatLists - (builtins.map - (crossSystem: - let - binaryName = "static-${crossSystem}"; - pkgsCrossStatic = - (import inputs.nixpkgs { - inherit system; - crossSystem = { - config = crossSystem; - }; - }).pkgsStatic; - in - [ - # An output for a statically-linked binary - { - name = binaryName; - value = package pkgsCrossStatic; - } - - # An output for an OCI image based on that binary - { - name = "oci-image-${crossSystem}"; - value = mkOciImage - pkgsCrossStatic - inputs.self.packages.${system}.${binaryName}; - } + "x86_64-unknown-linux-musl" + "aarch64-unknown-linux-musl" ] ) - [ - "x86_64-unknown-linux-musl" - "aarch64-unknown-linux-musl" - ] - ) - ); + ); - devShells.default = pkgsHost.mkShell { - env = env pkgsHost // { - # Rust Analyzer needs to be able to find the path to default crate - # sources, and it can read this environment variable to do so. The - # `rust-src` component is required in order for this to work. - RUST_SRC_PATH = "${toolchain}/lib/rustlib/src/rust/library"; - }; - - # Development tools - nativeBuildInputs = nativeBuildInputs pkgsHost ++ [ - # Always use nightly rustfmt because most of its options are unstable - # - # This needs to come before `toolchain` in this list, otherwise - # `$PATH` will have stable rustfmt instead. - inputs.fenix.packages.${system}.latest.rustfmt - - toolchain - ] ++ (with pkgsHost; [ - engage - - # Needed for producing Debian packages - cargo-deb - - # Needed for Complement - go - olm - - # Needed for our script for Complement - jq - - # Needed for finding broken markdown links - lychee - - # Useful for editing the book locally - mdbook - ]); - }; - }); + devShells.default = (mkScope pkgs).shell; + } + ); } diff --git a/nix/pkgs/book/default.nix b/nix/pkgs/book/default.nix new file mode 100644 index 00000000..cc0464d6 --- /dev/null +++ b/nix/pkgs/book/default.nix @@ -0,0 +1,34 @@ +# Keep sorted +{ default +, inputs +, mdbook +, stdenv +}: + +stdenv.mkDerivation { + pname = "${default.pname}-book"; + version = default.version; + + + src = let filter = inputs.nix-filter.lib; in filter { + root = inputs.self; + + # Keep sorted + include = [ + "book.toml" + "conduit-example.toml" + "debian/README.md" + "docs" + "README.md" + ]; + }; + + nativeBuildInputs = [ + mdbook + ]; + + buildPhase = '' + mdbook build + mv public $out + ''; +} diff --git a/nix/pkgs/default/cross-compilation-env.nix b/nix/pkgs/default/cross-compilation-env.nix new file mode 100644 index 00000000..fac85e02 --- /dev/null +++ b/nix/pkgs/default/cross-compilation-env.nix @@ -0,0 +1,100 @@ +{ lib +, pkgsBuildHost +, rust +, stdenv +}: + +lib.optionalAttrs stdenv.hostPlatform.isStatic { + ROCKSDB_STATIC = ""; +} +// +{ + CARGO_BUILD_RUSTFLAGS = + lib.concatStringsSep + " " + ([] + # This disables PIE for static builds, which isn't great in terms of + # security. Unfortunately, my hand is forced because nixpkgs' + # `libstdc++.a` is built without `-fPIE`, which precludes us from + # leaving PIE enabled. + ++ lib.optionals + stdenv.hostPlatform.isStatic + [ "-C" "relocation-model=static" ] + ++ lib.optionals + (stdenv.buildPlatform.config != stdenv.hostPlatform.config) + [ "-l" "c" ] + ++ lib.optionals + # This check has to match the one [here][0]. We only need to set + # these flags when using a different linker. Don't ask me why, though, + # because I don't know. All I know is it breaks otherwise. + # + # [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L37-L40 + ( + # Nixpkgs doesn't check for x86_64 here but we do, because I + # observed a failure building statically for x86_64 without + # including it here. Linkers are weird. + (stdenv.hostPlatform.isAarch64 || stdenv.hostPlatform.isx86_64) + && stdenv.hostPlatform.isStatic + && !stdenv.isDarwin + && !stdenv.cc.bintools.isLLVM + ) + [ + "-l" + "stdc++" + "-L" + "${stdenv.cc.cc.lib}/${stdenv.hostPlatform.config}/lib" + ] + ); +} + +# What follows is stolen from [here][0]. Its purpose is to properly configure +# compilers and linkers for various stages of the build, and even covers the +# case of build scripts that need native code compiled and run on the build +# platform (I think). +# +# [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L57-L80 +// +( + let + inherit (rust.lib) envVars; + in + lib.optionalAttrs + (stdenv.targetPlatform.rust.rustcTarget + != stdenv.hostPlatform.rust.rustcTarget) + ( + let + inherit (stdenv.targetPlatform.rust) cargoEnvVarTarget; + in + { + "CC_${cargoEnvVarTarget}" = envVars.ccForTarget; + "CXX_${cargoEnvVarTarget}" = envVars.cxxForTarget; + "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = + envVars.linkerForTarget; + } + ) + // + ( + let + inherit (stdenv.hostPlatform.rust) cargoEnvVarTarget rustcTarget; + in + { + "CC_${cargoEnvVarTarget}" = envVars.ccForHost; + "CXX_${cargoEnvVarTarget}" = envVars.cxxForHost; + "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForHost; + CARGO_BUILD_TARGET = rustcTarget; + } + ) + // + ( + let + inherit (stdenv.buildPlatform.rust) cargoEnvVarTarget; + in + { + "CC_${cargoEnvVarTarget}" = envVars.ccForBuild; + "CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild; + "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForBuild; + HOST_CC = "${pkgsBuildHost.stdenv.cc}/bin/cc"; + HOST_CXX = "${pkgsBuildHost.stdenv.cc}/bin/c++"; + } + ) +) diff --git a/nix/pkgs/default/default.nix b/nix/pkgs/default/default.nix new file mode 100644 index 00000000..75c87b42 --- /dev/null +++ b/nix/pkgs/default/default.nix @@ -0,0 +1,64 @@ +# Keep sorted +{ craneLib +, inputs +, lib +, pkgsBuildHost +, rocksdb +, rust +, stdenv +}: + +let + env = { + CONDUIT_VERSION_EXTRA = inputs.self.shortRev or inputs.self.dirtyShortRev; + ROCKSDB_INCLUDE_DIR = "${rocksdb}/include"; + ROCKSDB_LIB_DIR = "${rocksdb}/lib"; + } + // + (import ./cross-compilation-env.nix { + # Keep sorted + inherit + lib + pkgsBuildHost + rust + stdenv; + }); +in + +craneLib.buildPackage rec { + inherit + (craneLib.crateNameFromCargoToml { + cargoToml = "${inputs.self}/Cargo.toml"; + }) + pname + version; + + src = let filter = inputs.nix-filter.lib; in filter { + root = inputs.self; + + # Keep sorted + include = [ + "Cargo.lock" + "Cargo.toml" + "src" + ]; + }; + + # This is redundant with CI + doCheck = false; + + nativeBuildInputs = [ + # bindgen needs the build platform's libclang. Apparently due to "splicing + # weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't quite do the + # right thing here. + pkgsBuildHost.rustPlatform.bindgenHook + ]; + + inherit env; + + passthru = { + inherit env; + }; + + meta.mainProgram = pname; +} diff --git a/nix/pkgs/oci-image/default.nix b/nix/pkgs/oci-image/default.nix new file mode 100644 index 00000000..8b359ce4 --- /dev/null +++ b/nix/pkgs/oci-image/default.nix @@ -0,0 +1,25 @@ +# Keep sorted +{ default +, dockerTools +, lib +, tini +}: + +dockerTools.buildImage { + name = default.pname; + tag = "next"; + copyToRoot = [ + dockerTools.caCertificates + ]; + config = { + # Use the `tini` init system so that signals (e.g. ctrl+c/SIGINT) + # are handled as expected + Entrypoint = [ + "${lib.getExe' tini "tini"}" + "--" + ]; + Cmd = [ + "${lib.getExe default}" + ]; + }; +} diff --git a/nix/shell.nix b/nix/shell.nix new file mode 100644 index 00000000..8cfc1c44 --- /dev/null +++ b/nix/shell.nix @@ -0,0 +1,55 @@ +# Keep sorted +{ cargo-deb +, default +, engage +, go +, inputs +, jq +, lychee +, mdbook +, mkShell +, olm +, system +, toolchain +}: + +mkShell { + env = default.env // { + # Rust Analyzer needs to be able to find the path to default crate + # sources, and it can read this environment variable to do so. The + # `rust-src` component is required in order for this to work. + RUST_SRC_PATH = "${toolchain}/lib/rustlib/src/rust/library"; + }; + + # Development tools + nativeBuildInputs = default.nativeBuildInputs ++ [ + # Always use nightly rustfmt because most of its options are unstable + # + # This needs to come before `toolchain` in this list, otherwise + # `$PATH` will have stable rustfmt instead. + inputs.fenix.packages.${system}.latest.rustfmt + + # Keep sorted + engage + toolchain + + # Needed for producing Debian packages + cargo-deb + + # Needed for our script for Complement + jq + + # Needed for Complement + go + olm + + # Needed for our script for Complement + jq + + # Needed for finding broken markdown links + lychee + + # Useful for editing the book locally + mdbook + ]; +} From d796fe7cd32b6ff12906092a4a013cfb4cfe634a Mon Sep 17 00:00:00 2001 From: Charles Hall Date: Wed, 1 May 2024 00:38:55 -0700 Subject: [PATCH 4/8] make it easy to configure cargo features from nix Users of the nix package can now just use `.override` to choose what features they want. This also makes RocksDB automatically use jemalloc when Conduit is configured to use jemalloc. --- nix/pkgs/default/default.nix | 49 +++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/nix/pkgs/default/default.nix b/nix/pkgs/default/default.nix index 75c87b42..38771517 100644 --- a/nix/pkgs/default/default.nix +++ b/nix/pkgs/default/default.nix @@ -1,4 +1,4 @@ -# Keep sorted +# Dependencies (keep sorted) { craneLib , inputs , lib @@ -6,23 +6,34 @@ , rocksdb , rust , stdenv + +# Options (keep sorted) +, default-features ? true +, features ? [] }: let - env = { - CONDUIT_VERSION_EXTRA = inputs.self.shortRev or inputs.self.dirtyShortRev; - ROCKSDB_INCLUDE_DIR = "${rocksdb}/include"; - ROCKSDB_LIB_DIR = "${rocksdb}/lib"; - } - // - (import ./cross-compilation-env.nix { - # Keep sorted - inherit - lib - pkgsBuildHost - rust - stdenv; - }); + env = + let + rocksdb' = rocksdb.override { + enableJemalloc = builtins.elem "jemalloc" features; + }; + in + { + CONDUIT_VERSION_EXTRA = + inputs.self.shortRev or inputs.self.dirtyShortRev; + ROCKSDB_INCLUDE_DIR = "${rocksdb'}/include"; + ROCKSDB_LIB_DIR = "${rocksdb'}/lib"; + } + // + (import ./cross-compilation-env.nix { + # Keep sorted + inherit + lib + pkgsBuildHost + rust + stdenv; + }); in craneLib.buildPackage rec { @@ -44,6 +55,14 @@ craneLib.buildPackage rec { ]; }; + cargoExtraArgs = "--locked " + + lib.optionalString + (!default-features) + "--no-default-features " + + lib.optionalString + (features != []) + "--features " + (builtins.concatStringsSep "," features); + # This is redundant with CI doCheck = false; From 55259329a3c0371f63c74dae95982664b5699814 Mon Sep 17 00:00:00 2001 From: Charles Hall Date: Wed, 1 May 2024 12:08:42 -0700 Subject: [PATCH 5/8] make it easy to configure cargo profiles from nix This way you can easily build in debug mode with Nix. --- nix/pkgs/default/default.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nix/pkgs/default/default.nix b/nix/pkgs/default/default.nix index 38771517..79510f50 100644 --- a/nix/pkgs/default/default.nix +++ b/nix/pkgs/default/default.nix @@ -10,6 +10,7 @@ # Options (keep sorted) , default-features ? true , features ? [] +, profile ? "release" }: let @@ -73,6 +74,8 @@ craneLib.buildPackage rec { pkgsBuildHost.rustPlatform.bindgenHook ]; + CARGO_PROFILE = profile; + inherit env; passthru = { From 96cc1f6abdc6fccb1dcf14966584f21b6725bdac Mon Sep 17 00:00:00 2001 From: Charles Hall Date: Wed, 1 May 2024 01:05:24 -0700 Subject: [PATCH 6/8] only set `CONDUIT_VERSION_EXTRA` for final build This prevents us from needing to recompile the dependencies when that environment variable changes, which generally changes on every commit, which is far more frequently than the dependencies are actually changed. --- nix/pkgs/default/default.nix | 76 ++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/nix/pkgs/default/default.nix b/nix/pkgs/default/default.nix index 79510f50..560e310a 100644 --- a/nix/pkgs/default/default.nix +++ b/nix/pkgs/default/default.nix @@ -14,15 +14,13 @@ }: let - env = + buildDepsOnlyEnv = let rocksdb' = rocksdb.override { enableJemalloc = builtins.elem "jemalloc" features; }; in { - CONDUIT_VERSION_EXTRA = - inputs.self.shortRev or inputs.self.dirtyShortRev; ROCKSDB_INCLUDE_DIR = "${rocksdb'}/include"; ROCKSDB_LIB_DIR = "${rocksdb'}/lib"; } @@ -35,26 +33,45 @@ let rust stdenv; }); + + buildPackageEnv = { + CONDUIT_VERSION_EXTRA = inputs.self.shortRev or inputs.self.dirtyShortRev; + } // buildDepsOnlyEnv; + + commonAttrs = { + inherit + (craneLib.crateNameFromCargoToml { + cargoToml = "${inputs.self}/Cargo.toml"; + }) + pname + version; + + src = let filter = inputs.nix-filter.lib; in filter { + root = inputs.self; + + # Keep sorted + include = [ + "Cargo.lock" + "Cargo.toml" + "src" + ]; + }; + + nativeBuildInputs = [ + # bindgen needs the build platform's libclang. Apparently due to "splicing + # weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't quite do the + # right thing here. + pkgsBuildHost.rustPlatform.bindgenHook + ]; + + CARGO_PROFILE = profile; + }; in -craneLib.buildPackage rec { - inherit - (craneLib.crateNameFromCargoToml { - cargoToml = "${inputs.self}/Cargo.toml"; - }) - pname - version; - - src = let filter = inputs.nix-filter.lib; in filter { - root = inputs.self; - - # Keep sorted - include = [ - "Cargo.lock" - "Cargo.toml" - "src" - ]; - }; +craneLib.buildPackage ( commonAttrs // { + cargoArtifacts = craneLib.buildDepsOnly (commonAttrs // { + env = buildDepsOnlyEnv; + }); cargoExtraArgs = "--locked " + lib.optionalString @@ -67,20 +84,11 @@ craneLib.buildPackage rec { # This is redundant with CI doCheck = false; - nativeBuildInputs = [ - # bindgen needs the build platform's libclang. Apparently due to "splicing - # weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't quite do the - # right thing here. - pkgsBuildHost.rustPlatform.bindgenHook - ]; - - CARGO_PROFILE = profile; - - inherit env; + env = buildPackageEnv; passthru = { - inherit env; + env = buildPackageEnv; }; - meta.mainProgram = pname; -} + meta.mainProgram = commonAttrs.pname; +}) From 3d9d975a9fe508c64c14ce7377881649031ce98d Mon Sep 17 00:00:00 2001 From: Matthias Ahouansou Date: Sat, 4 May 2024 14:41:30 +0100 Subject: [PATCH 7/8] ci: avoid rebuilding bindgen and friends --- nix/pkgs/default/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/pkgs/default/default.nix b/nix/pkgs/default/default.nix index 560e310a..4577fea9 100644 --- a/nix/pkgs/default/default.nix +++ b/nix/pkgs/default/default.nix @@ -21,6 +21,7 @@ let }; in { + NIX_OUTPATH_USED_AS_RANDOM_SEED = "randomseed"; # https://crane.dev/faq/rebuilds-bindgen.html ROCKSDB_INCLUDE_DIR = "${rocksdb'}/include"; ROCKSDB_LIB_DIR = "${rocksdb'}/lib"; } From 2a2b9554c8eea2b64cb4f19f5418a90488b93dae Mon Sep 17 00:00:00 2001 From: Matthias Ahouansou Date: Sat, 4 May 2024 21:50:37 +0100 Subject: [PATCH 8/8] ci: temporarily disable CONDUIT_VERSION_EXTRA --- nix/pkgs/default/default.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nix/pkgs/default/default.nix b/nix/pkgs/default/default.nix index 4577fea9..2595346f 100644 --- a/nix/pkgs/default/default.nix +++ b/nix/pkgs/default/default.nix @@ -36,7 +36,8 @@ let }); buildPackageEnv = { - CONDUIT_VERSION_EXTRA = inputs.self.shortRev or inputs.self.dirtyShortRev; + # Temporarily disabled, see https://gitlab.com/famedly/conduit/-/merge_requests/662#note_1892753424 + # CONDUIT_VERSION_EXTRA = inputs.self.shortRev or inputs.self.dirtyShortRev; } // buildDepsOnlyEnv; commonAttrs = {