fix dynamic builds with liburing

The original implementation of this was really weird, so I restructed it
a lot while debugging, and am just gonna leave the restructured version.

Root cause of the segfault seems to be that upstream nixpkgs liburing
derivation is generating both static and dynamic libraries, causing
rocksdb to statically link liburing in a dynamic build, pulling in some
allocator stuff at the same time. I created a PR[1] to fix this upstream,
but it probably won't be available on nixos-unstable for quite a while,
so we can also patch it locally.

[1]: https://github.com/NixOS/nixpkgs/pull/314945
This commit is contained in:
Benjamin Lee 2024-05-25 18:04:17 -07:00 committed by June 🍓🦴
parent a08f90b161
commit b4cd8e9140
2 changed files with 27 additions and 21 deletions

View file

@ -42,6 +42,24 @@
(builtins.fromJSON (builtins.readFile ./flake.lock)) (builtins.fromJSON (builtins.readFile ./flake.lock))
.nodes.rocksdb.original.ref; .nodes.rocksdb.original.ref;
}); });
# TODO: remove once https://github.com/NixOS/nixpkgs/pull/314945 is available
liburing = pkgs.liburing.overrideAttrs (old: {
# the configure script doesn't support these, and unconditionally
# builds both static and dynamic libraries.
configureFlags = pkgs.lib.subtractLists
[ "--enable-static" "--disable-shared" ]
old.configureFlags;
postInstall = old.postInstall + ''
# we remove the extra outputs
#
# we need to do this to prevent rocksdb from trying to link the
# static library in a dynamic stdenv
rm $out/lib/liburing*${
if pkgs.stdenv.hostPlatform.isStatic then ".so*" else ".a"
}
'';
});
}); });
scopeHost = mkScope pkgsHost; scopeHost = mkScope pkgsHost;

View file

@ -9,9 +9,7 @@
, rocksdb , rocksdb
, rust , rust
, rust-jemalloc-sys , rust-jemalloc-sys
, snappy
, stdenv , stdenv
, pkgsStatic
# Options (keep sorted) # Options (keep sorted)
, default_features ? true , default_features ? true
@ -49,6 +47,8 @@ features'' = lib.subtractLists disable_features' features';
featureEnabled = feature : builtins.elem feature features''; featureEnabled = feature : builtins.elem feature features'';
enableLiburing = featureEnabled "io_uring" && stdenv.isLinux;
# This derivation will set the JEMALLOC_OVERRIDE variable, causing the # This derivation will set the JEMALLOC_OVERRIDE variable, causing the
# tikv-jemalloc-sys crate to use the nixpkgs jemalloc instead of building it's # tikv-jemalloc-sys crate to use the nixpkgs jemalloc instead of building it's
# own. In order for this to work, we need to set flags on the build that match # own. In order for this to work, we need to set flags on the build that match
@ -62,15 +62,9 @@ rust-jemalloc-sys' = (rust-jemalloc-sys.override {
# tikv-jemalloc-sys/profiling feature # tikv-jemalloc-sys/profiling feature
lib.optional (featureEnabled "jemalloc_prof") "--enable-prof"; lib.optional (featureEnabled "jemalloc_prof") "--enable-prof";
}); });
liburing' = pkgsStatic.liburing.overrideAttrs {
configureFlags = []; # liburing's configure file is handwritten so the default assumptions don't apply
isStatic = true;
};
buildDepsOnlyEnv = buildDepsOnlyEnv =
let let
uring = featureEnabled "io_uring" && stdenv.isLinux;
extraDeps = lib.optionals uring [ liburing'.dev liburing'.out];
rocksdb' = (rocksdb.override { rocksdb' = (rocksdb.override {
jemalloc = rust-jemalloc-sys'; jemalloc = rust-jemalloc-sys';
# rocksdb fails to build with prefixed jemalloc, which is required on # rocksdb fails to build with prefixed jemalloc, which is required on
@ -84,7 +78,8 @@ buildDepsOnlyEnv =
# TODO: static rocksdb fails to build on darwin # TODO: static rocksdb fails to build on darwin
# build log at <https://girlboss.ceo/~strawberry/pb/JjGH> # build log at <https://girlboss.ceo/~strawberry/pb/JjGH>
meta.broken = stdenv.hostPlatform.isStatic && stdenv.isDarwin; meta.broken = stdenv.hostPlatform.isStatic && stdenv.isDarwin;
propagatedBuildInputs = old.propagatedBuildInputs ++ extraDeps; # TODO: switch to enableUring option once https://github.com/NixOS/nixpkgs/pull/314945 is available
buildInputs = old.buildInputs ++ lib.optional enableLiburing liburing;
}); });
in in
{ {
@ -108,18 +103,11 @@ buildDepsOnlyEnv =
buildPackageEnv = { buildPackageEnv = {
CONDUWUIT_VERSION_EXTRA = inputs.self.shortRev or inputs.self.dirtyShortRev or ""; CONDUWUIT_VERSION_EXTRA = inputs.self.shortRev or inputs.self.dirtyShortRev or "";
} // buildDepsOnlyEnv // { } // buildDepsOnlyEnv // {
CARGO_BUILD_RUSTFLAGS = # Only needed in static stdenv because these are transitive dependencies of rocksdb
let CARGO_BUILD_RUSTFLAGS = buildDepsOnlyEnv.CARGO_BUILD_RUSTFLAGS
uring = featureEnabled "io_uring"; + lib.optionalString (enableLiburing && stdenv.hostPlatform.isStatic)
valid = (stdenv.hostPlatform.isAarch64 || stdenv.hostPlatform.isx86_64) " -L${lib.getLib liburing}/lib -luring";
&& stdenv.hostPlatform.isStatic };
&& !stdenv.isDarwin
&& !stdenv.cc.bintools.isLLVM;
in
buildDepsOnlyEnv.CARGO_BUILD_RUSTFLAGS
+ lib.optionalString (uring && valid) " -L${lib.getLib liburing'}/lib/ -luring"
+ " -L${lib.getLib snappy}/lib/ -lsnappy";
};