diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ab4581a..a1fa1895 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -302,7 +302,7 @@ jobs: with: cache-all-crates: "true" - - name: Build static ${{ matrix.target }} + - name: Build static ${{ matrix.target }}-all-features run: | if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]] then @@ -325,15 +325,35 @@ jobs: mv -v target/release/conduwuit static-${{ matrix.target }} mv -v target/release/${{ matrix.target }}.deb ${{ matrix.target }}.deb + - name: Build static x86_64-linux-musl-all-features-x86_64-haswell-optimised + if: ${{ matrix.target == "x86_64-linux-musl"}} + run: | + CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl" + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + + bin/nix-build-and-cache just .#static-x86_64-linux-musl-all-features-x86_64-haswell-optimised + + mkdir -v -p target/release/ + mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ + cp -v -f result/bin/conduit target/release/conduwuit + cp -v -f result/bin/conduit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit + # -p conduit is the main crate name + direnv exec . cargo deb --verbose --no-build --no-strip -p conduit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/x86_64-linux-musl-x86_64-haswell-optimised.deb + mv -v target/release/conduwuit static-x86_64-linux-musl-x86_64-haswell-optimised + mv -v target/release/x86_64-linux-musl-x86_64-haswell-optimised.deb x86_64-linux-musl-x86_64-haswell-optimised.deb + # quick smoke test of the x86_64 static release binary - - name: Run x86_64 static release binary + - name: Quick smoke test the x86_64 static release binary + if: ${{ matrix.target == "x86_64-linux-musl"}} run: | # GH actions default runners are x86_64 only if file result/bin/conduit | grep x86-64; then result/bin/conduit --version + result/bin/conduit --help + result/bin/conduit -Oserver_name="'$(date -u +%s).local'" -Odatabase_path="'/tmp/$(date -u +%s)'" --execute "server admin-notice awawawawawawawawawawa" --execute "server memory-usage" --execute "server shutdown" fi - - name: Build static debug ${{ matrix.target }} + - name: Build static debug ${{ matrix.target }}-all-features run: | if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]] then @@ -376,14 +396,14 @@ jobs: dpkg-deb --info ${{ matrix.target }}.deb dpkg-deb --info ${{ matrix.target }}-debug.deb - - name: Upload static-${{ matrix.target }} + - name: Upload static-${{ matrix.target }}-all-features uses: actions/upload-artifact@v4 with: name: static-${{ matrix.target }} path: static-${{ matrix.target }} if-no-files-found: error - - name: Upload deb ${{ matrix.target }} + - name: Upload deb ${{ matrix.target }}-all-features uses: actions/upload-artifact@v4 with: name: deb-${{ matrix.target }} @@ -391,14 +411,14 @@ jobs: if-no-files-found: error compression-level: 0 - - name: Upload static-${{ matrix.target }}-debug + - name: Upload static-${{ matrix.target }}-debug-all-features uses: actions/upload-artifact@v4 with: name: static-${{ matrix.target }}-debug path: static-${{ matrix.target }}-debug if-no-files-found: error - - name: Upload deb ${{ matrix.target }}-debug + - name: Upload deb ${{ matrix.target }}-debug-all-features uses: actions/upload-artifact@v4 with: name: deb-${{ matrix.target }}-debug @@ -406,19 +426,26 @@ jobs: if-no-files-found: error compression-level: 0 - - name: Build OCI image ${{ matrix.target }} + - name: Build OCI image ${{ matrix.target }}-all-features run: | bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features cp -v -f result oci-image-${{ matrix.target }}.tar.gz - - name: Build debug OCI image ${{ matrix.target }} + - name: Build OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised + if: ${{ matrix.target == "x86_64-linux-musl"}} + run: | + bin/nix-build-and-cache just .#oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised + + cp -v -f result oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz + + - name: Build debug OCI image ${{ matrix.target }}-all-features run: | bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features-debug cp -v -f result oci-image-${{ matrix.target }}-debug.tar.gz - - name: Upload OCI image ${{ matrix.target }} + - name: Upload OCI image ${{ matrix.target }}-all-features uses: actions/upload-artifact@v4 with: name: oci-image-${{ matrix.target }} @@ -426,7 +453,7 @@ jobs: if-no-files-found: error compression-level: 0 - - name: Upload OCI image ${{ matrix.target }}-debug + - name: Upload OCI image ${{ matrix.target }}-debug-all-features uses: actions/upload-artifact@v4 with: name: oci-image-${{ matrix.target }}-debug diff --git a/flake.nix b/flake.nix index e4ba8e3d..5041e468 100644 --- a/flake.nix +++ b/flake.nix @@ -72,9 +72,14 @@ "-DWITH_TESTS=1" # we use rust-rocksdb via C interface and dont need C++ RTTI "-DUSE_RTTI=1" + # this doesn't exist in RocksDB, and USE_SSE is deprecated for + # PORTABLE=$(march) + "-DFORCE_SSE42=1" ] old.cmakeFlags ++ [ + # no real reason to have snappy, no one uses this + "-DWITH_SNAPPY=0" # we dont need to use ldb or sst_dump (core_tools) "-DWITH_CORE_TOOLS=0" # we dont need trace tools @@ -272,6 +277,15 @@ value = scopeCrossStatic.main; } + # An output for a statically-linked binary with x86_64 haswell + # target optimisations + { + name = "${binaryName}-x86_64-haswell-optimised"; + value = scopeCrossStatic.main.override { + x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false); + }; + } + # An output for a statically-linked unstripped debug ("dev") binary { name = "${binaryName}-debug"; @@ -306,6 +320,22 @@ }; } + # An output for a statically-linked binary with `--all-features` and with x86_64 haswell + # target optimisations + { + name = "${binaryName}-all-features-x86_64-haswell-optimised"; + value = scopeCrossStatic.main.override { + all_features = true; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; + x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false); + }; + } + # An output for a statically-linked unstripped debug ("dev") binary with `--all-features` { name = "${binaryName}-all-features-debug"; @@ -337,6 +367,17 @@ value = scopeCrossStatic.oci-image; } + # An output for an OCI image based on that binary with x86_64 haswell + # target optimisations + { + name = "oci-image-${crossSystem}-x86_64-haswell-optimised"; + value = scopeCrossStatic.oci-image.override { + main = scopeCrossStatic.main.override { + x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false); + }; + }; + } + # An output for an OCI image based on that unstripped debug ("dev") binary { name = "oci-image-${crossSystem}-debug"; @@ -365,21 +406,39 @@ }; } + # An output for an OCI image based on that binary with `--all-features` and with x86_64 haswell + # target optimisations + { + name = "oci-image-${crossSystem}-all-features-x86_64-haswell-optimised"; + value = scopeCrossStatic.oci-image.override { + main = scopeCrossStatic.main.override { + all_features = true; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; + x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false); + }; + }; + } + # An output for an OCI image based on that unstripped debug ("dev") binary with `--all-features` { name = "oci-image-${crossSystem}-all-features-debug"; value = scopeCrossStatic.oci-image.override { main = scopeCrossStatic.main.override { - profile = "dev"; - all_features = true; - # debug build users expect full logs - disable_release_max_log_level = true; - disable_features = [ - # this is non-functional on nix for some reason - "hardened_malloc" - # dont include experimental features - "experimental" - ]; + profile = "dev"; + all_features = true; + # debug build users expect full logs + disable_release_max_log_level = true; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; }; }; } diff --git a/nix/pkgs/main/cross-compilation-env.nix b/nix/pkgs/main/cross-compilation-env.nix index 3fb94d41..0f326c92 100644 --- a/nix/pkgs/main/cross-compilation-env.nix +++ b/nix/pkgs/main/cross-compilation-env.nix @@ -45,7 +45,6 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic { ( let inherit (rust.lib) envVars; - #shouldUseLLD = platform: platform.isAarch64 && platform.isStatic && !stdenv.hostPlatform.isDarwin; in lib.optionalAttrs (stdenv.targetPlatform.rust.rustcTarget diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index abc85d82..32e11c06 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -13,12 +13,17 @@ , stdenv # Options (keep sorted) -, default_features ? true -, disable_release_max_log_level ? false , all_features ? false +, default_features ? true , disable_features ? [] +, disable_release_max_log_level ? false , features ? [] , profile ? "release" +# rocksdb compiled with -march=haswell and target-cpu=haswell rustflag +# haswell is pretty much any x86 cpu made in the last 12 years, and +# supports modern CPU extensions that rocksdb can make use of. +# disable if trying to make a portable x86_64 build for very old hardware +, x86_64_haswell_target_optimised ? false }: let @@ -79,6 +84,15 @@ buildDepsOnlyEnv = enableLiburing = enableLiburing; }).overrideAttrs (old: { enableLiburing = enableLiburing; + cmakeFlags = lib.optional x86_64_haswell_target_optimised (lib.subtractLists [ + # dont make a portable build if x86_64_haswell_target_optimised is enabled + "-DPORTABLE=1" + ] + old.cmakeFlags) + ++ lib.optionals x86_64_haswell_target_optimised [ + "-DPORTABLE=haswell" + ] + ++ old.cmakeFlags; }); in { @@ -105,7 +119,9 @@ buildPackageEnv = { # Only needed in static stdenv because these are transitive dependencies of rocksdb CARGO_BUILD_RUSTFLAGS = buildDepsOnlyEnv.CARGO_BUILD_RUSTFLAGS + lib.optionalString (enableLiburing && stdenv.hostPlatform.isStatic) - " -L${lib.getLib liburing}/lib -luring"; + " -L${lib.getLib liburing}/lib -luring" + + lib.optionalString x86_64_haswell_target_optimised + " -Ctarget-cpu=haswell"; };