start producing haswell target optimised x86 binaries for rocksdb fast CRC32 support

Signed-off-by: strawberry <strawberry@puppygock.gay>

.github/workflows/ci.yml

@@ -302,7 +302,7 @@ jobs:
               with:
                 cache-all-crates: "true"
 
-            - name: Build static ${{ matrix.target }}
+            - name: Build static ${{ matrix.target }}-all-features
               run: |
                   if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]]
                   then
@@ -325,15 +325,35 @@ jobs:
                   mv -v target/release/conduwuit static-${{ matrix.target }}
                   mv -v target/release/${{ matrix.target }}.deb ${{ matrix.target }}.deb
 
+            - name: Build static x86_64-linux-musl-all-features-x86_64-haswell-optimised
+              if: ${{ matrix.target == "x86_64-linux-musl"}}
+              run: |
+                  CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl"
+                  SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+
+                  bin/nix-build-and-cache just .#static-x86_64-linux-musl-all-features-x86_64-haswell-optimised
+
+                  mkdir -v -p target/release/
+                  mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/
+                  cp -v -f result/bin/conduit target/release/conduwuit
+                  cp -v -f result/bin/conduit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit
+                  # -p conduit is the main crate name
+                  direnv exec . cargo deb --verbose --no-build --no-strip -p conduit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/x86_64-linux-musl-x86_64-haswell-optimised.deb
+                  mv -v target/release/conduwuit static-x86_64-linux-musl-x86_64-haswell-optimised
+                  mv -v target/release/x86_64-linux-musl-x86_64-haswell-optimised.deb x86_64-linux-musl-x86_64-haswell-optimised.deb
+
             # quick smoke test of the x86_64 static release binary
-            - name: Run x86_64 static release binary
+            - name: Quick smoke test the x86_64 static release binary
+              if: ${{ matrix.target == "x86_64-linux-musl"}}
               run: |
                   # GH actions default runners are x86_64 only
                   if file result/bin/conduit | grep x86-64; then
                     result/bin/conduit --version
+                    result/bin/conduit --help
+                    result/bin/conduit -Oserver_name="'$(date -u +%s).local'" -Odatabase_path="'/tmp/$(date -u +%s)'" --execute "server admin-notice awawawawawawawawawawa" --execute "server memory-usage" --execute "server shutdown"
                   fi
 
-            - name: Build static debug ${{ matrix.target }}
+            - name: Build static debug ${{ matrix.target }}-all-features
               run: |
                   if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]]
                   then
@@ -376,14 +396,14 @@ jobs:
                   dpkg-deb --info ${{ matrix.target }}.deb
                   dpkg-deb --info ${{ matrix.target }}-debug.deb
 
-            - name: Upload static-${{ matrix.target }}
+            - name: Upload static-${{ matrix.target }}-all-features
               uses: actions/upload-artifact@v4
               with:
                   name: static-${{ matrix.target }}
                   path: static-${{ matrix.target }}
                   if-no-files-found: error
 
-            - name: Upload deb ${{ matrix.target }}
+            - name: Upload deb ${{ matrix.target }}-all-features
               uses: actions/upload-artifact@v4
               with:
                   name: deb-${{ matrix.target }}
@@ -391,14 +411,14 @@ jobs:
                   if-no-files-found: error
                   compression-level: 0
 
-            - name: Upload static-${{ matrix.target }}-debug
+            - name: Upload static-${{ matrix.target }}-debug-all-features
               uses: actions/upload-artifact@v4
               with:
                   name: static-${{ matrix.target }}-debug
                   path: static-${{ matrix.target }}-debug
                   if-no-files-found: error
 
-            - name: Upload deb ${{ matrix.target }}-debug
+            - name: Upload deb ${{ matrix.target }}-debug-all-features
               uses: actions/upload-artifact@v4
               with:
                   name: deb-${{ matrix.target }}-debug
@@ -406,19 +426,26 @@ jobs:
                   if-no-files-found: error
                   compression-level: 0
 
-            - name: Build OCI image ${{ matrix.target }}
+            - name: Build OCI image ${{ matrix.target }}-all-features
               run: |
                   bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features
 
                   cp -v -f result oci-image-${{ matrix.target }}.tar.gz
 
-            - name: Build debug OCI image ${{ matrix.target }}
+            - name: Build OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised
+              if: ${{ matrix.target == "x86_64-linux-musl"}}
+              run: |
+                  bin/nix-build-and-cache just .#oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised
+
+                  cp -v -f result oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz
+
+            - name: Build debug OCI image ${{ matrix.target }}-all-features
               run: |
                   bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features-debug
 
                   cp -v -f result oci-image-${{ matrix.target }}-debug.tar.gz
 
-            - name: Upload OCI image ${{ matrix.target }}
+            - name: Upload OCI image ${{ matrix.target }}-all-features
               uses: actions/upload-artifact@v4
               with:
                   name: oci-image-${{ matrix.target }}
@@ -426,7 +453,7 @@ jobs:
                   if-no-files-found: error
                   compression-level: 0
 
-            - name: Upload OCI image ${{ matrix.target }}-debug
+            - name: Upload OCI image ${{ matrix.target }}-debug-all-features
               uses: actions/upload-artifact@v4
               with:
                   name: oci-image-${{ matrix.target }}-debug

flake.nix

@@ -72,9 +72,14 @@
               "-DWITH_TESTS=1"
               # we use rust-rocksdb via C interface and dont need C++ RTTI
               "-DUSE_RTTI=1"
+              # this doesn't exist in RocksDB, and USE_SSE is deprecated for
+              # PORTABLE=$(march)
+              "-DFORCE_SSE42=1"
             ]
             old.cmakeFlags
             ++ [
+              # no real reason to have snappy, no one uses this
+              "-DWITH_SNAPPY=0"
               # we dont need to use ldb or sst_dump (core_tools)
               "-DWITH_CORE_TOOLS=0"
               # we dont need trace tools
@@ -272,6 +277,15 @@
                   value = scopeCrossStatic.main;
                 }
 
+                # An output for a statically-linked binary with x86_64 haswell
+                # target optimisations
+                {
+                  name = "${binaryName}-x86_64-haswell-optimised";
+                  value = scopeCrossStatic.main.override {
+                    x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false);
+                  };
+                }
+
                 # An output for a statically-linked unstripped debug ("dev") binary
                 {
                   name = "${binaryName}-debug";
@@ -306,6 +320,22 @@
                   };
                 }
 
+                # An output for a statically-linked binary with `--all-features` and with x86_64 haswell
+                # target optimisations
+                {
+                  name = "${binaryName}-all-features-x86_64-haswell-optimised";
+                  value = scopeCrossStatic.main.override {
+                    all_features = true;
+                    disable_features = [
+                        # this is non-functional on nix for some reason
+                        "hardened_malloc"
+                        # dont include experimental features
+                        "experimental"
+                    ];
+                    x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false);
+                  };
+                }
+
                 # An output for a statically-linked unstripped debug ("dev") binary with `--all-features`
                 {
                   name = "${binaryName}-all-features-debug";
@@ -337,6 +367,17 @@
                   value = scopeCrossStatic.oci-image;
                 }
 
+                # An output for an OCI image based on that binary with x86_64 haswell
+                # target optimisations
+                {
+                  name = "oci-image-${crossSystem}-x86_64-haswell-optimised";
+                  value = scopeCrossStatic.oci-image.override {
+                    main = scopeCrossStatic.main.override {
+                      x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false);
+                    };
+                  };
+                }
+
                 # An output for an OCI image based on that unstripped debug ("dev") binary
                 {
                   name = "oci-image-${crossSystem}-debug";
@@ -365,21 +406,39 @@
                   };
                 }
 
+                # An output for an OCI image based on that binary with `--all-features` and with x86_64 haswell
+                # target optimisations
+                {
+                  name = "oci-image-${crossSystem}-all-features-x86_64-haswell-optimised";
+                  value = scopeCrossStatic.oci-image.override {
+                    main = scopeCrossStatic.main.override {
+                      all_features = true;
+                      disable_features = [
+                          # this is non-functional on nix for some reason
+                          "hardened_malloc"
+                          # dont include experimental features
+                          "experimental"
+                      ];
+                      x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false);
+                    };
+                  };
+                }
+
                 # An output for an OCI image based on that unstripped debug ("dev") binary with `--all-features`
                 {
                   name = "oci-image-${crossSystem}-all-features-debug";
                   value = scopeCrossStatic.oci-image.override {
                     main = scopeCrossStatic.main.override {
-                        profile = "dev";
+                      profile = "dev";
-                        all_features = true;
+                      all_features = true;
-                        # debug build users expect full logs
+                      # debug build users expect full logs
-                        disable_release_max_log_level = true;
+                      disable_release_max_log_level = true;
-                        disable_features = [
+                      disable_features = [
-                            # this is non-functional on nix for some reason
+                          # this is non-functional on nix for some reason
-                            "hardened_malloc"
+                          "hardened_malloc"
-                            # dont include experimental features
+                          # dont include experimental features
-                            "experimental"
+                          "experimental"
-                        ];
+                      ];
                     };
                   };
                 }

nix/pkgs/main/cross-compilation-env.nix

@@ -45,7 +45,6 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic {
 (
   let
     inherit (rust.lib) envVars;
-    #shouldUseLLD = platform: platform.isAarch64 && platform.isStatic && !stdenv.hostPlatform.isDarwin;
   in
   lib.optionalAttrs
     (stdenv.targetPlatform.rust.rustcTarget

nix/pkgs/main/default.nix

@@ -13,12 +13,17 @@
 , stdenv
 
 # Options (keep sorted)
-, default_features ? true
-, disable_release_max_log_level ? false
 , all_features ? false
+, default_features ? true
 , disable_features ? []
+, disable_release_max_log_level ? false
 , features ? []
 , profile ? "release"
+# rocksdb compiled with -march=haswell and target-cpu=haswell rustflag
+# haswell is pretty much any x86 cpu made in the last 12 years, and
+# supports modern CPU extensions that rocksdb can make use of.
+# disable if trying to make a portable x86_64 build for very old hardware
+, x86_64_haswell_target_optimised ? false
 }:
 
 let
@@ -79,6 +84,15 @@ buildDepsOnlyEnv =
       enableLiburing = enableLiburing;
     }).overrideAttrs (old: {
       enableLiburing = enableLiburing;
+      cmakeFlags = lib.optional x86_64_haswell_target_optimised (lib.subtractLists [
+        # dont make a portable build if x86_64_haswell_target_optimised is enabled
+        "-DPORTABLE=1"
+      ]
+      old.cmakeFlags)
+      ++ lib.optionals x86_64_haswell_target_optimised [
+        "-DPORTABLE=haswell"
+      ]
+      ++ old.cmakeFlags;
     });
   in
   {
@@ -105,7 +119,9 @@ buildPackageEnv = {
   # Only needed in static stdenv because these are transitive dependencies of rocksdb
   CARGO_BUILD_RUSTFLAGS = buildDepsOnlyEnv.CARGO_BUILD_RUSTFLAGS
     + lib.optionalString (enableLiburing && stdenv.hostPlatform.isStatic)
-      " -L${lib.getLib liburing}/lib -luring";
+      " -L${lib.getLib liburing}/lib -luring"
+    + lib.optionalString x86_64_haswell_target_optimised
+      " -Ctarget-cpu=haswell";
 };