tune arc for meta data

share/common.nix

@@ -114,12 +114,9 @@ in
 
   # tweak ZFS
   boot.extraModprobeConfig = ''
-    options zfs zfetch_max_distance=268435456
-    options zfs zfs_arc_meta_limit_percent=75
+    options zfs zfs_arc_meta_limit_percent=100
     options zfs zfs_arc_min=134217728
     options zfs zfs_arc_max=4294967296
-    options zfs zfs_compressed_arc_enabled=0
-    options zfs zfs_abd_scatter_enabled=0
     options zfs zfs_txg_timeout=30
     options zfs zfs_vdev_scrub_min_active=1
     options zfs zfs_vdev_scrub_max_active=1
@@ -134,6 +131,18 @@ in
     options zfs zfs_vdev_def_queue_depth=128
   '';
 
+  # tune the ZFS pool for NVMe
+  system.activationScripts.zfsTuning = {
+    text = ''
+      # only one level of metadata caching
+      ${pkgs.zfs}/bin/zfs set primarycache=metadata zpool
+      ${pkgs.zfs}/bin/zfs set secondarycache=none zpool
+
+      # use always direct IO to avoid ARC overhead
+      ${pkgs.zfs}/bin/zfs set direct=always zpool
+    '';
+  };
+
   # Use the systemd-boot EFI boot loader.
   boot.loader.systemd-boot.enable = true;
   boot.loader.efi.canTouchEfiVariables = true;