more bcachefs stuff

neko/configuration.nix

@@ -1,7 +1,3 @@
-# Edit this configuration file to define what should be installed on
-# your system.  Help is available in the configuration.nix(5) man page
-# and in the NixOS manual (accessible by running `nixos-help`).
-
 { config, pkgs, ... }:
 
 {
@@ -14,9 +10,8 @@
       /data/nixos/share/common.nix
     ];
 
-  # our hostname and an ID for ZFS
+  # our hostname
   networking.hostName = "neko";
-  networking.hostId = "cf5a5ee6";
 
   # EurKey layout
   services.xserver.xkb.layout = "eu";

neko/hardware-configuration.nix

@@ -1,6 +1,3 @@
-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
 { config, lib, pkgs, modulesPath, ... }:
 
 {
@@ -18,9 +15,16 @@
 
   # /boot efi partition to boot in UEFI mode
   fileSystems."/boot" = {
-    device = "/dev/disk/by-id/nvme-Seagate_FireCuda_530_ZP4000GM30013_7VS01VBM-part1";
+    device = "UUID=CFA5-46EA";
     fsType = "vfat";
+    neededForBoot = true;
     options = [ "fmask=0022" "dmask=0022" ];
+  };
+
+  # /nix volume with the system & all persistent data
+  fileSystems."/nix" = {
+    device = "UUID=686a90a2-93ac-40a6-a01d-c7b61cc47750";
+    fsType = "bcachefs";
     neededForBoot = true;
   };
 }

neko/install.txt

@@ -29,6 +29,7 @@ HOST=neko
 DISK=/dev/disk/by-id/nvme-Seagate_FireCuda_530_ZP4000GM30013_7VS01VBM
 DISK2=/dev/disk/by-id/nvme-CT2000P5PSSD8_213330E4ED05
 DISK3=/dev/disk/by-id/nvme-Samsung_SSD_980_PRO_2TB_S69ENF0R846614L
+DISKS="$DISK $DISK2 $DISK3"
 
 # ensure 4k sector size
 nvme format --lbaf=1 --force $DISK
@@ -36,7 +37,8 @@ nvme id-ns -H $DISK
 sleep 5
 
 # create partition table on all disks and EFI partition
-for D in $DISK $DISK2 $DISK3; do
+RAID=""
+for D in $DISKS; do
     # kill old data
     sgdisk --zap-all $D
     blkdiscard -v -f $D
@@ -56,59 +58,22 @@ for D in $DISK $DISK2 $DISK3; do
     # boot partition after short sleep, needed on some machines
     sleep 5
     mkfs.vfat $D-part1
+
+    # add part2 to raid
+    RAID="$RAID $D-part2"
 done
 
 # take a look at the partitions
 lsblk
 
-# ZFS zpool creation with compression and encryption
+# create encrypted bcachefs
-zpool create \
+bcachefs format --encrypt -f $RAID
-    -o ashift=13 \
+nix-env -iA nixos.keyutils
-    -o autotrim=off \
+keyctl link @u @s
-    -O acltype=posixacl \
+bcachefs unlock $DISK-part2
-    -O atime=off \
-    -O canmount=off \
-    -O compression=on \
-    -O dnodesize=auto \
-    -O utf8only=on \
-    -O normalization=formD \
-    -O xattr=sa \
-    -O mountpoint=none \
-    -O encryption=on \
-    -O keylocation=prompt \
-    -O keyformat=passphrase \
-    zpool $DISK-part2 $DISK2-part2 $DISK3-part2
 
-sleep 5
+# prepare install, tmpfs root
-
+mount -t tmpfs none /mnt
-# take a look at the partitions
-lsblk
-
-# show the pool
-zpool status
-
-sleep 5
-
-# create all the volumes
-zfs create -o mountpoint=legacy zpool/data
-zfs create -o mountpoint=legacy zpool/nix
-zfs create -o mountpoint=legacy zpool/root
-
-sleep 5
-
-# show the pool
-zpool status
-
-sleep 5
-
-# create ZFS snapshot that we'll rollback to on boot
-# see https://ryanseipp.com/post/nixos-encrypted-root/
-zfs snapshot zpool/root@blank
-
-sleep 5
-
-# prepare install, root
-mount -t zfs zpool/root /mnt
 
 # Create directories to mount file systems on
 mkdir -p /mnt/{data,nix,boot,root,etc/nixos}
@@ -116,9 +81,12 @@ mkdir -p /mnt/{data,nix,boot,root,etc/nixos}
 # mount the ESP
 mount $DISK-part1 /mnt/boot
 
-# mount volumes
+# mount the /nix
-mount -t zfs zpool/data /mnt/data
+mount -t bcachefs $DISK-part2 /mnt/nix
-mount -t zfs zpool/nix /mnt/nix
+
+# mount the /data via bind mount
+mkdir /mnt/nix/data
+mount --bind /mnt/nix/data /mnt/data
 
 # bind mount persistent stuff to data
 mkdir -p /mnt/data/{root,nixos/$HOST}
@@ -135,7 +103,7 @@ mount
 # configure
 nixos-generate-config --root /mnt
 
-# check /mnt/etc/nixos/hardware-configuration.nix /mnt/etc/nixos/configuration.nix
+# check /mnt/etc/nixos/hardware-configuration.nix to get UUIDs
 
 # copy config data from another machine including secrets
 
@@ -148,7 +116,6 @@ nixos-install --option experimental-features 'nix-command flakes' --no-root-pass
 # unmount all stuff and sync
 
 umount -Rl /data /mnt
-zpool export -a
 sync
 
 # shutdown once

share/common.nix

@@ -26,8 +26,9 @@ in
   # atm all stuff is x86_64
   nixpkgs.hostPlatform = "x86_64-linux";
 
-  # enable ZFS
+  # enable bcachefs with latest kernel
-  boot.supportedFilesystems = [ "zfs" ];
+  boot.supportedFilesystems = [ "bcachefs" ];
+  boot.kernelPackages = pkgs.linuxPackages_latest;
 
   # my kernel parameters
   boot.kernelParams = [
@@ -35,13 +36,6 @@ in
     "quiet"
     "splash"
 
-    # no hibernate for ZFS systems
-    "nohibernate"
-
-    # make ARC fast
-    "init_on_alloc=0"
-    "init_on_free=0"
-
     # don't check for split locks, for KVM and Co.
     "split_lock_detect=off"
 
@@ -113,29 +107,6 @@ in
     };
   };
 
-  # tweak ZFS
-  boot.extraModprobeConfig = ''
-    # less scrub impact on other IO
-    options zfs zfs_scrub_delay=32
-    options zfs zfs_vdev_scrub_min_active=1
-    options zfs zfs_vdev_scrub_max_active=1
-  '';
-
-  # tune the ZFS pool for NVMe
-  system.activationScripts.zfsTuning = {
-    text = ''
-      # only one level of caching
-      ${pkgs.zfs}/bin/zfs set primarycache=all zpool
-      ${pkgs.zfs}/bin/zfs set secondarycache=none zpool
-
-      # I have backups and no real databases
-      ${pkgs.zfs}/bin/zfs set sync=disabled zpool
-
-      # use allow direct IO
-      ${pkgs.zfs}/bin/zfs set direct=standard zpool
-    '';
-  };
-
   # Use the systemd-boot EFI boot loader.
   boot.loader.systemd-boot.enable = true;
   boot.loader.efi.canTouchEfiVariables = true;
@@ -159,44 +130,26 @@ in
     themePackages = [ pkgs.adi1090x-plymouth-themes ];
   };
 
-  # root file system, we will rollback that on boot
+  # root file system, tmpfs
   fileSystems."/" = {
-    device = "zpool/root";
+    device = "none";
-    fsType = "zfs";
+    fsType = "tmpfs";
     neededForBoot = true;
-  };
+    options = [ "defaults" "size=25%" "mode=755" ];
-
-  # root rollback, see https://ryanseipp.com/post/nixos-encrypted-root/
-  boot.initrd.systemd.services.rollback = {
-    description = "Rollback root filesystem to a pristine state";
-    wantedBy = ["initrd.target"];
-    after = ["zfs-import-zpool.service"];
-    before = ["sysroot.mount"];
-    path = with pkgs; [zfs];
-    unitConfig.DefaultDependencies = "no";
-    serviceConfig.Type = "oneshot";
-    script = ''
-      zfs rollback -r zpool/root@blank && echo " >> >> Rollback Complete << <<"
-    '';
   };
 
   # my data
   fileSystems."/data" = {
-    device = "zpool/data";
+    device = "/nix/data";
-    fsType = "zfs";
+    fsType = "none";
-    neededForBoot = true;
-  };
-
-  # the system
-  fileSystems."/nix" = {
-    device = "zpool/nix";
-    fsType = "zfs";
     neededForBoot = true;
+    options = [ "bind" "x-gvfs-hide" ];
+    depends = [ "/nix" ];
   };
 
   # bind mount to have root home
-  fileSystems."/root" =
+  fileSystems."/root" = {
-    { device = "/data/root";
+    device = "/data/root";
     fsType = "none";
     neededForBoot = true;
     options = [ "bind" "x-gvfs-hide" ];
@@ -204,8 +157,8 @@ in
   };
 
   # bind mount to have NixOS configuration, different per host
-  fileSystems."/etc/nixos" =
+  fileSystems."/etc/nixos" = {
-    { device = "/data/nixos/${config.networking.hostName}";
+    device = "/data/nixos/${config.networking.hostName}";
     fsType = "none";
     neededForBoot = true;
     options = [ "bind" "x-gvfs-hide" ];
@@ -238,15 +191,6 @@ in
     ];
   };
 
-  # ensure our data is not rotting
-  services.zfs.autoScrub = {
-    enable = true;
-    interval = "weekly";
-  };
-
-  # trim the stuff, we use SSDs
-  services.zfs.trim.enable = true;
-
   # enable fast dbus
   services.dbus.implementation = "broker";
 
@@ -576,25 +520,6 @@ in
     mode = "0400";
   };
 
-  # send mails on ZFS events
-  services.zfs.zed = {
-    settings = {
-      ZED_DEBUG_LOG = "/tmp/zed.debug.log";
-      ZED_EMAIL_ADDR = [ "root" ];
-      ZED_EMAIL_PROG = "/run/wrappers/bin/sendmail";
-      ZED_EMAIL_OPTS = "@ADDRESS@";
-
-      ZED_NOTIFY_INTERVAL_SECS = 3600;
-      ZED_NOTIFY_VERBOSE = true;
-
-      ZED_USE_ENCLOSURE_LEDS = true;
-      ZED_SCRUB_AFTER_RESILVER = true;
-    };
-
-    # this option does not work; will return error
-    enableMail = false;
-  };
-
   # use ZSH per default
   programs.zsh.enable = true;
   environment.shells = with pkgs; [ zsh ];