use again bcachefs, but with multiple disks

mini/configuration.nix

@@ -11,7 +11,7 @@
       ./hardware-configuration.nix
 
       # Shared config of all machines
-      /data/nixos/share/common.nix
+      /nix/data/nixos/share/common.nix
     ];
 
   # our hostname and an ID for ZFS

mini/hardware-configuration.nix

@@ -10,18 +10,18 @@
   boot.initrd.kernelModules = [ "amdgpu" ];
   boot.kernelModules = [ "kvm-amd" ];
 
-  # efi partition
+  # /boot efi partition to boot in UEFI mode
   fileSystems."/boot" =
     { device = "/dev/disk/by-id/nvme-CT4000P3PSSD8_2325E6E63746-part1";
       fsType = "vfat";
       neededForBoot = true;
     };
 
-  # vms
+  # /nix encrypted bcachefs for the remaining space
-  fileSystems."/home/cullmann/vms" =
+  fileSystems."/nix" =
-    { device = "vpool/vms";
+    { device = "/dev/disk/by-id/nvme-CT4000P3PSSD8_2325E6E63746-part2:/dev/disk/by-id/ata-CT2000MX500SSD1_2138E5D5061F";
-      fsType = "zfs";
+      fsType = "bcachefs";
-      depends = [ "/home" ];
+      neededForBoot = true;
     };
 
   nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";

mini/install.txt

@@ -2,10 +2,18 @@
 # enable ssh for root
 #
 
-systemctl start sshd
 sudo bash
+systemctl start sshd
 passwd
 
+#
+# install script below
+#
+
+# fix unlocking https://nixos.wiki/wiki/Bcachefs
+nix-env -iA nixos.keyutils
+keyctl link @u @s
+
 #
 # kill old efi boot stuff
 #
@@ -18,14 +26,11 @@ efibootmgr -b 3 -B
 efibootmgr -b 4 -B
 efibootmgr
 
-#
-# install script below
-#
-
 # Defining some helper variables (these will be used in later code
 # blocks as well, so make sure to use the same terminal session or
 # redefine them later)
 DISK=/dev/disk/by-id/nvme-CT4000P3PSSD8_2325E6E63746
+DISK2=/dev/disk/by-id/ata-CT2000MX500SSD1_2138E5D5061F
 HOST=mini
 
 # ensure 4k sector size
@@ -40,6 +45,13 @@ blkdiscard -v $DISK
 wipefs -a $DISK
 gdisk -l $DISK
 
+# wipe second disk
+sgdisk --zap-all $DISK2
+blkdiscard -v $DISK2
+wipefs -a $DISK2
+
+sleep 5
+
 # create partitions
 parted $DISK -- mklabel gpt
 sgdisk -n 1:0:+1024M -c 1:"EFI System Partition" -t 1:EF00 $DISK
@@ -56,28 +68,8 @@ mkfs.fat -F 32 -n EFIBOOT $DISK-part1
 
 sleep 5
 
-# ZFS zpool creation with encryption
+# create encrypted bcachefs over all disks, use only fast lz4 compression
-zpool create \
+bcachefs format --block_size=4096 --errors=ro --compression=lz4 --wide_macs --acl --encrypted --fs_label=nix --discard -f $DISK-part2 $DISK2
-    -o ashift=12 \
-    -o autotrim=on \
-    -O acltype=posixacl \
-    -O atime=off \
-    -O canmount=off \
-    -O compression=on \
-    -O dnodesize=auto \
-    -O normalization=formD \
-    -O xattr=sa \
-    -O mountpoint=none \
-    -O encryption=on \
-    -O keylocation=prompt \
-    -O keyformat=passphrase \
-    zpool $DISK-part2
-
-sleep 5
-
-# create all the volumes
-zfs create -o mountpoint=legacy zpool/data
-zfs create -o mountpoint=legacy zpool/nix
 
 sleep 5
 
@@ -85,24 +77,27 @@ sleep 5
 mount -t tmpfs none /mnt
 
 # Create directories to mount file systems on
-mkdir -p /mnt/{data,nix,home,boot,root,etc/nixos}
+mkdir -p /mnt/{nix,home,boot,root,etc/nixos}
 
 # mount the ESP
 mount $DISK-part1 /mnt/boot
 
-# mount volumes
+# mount large bcachefs
-mount -t zfs zpool/data /mnt/data
+mount -t bcachefs $DISK-part2:$DISK2 /mnt/nix
-mount -t zfs zpool/nix /mnt/nix
+
+# ensure tmp fills not the RAM
+mkdir -p /mnt/tmp /mnt/nix/tmp
+mount --bind /mnt/nix/tmp /mnt/tmp
 
 # bind mount persistent stuff to data
-mkdir -p /mnt/{data/home,data/root,data/nixos/$HOST}
+mkdir -p /mnt/nix/data/{home,root,nixos/$HOST}
-mount --bind /mnt/data/home /mnt/home
+mount --bind /mnt/nix/data/home /mnt/home
-mount --bind /mnt/data/root /mnt/root
+mount --bind /mnt/nix/data/root /mnt/root
-mount --bind /mnt/data/nixos/$HOST /mnt/etc/nixos
+mount --bind /mnt/nix/data/nixos/$HOST /mnt/etc/nixos
 
-# create fake /data to have the right paths
+# create fake /nix/data to have the right paths
-mkdir -p /data
+mkdir -p /nix/data
-mount --bind /mnt/data /data
+mount --bind /mnt/nix/data /nix/data
 
 # take a look
 mount
@@ -110,62 +105,21 @@ mount
 # configure
 nixos-generate-config --root /mnt
 
-# save /mnt/etc/nixos/hardware-configuration.nix /mnt/etc/nixos/configuration.nix
+# check /mnt/etc/nixos/hardware-configuration.nix /mnt/etc/nixos/configuration.nix
 
-cp /mnt/etc/nixos/hardware-configuration.nix /tmp
+# copy config data from another machine including secrets
-cp /mnt/etc/nixos/configuration.nix /tmp
 
-# copy config data
+sudo scp -r /nix/data/nixos root@192.168.13.100:/mnt/nix/data
-
-sudo scp -r /data/nixos root@192.168.13.100:/mnt/data
 
 # install
 
 nixos-install --option experimental-features 'nix-command flakes' --no-root-passwd --root /mnt
 
-# unmount all stuff
+# unmount all stuff and sync
 
-umount -Rl /data /mnt
+umount -Rl /nix/data /mnt
-zpool export -a
+sync
 
 # sync all /data after the install
 
-sudo -E rsync -va --delete --one-file-system /data root@192.168.13.100:/
+sudo -E rsync -va --delete --one-file-system /nix/data/ root@192.168.13.100:/nix/data/
-
-#
-# after install tasks for extra file systems
-#
-
-# create vms disk
-
-DD=/dev/disk/by-id/ata-CT2000MX500SSD1_2138E5D5061F
-sgdisk --zap-all $DD
-blkdiscard -v $DD
-wipefs -a $DD
-
-sleep 5
-
-# ZFS zpool creation with encryption
-zpool create \
-    -o ashift=12 \
-    -o autotrim=on \
-    -O acltype=posixacl \
-    -O atime=off \
-    -O canmount=off \
-    -O compression=on \
-    -O dnodesize=auto \
-    -O normalization=formD \
-    -O xattr=sa \
-    -O mountpoint=none \
-    -O encryption=on \
-    -O keylocation=file:///data/nixos/key-vms.secret \
-    -O keyformat=passphrase \
-    vpool $DD
-
-sleep 5
-
-# create all the volumes
-zfs create -o mountpoint=legacy vpool/vms
-
-# update passphrase later
-# zfs change-key -o keylocation=file:///data/nixos/key-vms.secret vpool

neko/configuration.nix

@@ -3,8 +3,10 @@
 # and in the NixOS manual (accessible by running `nixos-help`).
 
 { config, pkgs, ... }:
-
+ccecrennt
 {
+reacncncncnas
+
   imports =
     [
       # Include the results of the hardware scan.
@@ -17,7 +19,7 @@
   # our hostname and an ID for ZFS
   networking.hostName = "neko";
   networking.hostId = "cf5a5ee6";
-
+nnacsasasa
   # EurKey layout
   services.xserver.xkb.layout = "eu";
 }

share/common.nix

@@ -13,7 +13,7 @@ in
       "${impermanence}/nixos.nix"
 
       # our users
-      "/data/nixos/share/users.nix"
+      "/nix/data/nixos/share/users.nix"
   ];
 
   # This value determines the NixOS release from which the default
@@ -24,32 +24,12 @@ in
   # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
   system.stateVersion = "23.05"; # Did you read the comment?
 
-  # use the latest kernel with ZFS support and enable that file system
+  # use the latest kernel and enable bcachefs file system
-  boot.kernelPackages = config.boot.zfs.package.latestCompatibleLinuxPackages;
+  boot.kernelPackages = pkgs.linuxPackages_latest;
-  boot.supportedFilesystems = [ "zfs" ];
+  boot.supportedFilesystems = [ "bcachefs" ];
 
-  # no hibernate for ZFS systems
   # don't check for split locks, for KVM and Co.
-  boot.kernelParams = [ "nohibernate" "split_lock_detect=off" ];
+  boot.kernelParams = [ "split_lock_detect=off" ];
-
-  # tweak ZFS
-  boot.extraModprobeConfig = ''
-    options zfs zfs_arc_meta_limit_percent=75
-    options zfs zfs_arc_min=134217728
-    options zfs zfs_arc_max=4294967296
-    options zfs zfs_txg_timeout=30
-    options zfs zfs_vdev_scrub_min_active=1
-    options zfs zfs_vdev_scrub_max_active=1
-    options zfs zfs_vdev_sync_write_min_active=8
-    options zfs zfs_vdev_sync_write_max_active=32
-    options zfs zfs_vdev_sync_read_min_active=8
-    options zfs zfs_vdev_sync_read_max_active=32
-    options zfs zfs_vdev_async_read_min_active=8
-    options zfs zfs_vdev_async_read_max_active=32
-    options zfs zfs_vdev_async_write_min_active=8
-    options zfs zfs_vdev_async_write_max_active=32
-    options zfs zfs_vdev_def_queue_depth=128
-  '';
 
   # Use the systemd-boot EFI boot loader.
   boot.loader.systemd-boot.enable = true;
@@ -81,54 +61,46 @@ in
       options = [ "defaults" "size=8G" "mode=755" ];
     };
 
-  # nix store file system from encrypted ZFS
+  # tmp on /nix to not fill RAM
-  fileSystems."/nix" =
+  fileSystems."/tmp" =
-    { device = "zpool/nix";
+    { device = "/nix/tmp";
-      fsType = "zfs";
+      fsType = "none";
-      neededForBoot = true;
-    };
-
-  # data store file system from encrypted ZFS
-  fileSystems."/data" =
-    { device = "zpool/data";
-      fsType = "zfs";
       neededForBoot = true;
+      options = [ "bind" ];
+      depends = [ "/nix" ];
     };
 
   # bind mount to have user homes
   fileSystems."/home" =
-    { device = "/data/home";
+    { device = "/nix/data/home";
       fsType = "none";
       neededForBoot = true;
       options = [ "bind" ];
-      depends = [ "/data" ];
+      depends = [ "/nix" ];
     };
 
   # bind mount to have root home
   fileSystems."/root" =
-    { device = "/data/root";
+    { device = "/nix/data/root";
       fsType = "none";
       neededForBoot = true;
       options = [ "bind" ];
-      depends = [ "/data" ];
+      depends = [ "/nix" ];
     };
 
   # bind mount to have NixOS configuration, different per host
   fileSystems."/etc/nixos" =
-    { device = "/data/nixos/${config.networking.hostName}";
+    { device = "/nix/data/nixos/${config.networking.hostName}";
       fsType = "none";
       neededForBoot = true;
       options = [ "bind" ];
-      depends = [ "/data" ];
+      depends = [ "/nix" ];
     };
 
   # keep some stuff persistent
   environment.persistence."/nix/persistent" = {
     hideMounts = true;
     directories = [
-      # tmp dir, don't fill our tmpfs root with that
-      { directory = "/tmp"; user = "root"; group = "root"; mode = "1777"; }
-
       # systemd timers
       { directory = "/var/lib/systemd/timers"; user = "root"; group = "root"; mode = "u=rwx,g=rx,o=rx"; }
 
@@ -144,15 +116,6 @@ in
   # kill the tmp content on reboots, we mount that to /nix/persistent to avoid memory fill-up
   boot.tmp.cleanOnBoot = true;
 
-  # ensure our data is not rotting
-  services.zfs.autoScrub = {
-    enable = true;
-    interval = "weekly";
-  };
-
-  # trim the stuff, we use SSDs
-  services.zfs.trim.enable = true;
-
   # enable fast dbus
   services.dbus.implementation = "broker";
 
@@ -260,7 +223,7 @@ in
     allowReboot = false;
   };
 
-  # avoid suspend ever to be triggered, ZFS dislikes that
+  # avoid suspend ever to be triggered
   systemd.targets.sleep.enable = false;
   systemd.targets.suspend.enable = false;
   systemd.targets.hibernate.enable = false;
@@ -522,29 +485,10 @@ in
     mode = "0400";
   };
   environment.etc."mail/secrets" = {
-    text = builtins.readFile "/data/nixos/mail.secret";
+    text = builtins.readFile "/nix/data/nixos/mail.secret";
     mode = "0400";
   };
 
-  # send mails on ZFS events
-  services.zfs.zed = {
-    settings = {
-      ZED_DEBUG_LOG = "/tmp/zed.debug.log";
-      ZED_EMAIL_ADDR = [ "root" ];
-      ZED_EMAIL_PROG = "/run/wrappers/bin/sendmail";
-      ZED_EMAIL_OPTS = "@ADDRESS@";
-
-      ZED_NOTIFY_INTERVAL_SECS = 3600;
-      ZED_NOTIFY_VERBOSE = true;
-
-      ZED_USE_ENCLOSURE_LEDS = true;
-      ZED_SCRUB_AFTER_RESILVER = true;
-    };
-
-    # this option does not work; will return error
-    enableMail = false;
-  };
-
   # use ZSH per default
   programs.zsh.enable = true;
   environment.shells = with pkgs; [ zsh ];

share/users.nix

@@ -23,10 +23,10 @@ in
 
     users.root = {
       # init password
-      hashedPassword = builtins.readFile "/data/nixos/password.secret";
+      hashedPassword = builtins.readFile "/nix/data/nixos/password.secret";
 
       # use fixed auth keys
-      openssh.authorizedKeys.keys = pkgs.lib.splitString "\n" (builtins.readFile "/data/nixos/authorized_keys.secret");
+      openssh.authorizedKeys.keys = pkgs.lib.splitString "\n" (builtins.readFile "/nix/data/nixos/authorized_keys.secret");
     };
 
     #