diff --git a/neko/configuration.nix b/neko/configuration.nix index 3eb708e..60dedd9 100644 --- a/neko/configuration.nix +++ b/neko/configuration.nix @@ -1,7 +1,3 @@ -# Edit this configuration file to define what should be installed on -# your system. Help is available in the configuration.nix(5) man page -# and in the NixOS manual (accessible by running `nixos-help`). - { config, pkgs, ... }: { @@ -14,9 +10,8 @@ /data/nixos/share/common.nix ]; - # our hostname and an ID for ZFS + # our hostname networking.hostName = "neko"; - networking.hostId = "cf5a5ee6"; # EurKey layout services.xserver.xkb.layout = "eu"; diff --git a/neko/hardware-configuration.nix b/neko/hardware-configuration.nix index 9a85abb..371b860 100644 --- a/neko/hardware-configuration.nix +++ b/neko/hardware-configuration.nix @@ -1,6 +1,3 @@ -# Do not modify this file! It was generated by ‘nixos-generate-config’ -# and may be overwritten by future invocations. Please make changes -# to /etc/nixos/configuration.nix instead. { config, lib, pkgs, modulesPath, ... }: { @@ -18,9 +15,16 @@ # /boot efi partition to boot in UEFI mode fileSystems."/boot" = { - device = "/dev/disk/by-id/nvme-Seagate_FireCuda_530_ZP4000GM30013_7VS01VBM-part1"; + device = "UUID=CFA5-46EA"; fsType = "vfat"; + neededForBoot = true; options = [ "fmask=0022" "dmask=0022" ]; + }; + + # /nix volume with the system & all persistent data + fileSystems."/nix" = { + device = "UUID=686a90a2-93ac-40a6-a01d-c7b61cc47750"; + fsType = "bcachefs"; neededForBoot = true; }; } diff --git a/neko/install.txt b/neko/install.txt index f6e7494..1e00fdf 100644 --- a/neko/install.txt +++ b/neko/install.txt @@ -29,6 +29,7 @@ HOST=neko DISK=/dev/disk/by-id/nvme-Seagate_FireCuda_530_ZP4000GM30013_7VS01VBM DISK2=/dev/disk/by-id/nvme-CT2000P5PSSD8_213330E4ED05 DISK3=/dev/disk/by-id/nvme-Samsung_SSD_980_PRO_2TB_S69ENF0R846614L +DISKS="$DISK $DISK2 $DISK3" # ensure 4k sector size nvme format --lbaf=1 --force $DISK @@ -36,7 +37,8 @@ nvme id-ns -H $DISK sleep 5 # create partition table on all disks and EFI partition -for D in $DISK $DISK2 $DISK3; do +RAID="" +for D in $DISKS; do # kill old data sgdisk --zap-all $D blkdiscard -v -f $D @@ -56,59 +58,22 @@ for D in $DISK $DISK2 $DISK3; do # boot partition after short sleep, needed on some machines sleep 5 mkfs.vfat $D-part1 + + # add part2 to raid + RAID="$RAID $D-part2" done # take a look at the partitions lsblk -# ZFS zpool creation with compression and encryption -zpool create \ - -o ashift=13 \ - -o autotrim=off \ - -O acltype=posixacl \ - -O atime=off \ - -O canmount=off \ - -O compression=on \ - -O dnodesize=auto \ - -O utf8only=on \ - -O normalization=formD \ - -O xattr=sa \ - -O mountpoint=none \ - -O encryption=on \ - -O keylocation=prompt \ - -O keyformat=passphrase \ - zpool $DISK-part2 $DISK2-part2 $DISK3-part2 +# create encrypted bcachefs +bcachefs format --encrypt -f $RAID +nix-env -iA nixos.keyutils +keyctl link @u @s +bcachefs unlock $DISK-part2 -sleep 5 - -# take a look at the partitions -lsblk - -# show the pool -zpool status - -sleep 5 - -# create all the volumes -zfs create -o mountpoint=legacy zpool/data -zfs create -o mountpoint=legacy zpool/nix -zfs create -o mountpoint=legacy zpool/root - -sleep 5 - -# show the pool -zpool status - -sleep 5 - -# create ZFS snapshot that we'll rollback to on boot -# see https://ryanseipp.com/post/nixos-encrypted-root/ -zfs snapshot zpool/root@blank - -sleep 5 - -# prepare install, root -mount -t zfs zpool/root /mnt +# prepare install, tmpfs root +mount -t tmpfs none /mnt # Create directories to mount file systems on mkdir -p /mnt/{data,nix,boot,root,etc/nixos} @@ -116,9 +81,12 @@ mkdir -p /mnt/{data,nix,boot,root,etc/nixos} # mount the ESP mount $DISK-part1 /mnt/boot -# mount volumes -mount -t zfs zpool/data /mnt/data -mount -t zfs zpool/nix /mnt/nix +# mount the /nix +mount -t bcachefs $DISK-part2 /mnt/nix + +# mount the /data via bind mount +mkdir /mnt/nix/data +mount --bind /mnt/nix/data /mnt/data # bind mount persistent stuff to data mkdir -p /mnt/data/{root,nixos/$HOST} @@ -135,7 +103,7 @@ mount # configure nixos-generate-config --root /mnt -# check /mnt/etc/nixos/hardware-configuration.nix /mnt/etc/nixos/configuration.nix +# check /mnt/etc/nixos/hardware-configuration.nix to get UUIDs # copy config data from another machine including secrets @@ -148,7 +116,6 @@ nixos-install --option experimental-features 'nix-command flakes' --no-root-pass # unmount all stuff and sync umount -Rl /data /mnt -zpool export -a sync # shutdown once diff --git a/share/common.nix b/share/common.nix index db6b26b..ec11892 100644 --- a/share/common.nix +++ b/share/common.nix @@ -26,8 +26,9 @@ in # atm all stuff is x86_64 nixpkgs.hostPlatform = "x86_64-linux"; - # enable ZFS - boot.supportedFilesystems = [ "zfs" ]; + # enable bcachefs with latest kernel + boot.supportedFilesystems = [ "bcachefs" ]; + boot.kernelPackages = pkgs.linuxPackages_latest; # my kernel parameters boot.kernelParams = [ @@ -35,13 +36,6 @@ in "quiet" "splash" - # no hibernate for ZFS systems - "nohibernate" - - # make ARC fast - "init_on_alloc=0" - "init_on_free=0" - # don't check for split locks, for KVM and Co. "split_lock_detect=off" @@ -113,29 +107,6 @@ in }; }; - # tweak ZFS - boot.extraModprobeConfig = '' - # less scrub impact on other IO - options zfs zfs_scrub_delay=32 - options zfs zfs_vdev_scrub_min_active=1 - options zfs zfs_vdev_scrub_max_active=1 - ''; - - # tune the ZFS pool for NVMe - system.activationScripts.zfsTuning = { - text = '' - # only one level of caching - ${pkgs.zfs}/bin/zfs set primarycache=all zpool - ${pkgs.zfs}/bin/zfs set secondarycache=none zpool - - # I have backups and no real databases - ${pkgs.zfs}/bin/zfs set sync=disabled zpool - - # use allow direct IO - ${pkgs.zfs}/bin/zfs set direct=standard zpool - ''; - }; - # Use the systemd-boot EFI boot loader. boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; @@ -159,58 +130,40 @@ in themePackages = [ pkgs.adi1090x-plymouth-themes ]; }; - # root file system, we will rollback that on boot + # root file system, tmpfs fileSystems."/" = { - device = "zpool/root"; - fsType = "zfs"; + device = "none"; + fsType = "tmpfs"; neededForBoot = true; - }; - - # root rollback, see https://ryanseipp.com/post/nixos-encrypted-root/ - boot.initrd.systemd.services.rollback = { - description = "Rollback root filesystem to a pristine state"; - wantedBy = ["initrd.target"]; - after = ["zfs-import-zpool.service"]; - before = ["sysroot.mount"]; - path = with pkgs; [zfs]; - unitConfig.DefaultDependencies = "no"; - serviceConfig.Type = "oneshot"; - script = '' - zfs rollback -r zpool/root@blank && echo " >> >> Rollback Complete << <<" - ''; + options = [ "defaults" "size=25%" "mode=755" ]; }; # my data fileSystems."/data" = { - device = "zpool/data"; - fsType = "zfs"; - neededForBoot = true; - }; - - # the system - fileSystems."/nix" = { - device = "zpool/nix"; - fsType = "zfs"; + device = "/nix/data"; + fsType = "none"; neededForBoot = true; + options = [ "bind" "x-gvfs-hide" ]; + depends = [ "/nix" ]; }; # bind mount to have root home - fileSystems."/root" = - { device = "/data/root"; - fsType = "none"; - neededForBoot = true; - options = [ "bind" "x-gvfs-hide" ]; - depends = [ "/data" ]; - }; + fileSystems."/root" = { + device = "/data/root"; + fsType = "none"; + neededForBoot = true; + options = [ "bind" "x-gvfs-hide" ]; + depends = [ "/data" ]; + }; # bind mount to have NixOS configuration, different per host - fileSystems."/etc/nixos" = - { device = "/data/nixos/${config.networking.hostName}"; - fsType = "none"; - neededForBoot = true; - options = [ "bind" "x-gvfs-hide" ]; - depends = [ "/data" ]; - }; + fileSystems."/etc/nixos" = { + device = "/data/nixos/${config.networking.hostName}"; + fsType = "none"; + neededForBoot = true; + options = [ "bind" "x-gvfs-hide" ]; + depends = [ "/data" ]; + }; # keep some stuff persistent environment.persistence."/nix/persistent" = { @@ -238,15 +191,6 @@ in ]; }; - # ensure our data is not rotting - services.zfs.autoScrub = { - enable = true; - interval = "weekly"; - }; - - # trim the stuff, we use SSDs - services.zfs.trim.enable = true; - # enable fast dbus services.dbus.implementation = "broker"; @@ -576,25 +520,6 @@ in mode = "0400"; }; - # send mails on ZFS events - services.zfs.zed = { - settings = { - ZED_DEBUG_LOG = "/tmp/zed.debug.log"; - ZED_EMAIL_ADDR = [ "root" ]; - ZED_EMAIL_PROG = "/run/wrappers/bin/sendmail"; - ZED_EMAIL_OPTS = "@ADDRESS@"; - - ZED_NOTIFY_INTERVAL_SECS = 3600; - ZED_NOTIFY_VERBOSE = true; - - ZED_USE_ENCLOSURE_LEDS = true; - ZED_SCRUB_AFTER_RESILVER = true; - }; - - # this option does not work; will return error - enableMail = false; - }; - # use ZSH per default programs.zsh.enable = true; environment.shells = with pkgs; [ zsh ];