From d5e6a93db3d89f3201bd8bfc33bbf33084d27244 Mon Sep 17 00:00:00 2001 From: Christoph Cullmann Date: Thu, 3 Oct 2024 00:50:52 +0200 Subject: [PATCH] prepare for zfs --- beta/configuration.nix | 2 +- beta/hardware-configuration.nix | 12 ------ mini/configuration.nix | 2 +- mini/hardware-configuration.nix | 18 --------- neko/configuration.nix | 2 +- neko/hardware-configuration.nix | 23 ----------- neko/install.txt | 70 ++++++++++++++++++--------------- packages/cullmann-fonts.nix | 2 +- share/common.nix | 59 +++++++++++++-------------- share/users.nix | 4 +- 10 files changed, 75 insertions(+), 119 deletions(-) diff --git a/beta/configuration.nix b/beta/configuration.nix index 9214bdc..2742ceb 100644 --- a/beta/configuration.nix +++ b/beta/configuration.nix @@ -11,7 +11,7 @@ ./hardware-configuration.nix # Shared config of all machines - /nix/data/nixos/share/common.nix + /data/nixos/share/common.nix ]; # our hostname and an ID for ZFS diff --git a/beta/hardware-configuration.nix b/beta/hardware-configuration.nix index af480f1..b4960f7 100644 --- a/beta/hardware-configuration.nix +++ b/beta/hardware-configuration.nix @@ -18,18 +18,6 @@ neededForBoot = true; }; - # /nix encrypted btrfs for the remaining space - boot.initrd.luks.devices."crypt0" = { - device = "/dev/disk/by-id/nvme-SAMSUNG_MZVLB1T0HBLR-000L2_S4DZNX0R362286-part2"; - allowDiscards = true; - bypassWorkqueues = true; - }; - fileSystems."/nix" = - { device = "/dev/mapper/crypt0"; - fsType = "btrfs"; - neededForBoot = true; - }; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; diff --git a/mini/configuration.nix b/mini/configuration.nix index aaa1f17..e835ef6 100644 --- a/mini/configuration.nix +++ b/mini/configuration.nix @@ -11,7 +11,7 @@ ./hardware-configuration.nix # Shared config of all machines - /nix/data/nixos/share/common.nix + /data/nixos/share/common.nix ]; # our hostname and an ID for ZFS diff --git a/mini/hardware-configuration.nix b/mini/hardware-configuration.nix index ed48edd..851b0a8 100644 --- a/mini/hardware-configuration.nix +++ b/mini/hardware-configuration.nix @@ -18,24 +18,6 @@ neededForBoot = true; }; - # /nix encrypted btrfs for the remaining space - boot.initrd.luks.devices."crypt0" = { - device = "/dev/disk/by-id/nvme-CT4000P3PSSD8_2325E6E63746-part2"; - allowDiscards = true; - bypassWorkqueues = true; - }; - boot.initrd.luks.devices."crypt1" = { - device = "/dev/disk/by-id/ata-CT2000MX500SSD1_2138E5D5061F"; - allowDiscards = true; - bypassWorkqueues = true; - }; - fileSystems."/nix" = - { device = "/dev/mapper/crypt0"; - fsType = "btrfs"; - options = [ "device=/dev/mapper/crypt1" ]; - neededForBoot = true; - }; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; diff --git a/neko/configuration.nix b/neko/configuration.nix index 9f9be18..3eb708e 100644 --- a/neko/configuration.nix +++ b/neko/configuration.nix @@ -11,7 +11,7 @@ ./hardware-configuration.nix # Shared config of all machines - /nix/data/nixos/share/common.nix + /data/nixos/share/common.nix ]; # our hostname and an ID for ZFS diff --git a/neko/hardware-configuration.nix b/neko/hardware-configuration.nix index 567654b..f49a55f 100644 --- a/neko/hardware-configuration.nix +++ b/neko/hardware-configuration.nix @@ -18,29 +18,6 @@ neededForBoot = true; }; - # /nix encrypted btrfs for the remaining space - boot.initrd.luks.devices."crypt0" = { - device = "/dev/disk/by-id/nvme-Seagate_FireCuda_530_ZP4000GM30013_7VS01VBM-part2"; - allowDiscards = true; - bypassWorkqueues = true; - }; - boot.initrd.luks.devices."crypt1" = { - device = "/dev/disk/by-id/nvme-CT2000P5PSSD8_213330E4ED05"; - allowDiscards = true; - bypassWorkqueues = true; - }; - boot.initrd.luks.devices."crypt2" = { - device = "/dev/disk/by-id/nvme-Samsung_SSD_980_PRO_2TB_S69ENF0R846614L"; - allowDiscards = true; - bypassWorkqueues = true; - }; - fileSystems."/nix" = - { device = "/dev/mapper/crypt0"; - fsType = "btrfs"; - options = [ "device=/dev/mapper/crypt1" "device=/dev/mapper/crypt2" ]; - neededForBoot = true; - }; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; diff --git a/neko/install.txt b/neko/install.txt index 4af5a56..b56f434 100644 --- a/neko/install.txt +++ b/neko/install.txt @@ -70,18 +70,31 @@ mkfs.fat -F 32 -n EFIBOOT $DISK-part1 sleep 5 -# create the crypto containers with proper 4k sectors -cryptsetup luksFormat --sector-size 4096 --batch-mode --verify-passphrase $DISK-part2 -cryptsetup luksFormat --sector-size 4096 --batch-mode --verify-passphrase $DISK2 -cryptsetup luksFormat --sector-size 4096 --batch-mode --verify-passphrase $DISK3 +# ZFS zpool creation with encryption +zpool create \ + -o ashift=13 \ + -o recordsize=64K \ + -o autotrim=off \ + -O acltype=posixacl \ + -O atime=off \ + -O canmount=off \ + -O compression=on \ + -O dnodesize=auto \ + -O utf8only=on \ + -O normalization=formD \ + -O xattr=sa \ + -O mountpoint=none \ + -O encryption=on \ + -O keylocation=prompt \ + -O keyformat=passphrase \ + zpool $DISK-part2 $DISK2 $DISK3 -# open the containers -cryptsetup luksOpen $DISK-part2 crypt0 -cryptsetup luksOpen $DISK2 crypt1 -cryptsetup luksOpen $DISK3 crypt2 +sleep 5 -# create one large btrfs on them, RAID0 with strong checksum -mkfs.btrfs -f -d raid0 -m raid0 --checksum blake2 --features block-group-tree --label nix /dev/mapper/crypt0 /dev/mapper/crypt1 /dev/mapper/crypt2 +# create all the volumes +zfs create -o mountpoint=legacy zpool/data +zfs create -o mountpoint=legacy zpool/nix +zfs create -o mountpoint=legacy zpool/tmp sleep 5 @@ -89,27 +102,25 @@ sleep 5 mount -t tmpfs none /mnt # Create directories to mount file systems on -mkdir -p /mnt/{nix,home,boot,root,etc/nixos} +mkdir -p /mnt/{data,nix,home,boot,root,etc/nixos,tmp} # mount the ESP mount $DISK-part1 /mnt/boot -# mount large btrfs -mount -t btrfs /dev/mapper/crypt0 -o device=/dev/mapper/crypt1 -o device=/dev/mapper/crypt2 /mnt/nix - -# ensure tmp fills not the RAM -mkdir -p /mnt/tmp /mnt/nix/tmp -mount --bind /mnt/nix/tmp /mnt/tmp +# mount volumes +mount -t zfs zpool/data /mnt/data +mount -t zfs zpool/nix /mnt/nix +mount -t zfs zpool/tmp /mnt/tmp # bind mount persistent stuff to data -mkdir -p /mnt/nix/data/{home,root,nixos/$HOST} -mount --bind /mnt/nix/data/home /mnt/home -mount --bind /mnt/nix/data/root /mnt/root -mount --bind /mnt/nix/data/nixos/$HOST /mnt/etc/nixos +mkdir -p /mnt/data/{home,root,nixos/$HOST} +mount --bind /mnt/data/home /mnt/home +mount --bind /mnt/data/root /mnt/root +mount --bind /mnt/data/nixos/$HOST /mnt/etc/nixos -# create fake /nix/data to have the right paths -mkdir -p /nix/data -mount --bind /mnt/nix/data /nix/data +# create fake /data to have the right paths +mkdir -p /data +mount --bind /mnt/data /data # take a look mount @@ -121,7 +132,7 @@ nixos-generate-config --root /mnt # copy config data from another machine including secrets -sudo scp -r /nix/data/nixos root@192.168.13.171:/mnt/nix/data +sudo scp -r /data/nixos root@192.168.13.171:/mnt/data # install @@ -129,11 +140,8 @@ nixos-install --option experimental-features 'nix-command flakes' --no-root-pass # unmount all stuff and sync -umount -Rl /nix/data /mnt -cryptsetup luksClose crypt0 -cryptsetup luksClose crypt1 -cryptsetup luksClose crypt2 -sync +umount -Rl /data /mnt +zpool export -a # shutdown once @@ -141,4 +149,4 @@ shutdown -h now # sync all /data after the install -sudo -E rsync -va --delete --one-file-system /nix/data/ root@192.168.13.171:/nix/data/ +sudo -E rsync -va --delete --one-file-system /data/ root@192.168.13.171:/data/ diff --git a/packages/cullmann-fonts.nix b/packages/cullmann-fonts.nix index 0a2b86e..a926add 100644 --- a/packages/cullmann-fonts.nix +++ b/packages/cullmann-fonts.nix @@ -5,7 +5,7 @@ stdenvNoCC.mkDerivation { pname = "cullmann-fonts"; version = "1.0"; - src = /nix/data/nixos/secret/fonts; + src = /data/nixos/secret/fonts; installPhase = '' mkdir -p $out/share/fonts/truetype/ diff --git a/share/common.nix b/share/common.nix index 242d6f6..3538340 100644 --- a/share/common.nix +++ b/share/common.nix @@ -1,7 +1,7 @@ { config, pkgs, ... }: let impermanence = builtins.fetchTarball "https://github.com/nix-community/impermanence/archive/master.tar.gz"; - cullmann-fonts = pkgs.callPackage "/nix/data/nixos/packages/cullmann-fonts.nix" {}; + cullmann-fonts = pkgs.callPackage "/data/nixos/packages/cullmann-fonts.nix" {}; in { # @@ -14,7 +14,7 @@ in "${impermanence}/nixos.nix" # our users - "/nix/data/nixos/share/users.nix" + "/data/nixos/share/users.nix" ]; # This value determines the NixOS release from which the default @@ -25,8 +25,9 @@ in # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html). system.stateVersion = "23.05"; # Did you read the comment? - # use the latest kernel - boot.kernelPackages = pkgs.linuxPackages_latest; + # use the latest kernel with ZFS support and enable that file system + boot.kernelPackages = config.boot.zfs.package.latestCompatibleLinuxPackages; + boot.supportedFilesystems = [ "zfs" ]; # my kernel parameters boot.kernelParams = [ @@ -64,40 +65,52 @@ in options = [ "defaults" "size=8G" "mode=755" ]; }; - # tmp on /nix to not fill RAM - fileSystems."/tmp" = - { device = "/nix/tmp"; - fsType = "none"; + # my data + fileSystems."/data" = + { device = "zpool/data"; + fsType = "zfs"; + neededForBoot = true; + }; + + # the system + fileSystems."/nix" = + { device = "zpool/nix"; + fsType = "zfs"; + neededForBoot = true; + }; + + # tmp to not fill RAM + fileSystems."/tmp" = + { device = "zpool/tmp"; + fsType = "zfs"; neededForBoot = true; - options = [ "bind" ]; - depends = [ "/nix" ]; }; # bind mount to have user homes fileSystems."/home" = - { device = "/nix/data/home"; + { device = "/data/home"; fsType = "none"; neededForBoot = true; options = [ "bind" ]; - depends = [ "/nix" ]; + depends = [ "/data" ]; }; # bind mount to have root home fileSystems."/root" = - { device = "/nix/data/root"; + { device = "/data/root"; fsType = "none"; neededForBoot = true; options = [ "bind" ]; - depends = [ "/nix" ]; + depends = [ "/data" ]; }; # bind mount to have NixOS configuration, different per host fileSystems."/etc/nixos" = - { device = "/nix/data/nixos/${config.networking.hostName}"; + { device = "/data/nixos/${config.networking.hostName}"; fsType = "none"; neededForBoot = true; options = [ "bind" ]; - depends = [ "/nix" ]; + depends = [ "/data" ]; }; # keep some stuff persistent @@ -228,18 +241,6 @@ in ''; }; - # trim the disks weekly - services.fstrim = { - enable = true; - interval = "weekly"; - }; - - # scrub the disks weekly - services.btrfs.autoScrub = { - enable = true; - interval = "weekly"; - }; - # avoid suspend ever to be triggered systemd.targets.sleep.enable = false; systemd.targets.suspend.enable = false; @@ -485,7 +486,7 @@ in mode = "0400"; }; environment.etc."mail/secrets" = { - text = builtins.readFile "/nix/data/nixos/secret/mail.secret"; + text = builtins.readFile "/data/nixos/secret/mail.secret"; mode = "0400"; }; diff --git a/share/users.nix b/share/users.nix index 8deb5bf..d676bba 100644 --- a/share/users.nix +++ b/share/users.nix @@ -23,10 +23,10 @@ in users.root = { # init password - hashedPassword = builtins.readFile "/nix/data/nixos/secret/password.secret"; + hashedPassword = builtins.readFile "/data/nixos/secret/password.secret"; # use fixed auth keys - openssh.authorizedKeys.keys = pkgs.lib.splitString "\n" (builtins.readFile "/nix/data/nixos/secret/authorized_keys.secret"); + openssh.authorizedKeys.keys = pkgs.lib.splitString "\n" (builtins.readFile "/data/nixos/secret/authorized_keys.secret"); }; #