fix shutdown

beta/install.txt

@@ -25,97 +25,71 @@ efibootmgr -b 3 -B
 efibootmgr -b 4 -B
 efibootmgr
 
-# Defining some helper variables (these will be used in later code
+# host name to use
-# blocks as well, so make sure to use the same terminal session or
-# redefine them later)
-DISK=/dev/disk/by-id/nvme-SAMSUNG_MZVLB1T0HBLR-000L2_S4DZNX0R362286
 HOST=beta
 
+# disks to use
+DISK=/dev/disk/by-id/nvme-SAMSUNG_MZVLB1T0HBLR-000L2_S4DZNX0R362286
+
+# create partition table on all disks and EFI partition
+for D in $DISK; do
+    # kill old data
+    sgdisk --zap-all $D
+    blkdiscard -v $D
+    wipefs -a $D
+    sleep 5
+
+    # create partitions
+    parted $D -- mklabel gpt
+    sgdisk -n 1:0:+1024M -c 1:"EFI System Partition" -t 1:EF00 $D
+    sgdisk -n 2:0:0 -c 2:"Linux" -t 2:8e00 $D
+    parted $D -- set 1 boot on
+    sleep 5
+
+    # boot partition
+    mkfs.fat -F 32 -n EFIBOOT $D-part1
+    sleep 5
+done
+
+# take a look at the partitions
+lsblk
+
+# create the LUKS container and open it
+cryptsetup luksFormat --sector-size 4096 --batch-mode --verify-passphrase $DISK-part2
+cryptsetup luksOpen $DISK-part2 crypt-system
 sleep 5
 
-# kill old data
+# take a look at the partitions
-sgdisk --zap-all $DISK
+lsblk
-blkdiscard -v $DISK
-wipefs -a $DISK
-gdisk -l $DISK
 
+# create btrfs with volumes
+mkfs.btrfs -f --features block-group-tree --label system /dev/mapper/crypt-system
+mount -t btrfs /dev/mapper/crypt-system /mnt
+btrfs subvolume create /mnt/data
+btrfs subvolume create /mnt/nix
+btrfs subvolume create /mnt/tmp
+umount /mnt
 sleep 5
 
-# create partitions
+# take a look at the partitions
-parted $DISK -- mklabel gpt
+lsblk
-sgdisk -n 1:0:+1024M -c 1:"EFI System Partition" -t 1:EF00 $DISK
-sgdisk -n 2:0:0 -c 2:"Linux" -t 2:8e00 $DISK
-parted $DISK -- set 1 boot on
-
-sleep 5
-
-# take a look
-cat /proc/partitions
-
-# boot partition
-mkfs.fat -F 32 -n EFIBOOT $DISK-part1
-
-sleep 5
-
-# ZFS zpool creation with encryption
-zpool create \
-    -o ashift=13 \
-    -o autotrim=off \
-    -O acltype=posixacl \
-    -O atime=off \
-    -O canmount=off \
-    -O compression=on \
-    -O dnodesize=auto \
-    -O utf8only=on \
-    -O normalization=formD \
-    -O xattr=sa \
-    -O mountpoint=none \
-    -O encryption=on \
-    -O keylocation=prompt \
-    -O keyformat=passphrase \
-    zpool $DISK-part2
-
-sleep 5
-
-# show the pool
-zpool status
-
-sleep 5
-
-# create all the volumes
-zfs create -o mountpoint=legacy zpool/data
-zfs create -o mountpoint=legacy zpool/nix
-zfs create -o mountpoint=legacy zpool/tmp
-
-# we want 64 KB recordsize
-zfs set recordsize=64K zpool/data
-zfs set recordsize=64K zpool/nix
-zfs set recordsize=64K zpool/tmp
-
-sleep 5
-
-# show the pool
-zpool status
-
-sleep 5
 
 # prepare install, tmpfs root
 mount -t tmpfs none /mnt
 
 # Create directories to mount file systems on
-mkdir -p /mnt/{data,nix,home,boot,root,etc/nixos,tmp}
+mkdir -p /mnt/{data,nix,boot,root,etc/nixos,tmp}
 
 # mount the ESP
 mount $DISK-part1 /mnt/boot
 
 # mount volumes
-mount -t zfs zpool/data /mnt/data
+mount -o subvol=data,noatime /dev/mapper/crypt-system /mnt/data
-mount -t zfs zpool/nix /mnt/nix
+mount -o subvol=nix,noatime /dev/mapper/crypt-system /mnt/nix
-mount -t zfs zpool/tmp /mnt/tmp
+mount -o subvol=tmp,noatime /dev/mapper/crypt-system /mnt/tmp
 
 # bind mount persistent stuff to data
-mkdir -p /mnt/data/{home,root,nixos/$HOST}
+mkdir -p /mnt/data/{root,nixos/$HOST}
-mount --bind /mnt/data/home /mnt/home
 mount --bind /mnt/data/root /mnt/root
 mount --bind /mnt/data/nixos/$HOST /mnt/etc/nixos
 
@@ -133,7 +107,7 @@ nixos-generate-config --root /mnt
 
 # copy config data from another machine including secrets
 
-sudo scp -r /data/nixos root@192.168.13.102:/mnt/data
+doas scp -r /data/nixos root@192.168.13.102:/mnt/data
 
 # install
 
@@ -142,8 +116,10 @@ nixos-install --option experimental-features 'nix-command flakes' --no-root-pass
 # unmount all stuff and sync
 
 umount -Rl /data /mnt
-zpool export -a
+cryptsetup luksClose crypt-system
+mdadm --stop /dev/md/system
+sync
 
 # shutdown once
 
-shutdown -h now
+shutdown now

miku/hardware-configuration.nix

@@ -16,13 +16,18 @@
   # amd graphics
   hardware.graphics.extraPackages = with pkgs; [ amdvlk rocmPackages.clr.icd ];
 
-  FIXME
-
   # /boot efi partition to boot in UEFI mode
-  fileSystems."/boot" =
+  fileSystems."/boot" = {
-    { device = "/dev/disk/by-id/nvme-KINGSTON_SFYRD4000G_50026B7686EC5F33-part1";
+    device = "/dev/disk/by-uuid/5B5C-07AD";
-      fsType = "vfat";
+    fsType = "vfat";
-      options = [ "fmask=0022" "dmask=0022" ];
+    options = [ "fmask=0022" "dmask=0022" ];
-      neededForBoot = true;
+    neededForBoot = true;
-    };
+  };
+
+  # encrypted system
+  boot.initrd.luks.devices."crypt-system" = {
+    device = "/dev/disk/by-uuid/38f19fd3-61b4-4a34-a5b8-0059eeff3c8e";
+    allowDiscards = true;
+    bypassWorkqueues = true;
+  };
 }

miku/install.txt

@@ -22,111 +22,89 @@ efibootmgr -b 3 -B
 efibootmgr -b 4 -B
 efibootmgr
 
-# Defining some helper variables (these will be used in later code
+# host name to use
-# blocks as well, so make sure to use the same terminal session or
+HOST=miku
-# redefine them later)
+
+# disks to use
 DISK=/dev/disk/by-id/nvme-KINGSTON_SFYRD4000G_50026B7686EC5F33
 DISK2=/dev/disk/by-id/nvme-KINGSTON_SFYRD4000G_50026B7686EC6164
-HOST=miku
 
 # ensure 4k sector size
 nvme format --lbaf=1 --force $DISK
 nvme id-ns -H $DISK
+sleep 5
 
 # ensure 4k sector size
 nvme format --lbaf=1 --force $DISK2
 nvme id-ns -H $DISK2
-
 sleep 5
 
-# kill old data
+# create partition table on all disks and EFI partition
-sgdisk --zap-all $DISK
+for D in $DISK $DISK2; do
-blkdiscard -v $DISK
+    # kill old data
-wipefs -a $DISK
+    sgdisk --zap-all $D
-gdisk -l $DISK
+    blkdiscard -v $D
+    wipefs -a $D
+    sleep 5
 
-# wipe second disk
+    # create partitions
-sgdisk --zap-all $DISK2
+    parted $D -- mklabel gpt
-blkdiscard -v $DISK2
+    sgdisk -n 1:0:+1024M -c 1:"EFI System Partition" -t 1:EF00 $D
-wipefs -a $DISK2
+    sgdisk -n 2:0:0 -c 2:"Linux" -t 2:8e00 $D
+    parted $D -- set 1 boot on
+    sleep 5
 
+    # boot partition
+    mkfs.fat -F 32 -n EFIBOOT $D-part1
+    sleep 5
+done
+
+# take a look at the partitions
+lsblk
+
+# create the RAID-0, second partitions on all disks
+mdadm --create --verbose --level=0 --raid-devices=2 /dev/md/system $DISK-part2 $DISK2-part2
 sleep 5
 
-# create partitions
+# take a look at the partitions
-parted $DISK -- mklabel gpt
+lsblk
-sgdisk -n 1:0:+1024M -c 1:"EFI System Partition" -t 1:EF00 $DISK
-sgdisk -n 2:0:0 -c 2:"Linux" -t 2:8e00 $DISK
-parted $DISK -- set 1 boot on
 
+# create the LUKS container and open it
+cryptsetup luksFormat --sector-size 4096 --batch-mode --verify-passphrase /dev/md/system
+cryptsetup luksOpen /dev/md/system crypt-system
 sleep 5
 
-# take a look
+# take a look at the partitions
-cat /proc/partitions
+lsblk
-
-# boot partition
-mkfs.fat -F 32 -n EFIBOOT $DISK-part1
 
+# create btrfs with volumes
+mkfs.btrfs -f --features block-group-tree --label system /dev/mapper/crypt-system
+mount -t btrfs /dev/mapper/crypt-system /mnt
+btrfs subvolume create /mnt/data
+btrfs subvolume create /mnt/nix
+btrfs subvolume create /mnt/tmp
+umount /mnt
 sleep 5
 
-# ZFS zpool creation with encryption
+# take a look at the partitions
-zpool create \
+lsblk
-    -o ashift=13 \
-    -o autotrim=off \
-    -O acltype=posixacl \
-    -O atime=off \
-    -O canmount=off \
-    -O compression=on \
-    -O dnodesize=auto \
-    -O utf8only=on \
-    -O normalization=formD \
-    -O xattr=sa \
-    -O mountpoint=none \
-    -O encryption=on \
-    -O keylocation=prompt \
-    -O keyformat=passphrase \
-    zpool $DISK-part2 $DISK2
-
-sleep 5
-
-# show the pool
-zpool status
-
-sleep 5
-
-# create all the volumes
-zfs create -o mountpoint=legacy zpool/data
-zfs create -o mountpoint=legacy zpool/nix
-zfs create -o mountpoint=legacy zpool/tmp
-
-# we want 64 KB recordsize
-zfs set recordsize=64K zpool/data
-zfs set recordsize=64K zpool/nix
-zfs set recordsize=64K zpool/tmp
-
-sleep 5
-
-# show the pool
-zpool status
-
-sleep 5
 
 # prepare install, tmpfs root
 mount -t tmpfs none /mnt
 
 # Create directories to mount file systems on
-mkdir -p /mnt/{data,nix,home,boot,root,etc/nixos,tmp}
+mkdir -p /mnt/{data,nix,boot,root,etc/nixos,tmp}
 
 # mount the ESP
 mount $DISK-part1 /mnt/boot
 
 # mount volumes
-mount -t zfs zpool/data /mnt/data
+mount -o subvol=data,noatime /dev/mapper/crypt-system /mnt/data
-mount -t zfs zpool/nix /mnt/nix
+mount -o subvol=nix,noatime /dev/mapper/crypt-system /mnt/nix
-mount -t zfs zpool/tmp /mnt/tmp
+mount -o subvol=tmp,noatime /dev/mapper/crypt-system /mnt/tmp
 
 # bind mount persistent stuff to data
-mkdir -p /mnt/data/{home,root,nixos/$HOST}
+mkdir -p /mnt/data/{root,nixos/$HOST}
-mount --bind /mnt/data/home /mnt/home
 mount --bind /mnt/data/root /mnt/root
 mount --bind /mnt/data/nixos/$HOST /mnt/etc/nixos
 
@@ -153,11 +131,13 @@ nixos-install --option experimental-features 'nix-command flakes' --no-root-pass
 # unmount all stuff and sync
 
 umount -Rl /data /mnt
-zpool export -a
+cryptsetup luksClose crypt-system
+mdadm --stop /dev/md/system
+sync
 
 # shutdown once
 
-shutdown -h now
+shutdown now
 
 # sync all /data after the install
 

neko/hardware-configuration.nix

@@ -16,13 +16,18 @@
   # intel graphics
   hardware.graphics.extraPackages = with pkgs; [ intel-media-driver intel-compute-runtime ];
 
-  FIXME
-
   # /boot efi partition to boot in UEFI mode
-  fileSystems."/boot" =
+  fileSystems."/boot" = {
-    { device = "/dev/disk/by-id/nvme-Seagate_FireCuda_530_ZP4000GM30013_7VS01VBM-part1";
+    device = "/dev/disk/by-uuid/554C-161A";
-      fsType = "vfat";
+    fsType = "vfat";
-      options = [ "fmask=0022" "dmask=0022" ];
+    options = [ "fmask=0022" "dmask=0022" ];
-      neededForBoot = true;
+    neededForBoot = true;
-    };
+  };
+
+  # encrypted system
+  boot.initrd.luks.devices."crypt-system" = {
+    device = "/dev/disk/by-uuid/91f98284-b0fa-40b9-8a32-37f71968b2dd";
+    allowDiscards = true;
+    bypassWorkqueues = true;
+  };
 }

share/common.nix

@@ -27,15 +27,21 @@ in
   boot.kernelPackages = pkgs.linuxPackages_latest;
 
   # allow md devices
-  boot.swraid.enable = true;
+  boot.swraid = {
-  boot.swraid.mdadmConf = ''
+    enable = true;
-    MAILADDR=christoph@cullmann.io
+    mdadmConf = ''
-  '';
+      MAILADDR=christoph@cullmann.io
+    '';
+  };
 
   # my kernel parameters
   boot.kernelParams = [
     # don't check for split locks, for KVM and Co.
     "split_lock_detect=off"
+
+    # fix igc 0000:0a:00.0 eno1: PCIe link lost, device now detached
+    "pcie_port_pm=off"
+    "pcie_aspm.policy=performance"
   ];
 
   # Use the systemd-boot EFI boot loader.
@@ -110,6 +116,18 @@ in
       depends = [ "/data" ];
     };
 
+  # trim the disks weekly
+  services.fstrim = {
+    enable = true;
+    interval = "weekly";
+  };
+
+  # scrub the disks weekly
+  services.btrfs.autoScrub = {
+    enable = true;
+    interval = "weekly";
+  };
+
   # keep some stuff persistent
   environment.persistence."/nix/persistent" = {
     hideMounts = true;

share/home.nix

@@ -38,6 +38,7 @@
       gc = "doas nix-collect-garbage --delete-older-than 7d";
       verify = "doas nix --extra-experimental-features nix-command store verify --all";
       optimize = "doas nix --extra-experimental-features nix-command store optimise";
+      scrub = "doas btrfs scrub start -B /data";
 
       # list latest files last
       ltr = "eza -l -s modified";
@@ -53,6 +54,8 @@
       mikuroot = "ssh root@miku.fritz.box";
       mini = "ssh mini.fritz.box";
       miniroot = "ssh root@mini.fritz.box";
+      moon = "ssh moon.babylon2k.com";
+      moonroot = "ssh root@moon.babylon2k.com";
       neko = "ssh neko.fritz.box";
       nekoroot = "ssh root@neko.fritz.box";
     };