From 584c54f8f9d491b58aa14e5d5deea86b0309f592 Mon Sep 17 00:00:00 2001 From: Christoph Cullmann Date: Sun, 18 Aug 2024 18:53:31 +0200 Subject: [PATCH] use again bcachefs, but with multiple disks --- mini/configuration.nix | 2 +- mini/hardware-configuration.nix | 12 +-- mini/install.txt | 126 ++++++++++---------------------- neko/configuration.nix | 6 +- share/common.nix | 94 +++++------------------- share/users.nix | 4 +- 6 files changed, 72 insertions(+), 172 deletions(-) diff --git a/mini/configuration.nix b/mini/configuration.nix index e835ef6..aaa1f17 100644 --- a/mini/configuration.nix +++ b/mini/configuration.nix @@ -11,7 +11,7 @@ ./hardware-configuration.nix # Shared config of all machines - /data/nixos/share/common.nix + /nix/data/nixos/share/common.nix ]; # our hostname and an ID for ZFS diff --git a/mini/hardware-configuration.nix b/mini/hardware-configuration.nix index e02d525..9adbd15 100644 --- a/mini/hardware-configuration.nix +++ b/mini/hardware-configuration.nix @@ -10,18 +10,18 @@ boot.initrd.kernelModules = [ "amdgpu" ]; boot.kernelModules = [ "kvm-amd" ]; - # efi partition + # /boot efi partition to boot in UEFI mode fileSystems."/boot" = { device = "/dev/disk/by-id/nvme-CT4000P3PSSD8_2325E6E63746-part1"; fsType = "vfat"; neededForBoot = true; }; - # vms - fileSystems."/home/cullmann/vms" = - { device = "vpool/vms"; - fsType = "zfs"; - depends = [ "/home" ]; + # /nix encrypted bcachefs for the remaining space + fileSystems."/nix" = + { device = "/dev/disk/by-id/nvme-CT4000P3PSSD8_2325E6E63746-part2:/dev/disk/by-id/ata-CT2000MX500SSD1_2138E5D5061F"; + fsType = "bcachefs"; + neededForBoot = true; }; nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; diff --git a/mini/install.txt b/mini/install.txt index 8e59841..e456d83 100644 --- a/mini/install.txt +++ b/mini/install.txt @@ -2,10 +2,18 @@ # enable ssh for root # -systemctl start sshd sudo bash +systemctl start sshd passwd +# +# install script below +# + +# fix unlocking https://nixos.wiki/wiki/Bcachefs +nix-env -iA nixos.keyutils +keyctl link @u @s + # # kill old efi boot stuff # @@ -18,14 +26,11 @@ efibootmgr -b 3 -B efibootmgr -b 4 -B efibootmgr -# -# install script below -# - # Defining some helper variables (these will be used in later code # blocks as well, so make sure to use the same terminal session or # redefine them later) DISK=/dev/disk/by-id/nvme-CT4000P3PSSD8_2325E6E63746 +DISK2=/dev/disk/by-id/ata-CT2000MX500SSD1_2138E5D5061F HOST=mini # ensure 4k sector size @@ -40,6 +45,13 @@ blkdiscard -v $DISK wipefs -a $DISK gdisk -l $DISK +# wipe second disk +sgdisk --zap-all $DISK2 +blkdiscard -v $DISK2 +wipefs -a $DISK2 + +sleep 5 + # create partitions parted $DISK -- mklabel gpt sgdisk -n 1:0:+1024M -c 1:"EFI System Partition" -t 1:EF00 $DISK @@ -56,28 +68,8 @@ mkfs.fat -F 32 -n EFIBOOT $DISK-part1 sleep 5 -# ZFS zpool creation with encryption -zpool create \ - -o ashift=12 \ - -o autotrim=on \ - -O acltype=posixacl \ - -O atime=off \ - -O canmount=off \ - -O compression=on \ - -O dnodesize=auto \ - -O normalization=formD \ - -O xattr=sa \ - -O mountpoint=none \ - -O encryption=on \ - -O keylocation=prompt \ - -O keyformat=passphrase \ - zpool $DISK-part2 - -sleep 5 - -# create all the volumes -zfs create -o mountpoint=legacy zpool/data -zfs create -o mountpoint=legacy zpool/nix +# create encrypted bcachefs over all disks, use only fast lz4 compression +bcachefs format --block_size=4096 --errors=ro --compression=lz4 --wide_macs --acl --encrypted --fs_label=nix --discard -f $DISK-part2 $DISK2 sleep 5 @@ -85,24 +77,27 @@ sleep 5 mount -t tmpfs none /mnt # Create directories to mount file systems on -mkdir -p /mnt/{data,nix,home,boot,root,etc/nixos} +mkdir -p /mnt/{nix,home,boot,root,etc/nixos} # mount the ESP mount $DISK-part1 /mnt/boot -# mount volumes -mount -t zfs zpool/data /mnt/data -mount -t zfs zpool/nix /mnt/nix +# mount large bcachefs +mount -t bcachefs $DISK-part2:$DISK2 /mnt/nix + +# ensure tmp fills not the RAM +mkdir -p /mnt/tmp /mnt/nix/tmp +mount --bind /mnt/nix/tmp /mnt/tmp # bind mount persistent stuff to data -mkdir -p /mnt/{data/home,data/root,data/nixos/$HOST} -mount --bind /mnt/data/home /mnt/home -mount --bind /mnt/data/root /mnt/root -mount --bind /mnt/data/nixos/$HOST /mnt/etc/nixos +mkdir -p /mnt/nix/data/{home,root,nixos/$HOST} +mount --bind /mnt/nix/data/home /mnt/home +mount --bind /mnt/nix/data/root /mnt/root +mount --bind /mnt/nix/data/nixos/$HOST /mnt/etc/nixos -# create fake /data to have the right paths -mkdir -p /data -mount --bind /mnt/data /data +# create fake /nix/data to have the right paths +mkdir -p /nix/data +mount --bind /mnt/nix/data /nix/data # take a look mount @@ -110,62 +105,21 @@ mount # configure nixos-generate-config --root /mnt -# save /mnt/etc/nixos/hardware-configuration.nix /mnt/etc/nixos/configuration.nix +# check /mnt/etc/nixos/hardware-configuration.nix /mnt/etc/nixos/configuration.nix -cp /mnt/etc/nixos/hardware-configuration.nix /tmp -cp /mnt/etc/nixos/configuration.nix /tmp +# copy config data from another machine including secrets -# copy config data - -sudo scp -r /data/nixos root@192.168.13.100:/mnt/data +sudo scp -r /nix/data/nixos root@192.168.13.100:/mnt/nix/data # install nixos-install --option experimental-features 'nix-command flakes' --no-root-passwd --root /mnt -# unmount all stuff +# unmount all stuff and sync -umount -Rl /data /mnt -zpool export -a +umount -Rl /nix/data /mnt +sync # sync all /data after the install -sudo -E rsync -va --delete --one-file-system /data root@192.168.13.100:/ - -# -# after install tasks for extra file systems -# - -# create vms disk - -DD=/dev/disk/by-id/ata-CT2000MX500SSD1_2138E5D5061F -sgdisk --zap-all $DD -blkdiscard -v $DD -wipefs -a $DD - -sleep 5 - -# ZFS zpool creation with encryption -zpool create \ - -o ashift=12 \ - -o autotrim=on \ - -O acltype=posixacl \ - -O atime=off \ - -O canmount=off \ - -O compression=on \ - -O dnodesize=auto \ - -O normalization=formD \ - -O xattr=sa \ - -O mountpoint=none \ - -O encryption=on \ - -O keylocation=file:///data/nixos/key-vms.secret \ - -O keyformat=passphrase \ - vpool $DD - -sleep 5 - -# create all the volumes -zfs create -o mountpoint=legacy vpool/vms - -# update passphrase later -# zfs change-key -o keylocation=file:///data/nixos/key-vms.secret vpool +sudo -E rsync -va --delete --one-file-system /nix/data/ root@192.168.13.100:/nix/data/ diff --git a/neko/configuration.nix b/neko/configuration.nix index 3eb708e..1ea3dfd 100644 --- a/neko/configuration.nix +++ b/neko/configuration.nix @@ -3,8 +3,10 @@ # and in the NixOS manual (accessible by running `nixos-help`). { config, pkgs, ... }: - +ccecrennt { +reacncncncnas + imports = [ # Include the results of the hardware scan. @@ -17,7 +19,7 @@ # our hostname and an ID for ZFS networking.hostName = "neko"; networking.hostId = "cf5a5ee6"; - +nnacsasasa # EurKey layout services.xserver.xkb.layout = "eu"; } diff --git a/share/common.nix b/share/common.nix index 6a501f9..48cf0ad 100644 --- a/share/common.nix +++ b/share/common.nix @@ -13,7 +13,7 @@ in "${impermanence}/nixos.nix" # our users - "/data/nixos/share/users.nix" + "/nix/data/nixos/share/users.nix" ]; # This value determines the NixOS release from which the default @@ -24,32 +24,12 @@ in # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html). system.stateVersion = "23.05"; # Did you read the comment? - # use the latest kernel with ZFS support and enable that file system - boot.kernelPackages = config.boot.zfs.package.latestCompatibleLinuxPackages; - boot.supportedFilesystems = [ "zfs" ]; + # use the latest kernel and enable bcachefs file system + boot.kernelPackages = pkgs.linuxPackages_latest; + boot.supportedFilesystems = [ "bcachefs" ]; - # no hibernate for ZFS systems # don't check for split locks, for KVM and Co. - boot.kernelParams = [ "nohibernate" "split_lock_detect=off" ]; - - # tweak ZFS - boot.extraModprobeConfig = '' - options zfs zfs_arc_meta_limit_percent=75 - options zfs zfs_arc_min=134217728 - options zfs zfs_arc_max=4294967296 - options zfs zfs_txg_timeout=30 - options zfs zfs_vdev_scrub_min_active=1 - options zfs zfs_vdev_scrub_max_active=1 - options zfs zfs_vdev_sync_write_min_active=8 - options zfs zfs_vdev_sync_write_max_active=32 - options zfs zfs_vdev_sync_read_min_active=8 - options zfs zfs_vdev_sync_read_max_active=32 - options zfs zfs_vdev_async_read_min_active=8 - options zfs zfs_vdev_async_read_max_active=32 - options zfs zfs_vdev_async_write_min_active=8 - options zfs zfs_vdev_async_write_max_active=32 - options zfs zfs_vdev_def_queue_depth=128 - ''; + boot.kernelParams = [ "split_lock_detect=off" ]; # Use the systemd-boot EFI boot loader. boot.loader.systemd-boot.enable = true; @@ -81,54 +61,46 @@ in options = [ "defaults" "size=8G" "mode=755" ]; }; - # nix store file system from encrypted ZFS - fileSystems."/nix" = - { device = "zpool/nix"; - fsType = "zfs"; - neededForBoot = true; - }; - - # data store file system from encrypted ZFS - fileSystems."/data" = - { device = "zpool/data"; - fsType = "zfs"; + # tmp on /nix to not fill RAM + fileSystems."/tmp" = + { device = "/nix/tmp"; + fsType = "none"; neededForBoot = true; + options = [ "bind" ]; + depends = [ "/nix" ]; }; # bind mount to have user homes fileSystems."/home" = - { device = "/data/home"; + { device = "/nix/data/home"; fsType = "none"; neededForBoot = true; options = [ "bind" ]; - depends = [ "/data" ]; + depends = [ "/nix" ]; }; # bind mount to have root home fileSystems."/root" = - { device = "/data/root"; + { device = "/nix/data/root"; fsType = "none"; neededForBoot = true; options = [ "bind" ]; - depends = [ "/data" ]; + depends = [ "/nix" ]; }; # bind mount to have NixOS configuration, different per host fileSystems."/etc/nixos" = - { device = "/data/nixos/${config.networking.hostName}"; + { device = "/nix/data/nixos/${config.networking.hostName}"; fsType = "none"; neededForBoot = true; options = [ "bind" ]; - depends = [ "/data" ]; + depends = [ "/nix" ]; }; # keep some stuff persistent environment.persistence."/nix/persistent" = { hideMounts = true; directories = [ - # tmp dir, don't fill our tmpfs root with that - { directory = "/tmp"; user = "root"; group = "root"; mode = "1777"; } - # systemd timers { directory = "/var/lib/systemd/timers"; user = "root"; group = "root"; mode = "u=rwx,g=rx,o=rx"; } @@ -144,15 +116,6 @@ in # kill the tmp content on reboots, we mount that to /nix/persistent to avoid memory fill-up boot.tmp.cleanOnBoot = true; - # ensure our data is not rotting - services.zfs.autoScrub = { - enable = true; - interval = "weekly"; - }; - - # trim the stuff, we use SSDs - services.zfs.trim.enable = true; - # enable fast dbus services.dbus.implementation = "broker"; @@ -260,7 +223,7 @@ in allowReboot = false; }; - # avoid suspend ever to be triggered, ZFS dislikes that + # avoid suspend ever to be triggered systemd.targets.sleep.enable = false; systemd.targets.suspend.enable = false; systemd.targets.hibernate.enable = false; @@ -522,29 +485,10 @@ in mode = "0400"; }; environment.etc."mail/secrets" = { - text = builtins.readFile "/data/nixos/mail.secret"; + text = builtins.readFile "/nix/data/nixos/mail.secret"; mode = "0400"; }; - # send mails on ZFS events - services.zfs.zed = { - settings = { - ZED_DEBUG_LOG = "/tmp/zed.debug.log"; - ZED_EMAIL_ADDR = [ "root" ]; - ZED_EMAIL_PROG = "/run/wrappers/bin/sendmail"; - ZED_EMAIL_OPTS = "@ADDRESS@"; - - ZED_NOTIFY_INTERVAL_SECS = 3600; - ZED_NOTIFY_VERBOSE = true; - - ZED_USE_ENCLOSURE_LEDS = true; - ZED_SCRUB_AFTER_RESILVER = true; - }; - - # this option does not work; will return error - enableMail = false; - }; - # use ZSH per default programs.zsh.enable = true; environment.shells = with pkgs; [ zsh ]; diff --git a/share/users.nix b/share/users.nix index fde40bf..687f0f5 100644 --- a/share/users.nix +++ b/share/users.nix @@ -23,10 +23,10 @@ in users.root = { # init password - hashedPassword = builtins.readFile "/data/nixos/password.secret"; + hashedPassword = builtins.readFile "/nix/data/nixos/password.secret"; # use fixed auth keys - openssh.authorizedKeys.keys = pkgs.lib.splitString "\n" (builtins.readFile "/data/nixos/authorized_keys.secret"); + openssh.authorizedKeys.keys = pkgs.lib.splitString "\n" (builtins.readFile "/nix/data/nixos/authorized_keys.secret"); }; #