more bcachefs stuff
This commit is contained in:
parent
539084beb4
commit
d7290ec6c1
4 changed files with 54 additions and 163 deletions
|
|
@ -1,7 +1,3 @@
|
|||
# Edit this configuration file to define what should be installed on
|
||||
# your system. Help is available in the configuration.nix(5) man page
|
||||
# and in the NixOS manual (accessible by running `nixos-help`).
|
||||
|
||||
{ config, pkgs, ... }:
|
||||
|
||||
{
|
||||
|
|
@ -14,9 +10,8 @@
|
|||
/data/nixos/share/common.nix
|
||||
];
|
||||
|
||||
# our hostname and an ID for ZFS
|
||||
# our hostname
|
||||
networking.hostName = "neko";
|
||||
networking.hostId = "cf5a5ee6";
|
||||
|
||||
# EurKey layout
|
||||
services.xserver.xkb.layout = "eu";
|
||||
|
|
|
|||
|
|
@ -1,6 +1,3 @@
|
|||
# Do not modify this file! It was generated by ‘nixos-generate-config’
|
||||
# and may be overwritten by future invocations. Please make changes
|
||||
# to /etc/nixos/configuration.nix instead.
|
||||
{ config, lib, pkgs, modulesPath, ... }:
|
||||
|
||||
{
|
||||
|
|
@ -18,9 +15,16 @@
|
|||
|
||||
# /boot efi partition to boot in UEFI mode
|
||||
fileSystems."/boot" = {
|
||||
device = "/dev/disk/by-id/nvme-Seagate_FireCuda_530_ZP4000GM30013_7VS01VBM-part1";
|
||||
device = "UUID=CFA5-46EA";
|
||||
fsType = "vfat";
|
||||
neededForBoot = true;
|
||||
options = [ "fmask=0022" "dmask=0022" ];
|
||||
};
|
||||
|
||||
# /nix volume with the system & all persistent data
|
||||
fileSystems."/nix" = {
|
||||
device = "UUID=686a90a2-93ac-40a6-a01d-c7b61cc47750";
|
||||
fsType = "bcachefs";
|
||||
neededForBoot = true;
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ HOST=neko
|
|||
DISK=/dev/disk/by-id/nvme-Seagate_FireCuda_530_ZP4000GM30013_7VS01VBM
|
||||
DISK2=/dev/disk/by-id/nvme-CT2000P5PSSD8_213330E4ED05
|
||||
DISK3=/dev/disk/by-id/nvme-Samsung_SSD_980_PRO_2TB_S69ENF0R846614L
|
||||
DISKS="$DISK $DISK2 $DISK3"
|
||||
|
||||
# ensure 4k sector size
|
||||
nvme format --lbaf=1 --force $DISK
|
||||
|
|
@ -36,7 +37,8 @@ nvme id-ns -H $DISK
|
|||
sleep 5
|
||||
|
||||
# create partition table on all disks and EFI partition
|
||||
for D in $DISK $DISK2 $DISK3; do
|
||||
RAID=""
|
||||
for D in $DISKS; do
|
||||
# kill old data
|
||||
sgdisk --zap-all $D
|
||||
blkdiscard -v -f $D
|
||||
|
|
@ -56,59 +58,22 @@ for D in $DISK $DISK2 $DISK3; do
|
|||
# boot partition after short sleep, needed on some machines
|
||||
sleep 5
|
||||
mkfs.vfat $D-part1
|
||||
|
||||
# add part2 to raid
|
||||
RAID="$RAID $D-part2"
|
||||
done
|
||||
|
||||
# take a look at the partitions
|
||||
lsblk
|
||||
|
||||
# ZFS zpool creation with compression and encryption
|
||||
zpool create \
|
||||
-o ashift=13 \
|
||||
-o autotrim=off \
|
||||
-O acltype=posixacl \
|
||||
-O atime=off \
|
||||
-O canmount=off \
|
||||
-O compression=on \
|
||||
-O dnodesize=auto \
|
||||
-O utf8only=on \
|
||||
-O normalization=formD \
|
||||
-O xattr=sa \
|
||||
-O mountpoint=none \
|
||||
-O encryption=on \
|
||||
-O keylocation=prompt \
|
||||
-O keyformat=passphrase \
|
||||
zpool $DISK-part2 $DISK2-part2 $DISK3-part2
|
||||
# create encrypted bcachefs
|
||||
bcachefs format --encrypt -f $RAID
|
||||
nix-env -iA nixos.keyutils
|
||||
keyctl link @u @s
|
||||
bcachefs unlock $DISK-part2
|
||||
|
||||
sleep 5
|
||||
|
||||
# take a look at the partitions
|
||||
lsblk
|
||||
|
||||
# show the pool
|
||||
zpool status
|
||||
|
||||
sleep 5
|
||||
|
||||
# create all the volumes
|
||||
zfs create -o mountpoint=legacy zpool/data
|
||||
zfs create -o mountpoint=legacy zpool/nix
|
||||
zfs create -o mountpoint=legacy zpool/root
|
||||
|
||||
sleep 5
|
||||
|
||||
# show the pool
|
||||
zpool status
|
||||
|
||||
sleep 5
|
||||
|
||||
# create ZFS snapshot that we'll rollback to on boot
|
||||
# see https://ryanseipp.com/post/nixos-encrypted-root/
|
||||
zfs snapshot zpool/root@blank
|
||||
|
||||
sleep 5
|
||||
|
||||
# prepare install, root
|
||||
mount -t zfs zpool/root /mnt
|
||||
# prepare install, tmpfs root
|
||||
mount -t tmpfs none /mnt
|
||||
|
||||
# Create directories to mount file systems on
|
||||
mkdir -p /mnt/{data,nix,boot,root,etc/nixos}
|
||||
|
|
@ -116,9 +81,12 @@ mkdir -p /mnt/{data,nix,boot,root,etc/nixos}
|
|||
# mount the ESP
|
||||
mount $DISK-part1 /mnt/boot
|
||||
|
||||
# mount volumes
|
||||
mount -t zfs zpool/data /mnt/data
|
||||
mount -t zfs zpool/nix /mnt/nix
|
||||
# mount the /nix
|
||||
mount -t bcachefs $DISK-part2 /mnt/nix
|
||||
|
||||
# mount the /data via bind mount
|
||||
mkdir /mnt/nix/data
|
||||
mount --bind /mnt/nix/data /mnt/data
|
||||
|
||||
# bind mount persistent stuff to data
|
||||
mkdir -p /mnt/data/{root,nixos/$HOST}
|
||||
|
|
@ -135,7 +103,7 @@ mount
|
|||
# configure
|
||||
nixos-generate-config --root /mnt
|
||||
|
||||
# check /mnt/etc/nixos/hardware-configuration.nix /mnt/etc/nixos/configuration.nix
|
||||
# check /mnt/etc/nixos/hardware-configuration.nix to get UUIDs
|
||||
|
||||
# copy config data from another machine including secrets
|
||||
|
||||
|
|
@ -148,7 +116,6 @@ nixos-install --option experimental-features 'nix-command flakes' --no-root-pass
|
|||
# unmount all stuff and sync
|
||||
|
||||
umount -Rl /data /mnt
|
||||
zpool export -a
|
||||
sync
|
||||
|
||||
# shutdown once
|
||||
|
|
|
|||
105
share/common.nix
105
share/common.nix
|
|
@ -26,8 +26,9 @@ in
|
|||
# atm all stuff is x86_64
|
||||
nixpkgs.hostPlatform = "x86_64-linux";
|
||||
|
||||
# enable ZFS
|
||||
boot.supportedFilesystems = [ "zfs" ];
|
||||
# enable bcachefs with latest kernel
|
||||
boot.supportedFilesystems = [ "bcachefs" ];
|
||||
boot.kernelPackages = pkgs.linuxPackages_latest;
|
||||
|
||||
# my kernel parameters
|
||||
boot.kernelParams = [
|
||||
|
|
@ -35,13 +36,6 @@ in
|
|||
"quiet"
|
||||
"splash"
|
||||
|
||||
# no hibernate for ZFS systems
|
||||
"nohibernate"
|
||||
|
||||
# make ARC fast
|
||||
"init_on_alloc=0"
|
||||
"init_on_free=0"
|
||||
|
||||
# don't check for split locks, for KVM and Co.
|
||||
"split_lock_detect=off"
|
||||
|
||||
|
|
@ -113,29 +107,6 @@ in
|
|||
};
|
||||
};
|
||||
|
||||
# tweak ZFS
|
||||
boot.extraModprobeConfig = ''
|
||||
# less scrub impact on other IO
|
||||
options zfs zfs_scrub_delay=32
|
||||
options zfs zfs_vdev_scrub_min_active=1
|
||||
options zfs zfs_vdev_scrub_max_active=1
|
||||
'';
|
||||
|
||||
# tune the ZFS pool for NVMe
|
||||
system.activationScripts.zfsTuning = {
|
||||
text = ''
|
||||
# only one level of caching
|
||||
${pkgs.zfs}/bin/zfs set primarycache=all zpool
|
||||
${pkgs.zfs}/bin/zfs set secondarycache=none zpool
|
||||
|
||||
# I have backups and no real databases
|
||||
${pkgs.zfs}/bin/zfs set sync=disabled zpool
|
||||
|
||||
# use allow direct IO
|
||||
${pkgs.zfs}/bin/zfs set direct=standard zpool
|
||||
'';
|
||||
};
|
||||
|
||||
# Use the systemd-boot EFI boot loader.
|
||||
boot.loader.systemd-boot.enable = true;
|
||||
boot.loader.efi.canTouchEfiVariables = true;
|
||||
|
|
@ -159,44 +130,26 @@ in
|
|||
themePackages = [ pkgs.adi1090x-plymouth-themes ];
|
||||
};
|
||||
|
||||
# root file system, we will rollback that on boot
|
||||
# root file system, tmpfs
|
||||
fileSystems."/" = {
|
||||
device = "zpool/root";
|
||||
fsType = "zfs";
|
||||
device = "none";
|
||||
fsType = "tmpfs";
|
||||
neededForBoot = true;
|
||||
};
|
||||
|
||||
# root rollback, see https://ryanseipp.com/post/nixos-encrypted-root/
|
||||
boot.initrd.systemd.services.rollback = {
|
||||
description = "Rollback root filesystem to a pristine state";
|
||||
wantedBy = ["initrd.target"];
|
||||
after = ["zfs-import-zpool.service"];
|
||||
before = ["sysroot.mount"];
|
||||
path = with pkgs; [zfs];
|
||||
unitConfig.DefaultDependencies = "no";
|
||||
serviceConfig.Type = "oneshot";
|
||||
script = ''
|
||||
zfs rollback -r zpool/root@blank && echo " >> >> Rollback Complete << <<"
|
||||
'';
|
||||
options = [ "defaults" "size=25%" "mode=755" ];
|
||||
};
|
||||
|
||||
# my data
|
||||
fileSystems."/data" = {
|
||||
device = "zpool/data";
|
||||
fsType = "zfs";
|
||||
neededForBoot = true;
|
||||
};
|
||||
|
||||
# the system
|
||||
fileSystems."/nix" = {
|
||||
device = "zpool/nix";
|
||||
fsType = "zfs";
|
||||
device = "/nix/data";
|
||||
fsType = "none";
|
||||
neededForBoot = true;
|
||||
options = [ "bind" "x-gvfs-hide" ];
|
||||
depends = [ "/nix" ];
|
||||
};
|
||||
|
||||
# bind mount to have root home
|
||||
fileSystems."/root" =
|
||||
{ device = "/data/root";
|
||||
fileSystems."/root" = {
|
||||
device = "/data/root";
|
||||
fsType = "none";
|
||||
neededForBoot = true;
|
||||
options = [ "bind" "x-gvfs-hide" ];
|
||||
|
|
@ -204,8 +157,8 @@ in
|
|||
};
|
||||
|
||||
# bind mount to have NixOS configuration, different per host
|
||||
fileSystems."/etc/nixos" =
|
||||
{ device = "/data/nixos/${config.networking.hostName}";
|
||||
fileSystems."/etc/nixos" = {
|
||||
device = "/data/nixos/${config.networking.hostName}";
|
||||
fsType = "none";
|
||||
neededForBoot = true;
|
||||
options = [ "bind" "x-gvfs-hide" ];
|
||||
|
|
@ -238,15 +191,6 @@ in
|
|||
];
|
||||
};
|
||||
|
||||
# ensure our data is not rotting
|
||||
services.zfs.autoScrub = {
|
||||
enable = true;
|
||||
interval = "weekly";
|
||||
};
|
||||
|
||||
# trim the stuff, we use SSDs
|
||||
services.zfs.trim.enable = true;
|
||||
|
||||
# enable fast dbus
|
||||
services.dbus.implementation = "broker";
|
||||
|
||||
|
|
@ -576,25 +520,6 @@ in
|
|||
mode = "0400";
|
||||
};
|
||||
|
||||
# send mails on ZFS events
|
||||
services.zfs.zed = {
|
||||
settings = {
|
||||
ZED_DEBUG_LOG = "/tmp/zed.debug.log";
|
||||
ZED_EMAIL_ADDR = [ "root" ];
|
||||
ZED_EMAIL_PROG = "/run/wrappers/bin/sendmail";
|
||||
ZED_EMAIL_OPTS = "@ADDRESS@";
|
||||
|
||||
ZED_NOTIFY_INTERVAL_SECS = 3600;
|
||||
ZED_NOTIFY_VERBOSE = true;
|
||||
|
||||
ZED_USE_ENCLOSURE_LEDS = true;
|
||||
ZED_SCRUB_AFTER_RESILVER = true;
|
||||
};
|
||||
|
||||
# this option does not work; will return error
|
||||
enableMail = false;
|
||||
};
|
||||
|
||||
# use ZSH per default
|
||||
programs.zsh.enable = true;
|
||||
environment.shells = with pkgs; [ zsh ];
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue