use again bcachefs, but with multiple disks
This commit is contained in:
parent
c67ff4d387
commit
584c54f8f9
|
|
@ -11,7 +11,7 @@
|
|||
./hardware-configuration.nix
|
||||
|
||||
# Shared config of all machines
|
||||
/data/nixos/share/common.nix
|
||||
/nix/data/nixos/share/common.nix
|
||||
];
|
||||
|
||||
# our hostname and an ID for ZFS
|
||||
|
|
|
|||
|
|
@ -10,18 +10,18 @@
|
|||
boot.initrd.kernelModules = [ "amdgpu" ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
|
||||
# efi partition
|
||||
# /boot efi partition to boot in UEFI mode
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-id/nvme-CT4000P3PSSD8_2325E6E63746-part1";
|
||||
fsType = "vfat";
|
||||
neededForBoot = true;
|
||||
};
|
||||
|
||||
# vms
|
||||
fileSystems."/home/cullmann/vms" =
|
||||
{ device = "vpool/vms";
|
||||
fsType = "zfs";
|
||||
depends = [ "/home" ];
|
||||
# /nix encrypted bcachefs for the remaining space
|
||||
fileSystems."/nix" =
|
||||
{ device = "/dev/disk/by-id/nvme-CT4000P3PSSD8_2325E6E63746-part2:/dev/disk/by-id/ata-CT2000MX500SSD1_2138E5D5061F";
|
||||
fsType = "bcachefs";
|
||||
neededForBoot = true;
|
||||
};
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
|
|
|
|||
126
mini/install.txt
126
mini/install.txt
|
|
@ -2,10 +2,18 @@
|
|||
# enable ssh for root
|
||||
#
|
||||
|
||||
systemctl start sshd
|
||||
sudo bash
|
||||
systemctl start sshd
|
||||
passwd
|
||||
|
||||
#
|
||||
# install script below
|
||||
#
|
||||
|
||||
# fix unlocking https://nixos.wiki/wiki/Bcachefs
|
||||
nix-env -iA nixos.keyutils
|
||||
keyctl link @u @s
|
||||
|
||||
#
|
||||
# kill old efi boot stuff
|
||||
#
|
||||
|
|
@ -18,14 +26,11 @@ efibootmgr -b 3 -B
|
|||
efibootmgr -b 4 -B
|
||||
efibootmgr
|
||||
|
||||
#
|
||||
# install script below
|
||||
#
|
||||
|
||||
# Defining some helper variables (these will be used in later code
|
||||
# blocks as well, so make sure to use the same terminal session or
|
||||
# redefine them later)
|
||||
DISK=/dev/disk/by-id/nvme-CT4000P3PSSD8_2325E6E63746
|
||||
DISK2=/dev/disk/by-id/ata-CT2000MX500SSD1_2138E5D5061F
|
||||
HOST=mini
|
||||
|
||||
# ensure 4k sector size
|
||||
|
|
@ -40,6 +45,13 @@ blkdiscard -v $DISK
|
|||
wipefs -a $DISK
|
||||
gdisk -l $DISK
|
||||
|
||||
# wipe second disk
|
||||
sgdisk --zap-all $DISK2
|
||||
blkdiscard -v $DISK2
|
||||
wipefs -a $DISK2
|
||||
|
||||
sleep 5
|
||||
|
||||
# create partitions
|
||||
parted $DISK -- mklabel gpt
|
||||
sgdisk -n 1:0:+1024M -c 1:"EFI System Partition" -t 1:EF00 $DISK
|
||||
|
|
@ -56,28 +68,8 @@ mkfs.fat -F 32 -n EFIBOOT $DISK-part1
|
|||
|
||||
sleep 5
|
||||
|
||||
# ZFS zpool creation with encryption
|
||||
zpool create \
|
||||
-o ashift=12 \
|
||||
-o autotrim=on \
|
||||
-O acltype=posixacl \
|
||||
-O atime=off \
|
||||
-O canmount=off \
|
||||
-O compression=on \
|
||||
-O dnodesize=auto \
|
||||
-O normalization=formD \
|
||||
-O xattr=sa \
|
||||
-O mountpoint=none \
|
||||
-O encryption=on \
|
||||
-O keylocation=prompt \
|
||||
-O keyformat=passphrase \
|
||||
zpool $DISK-part2
|
||||
|
||||
sleep 5
|
||||
|
||||
# create all the volumes
|
||||
zfs create -o mountpoint=legacy zpool/data
|
||||
zfs create -o mountpoint=legacy zpool/nix
|
||||
# create encrypted bcachefs over all disks, use only fast lz4 compression
|
||||
bcachefs format --block_size=4096 --errors=ro --compression=lz4 --wide_macs --acl --encrypted --fs_label=nix --discard -f $DISK-part2 $DISK2
|
||||
|
||||
sleep 5
|
||||
|
||||
|
|
@ -85,24 +77,27 @@ sleep 5
|
|||
mount -t tmpfs none /mnt
|
||||
|
||||
# Create directories to mount file systems on
|
||||
mkdir -p /mnt/{data,nix,home,boot,root,etc/nixos}
|
||||
mkdir -p /mnt/{nix,home,boot,root,etc/nixos}
|
||||
|
||||
# mount the ESP
|
||||
mount $DISK-part1 /mnt/boot
|
||||
|
||||
# mount volumes
|
||||
mount -t zfs zpool/data /mnt/data
|
||||
mount -t zfs zpool/nix /mnt/nix
|
||||
# mount large bcachefs
|
||||
mount -t bcachefs $DISK-part2:$DISK2 /mnt/nix
|
||||
|
||||
# ensure tmp fills not the RAM
|
||||
mkdir -p /mnt/tmp /mnt/nix/tmp
|
||||
mount --bind /mnt/nix/tmp /mnt/tmp
|
||||
|
||||
# bind mount persistent stuff to data
|
||||
mkdir -p /mnt/{data/home,data/root,data/nixos/$HOST}
|
||||
mount --bind /mnt/data/home /mnt/home
|
||||
mount --bind /mnt/data/root /mnt/root
|
||||
mount --bind /mnt/data/nixos/$HOST /mnt/etc/nixos
|
||||
mkdir -p /mnt/nix/data/{home,root,nixos/$HOST}
|
||||
mount --bind /mnt/nix/data/home /mnt/home
|
||||
mount --bind /mnt/nix/data/root /mnt/root
|
||||
mount --bind /mnt/nix/data/nixos/$HOST /mnt/etc/nixos
|
||||
|
||||
# create fake /data to have the right paths
|
||||
mkdir -p /data
|
||||
mount --bind /mnt/data /data
|
||||
# create fake /nix/data to have the right paths
|
||||
mkdir -p /nix/data
|
||||
mount --bind /mnt/nix/data /nix/data
|
||||
|
||||
# take a look
|
||||
mount
|
||||
|
|
@ -110,62 +105,21 @@ mount
|
|||
# configure
|
||||
nixos-generate-config --root /mnt
|
||||
|
||||
# save /mnt/etc/nixos/hardware-configuration.nix /mnt/etc/nixos/configuration.nix
|
||||
# check /mnt/etc/nixos/hardware-configuration.nix /mnt/etc/nixos/configuration.nix
|
||||
|
||||
cp /mnt/etc/nixos/hardware-configuration.nix /tmp
|
||||
cp /mnt/etc/nixos/configuration.nix /tmp
|
||||
# copy config data from another machine including secrets
|
||||
|
||||
# copy config data
|
||||
|
||||
sudo scp -r /data/nixos root@192.168.13.100:/mnt/data
|
||||
sudo scp -r /nix/data/nixos root@192.168.13.100:/mnt/nix/data
|
||||
|
||||
# install
|
||||
|
||||
nixos-install --option experimental-features 'nix-command flakes' --no-root-passwd --root /mnt
|
||||
|
||||
# unmount all stuff
|
||||
# unmount all stuff and sync
|
||||
|
||||
umount -Rl /data /mnt
|
||||
zpool export -a
|
||||
umount -Rl /nix/data /mnt
|
||||
sync
|
||||
|
||||
# sync all /data after the install
|
||||
|
||||
sudo -E rsync -va --delete --one-file-system /data root@192.168.13.100:/
|
||||
|
||||
#
|
||||
# after install tasks for extra file systems
|
||||
#
|
||||
|
||||
# create vms disk
|
||||
|
||||
DD=/dev/disk/by-id/ata-CT2000MX500SSD1_2138E5D5061F
|
||||
sgdisk --zap-all $DD
|
||||
blkdiscard -v $DD
|
||||
wipefs -a $DD
|
||||
|
||||
sleep 5
|
||||
|
||||
# ZFS zpool creation with encryption
|
||||
zpool create \
|
||||
-o ashift=12 \
|
||||
-o autotrim=on \
|
||||
-O acltype=posixacl \
|
||||
-O atime=off \
|
||||
-O canmount=off \
|
||||
-O compression=on \
|
||||
-O dnodesize=auto \
|
||||
-O normalization=formD \
|
||||
-O xattr=sa \
|
||||
-O mountpoint=none \
|
||||
-O encryption=on \
|
||||
-O keylocation=file:///data/nixos/key-vms.secret \
|
||||
-O keyformat=passphrase \
|
||||
vpool $DD
|
||||
|
||||
sleep 5
|
||||
|
||||
# create all the volumes
|
||||
zfs create -o mountpoint=legacy vpool/vms
|
||||
|
||||
# update passphrase later
|
||||
# zfs change-key -o keylocation=file:///data/nixos/key-vms.secret vpool
|
||||
sudo -E rsync -va --delete --one-file-system /nix/data/ root@192.168.13.100:/nix/data/
|
||||
|
|
|
|||
|
|
@ -3,8 +3,10 @@
|
|||
# and in the NixOS manual (accessible by running `nixos-help`).
|
||||
|
||||
{ config, pkgs, ... }:
|
||||
|
||||
ccecrennt
|
||||
{
|
||||
reacncncncnas
|
||||
|
||||
imports =
|
||||
[
|
||||
# Include the results of the hardware scan.
|
||||
|
|
@ -17,7 +19,7 @@
|
|||
# our hostname and an ID for ZFS
|
||||
networking.hostName = "neko";
|
||||
networking.hostId = "cf5a5ee6";
|
||||
|
||||
nnacsasasa
|
||||
# EurKey layout
|
||||
services.xserver.xkb.layout = "eu";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ in
|
|||
"${impermanence}/nixos.nix"
|
||||
|
||||
# our users
|
||||
"/data/nixos/share/users.nix"
|
||||
"/nix/data/nixos/share/users.nix"
|
||||
];
|
||||
|
||||
# This value determines the NixOS release from which the default
|
||||
|
|
@ -24,32 +24,12 @@ in
|
|||
# (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
|
||||
system.stateVersion = "23.05"; # Did you read the comment?
|
||||
|
||||
# use the latest kernel with ZFS support and enable that file system
|
||||
boot.kernelPackages = config.boot.zfs.package.latestCompatibleLinuxPackages;
|
||||
boot.supportedFilesystems = [ "zfs" ];
|
||||
# use the latest kernel and enable bcachefs file system
|
||||
boot.kernelPackages = pkgs.linuxPackages_latest;
|
||||
boot.supportedFilesystems = [ "bcachefs" ];
|
||||
|
||||
# no hibernate for ZFS systems
|
||||
# don't check for split locks, for KVM and Co.
|
||||
boot.kernelParams = [ "nohibernate" "split_lock_detect=off" ];
|
||||
|
||||
# tweak ZFS
|
||||
boot.extraModprobeConfig = ''
|
||||
options zfs zfs_arc_meta_limit_percent=75
|
||||
options zfs zfs_arc_min=134217728
|
||||
options zfs zfs_arc_max=4294967296
|
||||
options zfs zfs_txg_timeout=30
|
||||
options zfs zfs_vdev_scrub_min_active=1
|
||||
options zfs zfs_vdev_scrub_max_active=1
|
||||
options zfs zfs_vdev_sync_write_min_active=8
|
||||
options zfs zfs_vdev_sync_write_max_active=32
|
||||
options zfs zfs_vdev_sync_read_min_active=8
|
||||
options zfs zfs_vdev_sync_read_max_active=32
|
||||
options zfs zfs_vdev_async_read_min_active=8
|
||||
options zfs zfs_vdev_async_read_max_active=32
|
||||
options zfs zfs_vdev_async_write_min_active=8
|
||||
options zfs zfs_vdev_async_write_max_active=32
|
||||
options zfs zfs_vdev_def_queue_depth=128
|
||||
'';
|
||||
boot.kernelParams = [ "split_lock_detect=off" ];
|
||||
|
||||
# Use the systemd-boot EFI boot loader.
|
||||
boot.loader.systemd-boot.enable = true;
|
||||
|
|
@ -81,54 +61,46 @@ in
|
|||
options = [ "defaults" "size=8G" "mode=755" ];
|
||||
};
|
||||
|
||||
# nix store file system from encrypted ZFS
|
||||
fileSystems."/nix" =
|
||||
{ device = "zpool/nix";
|
||||
fsType = "zfs";
|
||||
neededForBoot = true;
|
||||
};
|
||||
|
||||
# data store file system from encrypted ZFS
|
||||
fileSystems."/data" =
|
||||
{ device = "zpool/data";
|
||||
fsType = "zfs";
|
||||
# tmp on /nix to not fill RAM
|
||||
fileSystems."/tmp" =
|
||||
{ device = "/nix/tmp";
|
||||
fsType = "none";
|
||||
neededForBoot = true;
|
||||
options = [ "bind" ];
|
||||
depends = [ "/nix" ];
|
||||
};
|
||||
|
||||
# bind mount to have user homes
|
||||
fileSystems."/home" =
|
||||
{ device = "/data/home";
|
||||
{ device = "/nix/data/home";
|
||||
fsType = "none";
|
||||
neededForBoot = true;
|
||||
options = [ "bind" ];
|
||||
depends = [ "/data" ];
|
||||
depends = [ "/nix" ];
|
||||
};
|
||||
|
||||
# bind mount to have root home
|
||||
fileSystems."/root" =
|
||||
{ device = "/data/root";
|
||||
{ device = "/nix/data/root";
|
||||
fsType = "none";
|
||||
neededForBoot = true;
|
||||
options = [ "bind" ];
|
||||
depends = [ "/data" ];
|
||||
depends = [ "/nix" ];
|
||||
};
|
||||
|
||||
# bind mount to have NixOS configuration, different per host
|
||||
fileSystems."/etc/nixos" =
|
||||
{ device = "/data/nixos/${config.networking.hostName}";
|
||||
{ device = "/nix/data/nixos/${config.networking.hostName}";
|
||||
fsType = "none";
|
||||
neededForBoot = true;
|
||||
options = [ "bind" ];
|
||||
depends = [ "/data" ];
|
||||
depends = [ "/nix" ];
|
||||
};
|
||||
|
||||
# keep some stuff persistent
|
||||
environment.persistence."/nix/persistent" = {
|
||||
hideMounts = true;
|
||||
directories = [
|
||||
# tmp dir, don't fill our tmpfs root with that
|
||||
{ directory = "/tmp"; user = "root"; group = "root"; mode = "1777"; }
|
||||
|
||||
# systemd timers
|
||||
{ directory = "/var/lib/systemd/timers"; user = "root"; group = "root"; mode = "u=rwx,g=rx,o=rx"; }
|
||||
|
||||
|
|
@ -144,15 +116,6 @@ in
|
|||
# kill the tmp content on reboots, we mount that to /nix/persistent to avoid memory fill-up
|
||||
boot.tmp.cleanOnBoot = true;
|
||||
|
||||
# ensure our data is not rotting
|
||||
services.zfs.autoScrub = {
|
||||
enable = true;
|
||||
interval = "weekly";
|
||||
};
|
||||
|
||||
# trim the stuff, we use SSDs
|
||||
services.zfs.trim.enable = true;
|
||||
|
||||
# enable fast dbus
|
||||
services.dbus.implementation = "broker";
|
||||
|
||||
|
|
@ -260,7 +223,7 @@ in
|
|||
allowReboot = false;
|
||||
};
|
||||
|
||||
# avoid suspend ever to be triggered, ZFS dislikes that
|
||||
# avoid suspend ever to be triggered
|
||||
systemd.targets.sleep.enable = false;
|
||||
systemd.targets.suspend.enable = false;
|
||||
systemd.targets.hibernate.enable = false;
|
||||
|
|
@ -522,29 +485,10 @@ in
|
|||
mode = "0400";
|
||||
};
|
||||
environment.etc."mail/secrets" = {
|
||||
text = builtins.readFile "/data/nixos/mail.secret";
|
||||
text = builtins.readFile "/nix/data/nixos/mail.secret";
|
||||
mode = "0400";
|
||||
};
|
||||
|
||||
# send mails on ZFS events
|
||||
services.zfs.zed = {
|
||||
settings = {
|
||||
ZED_DEBUG_LOG = "/tmp/zed.debug.log";
|
||||
ZED_EMAIL_ADDR = [ "root" ];
|
||||
ZED_EMAIL_PROG = "/run/wrappers/bin/sendmail";
|
||||
ZED_EMAIL_OPTS = "@ADDRESS@";
|
||||
|
||||
ZED_NOTIFY_INTERVAL_SECS = 3600;
|
||||
ZED_NOTIFY_VERBOSE = true;
|
||||
|
||||
ZED_USE_ENCLOSURE_LEDS = true;
|
||||
ZED_SCRUB_AFTER_RESILVER = true;
|
||||
};
|
||||
|
||||
# this option does not work; will return error
|
||||
enableMail = false;
|
||||
};
|
||||
|
||||
# use ZSH per default
|
||||
programs.zsh.enable = true;
|
||||
environment.shells = with pkgs; [ zsh ];
|
||||
|
|
|
|||
|
|
@ -23,10 +23,10 @@ in
|
|||
|
||||
users.root = {
|
||||
# init password
|
||||
hashedPassword = builtins.readFile "/data/nixos/password.secret";
|
||||
hashedPassword = builtins.readFile "/nix/data/nixos/password.secret";
|
||||
|
||||
# use fixed auth keys
|
||||
openssh.authorizedKeys.keys = pkgs.lib.splitString "\n" (builtins.readFile "/data/nixos/authorized_keys.secret");
|
||||
openssh.authorizedKeys.keys = pkgs.lib.splitString "\n" (builtins.readFile "/nix/data/nixos/authorized_keys.secret");
|
||||
};
|
||||
|
||||
#
|
||||
|
|
|
|||
Loading…
Reference in a new issue