Compare commits

..

12 Commits

Author SHA1 Message Date
7a3885a036 rename: temp_16tb -> Poup_16T throughout
- Module renamed to poup-16t-disk.nix
- Option namespace: gortium.poup16t (was gortium.temp16tb)
- LUKS name: poup_16t (was temp_16tb)
- Mount point: /mnt/Poup_16T (was /mnt/temp_16tb)
- btrbk instance: poup16t (was temp16tb)
- flake.nix import and host config updated
2026-06-17 14:57:37 -04:00
44b8121edb feat: add btrfs + btrbk module for 16TB data disk
- New module: modules/nixos/filesystem/temp-16tb-disk.nix with btrfs kernel support, btrfs-progs + btrbk, LUKS2 auto-unlock, nofail mount, btrbk daily snapshots (14d/4w/3m)
- Registered in flake.nix for lazyworkhorse
- Host config with commented-out enable block (fill UUID when disk reconnected)
2026-06-17 14:51:47 -04:00
5aca97e057 Merge pull request 'feat: add uConsole CM5 host with Reticulum mesh network support' (#61) from feat/uconsole-cm5-v3 into master
Reviewed-on: #61
2026-06-06 13:19:19 +00:00
a51e095717 feat: enable aarch64 cross-build on lazyworkhorse (QEMU binfmt + extra-platforms) 2026-06-06 09:16:23 -04:00
9ebbb1c0c6 fix: bump nixos-raspberrypi to v1.20260517.0 (matches nixos-uconsole tested version) 2026-06-05 23:38:21 -04:00
7f11da1878 fix: let nixos-raspberrypi manage kernel version (patches incompatible with linuxPackages_latest) 2026-06-05 23:33:10 -04:00
29cc20bb04 fix: add wants=network-online.target to rnsd and kismet services to silence eval warnings 2026-06-05 22:58:09 -04:00
1617ac9149 fix: migrate from deprecated kernelboot to kernel bootloader for nixos-raspberrypi 2026-06-05 22:57:26 -04:00
24f15c98cd fix: add format=setuptools to all reticulum overlay python packages 2026-06-05 22:46:54 -04:00
bdd6d03739 fix: use mkForce for PermitRootLogin to override upstream module default 2026-06-05 22:45:59 -04:00
a0a6663793 fix: use mkForce for PasswordAuthentication to override upstream module default 2026-06-05 22:45:30 -04:00
b66ffadb79 fix: add missing 'keys' to uConsole module args 2026-06-05 22:43:53 -04:00
8 changed files with 149 additions and 622 deletions

View File

@@ -17,7 +17,7 @@
inputs.nixpkgs.follows = "nixpkgs";
};
nixos-raspberrypi = {
url = "github:nvmd/nixos-raspberrypi/v1.20260317.0";
url = "github:nvmd/nixos-raspberrypi/v1.20260517.0";
inputs.nixpkgs.follows = "nixpkgs";
};
self.submodules = true;
@@ -65,11 +65,11 @@
./hosts/lazyworkhorse/configuration.nix
./hosts/lazyworkhorse/hardware-configuration.nix
./modules/nixos/filesystem/hoardingcow-mount.nix
./modules/nixos/filesystem/poup-16t-disk.nix
./modules/nixos/services/docker_manager.nix
./modules/nixos/services/open_code_server.nix
./modules/nixos/services/ollama_init_custom_models.nix
./modules/nixos/services/openclaw_node.nix
./modules/nixos/services/rollback-sentinel.nix
./modules/nixos/security/ai-worker-restricted.nix
./users/gortium.nix
./users/ai-worker.nix

View File

@@ -8,9 +8,21 @@
# NAS Mounting
hoardingcow-mount.enable = true;
# 16TB btrfs storage disk (WD Red Pro — Poup_16T — LUKS2 + btrfs + btrbk snapshots)
# ⚠ SETUP REQUIRED: Connect the disk, get the LUKS UUID with 'blkid /dev/sdb',
# then set gortium.poup16t.luksUuid here and deploy.
# gortium.poup16t = {
# enable = true;
# luksUuid = "REPLACE_ME_WITH_REAL_UUID";
# };
# Flakesss
nix.settings.experimental-features = [ "nix-command" "flakes" "flake-self-attrs" ];
nix.settings.trusted-users = [ "root" "gortium" ];
nix.settings.extra-platforms = [ "aarch64-linux" ];
# QEMU binfmt for cross-building aarch64 NixOS targets
boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
# Garbage collection
nix.gc = {
@@ -321,40 +333,10 @@
environment.etc."ssh/ssh_host_ed25519_key.pub".text =
"${keys.hosts.lazyworkhorse.main}";
# ── Boot sentinel: auto-rollback on critical service failure ───────────────
services.rollbackSentinel.enable = true;
# Tier-1: failure triggers rollback
services.rollbackSentinel.tier1Services = [
"sshd" "docker" "traefik" "authelia"
];
# Tier-2: warn only
services.rollbackSentinel.tier2Services = [
"gitea" "hermes" "ollama" "synapse" "nextcloud"
"vaultwarden" "wireguard" "homeassistant" "fail2ban"
];
# Wait 2 minutes after boot before checking (lets services initialize)
services.rollbackSentinel.bootDelay = "120";
# Change boot default only (not --rollback-now) for safety
services.rollbackSentinel.rollbackMode = "set-default";
services.fstrim.enable = true;
services.zfs.autoSnapshot.enable = true;
services.zfs.autoScrub.enable = true;
# Ensure com.sun:auto-snapshot is set on ZFS datasets so auto-snapshots actually run
systemd.services."zfs-set-auto-snapshot" = {
description = "Set com.sun:auto-snapshot=true on ZFS datasets";
after = [ "zfs-import.target" ];
wants = [ "zfs-import.target" ];
wantedBy = [ "multi-user.target" ];
path = with pkgs; [ zfs ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = "${pkgs.zfs}/bin/zfs set -r com.sun:auto-snapshot=true rpool";
};
};
# Mi50 config
hardware.graphics = {

View File

@@ -1,4 +1,4 @@
{ config, lib, pkgs, paths, self, ... }:
{ config, lib, pkgs, paths, self, keys, ... }:
{
# Basic Host Info
@@ -9,15 +9,17 @@
# System State
system.stateVersion = "25.05";
# Boot & Hardware (uconsole-cm5 module handles boot.loader)
boot.kernelPackages = pkgs.linuxPackages_latest;
# Boot & Hardware (migrated to kernel bootloader per nixos-raspberrypi deprecation notice)
boot.loader.raspberry-pi.bootloader = "kernel";
# kernel managed by nixos-raspberrypi module — don't override, patches are version-specific
# boot.kernelPackages = pkgs.linuxPackages_latest;
# Networking
networking.networkmanager.enable = true;
services.openssh = {
enable = true;
settings.PermitRootLogin = "prohibit-password";
settings.PasswordAuthentication = false;
settings.PermitRootLogin = lib.mkForce "prohibit-password";
settings.PasswordAuthentication = lib.mkForce false;
};
# User
@@ -106,6 +108,7 @@
# ============================================================
systemd.services.rnsd = {
description = "Reticulum Network Stack Daemon";
wants = [ "network-online.target" ];
after = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
@@ -123,6 +126,7 @@
# ============================================================
systemd.services.kismet = {
description = "Kismet Wi-Fi Monitor & IDS";
wants = [ "network-online.target" ];
after = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {

View File

@@ -0,0 +1,121 @@
{ config, lib, pkgs, ... }:
let
cfg = config.gortium.poup16t;
luksName = cfg.luksName;
in
with lib;
{
options.gortium.poup16t = {
enable = mkEnableOption "Poup_16T storage disk (btrfs + LUKS + btrbk snapshots)";
luksUuid = mkOption {
type = types.str;
description = ''
UUID of the LUKS partition on the 16TB disk (WD Red Pro).
Find this by running as root when the disk is connected:
blkid /dev/sdb # or wherever the disk appears
lsblk -o NAME,SIZE,FSTYPE,UUID
Since btrfs is inside LUKS, the FS UUID is hidden use the
LUKS partition UUID from blkid (it'll show TYPE=\"crypto_LUKS\").
'';
example = "00000000-0000-0000-0000-000000000000";
};
luksName = mkOption {
type = types.str;
default = "poup_16t";
description = "Name for the LUKS /dev/mapper/ mapping";
};
mountPoint = mkOption {
type = types.str;
default = "/mnt/Poup_16T";
description = "Mount point for the 16TB data disk";
};
btrfsOptions = mkOption {
type = types.listOf types.str;
default = [ "defaults" "noatime" "compress=zstd:3" "nofail" ];
description = "Mount options for the btrfs filesystem. 'nofail' ensures boot succeeds when disk is disconnected.";
};
btrbk = {
enable = mkOption {
type = types.bool;
default = true;
description = "Enable btrbk snapshot management on this volume";
};
schedule = mkOption {
type = types.str;
default = "daily";
description = "systemd calendar event for btrbk (e.g. 'daily', 'hourly', '*-*-* 00:00:00')";
};
preserveMin = mkOption {
type = types.str;
default = "2d";
description = "btrbk snapshot_preserve_min minimum age before pruning";
};
preserve = mkOption {
type = types.str;
default = "14d 4w 3m";
description = "btrbk snapshot_preserve retention policy (daily, weekly, monthly)";
};
snapshotDir = mkOption {
type = types.str;
default = ".snapshots";
description = "Directory name for snapshots relative to volume root";
};
};
};
config = mkIf cfg.enable {
# Enable btrfs kernel support (no DKMS needed — it's in-tree)
boot.supportedFilesystems = [ "btrfs" ];
# Install btrfs administration tools
environment.systemPackages = with pkgs; [
btrfs-progs # mkfs.btrfs, btrfs, fsck, balance, scrub
btrbk # Snapshot management + rotation
];
# LUKS2 unlock at boot (uses keyfile or prompts if unavailable)
# Since the disk may be disconnected, initrd times out gracefully (~30s)
boot.initrd.luks.devices.${luksName} = {
device = "/dev/disk/by-uuid/${cfg.luksUuid}";
preLVM = false;
allowDiscards = true;
};
# Mount the unlocked mapper device as btrfs
fileSystems.${cfg.mountPoint} = {
device = "/dev/mapper/${luksName}";
fsType = "btrfs";
options = cfg.btrfsOptions;
};
# btrbk — automated snapshot creation and rotation
services.btrbk = mkIf cfg.btrbk.enable {
instances.poup16t = {
onCalendar = cfg.btrbk.schedule;
settings = {
snapshot_preserve_min = cfg.btrbk.preserveMin;
snapshot_preserve = cfg.btrbk.preserve;
volume.${cfg.mountPoint} = {
snapshot_create = "always";
snapshot_dir = cfg.btrbk.snapshotDir;
subvolume = ".";
};
};
};
};
};
}

View File

@@ -1,400 +0,0 @@
#!/usr/bin/env bash
# =============================================================================
# nixos-rollback.sh — NixOS systemd-boot Rollback Script
#
# Detects a failed NixOS generation (critical services not starting) and sets
# the previous generation as the default boot option for systemd-boot.
# Logs all actions to syslog/journald and a local logfile. Fails safely when
# no previous generation exists or required files are missing.
#
# Integration with the boot sentinel:
# sentinel-check.sh → detects Tier-1 service failures (sshd, docker,
# traefik, authelia) after a boot
# nixos-rollback.sh ← called when sentinel exits nonzero; sets previous
# generation as default for next boot
#
# Usage:
# nixos-rollback.sh # auto-detect & set previous gen
# nixos-rollback.sh --dry-run # show what would be done
# nixos-rollback.sh --rollback-now # also run nixos-rebuild switch
# # --rollback for immediate fix
# nixos-rollback.sh --help # full help text
#
# Exit codes:
# 0 — rollback applied (or dry-run would apply)
# 1 — preflight failure (missing files, permissions)
# 2 — no previous generation available
# 3 — nixos-rebuild --rollback failed (only with --rollback-now)
#
# Installation on NixOS:
# Place in /usr/local/bin/nixos-rollback.sh and make executable.
# Add a systemd oneshot service to run it after sentinel-check detects
# failures, or invoke directly from a sentinel timer.
# =============================================================================
set -euo pipefail
# ── Configuration ────────────────────────────────────────────────────────────
# These can be overridden via environment variables for testing.
LOADER_CONF="${NIXOS_ROLLBACK_LOADER_CONF:-/boot/loader/loader.conf}"
ENTRIES_DIR="${NIXOS_ROLLBACK_ENTRIES_DIR:-/boot/loader/entries}"
LOGFILE="${NIXOS_ROLLBACK_LOGFILE:-/var/log/nixos-rollback.log}"
SYSLOG_IDENT="nixos-rollback"
# ── CLI flags ────────────────────────────────────────────────────────────────
DRY_RUN=false
ROLLBACK_NOW=false
# ── Colors (disabled when not a terminal) ────────────────────────────────────
if [ -t 1 ]; then
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
NC='\033[0m' # No Color
else
RED=''; GREEN=''; YELLOW=''; CYAN=''; NC=''
fi
# =============================================================================
# Help
# =============================================================================
usage() {
cat <<EOF
${CYAN}nixos-rollback.sh${NC} — Set the previous NixOS generation as systemd-boot default
${CYAN}USAGE${NC}
nixos-rollback.sh [OPTIONS]
${CYAN}OPTIONS${NC}
--dry-run Show what would be done without making changes
--rollback-now Also run 'nixos-rebuild switch --rollback' for
immediate fix of the running system (requires
nixos-rebuild on PATH)
-h, --help Show this help text
${CYAN}DESCRIPTION${NC}
Reads the current default boot entry from ${LOADER_CONF},
determines the previous generation number, and writes it as the
new default. The script only modifies systemd-boot config —
it does NOT touch the Nix store or system profile unless
--rollback-now is passed.
Designed as the rollback half of a boot sentinel:
1. System boots into generation N
2. sentinel-check.sh detects Tier-1 service failures
3. nixos-rollback.sh sets default to generation N-1
4. Next reboot uses the working generation
${CYAN}EXIT CODES${NC}
0 Rollback applied (or dry-run would apply)
1 Preflight failure (missing files, permissions)
2 No previous generation available (only one generation)
3 nixos-rebuild --rollback failed (with --rollback-now)
${CYAN}FILES${NC}
${LOADER_CONF} systemd-boot loader configuration
${ENTRIES_DIR}/ generation entry .conf files
${LOGFILE} action log (append-only)
EOF
}
# =============================================================================
# Logging
# =============================================================================
log() {
local level="$1"; shift
local msg="$*"
local timestamp
timestamp="$(date '+%Y-%m-%d %H:%M:%S')"
echo "${timestamp} [${level}] ${msg}" >> "${LOGFILE}"
logger -t "${SYSLOG_IDENT}" -p "user.${level}" "${msg}"
# Also print to stderr for ERROR/WARN, stdout for INFO
case "${level}" in
ERROR) echo >&2 "${RED}[ERROR]${NC} ${msg}" ;;
WARN) echo >&2 "${YELLOW}[WARN]${NC} ${msg}" ;;
INFO) echo " ${GREEN}[INFO]${NC} ${msg}" ;;
esac
}
info() { log "INFO" "$@"; }
warn() { log "WARN" "$@"; }
error() { log "ERROR" "$@"; }
# =============================================================================
# Preflight checks
# =============================================================================
preflight() {
# Must run as root (need to write to /boot), unless overridden for testing
if [ -z "${NIXOS_ROLLBACK_SKIP_ROOT_CHECK:-}" ] && [ "$(id -u)" -ne 0 ]; then
error "This script must be run as root (needs write access to /boot/loader)"
error "Set NIXOS_ROLLBACK_SKIP_ROOT_CHECK=1 for testing against mock paths."
exit 1
fi
# Directories and files
if [ ! -d "${ENTRIES_DIR}" ]; then
error "Boot entries directory not found: ${ENTRIES_DIR}"
exit 1
fi
if [ ! -f "${LOADER_CONF}" ]; then
error "Loader config not found: ${LOADER_CONF}"
exit 1
fi
if [ ! -r "${LOADER_CONF}" ]; then
error "Cannot read loader config: ${LOADER_CONF}"
exit 1
fi
# Check write access to /boot/loader (parent of loader.conf)
local loader_dir
loader_dir="$(dirname "${LOADER_CONF}")"
if [ ! -w "${loader_dir}" ]; then
error "Cannot write to ${loader_dir} (insufficient permissions)"
exit 1
fi
# Logfile directory must exist
local log_dir
log_dir="$(dirname "${LOGFILE}")"
if [ ! -d "${log_dir}" ]; then
warn "Log directory ${log_dir} does not exist, creating it"
mkdir -p "${log_dir}" 2>/dev/null || {
error "Cannot create log directory ${log_dir}"
exit 1
}
fi
# Check --rollback-now dependencies
if [ "${ROLLBACK_NOW}" = true ]; then
if ! command -v nixos-rebuild &>/dev/null; then
error "nixos-rebuild not found on PATH (required for --rollback-now)"
exit 1
fi
fi
}
# =============================================================================
# Generation helpers
# =============================================================================
# get_current_default: reads the current default entry from loader.conf
# Returns: "nixos-generation-N.conf" or empty string
get_current_default() {
grep -E '^default\s+' "${LOADER_CONF}" 2>/dev/null \
| awk '{print $2}' \
|| true
}
# extract_gen_number: extracts the numeric generation from a conf filename
# Input: "nixos-generation-367.conf"
# Output: 367
extract_gen_number() {
echo "$1" | sed 's/nixos-generation-//;s/\.conf//'
}
# get_all_gen_numbers: returns sorted list of generation numbers from entries dir
get_all_gen_numbers() {
local -a gens=()
local f n
for f in "${ENTRIES_DIR}"/nixos-generation-*.conf; do
[ -f "${f}" ] || continue
n="$(basename "${f}" | sed 's/nixos-generation-//;s/\.conf//')"
gens+=("${n}")
done
if [ "${#gens[@]}" -eq 0 ]; then
return 1
fi
# Sort numerically and output
printf '%s\n' "${gens[@]}" | sort -n
}
# get_previous_gen: given current generation number, find the previous one
# from the list of all available generations
get_previous_gen() {
local current="$1"
shift
local -a gens=("$@")
local prev=""
local g
for g in "${gens[@]}"; do
if [ "${g}" -lt "${current}" ]; then
prev="${g}"
fi
done
if [ -z "${prev}" ]; then
return 1
fi
echo "${prev}"
}
# =============================================================================
# Main rollback logic
# =============================================================================
do_rollback() {
# Step 1: Read current default
local current_entry
current_entry="$(get_current_default)"
if [ -z "${current_entry}" ]; then
error "No 'default' entry found in ${LOADER_CONF}"
error "Cannot determine current generation — aborting"
exit 1
fi
info "Current default boot entry: ${current_entry}"
# Step 2: Build sorted list of all available generations
local -a all_gens=()
local line
while IFS= read -r line; do
all_gens+=("${line}")
done < <(get_all_gen_numbers || true)
if [ "${#all_gens[@]}" -eq 0 ]; then
error "No NixOS generation .conf files found in ${ENTRIES_DIR}"
exit 1
fi
info "Available generations: ${all_gens[*]}"
# Step 3: Find current generation number
local current_gen
current_gen="$(extract_gen_number "${current_entry}")"
# Verify current_gen is a valid number
if ! [[ "${current_gen}" =~ ^[0-9]+$ ]]; then
error "Could not parse generation number from '${current_entry}'"
exit 1
fi
# Step 4: Find the previous generation
local prev_gen
prev_gen="$(get_previous_gen "${current_gen}" "${all_gens[@]}")" || {
error "No previous generation found before generation ${current_gen}"
error "This is the oldest available generation — cannot roll back further"
exit 2
}
local prev_entry="nixos-generation-${prev_gen}.conf"
local prev_conf_path="${ENTRIES_DIR}/${prev_entry}"
if [ ! -f "${prev_conf_path}" ]; then
error "Previous generation entry not found: ${prev_conf_path}"
error "The .conf file for generation ${prev_gen} is missing — cannot roll back"
exit 1
fi
info "Target rollback generation: ${prev_gen}${prev_entry}"
# Step 5: Apply the rollback
if [ "${DRY_RUN}" = true ]; then
echo ""
echo " ${CYAN}[DRY RUN]${NC} Would change ${LOADER_CONF}:"
echo " ${YELLOW}-${NC} default ${current_entry}"
echo " ${GREEN}+${NC} default ${prev_entry}"
echo ""
info "DRY RUN — no changes made"
exit 0
fi
# Write new default
# Use sed with a backup (.bak)
sed -i.bak "s/^default\s\+${current_entry}/default ${prev_entry}/" "${LOADER_CONF}"
# Verify the change was applied
local new_default
new_default="$(get_current_default)"
if [ "${new_default}" != "${prev_entry}" ]; then
error "Failed to set default boot entry to ${prev_entry}"
error "Current default is still: ${new_default}"
# Attempt to restore backup
if [ -f "${LOADER_CONF}.bak" ]; then
cp "${LOADER_CONF}.bak" "${LOADER_CONF}"
info "Restored backup from ${LOADER_CONF}.bak"
fi
exit 1
fi
info "Successfully set default boot entry to ${prev_entry} (generation ${prev_gen})"
info "Backup of previous config saved to ${LOADER_CONF}.bak"
# Step 6: Optionally run nixos-rebuild switch --rollback
if [ "${ROLLBACK_NOW}" = true ]; then
echo ""
info "Running nixos-rebuild switch --rollback for immediate effect..."
if nixos-rebuild switch --rollback 2>&1 | while IFS= read -r line; do
logger -t "${SYSLOG_IDENT}" "nixos-rebuild: ${line}"
echo " ${line}"
done; then
info "nixos-rebuild switch --rollback completed successfully"
else
local rc=$?
error "nixos-rebuild switch --rollback failed with exit code ${rc}"
error "The boot default has been changed but the current system was NOT rolled back"
error "Reboot to apply the rollback"
exit 3
fi
fi
info "Rollback complete. Next boot will use generation ${prev_gen}."
if [ "${ROLLBACK_NOW}" = false ]; then
echo ""
echo " ${YELLOW}NOTE:${NC} The current running system is unchanged."
echo " Reboot to boot into generation ${prev_gen}."
echo " Or re-run with --rollback-now for immediate effect."
fi
}
# =============================================================================
# Main
# =============================================================================
main() {
# Parse arguments
while [ $# -gt 0 ]; do
case "$1" in
--dry-run)
DRY_RUN=true
shift
;;
--rollback-now)
ROLLBACK_NOW=true
shift
;;
-h|--help)
usage
exit 0
;;
*)
echo >&2 "Unknown option: $1"
echo >&2 "Use --help for usage information."
exit 1
;;
esac
done
echo ""
echo " ${CYAN}═══ NixOS systemd-boot Rollback ═══${NC}"
echo ""
preflight
if [ "${DRY_RUN}" = true ]; then
info "DRY RUN mode — no changes will be made"
fi
if [ "${ROLLBACK_NOW}" = true ]; then
info "ROLLBACK NOW mode — will also run nixos-rebuild switch --rollback"
fi
echo ""
do_rollback
}
main "$@"

View File

@@ -1,184 +0,0 @@
{ config, pkgs, lib, ... }:
with lib;
let
cfg = config.services.rollbackSentinel;
# ── Scripts ────────────────────────────────────────────────────────────────
# Sentinel check — verifies Tier-1 services are active after boot.
# Exits nonzero when any Tier-1 service is down, which triggers the rollback.
sentinelCheck = pkgs.writeShellScriptBin "sentinel-check.sh" ''
#!/usr/bin/env bash
set -euo pipefail
SYSLOG_IDENT="nixos-sentinel"
LOGFILE="/var/log/nixos-sentinel.log"
echo "=== NixOS Sentinel Check ==="
echo "Tier-1 services: ${builtins.toString cfg.tier1Services}"
echo "Tier-2 services: ${builtins.toString cfg.tier2Services}"
FAILED=0
# Check Tier-1 services any failure means rollback
for svc in ${builtins.toString cfg.tier1Services}; do
if systemctl is-active --quiet "$svc" 2>/dev/null; then
echo " [OK] Tier-1: $svc"
else
echo " [FAIL] Tier-1: $svc is NOT active"
logger -t "$SYSLOG_IDENT" -p user.err "Tier-1 FAILURE: $svc is not active"
FAILED=1
fi
done
# Check Tier-2 services warn only
for svc in ${builtins.toString cfg.tier2Services}; do
if systemctl is-active --quiet "$svc" 2>/dev/null; then
echo " [OK] Tier-2: $svc"
else
echo " [WARN] Tier-2: $svc is NOT active"
logger -t "$SYSLOG_IDENT" -p user.warn "Tier-2 WARNING: $svc is not active"
fi
done
echo "=== Sentinel result: $([ "$FAILED" -eq 0 ] && echo 'PASS' || echo 'FAIL') ==="
echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] sentinel $([ "$FAILED" -eq 0 ] && echo 'PASS' || echo 'FAIL')" >> "$LOGFILE"
exit $FAILED
'';
# Rollback script — package the companion shell script from this directory.
# Uses builtins.readFile to embed the content at evaluation time.
rollbackScript = pkgs.writeShellScriptBin "nixos-rollback.sh" (builtins.readFile ./nixos-rollback.sh);
# Resolve rollback flags from config
rollbackFlags =
if cfg.rollbackMode == "dry-run" then "--dry-run"
else if cfg.rollbackMode == "rollback-now" then "--rollback-now"
else "";
in {
options.services.rollbackSentinel = {
enable = mkEnableOption "NixOS Rollback Sentinel auto-rollback on critical service failure";
tier1Services = mkOption {
type = types.listOf types.str;
default = [ "sshd" "docker" "traefik" "authelia" ];
description = ''
Tier-1 services whose failure triggers an automatic systemd-boot rollback.
On boot, the sentinel waits ${cfg.bootDelay} seconds, then checks each
service. If ANY service in this list is inactive, it runs the rollback
script which sets the previous NixOS generation as the default boot entry.
'';
};
tier2Services = mkOption {
type = types.listOf types.str;
default = [
"gitea" "hermes" "ollama" "synapse" "nextcloud"
"vaultwarden" "wireguard" "homeassistant" "fail2ban"
];
description = ''
Tier-2 services whose failure is logged as a warning but does NOT trigger
an automatic rollback. Useful for detecting non-critical service issues.
'';
};
tier3InfoServices = mkOption {
type = types.listOf types.str;
default = [
"act_runner" "syncthing" "restic" "fava"
"homer" "cups" "fstrim"
];
description = ''
Tier-3 informational checks (log-only, no warning). These are services
that the sentinel will note the status of for diagnostics.
'';
};
bootDelay = mkOption {
type = types.str;
default = "120";
description = ''
Seconds to wait after multi-user.target before running the boot-time
sentinel check. This gives Tier-1 services time to start before
the sentinel decides they've failed.
'';
};
rollbackMode = mkOption {
type = types.enum [ "set-default" "rollback-now" "dry-run" ];
default = "set-default";
description = ''
Rollback strategy when Tier-1 failures are detected:
- set-default: Write the previous generation to loader.conf (next reboot).
- rollback-now: Also run nixos-rebuild switch --rollback for immediate fix.
- dry-run: Log what would happen but take no action (testing).
'';
};
enablePostRebuild = mkOption {
type = types.bool;
default = true;
description = ''
When enabled, the sentinel check runs after every nixos-rebuild switch
activation. If a newly deployed generation has Tier-1 failures, it
triggers rollback immediately.
'';
};
};
config = mkIf cfg.enable {
# ── Deploy scripts to PATH ───────────────────────────────────────────────
environment.systemPackages = [ sentinelCheck rollbackScript ];
# Ensure log directory exists
systemd.tmpfiles.rules = [
"d /var/log/nixos-sentinel 0755 root root -"
];
# ── Boot-time sentinel service ───────────────────────────────────────────
# Runs after multi-user.target with a configurable delay, checks Tier-1
# services, and triggers rollback if any are down.
systemd.services.nixos-sentinel = {
description = "NixOS Boot Sentinel check critical services, roll back on failure";
after = [ "network.target" "multi-user.target" ];
wants = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
path = with pkgs; [ coreutils gawk gnused systemd ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStartPre = "${pkgs.coreutils}/bin/sleep ${cfg.bootDelay}";
ExecStart = "${sentinelCheck}/bin/sentinel-check.sh";
ExecStartPost = "${rollbackScript}/bin/nixos-rollback.sh ${rollbackFlags}";
};
};
# ── Post-rebuild sentinel service (triggered by activation script) ──────
systemd.services.nixos-sentinel-rebuild = mkIf cfg.enablePostRebuild {
description = "NixOS Post-Rebuild Sentinel check services after nixos-rebuild";
after = [ "network.target" ];
path = with pkgs; [ coreutils gawk gnused systemd ];
serviceConfig = {
Type = "oneshot";
ExecStart = "${sentinelCheck}/bin/sentinel-check.sh";
ExecStartPost = "${rollbackScript}/bin/nixos-rollback.sh ${rollbackFlags}";
};
};
# Activation script — fires after every nixos-rebuild switch
system.activationScripts.rollback-sentinel = mkIf cfg.enablePostRebuild ''
# Start the post-rebuild sentinel in the background.
# This runs on every activation (boot + nixos-rebuild). On boot the
# boot-time service handles it, so this is primarily for nixos-rebuild,
# but running twice is safe (idempotent rollback).
systemctl start nixos-sentinel-rebuild.service --no-block 2>/dev/null || true
'';
};
}

View File

@@ -5,6 +5,7 @@ in {
reticulumStack = python3.pkgs.buildPythonApplication rec {
pname = "reticulum";
version = "1.2.9";
format = "setuptools";
src = pyPkgs.fetchPypi {
pname = "rns";
inherit version;
@@ -24,6 +25,7 @@ in {
lxmf = python3.pkgs.buildPythonApplication rec {
pname = "lxmf";
version = "0.9.8";
format = "setuptools";
src = pyPkgs.fetchPypi {
inherit pname version;
sha256 = "30f39f3a975a049c12ee2cfceb3261d24cb5adec881c6821f7354464b3f3650c";
@@ -42,6 +44,7 @@ in {
nomadnet = python3.pkgs.buildPythonApplication rec {
pname = "nomadnet";
version = "1.1.1";
format = "setuptools";
src = pyPkgs.fetchPypi {
inherit pname version;
sha256 = "fa13b64a10e75b705a58024815ab72451700aa726af96d415ba99dec28dfc40a";
@@ -60,6 +63,7 @@ in {
rnsh = python3.pkgs.buildPythonApplication rec {
pname = "rnsh";
version = "0.1.7";
format = "setuptools";
src = pyPkgs.fetchPypi {
inherit pname version;
sha256 = "9cb72f25abb1c6d300f8014b264184ff78f592fe88e36094938012990b797c93";