Compare commits

..

11 Commits

Author SHA1 Message Date
15f70019d5 feat: integrate rollback sentinel as NixOS module
Add rollback-sentinel NixOS module that:
- Deploys sentinel-check.sh (inline) and nixos-rollback.sh (from file) as
  system packages
- Runs a boot-time systemd oneshot service after multi-user.target with
  configurable delay — checks Tier-1 services, triggers rollback on failure
- Runs a post-rebuild service via activation script after every
  nixos-rebuild switch
- Exposes options for tier1Services, tier2Services, tier3InfoServices,
  bootDelay, rollbackMode (set-default/rollback-now/dry-run), and
  enablePostRebuild

Module wired into flake.nix for lazyworkhorse and enabled in
configuration.nix with standard Tier-1/2 service lists and 120s delay.
2026-05-25 00:08:39 -04:00
e216c8edac fix: update compose submodule - honcho embedding config fix + deriver in CMD 2026-05-23 18:54:54 -04:00
2b4b2e2216 fix: update compose submodule - honcho config fixes 2026-05-23 16:51:24 -04:00
6be4076372 fix: update compose submodule to fix honcho config corrupted values 2026-05-23 16:43:22 -04:00
690873d0e4 fix: update compose submodule for honcho auth + LLM config 2026-05-23 08:34:46 -04:00
cd817c7fd2 chore: update compose submodule (fix nginx permissions - run as root) 2026-05-23 01:21:31 -04:00
33d1d860fb chore: update compose submodule (fix nginx permissions) 2026-05-23 00:31:46 -04:00
8d7afecb6e chore: update compose submodule (merge Honcho+OpenConcho into single container) 2026-05-23 00:00:27 -04:00
6f18071198 chore: update compose submodule (expose Honcho via Traefik + fix VECTOR_STORE_DIMENSIONS) 2026-05-22 23:54:35 -04:00
b43c6794b3 chore: update compose submodule (fix VECTOR_STORE_DIMENSIONS empty value) 2026-05-22 23:52:24 -04:00
8b9a144254 fix: update compose submodule for Matrix bridge deps + persistent venv
Updates assets/compose submodule to 8adbbf0 (compose fix/matrix-bridge-deps).

Compose commit adds:
- uv pip install openai mautrix[encryption] to hermes entrypoint
- Persistent venv volume at /opt/hermes/.venv
- Empty-volume first-boot handling (venv recreation)

This ensures Matrix bridge dependencies survive container recreation.
2026-05-20 14:36:42 -04:00
9 changed files with 706 additions and 296 deletions

31
flake.nix Executable file → Normal file
View File

@@ -2,34 +2,28 @@
description = "Gortium infra flake";
inputs = {
nixpkgs.url = "github:nixos/nixpkgs?ref=25.11";
nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-unstable";
agenix = {
url = "github:ryantm/agenix";
inputs.darwin.follows = "";
inputs.nixpkgs.follows = "nixpkgs";
};
disko = {
url = "github:nix-community/disko";
inputs.nixpkgs.follows = "nixpkgs";
};
lix = {
url = "git+https://git.lix.systems/lix-project/lix?ref=main";
inputs.nixpkgs.follows = "nixpkgs";
};
nixpkgs-uconsole = {
url = "github:nixos/nixpkgs/54170c54449ea4d6725efd30d719c5e505f1c10e";
};
nixos-uconsole = {
url = "github:nixos-uconsole/nixos-uconsole/v1.1.0";
inputs.nixpkgs.follows = "nixpkgs-uconsole";
url = "github:nixos-uconsole/nixos-uconsole";
inputs.nixpkgs.follows = "nixpkgs";
};
nixos-raspberrypi = {
url = "github:nvmd/nixos-raspberrypi/v1.20260317.0";
inputs.nixpkgs.follows = "nixos-uconsole/nixpkgs";
inputs.nixpkgs.follows = "nixpkgs";
};
self.submodules = true;
};
outputs = { self, nixpkgs, agenix, disko, lix, nixos-uconsole, nixos-raspberrypi, ... }@inputs:
outputs = { self, nixpkgs, agenix, lix, nixos-uconsole, nixos-raspberrypi, ... }@inputs:
let
system = "x86_64-linux";
keys = import ./lib/keys.nix;
@@ -75,6 +69,7 @@
./modules/nixos/services/open_code_server.nix
./modules/nixos/services/ollama_init_custom_models.nix
./modules/nixos/services/openclaw_node.nix
./modules/nixos/services/rollback-sentinel.nix
./modules/nixos/security/ai-worker-restricted.nix
./users/gortium.nix
./users/ai-worker.nix
@@ -95,21 +90,19 @@
];
};
uConsole = nixos-uconsole.lib.mkUConsoleSystem {
variant = "cm5";
uConsole = nixos-raspberrypi.lib.nixosSystem {
specialArgs = { inherit self keys paths inputs nixos-raspberrypi; };
modules = [
{
nixpkgs.overlays = overlays;
nixpkgs.config.allowUnfree = true;
nixpkgs.config.permittedInsecurePackages = [
"openclaw-2026.3.12"
];
nixpkgs.hostPlatform = "aarch64-linux";
nix.package = lix.packages."aarch64-linux".default;
}
disko.nixosModules.disko
nixos-raspberrypi.nixosModules.raspberry-pi-5.base
nixos-uconsole.nixosModules.uconsole-cm5
./hosts/uConsole/configuration.nix
./hosts/uConsole/hardware-configuration.nix
./hosts/uConsole/disko-config.nix
];
};
};

View File

@@ -11,10 +11,6 @@
# Flakesss
nix.settings.experimental-features = [ "nix-command" "flakes" "flake-self-attrs" ];
nix.settings.trusted-users = [ "root" "gortium" ];
nix.settings.extra-platforms = [ "aarch64-linux" ];
# QEMU binfmt for cross-building aarch64 NixOS targets
boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
# Garbage collection
nix.gc = {
@@ -325,10 +321,40 @@
environment.etc."ssh/ssh_host_ed25519_key.pub".text =
"${keys.hosts.lazyworkhorse.main}";
# ── Boot sentinel: auto-rollback on critical service failure ───────────────
services.rollbackSentinel.enable = true;
# Tier-1: failure triggers rollback
services.rollbackSentinel.tier1Services = [
"sshd" "docker" "traefik" "authelia"
];
# Tier-2: warn only
services.rollbackSentinel.tier2Services = [
"gitea" "hermes" "ollama" "synapse" "nextcloud"
"vaultwarden" "wireguard" "homeassistant" "fail2ban"
];
# Wait 2 minutes after boot before checking (lets services initialize)
services.rollbackSentinel.bootDelay = "120";
# Change boot default only (not --rollback-now) for safety
services.rollbackSentinel.rollbackMode = "set-default";
services.fstrim.enable = true;
services.zfs.autoSnapshot.enable = true;
services.zfs.autoScrub.enable = true;
# Ensure com.sun:auto-snapshot is set on ZFS datasets so auto-snapshots actually run
systemd.services."zfs-set-auto-snapshot" = {
description = "Set com.sun:auto-snapshot=true on ZFS datasets";
after = [ "zfs-import.target" ];
wants = [ "zfs-import.target" ];
wantedBy = [ "multi-user.target" ];
path = with pkgs; [ zfs ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = "${pkgs.zfs}/bin/zfs set -r com.sun:auto-snapshot=true rpool";
};
};
# Mi50 config
hardware.graphics = {

135
hosts/uConsole/configuration.nix Executable file → Normal file
View File

@@ -1,18 +1,4 @@
{ config, lib, pkgs, paths, self, keys, ... }:
let
# Backlight fallback for CM5 display quirk
# The kernel driver usually handles this, but some boots need a kick
backlightFixScript = pkgs.writeShellScript "backlight-fix" ''
# Try sysfs backlight control
for bl in /sys/class/backlight/*/brightness; do
if [ -f "$bl" ]; then
max=$(cat "$(dirname "$bl")/max_brightness" 2>/dev/null || echo 100)
echo "$max" > "$bl" 2>/dev/null || true
fi
done
'';
in
{ config, lib, pkgs, paths, self, ... }:
{
# Basic Host Info
@@ -21,37 +7,17 @@ in
i18n.defaultLocale = "en_CA.UTF-8";
# System State
system.stateVersion = "25.11";
system.stateVersion = "25.05";
# Boot & Hardware (migrated to kernel bootloader per nixos-raspberrypi deprecation notice)
boot.loader.raspberry-pi.bootloader = "kernel";
# kernel managed by nixos-raspberrypi module — don't override, patches are version-specific
# boot.kernelPackages = pkgs.linuxPackages_latest;
# Kernel parameters matching nixos-uconsole CM5 module
# console=tty1 is critical — without it, console output goes to ttyAMA0 not fb0
boot.kernelParams = [
"8250.nr_uarts=1"
"console=tty1"
];
# Enable Mesa GPU drivers — REQUIRED for VC4 display pipeline to initialize
hardware.graphics.enable = true;
# Console font sized for the 5" 720x1280 display (from nixos-uconsole base module)
console = {
earlySetup = true;
font = "ter-v24n";
packages = with pkgs; [ terminus_font ];
};
# Boot & Hardware (uconsole-cm5 module handles boot.loader)
boot.kernelPackages = pkgs.linuxPackages_latest;
# Networking
networking.networkmanager.enable = true;
services.openssh = {
enable = true;
# TODO: lock down after first deployment
settings.PermitRootLogin = lib.mkForce "yes";
settings.PasswordAuthentication = lib.mkForce true;
settings.PermitRootLogin = "prohibit-password";
settings.PasswordAuthentication = false;
};
# User
@@ -88,7 +54,6 @@ in
htop
tmux
neovim
libgpiod # GPIO control (for internal USB hub, AIO modules)
# ===== HAM Radio =====
js8call
@@ -108,8 +73,7 @@ in
soapysdr-with-plugins # SoapySDR + hardware support plugins
# ===== Mesh / LoRa =====
# meshtastic not available in nixpkgs 25.11 stable; install manually:
# nix shell nixpkgs#meshtastic -c meshtastic
meshtastic # Python CLI for Meshtastic devices
reticulumStack # Reticulum Network Stack (rnsd, rnsh, rncp, rnx, rnpath, etc.)
lxmf # LXMF messaging protocol
nomadnet # Nomad Network client
@@ -142,7 +106,6 @@ in
# ============================================================
systemd.services.rnsd = {
description = "Reticulum Network Stack Daemon";
wants = [ "network-online.target" ];
after = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
@@ -160,7 +123,6 @@ in
# ============================================================
systemd.services.kismet = {
description = "Kismet Wi-Fi Monitor & IDS";
wants = [ "network-online.target" ];
after = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
@@ -173,33 +135,22 @@ in
};
# ============================================================
# Kernel modules for SDR, radio, and WiFi
# Kernel modules for SDR and radio
# ============================================================
boot.kernelModules = [
"mt7921u" # MediaTek MT7921 USB WiFi (uConsole AC1200)
"88x2bu" # Realtek 8812/8821BU USB WiFi (common adapter)
"rtl8xxxu" # RTL8188/8192/8723 USB WiFi
"rtl2832_sdr" # RTL-SDR kernel module
"dvb_usb_rtl28xxu" # RTL-SDR DVB-T
# Display drivers — loaded AFTER RP1 PCIe southbridge init (~12s)
# NOTHING in initrd — ALL RP1 hardware is behind PCIe
"panel_cwu50" # uConsole DSI panel driver
"vc4" # VideoCore 4 KMS GPU driver
"rp1_dsi" # RP1 DSI bridge driver
];
boot.blacklistedKernelModules = [ ];
# Rien dans initrd pour le display — tout RP1 est derrière PCIe
boot.initrd.kernelModules = lib.mkForce [ ];
# ============================================================
# Extra udev rules for SDR and HAM radio devices
# ============================================================
services.udev.packages = with pkgs; [ rtl-sdr ];
# ============================================================
# Enable IPv6 for Reticulum mesh
# ============================================================
@@ -213,74 +164,4 @@ in
# Reticulum uses its own encryption and doesn't need open ports
# for basic mesh operations (peer-to-peer discovery).
# For TCP interfaces, open additional ports as needed.
# ============================================================
# Hyprland Wayland compositor (manual start)
# No SDDM — boot to console, user starts Hyprland with command
# Display modules (vc4/panel_cwu50) load late after RP1 PCIe init
# ============================================================
programs.hyprland = {
enable = true;
xwayland.enable = true;
};
# SDDM disabled — was blocking boot when display isn't ready
# services.displayManager.sddm = {
# enable = true;
# wayland.enable = true;
# };
# ============================================================
# CM5 Config.txt Fix: use [pi5] section (not [cm5])
# Rex's images use [pi5], the CM5 firmware may not detect [cm5]
# ============================================================
# Merge nixos-uconsole GPIO config with our [pi5] overrides
# GPIO 10/11 are from nixos-uconsole configtxt.nix (audio amplifier)
# [pi5] section fixes the CM5 detection issue — firmware matches [pi5] not [cm5]
hardware.raspberry-pi.extra-config = ''
[all]
gpio=10=ip,np
gpio=11=op,dh
[pi5]
dtparam=pciex1=off
dtoverlay=clockworkpi-uconsole-cm5
dtoverlay=dwc2,dr_mode=host
dtoverlay=vc4-kms-v3d-pi5,cma-384
dtparam=nohdmi1=off
'';
# ============================================================
# CM5 Display Backlight Fix
# The kernel driver initializes backlight, but some boots fail.
# This service kicks it after boot as a reliable fallback.
# ============================================================
systemd.services.cm5-backlight-fix = {
description = "CM5 Display Backlight Fix";
after = [ "multi-user.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
ExecStart = "${backlightFixScript}";
};
};
# ============================================================
# Internal USB Hub Enable (GPIO 23) — DISABLED
# This service freeze the CM5 because gpioset 0 23=1 writes
# to the wrong GPIO chip (BCM2712 native, not RP1).
# Enable manually after boot once the correct chip is confirmed:
# gpioset 0 23=1 # on chip 0 (BCM2712, CORE_VOLT or critical)
# gpioset 512 23=1 # on chip 512 (RP1, likely correct)
# ============================================================
# systemd.services.enable-gpio23-usb-hub = {
# description = "Enable Internal USB Hub (GPIO 23)";
# before = [ "network.target" ];
# wantedBy = [ "multi-user.target" ];
# serviceConfig = {
# Type = "oneshot";
# RemainAfterExit = true;
# ExecStart = "${pkgs.libgpiod}/bin/gpioset 0 23=1";
# ExecStop = "${pkgs.libgpiod}/bin/gpioset 0 23=0";
# };
# };
}

View File

@@ -1,46 +0,0 @@
{ lib, ... }:
{
disko.devices.disk.main = {
type = "disk";
device = "/dev/mmcblk0";
content = {
type = "gpt";
partitions = {
boot = {
name = "FIRMWARE";
size = "1G";
type = "EF00";
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot/firmware";
mountOptions = [
"fmask=0022"
"dmask=0022"
];
};
};
root = {
name = "NIXOS_UCM5";
size = "30G";
content = {
type = "filesystem";
format = "ext4";
mountpoint = "/";
mountOptions = [ "noatime" ];
};
};
home = {
name = "NIXOS_HOME";
size = "100%";
content = {
type = "filesystem";
format = "ext4";
mountpoint = "/home";
mountOptions = [ "noatime" ];
};
};
};
};
};
}

View File

@@ -9,28 +9,15 @@
boot.initrd.kernelModules = [ ];
boot.extraModulePackages = [ ];
# Filesystems for NixOS install.
# mkForce overrides disko's auto-generated paths so we can use
# filesystem labels (by-label) which work with loop device installs.
# Disko will set its own paths when nixos-anywhere is used.
fileSystems."/" = lib.mkForce {
# uConsole CM5 uses NVMe or eMMC for boot storage
# The uconsole-cm5 module sets up /boot/firmware and default /
# Override device label here if using different storage
fileSystems."/" = lib.mkDefault {
device = "/dev/disk/by-label/NIXOS_UCM5";
fsType = "ext4";
options = [ "noatime" ];
};
fileSystems."/boot/firmware" = lib.mkForce {
device = "/dev/disk/by-label/FIRMWARE";
fsType = "vfat";
options = [ "fmask=0022" "dmask=0022" ];
};
fileSystems."/home" = lib.mkForce {
device = "/dev/disk/by-label/NIXOS_HOME";
fsType = "ext4";
options = [ "noatime" ];
};
swapDevices = [ ];
nixpkgs.hostPlatform = lib.mkDefault "aarch64-linux";

View File

@@ -0,0 +1,400 @@
#!/usr/bin/env bash
# =============================================================================
# nixos-rollback.sh — NixOS systemd-boot Rollback Script
#
# Detects a failed NixOS generation (critical services not starting) and sets
# the previous generation as the default boot option for systemd-boot.
# Logs all actions to syslog/journald and a local logfile. Fails safely when
# no previous generation exists or required files are missing.
#
# Integration with the boot sentinel:
# sentinel-check.sh → detects Tier-1 service failures (sshd, docker,
# traefik, authelia) after a boot
# nixos-rollback.sh ← called when sentinel exits nonzero; sets previous
# generation as default for next boot
#
# Usage:
# nixos-rollback.sh # auto-detect & set previous gen
# nixos-rollback.sh --dry-run # show what would be done
# nixos-rollback.sh --rollback-now # also run nixos-rebuild switch
# # --rollback for immediate fix
# nixos-rollback.sh --help # full help text
#
# Exit codes:
# 0 — rollback applied (or dry-run would apply)
# 1 — preflight failure (missing files, permissions)
# 2 — no previous generation available
# 3 — nixos-rebuild --rollback failed (only with --rollback-now)
#
# Installation on NixOS:
# Place in /usr/local/bin/nixos-rollback.sh and make executable.
# Add a systemd oneshot service to run it after sentinel-check detects
# failures, or invoke directly from a sentinel timer.
# =============================================================================
set -euo pipefail
# ── Configuration ────────────────────────────────────────────────────────────
# These can be overridden via environment variables for testing.
LOADER_CONF="${NIXOS_ROLLBACK_LOADER_CONF:-/boot/loader/loader.conf}"
ENTRIES_DIR="${NIXOS_ROLLBACK_ENTRIES_DIR:-/boot/loader/entries}"
LOGFILE="${NIXOS_ROLLBACK_LOGFILE:-/var/log/nixos-rollback.log}"
SYSLOG_IDENT="nixos-rollback"
# ── CLI flags ────────────────────────────────────────────────────────────────
DRY_RUN=false
ROLLBACK_NOW=false
# ── Colors (disabled when not a terminal) ────────────────────────────────────
if [ -t 1 ]; then
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
NC='\033[0m' # No Color
else
RED=''; GREEN=''; YELLOW=''; CYAN=''; NC=''
fi
# =============================================================================
# Help
# =============================================================================
usage() {
cat <<EOF
${CYAN}nixos-rollback.sh${NC} — Set the previous NixOS generation as systemd-boot default
${CYAN}USAGE${NC}
nixos-rollback.sh [OPTIONS]
${CYAN}OPTIONS${NC}
--dry-run Show what would be done without making changes
--rollback-now Also run 'nixos-rebuild switch --rollback' for
immediate fix of the running system (requires
nixos-rebuild on PATH)
-h, --help Show this help text
${CYAN}DESCRIPTION${NC}
Reads the current default boot entry from ${LOADER_CONF},
determines the previous generation number, and writes it as the
new default. The script only modifies systemd-boot config —
it does NOT touch the Nix store or system profile unless
--rollback-now is passed.
Designed as the rollback half of a boot sentinel:
1. System boots into generation N
2. sentinel-check.sh detects Tier-1 service failures
3. nixos-rollback.sh sets default to generation N-1
4. Next reboot uses the working generation
${CYAN}EXIT CODES${NC}
0 Rollback applied (or dry-run would apply)
1 Preflight failure (missing files, permissions)
2 No previous generation available (only one generation)
3 nixos-rebuild --rollback failed (with --rollback-now)
${CYAN}FILES${NC}
${LOADER_CONF} systemd-boot loader configuration
${ENTRIES_DIR}/ generation entry .conf files
${LOGFILE} action log (append-only)
EOF
}
# =============================================================================
# Logging
# =============================================================================
log() {
local level="$1"; shift
local msg="$*"
local timestamp
timestamp="$(date '+%Y-%m-%d %H:%M:%S')"
echo "${timestamp} [${level}] ${msg}" >> "${LOGFILE}"
logger -t "${SYSLOG_IDENT}" -p "user.${level}" "${msg}"
# Also print to stderr for ERROR/WARN, stdout for INFO
case "${level}" in
ERROR) echo >&2 "${RED}[ERROR]${NC} ${msg}" ;;
WARN) echo >&2 "${YELLOW}[WARN]${NC} ${msg}" ;;
INFO) echo " ${GREEN}[INFO]${NC} ${msg}" ;;
esac
}
info() { log "INFO" "$@"; }
warn() { log "WARN" "$@"; }
error() { log "ERROR" "$@"; }
# =============================================================================
# Preflight checks
# =============================================================================
preflight() {
# Must run as root (need to write to /boot), unless overridden for testing
if [ -z "${NIXOS_ROLLBACK_SKIP_ROOT_CHECK:-}" ] && [ "$(id -u)" -ne 0 ]; then
error "This script must be run as root (needs write access to /boot/loader)"
error "Set NIXOS_ROLLBACK_SKIP_ROOT_CHECK=1 for testing against mock paths."
exit 1
fi
# Directories and files
if [ ! -d "${ENTRIES_DIR}" ]; then
error "Boot entries directory not found: ${ENTRIES_DIR}"
exit 1
fi
if [ ! -f "${LOADER_CONF}" ]; then
error "Loader config not found: ${LOADER_CONF}"
exit 1
fi
if [ ! -r "${LOADER_CONF}" ]; then
error "Cannot read loader config: ${LOADER_CONF}"
exit 1
fi
# Check write access to /boot/loader (parent of loader.conf)
local loader_dir
loader_dir="$(dirname "${LOADER_CONF}")"
if [ ! -w "${loader_dir}" ]; then
error "Cannot write to ${loader_dir} (insufficient permissions)"
exit 1
fi
# Logfile directory must exist
local log_dir
log_dir="$(dirname "${LOGFILE}")"
if [ ! -d "${log_dir}" ]; then
warn "Log directory ${log_dir} does not exist, creating it"
mkdir -p "${log_dir}" 2>/dev/null || {
error "Cannot create log directory ${log_dir}"
exit 1
}
fi
# Check --rollback-now dependencies
if [ "${ROLLBACK_NOW}" = true ]; then
if ! command -v nixos-rebuild &>/dev/null; then
error "nixos-rebuild not found on PATH (required for --rollback-now)"
exit 1
fi
fi
}
# =============================================================================
# Generation helpers
# =============================================================================
# get_current_default: reads the current default entry from loader.conf
# Returns: "nixos-generation-N.conf" or empty string
get_current_default() {
grep -E '^default\s+' "${LOADER_CONF}" 2>/dev/null \
| awk '{print $2}' \
|| true
}
# extract_gen_number: extracts the numeric generation from a conf filename
# Input: "nixos-generation-367.conf"
# Output: 367
extract_gen_number() {
echo "$1" | sed 's/nixos-generation-//;s/\.conf//'
}
# get_all_gen_numbers: returns sorted list of generation numbers from entries dir
get_all_gen_numbers() {
local -a gens=()
local f n
for f in "${ENTRIES_DIR}"/nixos-generation-*.conf; do
[ -f "${f}" ] || continue
n="$(basename "${f}" | sed 's/nixos-generation-//;s/\.conf//')"
gens+=("${n}")
done
if [ "${#gens[@]}" -eq 0 ]; then
return 1
fi
# Sort numerically and output
printf '%s\n' "${gens[@]}" | sort -n
}
# get_previous_gen: given current generation number, find the previous one
# from the list of all available generations
get_previous_gen() {
local current="$1"
shift
local -a gens=("$@")
local prev=""
local g
for g in "${gens[@]}"; do
if [ "${g}" -lt "${current}" ]; then
prev="${g}"
fi
done
if [ -z "${prev}" ]; then
return 1
fi
echo "${prev}"
}
# =============================================================================
# Main rollback logic
# =============================================================================
do_rollback() {
# Step 1: Read current default
local current_entry
current_entry="$(get_current_default)"
if [ -z "${current_entry}" ]; then
error "No 'default' entry found in ${LOADER_CONF}"
error "Cannot determine current generation — aborting"
exit 1
fi
info "Current default boot entry: ${current_entry}"
# Step 2: Build sorted list of all available generations
local -a all_gens=()
local line
while IFS= read -r line; do
all_gens+=("${line}")
done < <(get_all_gen_numbers || true)
if [ "${#all_gens[@]}" -eq 0 ]; then
error "No NixOS generation .conf files found in ${ENTRIES_DIR}"
exit 1
fi
info "Available generations: ${all_gens[*]}"
# Step 3: Find current generation number
local current_gen
current_gen="$(extract_gen_number "${current_entry}")"
# Verify current_gen is a valid number
if ! [[ "${current_gen}" =~ ^[0-9]+$ ]]; then
error "Could not parse generation number from '${current_entry}'"
exit 1
fi
# Step 4: Find the previous generation
local prev_gen
prev_gen="$(get_previous_gen "${current_gen}" "${all_gens[@]}")" || {
error "No previous generation found before generation ${current_gen}"
error "This is the oldest available generation — cannot roll back further"
exit 2
}
local prev_entry="nixos-generation-${prev_gen}.conf"
local prev_conf_path="${ENTRIES_DIR}/${prev_entry}"
if [ ! -f "${prev_conf_path}" ]; then
error "Previous generation entry not found: ${prev_conf_path}"
error "The .conf file for generation ${prev_gen} is missing — cannot roll back"
exit 1
fi
info "Target rollback generation: ${prev_gen}${prev_entry}"
# Step 5: Apply the rollback
if [ "${DRY_RUN}" = true ]; then
echo ""
echo " ${CYAN}[DRY RUN]${NC} Would change ${LOADER_CONF}:"
echo " ${YELLOW}-${NC} default ${current_entry}"
echo " ${GREEN}+${NC} default ${prev_entry}"
echo ""
info "DRY RUN — no changes made"
exit 0
fi
# Write new default
# Use sed with a backup (.bak)
sed -i.bak "s/^default\s\+${current_entry}/default ${prev_entry}/" "${LOADER_CONF}"
# Verify the change was applied
local new_default
new_default="$(get_current_default)"
if [ "${new_default}" != "${prev_entry}" ]; then
error "Failed to set default boot entry to ${prev_entry}"
error "Current default is still: ${new_default}"
# Attempt to restore backup
if [ -f "${LOADER_CONF}.bak" ]; then
cp "${LOADER_CONF}.bak" "${LOADER_CONF}"
info "Restored backup from ${LOADER_CONF}.bak"
fi
exit 1
fi
info "Successfully set default boot entry to ${prev_entry} (generation ${prev_gen})"
info "Backup of previous config saved to ${LOADER_CONF}.bak"
# Step 6: Optionally run nixos-rebuild switch --rollback
if [ "${ROLLBACK_NOW}" = true ]; then
echo ""
info "Running nixos-rebuild switch --rollback for immediate effect..."
if nixos-rebuild switch --rollback 2>&1 | while IFS= read -r line; do
logger -t "${SYSLOG_IDENT}" "nixos-rebuild: ${line}"
echo " ${line}"
done; then
info "nixos-rebuild switch --rollback completed successfully"
else
local rc=$?
error "nixos-rebuild switch --rollback failed with exit code ${rc}"
error "The boot default has been changed but the current system was NOT rolled back"
error "Reboot to apply the rollback"
exit 3
fi
fi
info "Rollback complete. Next boot will use generation ${prev_gen}."
if [ "${ROLLBACK_NOW}" = false ]; then
echo ""
echo " ${YELLOW}NOTE:${NC} The current running system is unchanged."
echo " Reboot to boot into generation ${prev_gen}."
echo " Or re-run with --rollback-now for immediate effect."
fi
}
# =============================================================================
# Main
# =============================================================================
main() {
# Parse arguments
while [ $# -gt 0 ]; do
case "$1" in
--dry-run)
DRY_RUN=true
shift
;;
--rollback-now)
ROLLBACK_NOW=true
shift
;;
-h|--help)
usage
exit 0
;;
*)
echo >&2 "Unknown option: $1"
echo >&2 "Use --help for usage information."
exit 1
;;
esac
done
echo ""
echo " ${CYAN}═══ NixOS systemd-boot Rollback ═══${NC}"
echo ""
preflight
if [ "${DRY_RUN}" = true ]; then
info "DRY RUN mode — no changes will be made"
fi
if [ "${ROLLBACK_NOW}" = true ]; then
info "ROLLBACK NOW mode — will also run nixos-rebuild switch --rollback"
fi
echo ""
do_rollback
}
main "$@"

View File

@@ -0,0 +1,184 @@
{ config, pkgs, lib, ... }:
with lib;
let
cfg = config.services.rollbackSentinel;
# ── Scripts ────────────────────────────────────────────────────────────────
# Sentinel check — verifies Tier-1 services are active after boot.
# Exits nonzero when any Tier-1 service is down, which triggers the rollback.
sentinelCheck = pkgs.writeShellScriptBin "sentinel-check.sh" ''
#!/usr/bin/env bash
set -euo pipefail
SYSLOG_IDENT="nixos-sentinel"
LOGFILE="/var/log/nixos-sentinel.log"
echo "=== NixOS Sentinel Check ==="
echo "Tier-1 services: ${builtins.toString cfg.tier1Services}"
echo "Tier-2 services: ${builtins.toString cfg.tier2Services}"
FAILED=0
# Check Tier-1 services any failure means rollback
for svc in ${builtins.toString cfg.tier1Services}; do
if systemctl is-active --quiet "$svc" 2>/dev/null; then
echo " [OK] Tier-1: $svc"
else
echo " [FAIL] Tier-1: $svc is NOT active"
logger -t "$SYSLOG_IDENT" -p user.err "Tier-1 FAILURE: $svc is not active"
FAILED=1
fi
done
# Check Tier-2 services warn only
for svc in ${builtins.toString cfg.tier2Services}; do
if systemctl is-active --quiet "$svc" 2>/dev/null; then
echo " [OK] Tier-2: $svc"
else
echo " [WARN] Tier-2: $svc is NOT active"
logger -t "$SYSLOG_IDENT" -p user.warn "Tier-2 WARNING: $svc is not active"
fi
done
echo "=== Sentinel result: $([ "$FAILED" -eq 0 ] && echo 'PASS' || echo 'FAIL') ==="
echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] sentinel $([ "$FAILED" -eq 0 ] && echo 'PASS' || echo 'FAIL')" >> "$LOGFILE"
exit $FAILED
'';
# Rollback script — package the companion shell script from this directory.
# Uses builtins.readFile to embed the content at evaluation time.
rollbackScript = pkgs.writeShellScriptBin "nixos-rollback.sh" (builtins.readFile ./nixos-rollback.sh);
# Resolve rollback flags from config
rollbackFlags =
if cfg.rollbackMode == "dry-run" then "--dry-run"
else if cfg.rollbackMode == "rollback-now" then "--rollback-now"
else "";
in {
options.services.rollbackSentinel = {
enable = mkEnableOption "NixOS Rollback Sentinel auto-rollback on critical service failure";
tier1Services = mkOption {
type = types.listOf types.str;
default = [ "sshd" "docker" "traefik" "authelia" ];
description = ''
Tier-1 services whose failure triggers an automatic systemd-boot rollback.
On boot, the sentinel waits ${cfg.bootDelay} seconds, then checks each
service. If ANY service in this list is inactive, it runs the rollback
script which sets the previous NixOS generation as the default boot entry.
'';
};
tier2Services = mkOption {
type = types.listOf types.str;
default = [
"gitea" "hermes" "ollama" "synapse" "nextcloud"
"vaultwarden" "wireguard" "homeassistant" "fail2ban"
];
description = ''
Tier-2 services whose failure is logged as a warning but does NOT trigger
an automatic rollback. Useful for detecting non-critical service issues.
'';
};
tier3InfoServices = mkOption {
type = types.listOf types.str;
default = [
"act_runner" "syncthing" "restic" "fava"
"homer" "cups" "fstrim"
];
description = ''
Tier-3 informational checks (log-only, no warning). These are services
that the sentinel will note the status of for diagnostics.
'';
};
bootDelay = mkOption {
type = types.str;
default = "120";
description = ''
Seconds to wait after multi-user.target before running the boot-time
sentinel check. This gives Tier-1 services time to start before
the sentinel decides they've failed.
'';
};
rollbackMode = mkOption {
type = types.enum [ "set-default" "rollback-now" "dry-run" ];
default = "set-default";
description = ''
Rollback strategy when Tier-1 failures are detected:
- set-default: Write the previous generation to loader.conf (next reboot).
- rollback-now: Also run nixos-rebuild switch --rollback for immediate fix.
- dry-run: Log what would happen but take no action (testing).
'';
};
enablePostRebuild = mkOption {
type = types.bool;
default = true;
description = ''
When enabled, the sentinel check runs after every nixos-rebuild switch
activation. If a newly deployed generation has Tier-1 failures, it
triggers rollback immediately.
'';
};
};
config = mkIf cfg.enable {
# ── Deploy scripts to PATH ───────────────────────────────────────────────
environment.systemPackages = [ sentinelCheck rollbackScript ];
# Ensure log directory exists
systemd.tmpfiles.rules = [
"d /var/log/nixos-sentinel 0755 root root -"
];
# ── Boot-time sentinel service ───────────────────────────────────────────
# Runs after multi-user.target with a configurable delay, checks Tier-1
# services, and triggers rollback if any are down.
systemd.services.nixos-sentinel = {
description = "NixOS Boot Sentinel check critical services, roll back on failure";
after = [ "network.target" "multi-user.target" ];
wants = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
path = with pkgs; [ coreutils gawk gnused systemd ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStartPre = "${pkgs.coreutils}/bin/sleep ${cfg.bootDelay}";
ExecStart = "${sentinelCheck}/bin/sentinel-check.sh";
ExecStartPost = "${rollbackScript}/bin/nixos-rollback.sh ${rollbackFlags}";
};
};
# ── Post-rebuild sentinel service (triggered by activation script) ──────
systemd.services.nixos-sentinel-rebuild = mkIf cfg.enablePostRebuild {
description = "NixOS Post-Rebuild Sentinel check services after nixos-rebuild";
after = [ "network.target" ];
path = with pkgs; [ coreutils gawk gnused systemd ];
serviceConfig = {
Type = "oneshot";
ExecStart = "${sentinelCheck}/bin/sentinel-check.sh";
ExecStartPost = "${rollbackScript}/bin/nixos-rollback.sh ${rollbackFlags}";
};
};
# Activation script — fires after every nixos-rebuild switch
system.activationScripts.rollback-sentinel = mkIf cfg.enablePostRebuild ''
# Start the post-rebuild sentinel in the background.
# This runs on every activation (boot + nixos-rebuild). On boot the
# boot-time service handles it, so this is primarily for nixos-rebuild,
# but running twice is safe (idempotent rollback).
systemctl start nixos-sentinel-rebuild.service --no-block 2>/dev/null || true
'';
};
}

149
overlays/reticulum.nix Executable file → Normal file
View File

@@ -1,92 +1,77 @@
final: prev: let
python3 = final.python3;
pyPkgs = python3.pkgs;
in
{
reticulumStack = python3.pkgs.buildPythonApplication rec {
pname = "reticulum";
version = "1.2.9";
format = "setuptools";
src = pyPkgs.fetchPypi {
pname = "rns";
inherit version;
sha256 = "554814231c237b9caacf8df669312e57dd7d3f84b6d4810125087d1a79a75d75";
};
patchPhase = ''
# Fix license_files syntax: ("LICENSE") is a string not tuple
# Newer setuptools iterates over it char by char, fails on 'S'
substituteInPlace setup.py \
--replace-fail 'license_files = ("LICENSE")' 'license_files = ("LICENSE",)'
'';
propagatedBuildInputs = with pyPkgs; [ cryptography pyserial ];
doCheck = false;
pythonImportsCheck = [ "RNS" ];
meta = with final.lib; {
description = "Self-configuring, encrypted and resilient mesh networking stack";
homepage = "https://reticulum.network/";
license = licenses.mit;
platforms = platforms.linux;
};
in {
reticulumStack = python3.pkgs.buildPythonApplication rec {
pname = "reticulum";
version = "1.2.9";
src = pyPkgs.fetchPypi {
pname = "rns";
inherit version;
sha256 = "554814231c237b9caacf8df669312e57dd7d3f84b6d4810125087d1a79a75d75";
};
propagatedBuildInputs = with pyPkgs; [ cryptography pyserial ];
doCheck = false;
pythonImportsCheck = [ "RNS" ];
meta = with final.lib; {
description = "Self-configuring, encrypted and resilient mesh networking stack";
homepage = "https://reticulum.network/";
license = licenses.mit;
platforms = platforms.linux;
};
};
lxmf = python3.pkgs.buildPythonApplication rec {
pname = "lxmf";
version = "0.9.8";
format = "setuptools";
src = pyPkgs.fetchPypi {
inherit pname version;
sha256 = "30f39f3a975a049c12ee2cfceb3261d24cb5adec881c6821f7354464b3f3650c";
};
propagatedBuildInputs = [ final.reticulumStack ];
doCheck = false;
pythonImportsCheck = [ "LXMF" ];
meta = with final.lib; {
description = "Lightweight Extensible Message Format for Reticulum";
homepage = "https://github.com/markqvist/lxmf";
license = licenses.mit;
platforms = platforms.linux;
};
lxmf = python3.pkgs.buildPythonApplication rec {
pname = "lxmf";
version = "0.9.8";
src = pyPkgs.fetchPypi {
inherit pname version;
sha256 = "30f39f3a975a049c12ee2cfceb3261d24cb5adec881c6821f7354464b3f3650c";
};
propagatedBuildInputs = [ final.reticulumStack ];
doCheck = false;
pythonImportsCheck = [ "LXMF" ];
meta = with final.lib; {
description = "Lightweight Extensible Message Format for Reticulum";
homepage = "https://github.com/markqvist/lxmf";
license = licenses.mit;
platforms = platforms.linux;
};
};
nomadnet = python3.pkgs.buildPythonApplication rec {
pname = "nomadnet";
version = "1.1.1";
format = "setuptools";
src = pyPkgs.fetchPypi {
inherit pname version;
sha256 = "fa13b64a10e75b705a58024815ab72451700aa726af96d415ba99dec28dfc40a";
};
propagatedBuildInputs = with pyPkgs; [ final.reticulumStack final.lxmf urwid qrcode ];
doCheck = false;
pythonImportsCheck = [ "nomadnet" ];
meta = with final.lib; {
description = "Nomad Network resilient mesh communications platform";
homepage = "https://github.com/markqvist/NomadNet";
license = licenses.mit;
platforms = platforms.linux;
};
nomadnet = python3.pkgs.buildPythonApplication rec {
pname = "nomadnet";
version = "1.1.1";
src = pyPkgs.fetchPypi {
inherit pname version;
sha256 = "fa13b64a10e75b705a58024815ab72451700aa726af96d415ba99dec28dfc40a";
};
propagatedBuildInputs = with pyPkgs; [ final.reticulumStack final.lxmf urwid qrcode ];
doCheck = false;
pythonImportsCheck = [ "nomadnet" ];
meta = with final.lib; {
description = "Nomad Network resilient mesh communications platform";
homepage = "https://github.com/markqvist/NomadNet";
license = licenses.mit;
platforms = platforms.linux;
};
};
rnsh = python3.pkgs.buildPythonApplication rec {
pname = "rnsh";
version = "0.1.7";
format = "setuptools";
src = pyPkgs.fetchPypi {
inherit pname version;
sha256 = "9cb72f25abb1c6d300f8014b264184ff78f592fe88e36094938012990b797c93";
};
propagatedBuildInputs = [ final.reticulumStack ];
doCheck = false;
pythonImportsCheck = [ "rnsh" ];
meta = with final.lib; {
description = "Remote shell over Reticulum";
homepage = "https://github.com/acehoss/rnsh";
license = licenses.mit;
platforms = platforms.linux;
};
rnsh = python3.pkgs.buildPythonApplication rec {
pname = "rnsh";
version = "0.1.7";
src = pyPkgs.fetchPypi {
inherit pname version;
sha256 = "9cb72f25abb1c6d300f8014b264184ff78f592fe88e36094938012990b797c93";
};
}
# meshtastic may not exist in all nixpkgs versions (e.g. not in 25.11)
// prev.lib.optionalAttrs (prev ? meshtastic) {
inherit (prev) meshtastic;
}
propagatedBuildInputs = [ final.reticulumStack ];
doCheck = false;
pythonImportsCheck = [ "rnsh" ];
meta = with final.lib; {
description = "Remote shell over Reticulum";
homepage = "https://github.com/acehoss/rnsh";
license = licenses.mit;
platforms = platforms.linux;
};
};
}