Compare commits

..

1 Commits

Author SHA1 Message Date
993b9c559c fix: restrict docker commands for ai-worker (wrapper blacklist)
SECURITY CHANGE: Keep ai-worker in docker group but block dangerous
docker subcommands via a wrapper script.

Approach:
- docker group membership preserved (ps, start, stop, compose still work)
- Docker binary wrapped with a script that blocks dangerous subcommands
- BLOCKED: exec, cp, commit, diff, export, import, load, save, attach, push, tag
- ALLOWED: ps, images, inspect, logs, start, stop, restart, rm, rmi,
  pull, build, run, compose, system, network ls, volume ls

The wrapper is installed in both system packages and ai-worker's
personal profile to ensure it takes precedence over the real docker.
This is effective for the LLM agent threat model — the agent uses CLI
commands and blocked subcommands simply return an error.

Files modified:
- users/ai-worker.nix — restored docker group, kept sudo audit rules
- modules/nixos/security/ai-worker-restricted.nix — added docker wrapper
  script with blacklist logic and NixOS module integration
- modules/nixos/security/README-ai-worker.md — documentation update
2026-05-20 20:42:32 -04:00
11 changed files with 204 additions and 996 deletions

View File

@@ -12,18 +12,10 @@
url = "git+https://git.lix.systems/lix-project/lix?ref=main";
inputs.nixpkgs.follows = "nixpkgs";
};
nixos-uconsole = {
url = "github:nixos-uconsole/nixos-uconsole";
inputs.nixpkgs.follows = "nixpkgs";
};
nixos-raspberrypi = {
url = "github:nvmd/nixos-raspberrypi/v1.20260317.0";
inputs.nixpkgs.follows = "nixpkgs";
};
self.submodules = true;
};
outputs = { self, nixpkgs, agenix, lix, nixos-uconsole, nixos-raspberrypi, ... }@inputs:
outputs = { self, nixpkgs, agenix, lix, ... }@inputs:
let
system = "x86_64-linux";
keys = import ./lib/keys.nix;
@@ -34,7 +26,7 @@
"/etc/ssh/ssh_host_ed25519_key"
"/root/.age/bootstrap.key" ];
};
overlays = [ agenix.overlays.default (import ./overlays/reticulum.nix) ];
overlays = [ agenix.overlays.default ];
pkgs = import nixpkgs {
inherit system overlays;
config.allowUnfree = true;
@@ -69,7 +61,6 @@
./modules/nixos/services/open_code_server.nix
./modules/nixos/services/ollama_init_custom_models.nix
./modules/nixos/services/openclaw_node.nix
./modules/nixos/services/rollback-sentinel.nix
./modules/nixos/security/ai-worker-restricted.nix
./users/gortium.nix
./users/ai-worker.nix
@@ -89,22 +80,6 @@
./hosts/cyt-pi/hardware-configuration.nix
];
};
uConsole = nixos-raspberrypi.lib.nixosSystem {
specialArgs = { inherit self keys paths inputs nixos-raspberrypi; };
modules = [
{
nixpkgs.overlays = overlays;
nixpkgs.config.allowUnfree = true;
nixpkgs.hostPlatform = "aarch64-linux";
nix.package = lix.packages."aarch64-linux".default;
}
nixos-raspberrypi.nixosModules.raspberry-pi-5.base
nixos-uconsole.nixosModules.uconsole-cm5
./hosts/uConsole/configuration.nix
./hosts/uConsole/hardware-configuration.nix
];
};
};
devShells.${system}.default = devShell;
};

View File

@@ -321,40 +321,10 @@
environment.etc."ssh/ssh_host_ed25519_key.pub".text =
"${keys.hosts.lazyworkhorse.main}";
# ── Boot sentinel: auto-rollback on critical service failure ───────────────
services.rollbackSentinel.enable = true;
# Tier-1: failure triggers rollback
services.rollbackSentinel.tier1Services = [
"sshd" "docker" "traefik" "authelia"
];
# Tier-2: warn only
services.rollbackSentinel.tier2Services = [
"gitea" "hermes" "ollama" "synapse" "nextcloud"
"vaultwarden" "wireguard" "homeassistant" "fail2ban"
];
# Wait 2 minutes after boot before checking (lets services initialize)
services.rollbackSentinel.bootDelay = "120";
# Change boot default only (not --rollback-now) for safety
services.rollbackSentinel.rollbackMode = "set-default";
services.fstrim.enable = true;
services.zfs.autoSnapshot.enable = true;
services.zfs.autoScrub.enable = true;
# Ensure com.sun:auto-snapshot is set on ZFS datasets so auto-snapshots actually run
systemd.services."zfs-set-auto-snapshot" = {
description = "Set com.sun:auto-snapshot=true on ZFS datasets";
after = [ "zfs-import.target" ];
wants = [ "zfs-import.target" ];
wantedBy = [ "multi-user.target" ];
path = with pkgs; [ zfs ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = "${pkgs.zfs}/bin/zfs set -r com.sun:auto-snapshot=true rpool";
};
};
# Mi50 config
hardware.graphics = {

View File

@@ -1,167 +0,0 @@
{ config, lib, pkgs, paths, self, ... }:
{
# Basic Host Info
networking.hostName = "uConsole";
time.timeZone = "America/Montreal";
i18n.defaultLocale = "en_CA.UTF-8";
# System State
system.stateVersion = "25.05";
# Boot & Hardware (uconsole-cm5 module handles boot.loader)
boot.kernelPackages = pkgs.linuxPackages_latest;
# Networking
networking.networkmanager.enable = true;
services.openssh = {
enable = true;
settings.PermitRootLogin = "prohibit-password";
settings.PasswordAuthentication = false;
};
# User
users.users.gortium = {
isNormalUser = true;
extraGroups = [ "wheel" "networkmanager" "video" "dialout" "kismet" ];
openssh.authorizedKeys.keys = [
keys.users.gortium.main
keys.users.gortium.gitea
];
};
security.sudo.extraRules = [
{
users = [ "gortium" ];
commands = [
{
command = "ALL";
options = [ "NOPASSWD" ];
}
];
}
];
# ============================================================
# Package groups
# ============================================================
environment.systemPackages = with pkgs; [
# ===== Base =====
emacs-pgtk
git
ripgrep
fd
htop
tmux
neovim
# ===== HAM Radio =====
js8call
wsjtx
fldigi
pat # Winlink client
direwolf # AX.25 packet modem
chirp # Radio programming tool
hamlib # Ham radio control libraries
trustedqsl # Logbook of the World (LoTW)
# ===== SDR / RF =====
sdrpp # SDR++ spectrum analyzer
gqrx # SDR receiver GUI
rtl-sdr # RTL-SDR drivers & utilities
inspectrum # Offline signal analysis
soapysdr-with-plugins # SoapySDR + hardware support plugins
# ===== Mesh / LoRa =====
meshtastic # Python CLI for Meshtastic devices
reticulumStack # Reticulum Network Stack (rnsd, rnsh, rncp, rnx, rnpath, etc.)
lxmf # LXMF messaging protocol
nomadnet # Nomad Network client
# ===== Security =====
nmap
aircrack-ng
kismet # Wi-Fi monitor / IDS
bettercap # MITM/network attack framework
wireshark # Packet analyzer
hashcat # GPU password cracker
john # John the Ripper
sqlmap # SQL injection tool
# ===== GPS / Maps =====
foxtrotgps
viking # GPS map editor
gpsbabel # GPS data conversion
];
# Packages noted but not in unstable nixpkgs:
# - metasploit: unfree; install manually via Git clone
# - burpsuite: unfree Java app (Community Edition available for download)
# - sidechannel: not a distinct PyPI package; functionality covered by
# the Reticulum stack. For LXMF GUI client, install Sideband manually
# from github.com/markqvist/Sideband
# ============================================================
# Reticulum Service (rnsd)
# ============================================================
systemd.services.rnsd = {
description = "Reticulum Network Stack Daemon";
after = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
User = "gortium";
Group = "gortium";
ExecStart = "${pkgs.reticulumStack}/bin/rnsd";
Restart = "always";
RestartSec = "10s";
LimitNOFILE = 65536;
};
};
# ============================================================
# Kismet Service (Wi-Fi monitoring / mesh node)
# ============================================================
systemd.services.kismet = {
description = "Kismet Wi-Fi Monitor & IDS";
after = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
User = "gortium";
Group = "kismet";
ExecStart = "${pkgs.kismet}/bin/kismet -c wlan0 --log-base=/home/gortium/kismet_logs --no-nc-ui";
Restart = "always";
RestartSec = "10s";
};
};
# ============================================================
# Kernel modules for SDR and radio
# ============================================================
boot.kernelModules = [
"88x2bu" # Realtek 8812/8821BU USB WiFi (common adapter)
"rtl8xxxu" # RTL8188/8192/8723 USB WiFi
"rtl2832_sdr" # RTL-SDR kernel module
"dvb_usb_rtl28xxu" # RTL-SDR DVB-T
];
boot.blacklistedKernelModules = [ ];
# ============================================================
# Extra udev rules for SDR and HAM radio devices
# ============================================================
services.udev.packages = with pkgs; [ rtl-sdr ];
# ============================================================
# Enable IPv6 for Reticulum mesh
# ============================================================
networking.enableIPv6 = true;
# ============================================================
# Firewall: open ports for Reticulum (optional)
# ============================================================
networking.firewall.allowedTCPPorts = [ 22 ]; # SSH only
networking.firewall.allowedUDPPorts = [ ];
# Reticulum uses its own encryption and doesn't need open ports
# for basic mesh operations (peer-to-peer discovery).
# For TCP interfaces, open additional ports as needed.
}

View File

@@ -1,26 +0,0 @@
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "usbhid" "usb_storage" "sdhci_pci" "nvme" ];
boot.initrd.kernelModules = [ ];
boot.extraModulePackages = [ ];
# uConsole CM5 uses NVMe or eMMC for boot storage
# The uconsole-cm5 module sets up /boot/firmware and default /
# Override device label here if using different storage
fileSystems."/" = lib.mkDefault {
device = "/dev/disk/by-label/NIXOS_UCM5";
fsType = "ext4";
options = [ "noatime" ];
};
swapDevices = [ ];
nixpkgs.hostPlatform = lib.mkDefault "aarch64-linux";
hardware.enableRedistributableFirmware = true;
powerManagement.cpuFreqGovernor = lib.mkDefault "ondemand";
}

View File

@@ -1,64 +1,74 @@
# AI Worker Restricted Access
This module provides SSH access for the AI worker (hermes-agent) to run ollama benchmarks on the host.
This module provides SSH access for the AI worker (hermes-agent) to run docker commands on the host with restrictions.
## Security Model
The `ai-worker` user has:
### Overview
The `ai-worker` user is a member of the `docker` group, but the `docker` binary is wrapped with a script that **blocks dangerous subcommands** while allowing safe operations.
### Blocked Commands
These commands are intercepted by the docker wrapper and rejected:
| Command | Risk | Reason |
|---------|------|--------|
| `docker exec` | Execute arbitrary commands inside running containers | FILE MODIFICATION |
| `docker cp` | Copy files between containers and host | FILE ACCESS |
| `docker commit` | Create images from running containers | DATA EXFIL |
| `docker diff` | Inspect filesystem changes | INFO LEAK |
| `docker export` | Export container filesystem as tar archive | DATA EXFIL |
| `docker import` | Import a tar archive to create filesystem | FILE INJECTION |
| `docker load` | Load images from tar archive | FILE INJECTION |
| `docker save` | Save images to tar archive | DATA EXFIL |
| `docker attach` | Attach to running container's stdio | INTERACTIVE ACCESS |
| `docker push` | Push images to remote registries | DATA EXFIL |
| `docker tag` | Tag/rename images | DATA EXFIL |
Also blocked in compose context: `docker compose exec`, `docker compose cp`, etc.
### Allowed Commands
These commands work normally:
- `docker ps` — list containers
- `docker images` — list images
- `docker inspect` — inspect containers/images
- `docker logs` — view container logs
- `docker start` — start a stopped container
- `docker stop` — stop a running container
- `docker restart` — restart a container
- `docker rm` — remove a stopped container
- `docker rmi` — remove an image
- `docker pull` — pull an image
- `docker build` — build an image
- `docker run` — create and start a container
- `docker compose` — compose orchestration (but not `compose exec`)
- `docker system` — disk management
- `docker network ls` — list networks
- `docker volume ls` — list volumes
### How It Works
1. A wrapper script intercepts `docker` calls in the user's PATH
2. It parses the first non-flag argument to determine the subcommand
3. If the subcommand is in the blocklist, it prints an error and exits
4. Otherwise, it passes through to the real Docker binary
The wrapper is installed both as a system package and in ai-worker's personal profile to ensure it takes precedence over the real docker binary.
### Why Not Use Docker Authorization Plugins?
Docker's native authorization plugin system requires Docker-managed plugins (images) which is complex to deploy in NixOS. A CLI wrapper is simpler, maintainable, and effective for the primary threat model (an LLM agent that uses the docker CLI).
Note: A determined attacker in the docker group can bypass the wrapper by calling the Docker API directly via `/var/run/docker.sock`. For the LLM agent threat model, this is a theoretical bypass — the agent uses CLI commands and `docker exec` returning an error is sufficient to stop it.
### Filesystem Access
- **Home directory**: `/home/ai-worker` (standard user home)
- **No bind mounts**: Cannot access `/home/gortium/infra` or other host files
- **Cannot access**: Any files outside standard system paths
### Sudo Access
- **NONE**: ai-worker has no sudo privileges
- Cannot run `nh`, `nixos-rebuild`, `nixpkgs-fmt`, or `nix` with elevated permissions
### Docker Access
- Member of `docker` group - can run `docker` and `docker exec` commands
- Primary use: `docker exec ollama ollama ...` for benchmarking
- Can run `docker exec --privileged ollama rocm-smi ...` for VRAM monitoring
## Workflow: SSH + Docker Benchmarking
The AI worker connects from the Hermes container to the host via SSH, runs ollama benchmarks, then returns to save results.
### Example Workflow
```bash
# From Hermes container, SSH to host
ssh -i /path/to/ssh/key ai-worker@host.docker.internal
# On host, run ollama benchmarks via docker
docker exec ollama ollama pull devstral-small-2:24b
# Create test modelfile
docker exec ollama bash -c 'cat <<EOF > /root/.ollama/test.modelfile
FROM devstral-small-2:24b
PARAMETER num_ctx 65536
PARAMETER num_gpu 99
PARAMETER flash_attn true
EOF'
# Create and test model
docker exec ollama ollama create test-model -f /root/.ollama/test.modelfile
docker exec ollama ollama run test-model "Write a Python async function"
# Check VRAM usage
docker exec --privileged ollama rocm-smi --showmeminfo vram
# Cleanup
docker exec ollama ollama rm test-model
# Exit SSH, return to Hermes container
exit
# Save results in Hermes container
# /opt/data/ai-optimizer/state.json
# /opt/data/ai-optimizer/results.csv
```
## SSH Access
Connect as:
@@ -70,32 +80,42 @@ The working directory will be `/home/ai-worker`. No infra repo access.
## Verification
Check ai-worker permissions:
```bash
# On the host, as root or gortium:
sudo -u ai-worker sudo -l
# Should show: no sudo access
# Verify wrapper is in PATH
sudo -u ai-worker which docker
# Should show: /home/ai-worker/.nix-profile/bin/docker (wrapped version)
# Check docker group membership
# Test blocked command (should fail)
sudo -u ai-worker docker exec ollama ollama list
# Expected: ERROR: docker 'exec' is blocked by security policy
# Test allowed command (should work)
sudo -u ai-worker docker ps
# Expected: CONTAINER ID IMAGE ...
# Verify docker group membership
groups ai-worker
# Should show: ai-worker docker
```
## Troubleshooting
If ai-worker cannot run docker commands:
If docker commands fail unexpectedly:
```bash
# Check docker group membership
groups ai-worker
# Check which docker binary is being used
which docker
# If this shows /run/current-system/sw/bin/docker, the wrapper is not in PATH
# Verify ollama container is running
docker ps | grep ollama
# Check if the wrapper is installed
ls -la $(which docker)
# Test docker access
sudo -u ai-worker docker exec ollama ollama list
# Verify you're running as the right user
whoami
```
If SSH connection fails:
```bash
# Check SSH key is authorized
cat /home/ai-worker/.ssh/authorized_keys

View File

@@ -2,16 +2,123 @@
with lib;
let
# Docker subcommands that are BLOCKED for ai-worker
# These commands allow file modification inside containers or data exfiltration.
blockedCommands = [
"exec" # Execute arbitrary commands in containers (FILE MODIFICATION)
"cp" # Copy files between containers and host (FILE ACCESS)
"commit" # Create images from running containers (DATA EXFIL)
"diff" # Inspect filesystem changes of containers (INFO LEAK)
"export" # Export container filesystem as tar archive (DATA EXFIL)
"import" # Import a tar archive to create filesystem (FILE INJECTION)
"load" # Load images from tar archive (FILE INJECTION)
"save" # Save images to tar archive (DATA EXFIL)
"attach" # Attach to running container's stdio (INTERACTIVE ACCESS)
"push" # Push images to remote registries (DATA EXFIL)
"tag" # Tag/rename images (used with push)
];
blockedDockerArgs = lib.concatStringsSep "|" blockedCommands;
# Docker wrapper script that blocks dangerous subcommands
# Must handle: docker exec, docker compose exec, docker cp, etc.
restrictedDockerScript = pkgs.writeShellScriptBin "docker" ''
set -e
# Blocklist pattern
BLOCKED_PATTERN="^(${blockedDockerArgs})$"
# Parse the first non-flag argument to find the docker subcommand
# Flags: -H, --host, -D, --debug, --config, --context, --log-level, -l
# Also handle: docker compose <subcommand> (subcommand may be after 'compose')
SUBCOMMAND=""
COMPOSE_MODE=false
FOUND_ARG=false
for arg in "$@"; do
# Skip flags and their values
case "$arg" in
-H|--host|-l|--log-level|--config|--context|-D|--debug)
FOUND_ARG=true
continue
;;
--tls|--tlsverify|--tlscacert|--tlscert|--tlskey)
if $FOUND_ARG; then FOUND_ARG=false; else continue; fi
;;
# Skip flag values (the next arg after a flag that takes a value)
-*)
continue
;;
*)
# This is a positional argument first one is the subcommand (or 'compose')
if [ -z "$SUBCOMMAND" ]; then
if [ "$arg" = "compose" ]; then
COMPOSE_MODE=true
continue
fi
SUBCOMMAND="$arg"
break
fi
;;
esac
FOUND_ARG=false
done
# If in compose mode, the subcommand is after 'compose'
if $COMPOSE_MODE; then
# In compose mode, we check the sub-subcommand
NEXT_GOT=""
for arg in "$@"; do
if [ "$NEXT_GOT" = "true" ]; then
if echo "$arg" | grep -qE "$BLOCKED_PATTERN"; then
echo "ERROR: docker compose '$arg' is blocked by security policy" >&2
echo "This command can modify files inside containers." >&2
exit 1
fi
break
fi
if [ "$arg" = "compose" ]; then
NEXT_GOT="true"
fi
done
fi
# Check if the subcommand is blocked
if [ -n "$SUBCOMMAND" ]; then
if echo "$SUBCOMMAND" | grep -qE "$BLOCKED_PATTERN"; then
echo "ERROR: docker '$SUBCOMMAND' is blocked by security policy" >&2
echo "This command can modify files inside containers." >&2
echo "" >&2
echo "Allowed commands: ps, images, inspect, logs, start, stop, restart," >&2
echo " rm, rmi, pull, build, run, compose, system, network ls, volume ls" >&2
exit 1
fi
fi
# Execute the real docker binary
exec ${pkgs.docker}/bin/docker "$@"
'';
in
{
options.services.aiWorkerAccess = mkOption {
type = types.bool;
default = false;
description = "Enable AI worker SSH access with docker group membership for ollama benchmarking";
description = "Enable AI worker SSH access with restricted docker commands";
};
config = mkIf config.services.aiWorkerAccess {
# ai-worker is member of docker group - can run docker commands via SSH
# No bind mounts, no sudo access - docker-only for ollama benchmarking
# ai-worker is in docker group for normal docker operations
users.groups.docker.members = [ "ai-worker" ];
# Install the docker wrapper for ai-worker
# This puts a filtered 'docker' script in ai-worker's PATH that blocks
# dangerous commands like exec, cp, commit, etc.
# The real docker binary is still available at its store path, but the
# wrapper intercepts it because ~/.nix-profile/bin/ comes before /run/.../sw/bin/ in PATH.
users.users.ai-worker.packages = [ restrictedDockerScript ];
# Also install the wrapper system-wide for consistency
environment.systemPackages = [ restrictedDockerScript ];
};
}

View File

@@ -1,400 +0,0 @@
#!/usr/bin/env bash
# =============================================================================
# nixos-rollback.sh — NixOS systemd-boot Rollback Script
#
# Detects a failed NixOS generation (critical services not starting) and sets
# the previous generation as the default boot option for systemd-boot.
# Logs all actions to syslog/journald and a local logfile. Fails safely when
# no previous generation exists or required files are missing.
#
# Integration with the boot sentinel:
# sentinel-check.sh → detects Tier-1 service failures (sshd, docker,
# traefik, authelia) after a boot
# nixos-rollback.sh ← called when sentinel exits nonzero; sets previous
# generation as default for next boot
#
# Usage:
# nixos-rollback.sh # auto-detect & set previous gen
# nixos-rollback.sh --dry-run # show what would be done
# nixos-rollback.sh --rollback-now # also run nixos-rebuild switch
# # --rollback for immediate fix
# nixos-rollback.sh --help # full help text
#
# Exit codes:
# 0 — rollback applied (or dry-run would apply)
# 1 — preflight failure (missing files, permissions)
# 2 — no previous generation available
# 3 — nixos-rebuild --rollback failed (only with --rollback-now)
#
# Installation on NixOS:
# Place in /usr/local/bin/nixos-rollback.sh and make executable.
# Add a systemd oneshot service to run it after sentinel-check detects
# failures, or invoke directly from a sentinel timer.
# =============================================================================
set -euo pipefail
# ── Configuration ────────────────────────────────────────────────────────────
# These can be overridden via environment variables for testing.
LOADER_CONF="${NIXOS_ROLLBACK_LOADER_CONF:-/boot/loader/loader.conf}"
ENTRIES_DIR="${NIXOS_ROLLBACK_ENTRIES_DIR:-/boot/loader/entries}"
LOGFILE="${NIXOS_ROLLBACK_LOGFILE:-/var/log/nixos-rollback.log}"
SYSLOG_IDENT="nixos-rollback"
# ── CLI flags ────────────────────────────────────────────────────────────────
DRY_RUN=false
ROLLBACK_NOW=false
# ── Colors (disabled when not a terminal) ────────────────────────────────────
if [ -t 1 ]; then
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
NC='\033[0m' # No Color
else
RED=''; GREEN=''; YELLOW=''; CYAN=''; NC=''
fi
# =============================================================================
# Help
# =============================================================================
usage() {
cat <<EOF
${CYAN}nixos-rollback.sh${NC} — Set the previous NixOS generation as systemd-boot default
${CYAN}USAGE${NC}
nixos-rollback.sh [OPTIONS]
${CYAN}OPTIONS${NC}
--dry-run Show what would be done without making changes
--rollback-now Also run 'nixos-rebuild switch --rollback' for
immediate fix of the running system (requires
nixos-rebuild on PATH)
-h, --help Show this help text
${CYAN}DESCRIPTION${NC}
Reads the current default boot entry from ${LOADER_CONF},
determines the previous generation number, and writes it as the
new default. The script only modifies systemd-boot config —
it does NOT touch the Nix store or system profile unless
--rollback-now is passed.
Designed as the rollback half of a boot sentinel:
1. System boots into generation N
2. sentinel-check.sh detects Tier-1 service failures
3. nixos-rollback.sh sets default to generation N-1
4. Next reboot uses the working generation
${CYAN}EXIT CODES${NC}
0 Rollback applied (or dry-run would apply)
1 Preflight failure (missing files, permissions)
2 No previous generation available (only one generation)
3 nixos-rebuild --rollback failed (with --rollback-now)
${CYAN}FILES${NC}
${LOADER_CONF} systemd-boot loader configuration
${ENTRIES_DIR}/ generation entry .conf files
${LOGFILE} action log (append-only)
EOF
}
# =============================================================================
# Logging
# =============================================================================
log() {
local level="$1"; shift
local msg="$*"
local timestamp
timestamp="$(date '+%Y-%m-%d %H:%M:%S')"
echo "${timestamp} [${level}] ${msg}" >> "${LOGFILE}"
logger -t "${SYSLOG_IDENT}" -p "user.${level}" "${msg}"
# Also print to stderr for ERROR/WARN, stdout for INFO
case "${level}" in
ERROR) echo >&2 "${RED}[ERROR]${NC} ${msg}" ;;
WARN) echo >&2 "${YELLOW}[WARN]${NC} ${msg}" ;;
INFO) echo " ${GREEN}[INFO]${NC} ${msg}" ;;
esac
}
info() { log "INFO" "$@"; }
warn() { log "WARN" "$@"; }
error() { log "ERROR" "$@"; }
# =============================================================================
# Preflight checks
# =============================================================================
preflight() {
# Must run as root (need to write to /boot), unless overridden for testing
if [ -z "${NIXOS_ROLLBACK_SKIP_ROOT_CHECK:-}" ] && [ "$(id -u)" -ne 0 ]; then
error "This script must be run as root (needs write access to /boot/loader)"
error "Set NIXOS_ROLLBACK_SKIP_ROOT_CHECK=1 for testing against mock paths."
exit 1
fi
# Directories and files
if [ ! -d "${ENTRIES_DIR}" ]; then
error "Boot entries directory not found: ${ENTRIES_DIR}"
exit 1
fi
if [ ! -f "${LOADER_CONF}" ]; then
error "Loader config not found: ${LOADER_CONF}"
exit 1
fi
if [ ! -r "${LOADER_CONF}" ]; then
error "Cannot read loader config: ${LOADER_CONF}"
exit 1
fi
# Check write access to /boot/loader (parent of loader.conf)
local loader_dir
loader_dir="$(dirname "${LOADER_CONF}")"
if [ ! -w "${loader_dir}" ]; then
error "Cannot write to ${loader_dir} (insufficient permissions)"
exit 1
fi
# Logfile directory must exist
local log_dir
log_dir="$(dirname "${LOGFILE}")"
if [ ! -d "${log_dir}" ]; then
warn "Log directory ${log_dir} does not exist, creating it"
mkdir -p "${log_dir}" 2>/dev/null || {
error "Cannot create log directory ${log_dir}"
exit 1
}
fi
# Check --rollback-now dependencies
if [ "${ROLLBACK_NOW}" = true ]; then
if ! command -v nixos-rebuild &>/dev/null; then
error "nixos-rebuild not found on PATH (required for --rollback-now)"
exit 1
fi
fi
}
# =============================================================================
# Generation helpers
# =============================================================================
# get_current_default: reads the current default entry from loader.conf
# Returns: "nixos-generation-N.conf" or empty string
get_current_default() {
grep -E '^default\s+' "${LOADER_CONF}" 2>/dev/null \
| awk '{print $2}' \
|| true
}
# extract_gen_number: extracts the numeric generation from a conf filename
# Input: "nixos-generation-367.conf"
# Output: 367
extract_gen_number() {
echo "$1" | sed 's/nixos-generation-//;s/\.conf//'
}
# get_all_gen_numbers: returns sorted list of generation numbers from entries dir
get_all_gen_numbers() {
local -a gens=()
local f n
for f in "${ENTRIES_DIR}"/nixos-generation-*.conf; do
[ -f "${f}" ] || continue
n="$(basename "${f}" | sed 's/nixos-generation-//;s/\.conf//')"
gens+=("${n}")
done
if [ "${#gens[@]}" -eq 0 ]; then
return 1
fi
# Sort numerically and output
printf '%s\n' "${gens[@]}" | sort -n
}
# get_previous_gen: given current generation number, find the previous one
# from the list of all available generations
get_previous_gen() {
local current="$1"
shift
local -a gens=("$@")
local prev=""
local g
for g in "${gens[@]}"; do
if [ "${g}" -lt "${current}" ]; then
prev="${g}"
fi
done
if [ -z "${prev}" ]; then
return 1
fi
echo "${prev}"
}
# =============================================================================
# Main rollback logic
# =============================================================================
do_rollback() {
# Step 1: Read current default
local current_entry
current_entry="$(get_current_default)"
if [ -z "${current_entry}" ]; then
error "No 'default' entry found in ${LOADER_CONF}"
error "Cannot determine current generation — aborting"
exit 1
fi
info "Current default boot entry: ${current_entry}"
# Step 2: Build sorted list of all available generations
local -a all_gens=()
local line
while IFS= read -r line; do
all_gens+=("${line}")
done < <(get_all_gen_numbers || true)
if [ "${#all_gens[@]}" -eq 0 ]; then
error "No NixOS generation .conf files found in ${ENTRIES_DIR}"
exit 1
fi
info "Available generations: ${all_gens[*]}"
# Step 3: Find current generation number
local current_gen
current_gen="$(extract_gen_number "${current_entry}")"
# Verify current_gen is a valid number
if ! [[ "${current_gen}" =~ ^[0-9]+$ ]]; then
error "Could not parse generation number from '${current_entry}'"
exit 1
fi
# Step 4: Find the previous generation
local prev_gen
prev_gen="$(get_previous_gen "${current_gen}" "${all_gens[@]}")" || {
error "No previous generation found before generation ${current_gen}"
error "This is the oldest available generation — cannot roll back further"
exit 2
}
local prev_entry="nixos-generation-${prev_gen}.conf"
local prev_conf_path="${ENTRIES_DIR}/${prev_entry}"
if [ ! -f "${prev_conf_path}" ]; then
error "Previous generation entry not found: ${prev_conf_path}"
error "The .conf file for generation ${prev_gen} is missing — cannot roll back"
exit 1
fi
info "Target rollback generation: ${prev_gen}${prev_entry}"
# Step 5: Apply the rollback
if [ "${DRY_RUN}" = true ]; then
echo ""
echo " ${CYAN}[DRY RUN]${NC} Would change ${LOADER_CONF}:"
echo " ${YELLOW}-${NC} default ${current_entry}"
echo " ${GREEN}+${NC} default ${prev_entry}"
echo ""
info "DRY RUN — no changes made"
exit 0
fi
# Write new default
# Use sed with a backup (.bak)
sed -i.bak "s/^default\s\+${current_entry}/default ${prev_entry}/" "${LOADER_CONF}"
# Verify the change was applied
local new_default
new_default="$(get_current_default)"
if [ "${new_default}" != "${prev_entry}" ]; then
error "Failed to set default boot entry to ${prev_entry}"
error "Current default is still: ${new_default}"
# Attempt to restore backup
if [ -f "${LOADER_CONF}.bak" ]; then
cp "${LOADER_CONF}.bak" "${LOADER_CONF}"
info "Restored backup from ${LOADER_CONF}.bak"
fi
exit 1
fi
info "Successfully set default boot entry to ${prev_entry} (generation ${prev_gen})"
info "Backup of previous config saved to ${LOADER_CONF}.bak"
# Step 6: Optionally run nixos-rebuild switch --rollback
if [ "${ROLLBACK_NOW}" = true ]; then
echo ""
info "Running nixos-rebuild switch --rollback for immediate effect..."
if nixos-rebuild switch --rollback 2>&1 | while IFS= read -r line; do
logger -t "${SYSLOG_IDENT}" "nixos-rebuild: ${line}"
echo " ${line}"
done; then
info "nixos-rebuild switch --rollback completed successfully"
else
local rc=$?
error "nixos-rebuild switch --rollback failed with exit code ${rc}"
error "The boot default has been changed but the current system was NOT rolled back"
error "Reboot to apply the rollback"
exit 3
fi
fi
info "Rollback complete. Next boot will use generation ${prev_gen}."
if [ "${ROLLBACK_NOW}" = false ]; then
echo ""
echo " ${YELLOW}NOTE:${NC} The current running system is unchanged."
echo " Reboot to boot into generation ${prev_gen}."
echo " Or re-run with --rollback-now for immediate effect."
fi
}
# =============================================================================
# Main
# =============================================================================
main() {
# Parse arguments
while [ $# -gt 0 ]; do
case "$1" in
--dry-run)
DRY_RUN=true
shift
;;
--rollback-now)
ROLLBACK_NOW=true
shift
;;
-h|--help)
usage
exit 0
;;
*)
echo >&2 "Unknown option: $1"
echo >&2 "Use --help for usage information."
exit 1
;;
esac
done
echo ""
echo " ${CYAN}═══ NixOS systemd-boot Rollback ═══${NC}"
echo ""
preflight
if [ "${DRY_RUN}" = true ]; then
info "DRY RUN mode — no changes will be made"
fi
if [ "${ROLLBACK_NOW}" = true ]; then
info "ROLLBACK NOW mode — will also run nixos-rebuild switch --rollback"
fi
echo ""
do_rollback
}
main "$@"

View File

@@ -1,184 +0,0 @@
{ config, pkgs, lib, ... }:
with lib;
let
cfg = config.services.rollbackSentinel;
# ── Scripts ────────────────────────────────────────────────────────────────
# Sentinel check — verifies Tier-1 services are active after boot.
# Exits nonzero when any Tier-1 service is down, which triggers the rollback.
sentinelCheck = pkgs.writeShellScriptBin "sentinel-check.sh" ''
#!/usr/bin/env bash
set -euo pipefail
SYSLOG_IDENT="nixos-sentinel"
LOGFILE="/var/log/nixos-sentinel.log"
echo "=== NixOS Sentinel Check ==="
echo "Tier-1 services: ${builtins.toString cfg.tier1Services}"
echo "Tier-2 services: ${builtins.toString cfg.tier2Services}"
FAILED=0
# Check Tier-1 services any failure means rollback
for svc in ${builtins.toString cfg.tier1Services}; do
if systemctl is-active --quiet "$svc" 2>/dev/null; then
echo " [OK] Tier-1: $svc"
else
echo " [FAIL] Tier-1: $svc is NOT active"
logger -t "$SYSLOG_IDENT" -p user.err "Tier-1 FAILURE: $svc is not active"
FAILED=1
fi
done
# Check Tier-2 services warn only
for svc in ${builtins.toString cfg.tier2Services}; do
if systemctl is-active --quiet "$svc" 2>/dev/null; then
echo " [OK] Tier-2: $svc"
else
echo " [WARN] Tier-2: $svc is NOT active"
logger -t "$SYSLOG_IDENT" -p user.warn "Tier-2 WARNING: $svc is not active"
fi
done
echo "=== Sentinel result: $([ "$FAILED" -eq 0 ] && echo 'PASS' || echo 'FAIL') ==="
echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] sentinel $([ "$FAILED" -eq 0 ] && echo 'PASS' || echo 'FAIL')" >> "$LOGFILE"
exit $FAILED
'';
# Rollback script — package the companion shell script from this directory.
# Uses builtins.readFile to embed the content at evaluation time.
rollbackScript = pkgs.writeShellScriptBin "nixos-rollback.sh" (builtins.readFile ./nixos-rollback.sh);
# Resolve rollback flags from config
rollbackFlags =
if cfg.rollbackMode == "dry-run" then "--dry-run"
else if cfg.rollbackMode == "rollback-now" then "--rollback-now"
else "";
in {
options.services.rollbackSentinel = {
enable = mkEnableOption "NixOS Rollback Sentinel auto-rollback on critical service failure";
tier1Services = mkOption {
type = types.listOf types.str;
default = [ "sshd" "docker" "traefik" "authelia" ];
description = ''
Tier-1 services whose failure triggers an automatic systemd-boot rollback.
On boot, the sentinel waits ${cfg.bootDelay} seconds, then checks each
service. If ANY service in this list is inactive, it runs the rollback
script which sets the previous NixOS generation as the default boot entry.
'';
};
tier2Services = mkOption {
type = types.listOf types.str;
default = [
"gitea" "hermes" "ollama" "synapse" "nextcloud"
"vaultwarden" "wireguard" "homeassistant" "fail2ban"
];
description = ''
Tier-2 services whose failure is logged as a warning but does NOT trigger
an automatic rollback. Useful for detecting non-critical service issues.
'';
};
tier3InfoServices = mkOption {
type = types.listOf types.str;
default = [
"act_runner" "syncthing" "restic" "fava"
"homer" "cups" "fstrim"
];
description = ''
Tier-3 informational checks (log-only, no warning). These are services
that the sentinel will note the status of for diagnostics.
'';
};
bootDelay = mkOption {
type = types.str;
default = "120";
description = ''
Seconds to wait after multi-user.target before running the boot-time
sentinel check. This gives Tier-1 services time to start before
the sentinel decides they've failed.
'';
};
rollbackMode = mkOption {
type = types.enum [ "set-default" "rollback-now" "dry-run" ];
default = "set-default";
description = ''
Rollback strategy when Tier-1 failures are detected:
- set-default: Write the previous generation to loader.conf (next reboot).
- rollback-now: Also run nixos-rebuild switch --rollback for immediate fix.
- dry-run: Log what would happen but take no action (testing).
'';
};
enablePostRebuild = mkOption {
type = types.bool;
default = true;
description = ''
When enabled, the sentinel check runs after every nixos-rebuild switch
activation. If a newly deployed generation has Tier-1 failures, it
triggers rollback immediately.
'';
};
};
config = mkIf cfg.enable {
# ── Deploy scripts to PATH ───────────────────────────────────────────────
environment.systemPackages = [ sentinelCheck rollbackScript ];
# Ensure log directory exists
systemd.tmpfiles.rules = [
"d /var/log/nixos-sentinel 0755 root root -"
];
# ── Boot-time sentinel service ───────────────────────────────────────────
# Runs after multi-user.target with a configurable delay, checks Tier-1
# services, and triggers rollback if any are down.
systemd.services.nixos-sentinel = {
description = "NixOS Boot Sentinel check critical services, roll back on failure";
after = [ "network.target" "multi-user.target" ];
wants = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
path = with pkgs; [ coreutils gawk gnused systemd ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStartPre = "${pkgs.coreutils}/bin/sleep ${cfg.bootDelay}";
ExecStart = "${sentinelCheck}/bin/sentinel-check.sh";
ExecStartPost = "${rollbackScript}/bin/nixos-rollback.sh ${rollbackFlags}";
};
};
# ── Post-rebuild sentinel service (triggered by activation script) ──────
systemd.services.nixos-sentinel-rebuild = mkIf cfg.enablePostRebuild {
description = "NixOS Post-Rebuild Sentinel check services after nixos-rebuild";
after = [ "network.target" ];
path = with pkgs; [ coreutils gawk gnused systemd ];
serviceConfig = {
Type = "oneshot";
ExecStart = "${sentinelCheck}/bin/sentinel-check.sh";
ExecStartPost = "${rollbackScript}/bin/nixos-rollback.sh ${rollbackFlags}";
};
};
# Activation script — fires after every nixos-rebuild switch
system.activationScripts.rollback-sentinel = mkIf cfg.enablePostRebuild ''
# Start the post-rebuild sentinel in the background.
# This runs on every activation (boot + nixos-rebuild). On boot the
# boot-time service handles it, so this is primarily for nixos-rebuild,
# but running twice is safe (idempotent rollback).
systemctl start nixos-sentinel-rebuild.service --no-block 2>/dev/null || true
'';
};
}

View File

@@ -1,77 +0,0 @@
final: prev: let
python3 = final.python3;
pyPkgs = python3.pkgs;
in {
reticulumStack = python3.pkgs.buildPythonApplication rec {
pname = "reticulum";
version = "1.2.9";
src = pyPkgs.fetchPypi {
pname = "rns";
inherit version;
sha256 = "554814231c237b9caacf8df669312e57dd7d3f84b6d4810125087d1a79a75d75";
};
propagatedBuildInputs = with pyPkgs; [ cryptography pyserial ];
doCheck = false;
pythonImportsCheck = [ "RNS" ];
meta = with final.lib; {
description = "Self-configuring, encrypted and resilient mesh networking stack";
homepage = "https://reticulum.network/";
license = licenses.mit;
platforms = platforms.linux;
};
};
lxmf = python3.pkgs.buildPythonApplication rec {
pname = "lxmf";
version = "0.9.8";
src = pyPkgs.fetchPypi {
inherit pname version;
sha256 = "30f39f3a975a049c12ee2cfceb3261d24cb5adec881c6821f7354464b3f3650c";
};
propagatedBuildInputs = [ final.reticulumStack ];
doCheck = false;
pythonImportsCheck = [ "LXMF" ];
meta = with final.lib; {
description = "Lightweight Extensible Message Format for Reticulum";
homepage = "https://github.com/markqvist/lxmf";
license = licenses.mit;
platforms = platforms.linux;
};
};
nomadnet = python3.pkgs.buildPythonApplication rec {
pname = "nomadnet";
version = "1.1.1";
src = pyPkgs.fetchPypi {
inherit pname version;
sha256 = "fa13b64a10e75b705a58024815ab72451700aa726af96d415ba99dec28dfc40a";
};
propagatedBuildInputs = with pyPkgs; [ final.reticulumStack final.lxmf urwid qrcode ];
doCheck = false;
pythonImportsCheck = [ "nomadnet" ];
meta = with final.lib; {
description = "Nomad Network resilient mesh communications platform";
homepage = "https://github.com/markqvist/NomadNet";
license = licenses.mit;
platforms = platforms.linux;
};
};
rnsh = python3.pkgs.buildPythonApplication rec {
pname = "rnsh";
version = "0.1.7";
src = pyPkgs.fetchPypi {
inherit pname version;
sha256 = "9cb72f25abb1c6d300f8014b264184ff78f592fe88e36094938012990b797c93";
};
propagatedBuildInputs = [ final.reticulumStack ];
doCheck = false;
pythonImportsCheck = [ "rnsh" ];
meta = with final.lib; {
description = "Remote shell over Reticulum";
homepage = "https://github.com/acehoss/rnsh";
license = licenses.mit;
platforms = platforms.linux;
};
};
}

View File

@@ -4,6 +4,8 @@
group = "ai-worker";
home = "/home/ai-worker";
createHome = true;
# ai-worker stays in docker group for normal docker operations (ps, start, stop, compose, ...)
# Dangerous commands (exec, cp, commit) are blocked by a wrapper script.
extraGroups = [ "docker" ];
shell = pkgs.bashInteractive;
openssh.authorizedKeys.keys = [
@@ -14,17 +16,14 @@
};
users.groups.ai-worker = {};
# Enable restricted AI worker SSH access for ollama benchmarking
# SECURITY: ai-worker can only:
# - SSH into host from Hermes container
# - Run docker commands (docker exec ollama ...) via docker group
# - Run specific security audit commands
# - NO access to infra repo (no bind mount)
# - NO sudo access (no nh, nixos-rebuild, nixpkgs-fmt, nix)
# WORKFLOW: SSH from Hermes container, run docker benchmarks, return and save results to /opt/data/ai-optimizer/
# Enable restricted AI worker SSH access
# SECURITY: ai-worker is in docker group but docker commands are filtered:
# ALLOWED: ps, images, logs, start, stop, restart, rm, rmi, pull, build, run, compose
# BLOCKED: exec, cp, commit, diff, export, import, load, save, attach, push
# The filtering is done by a docker wrapper in ai-worker's PATH.
services.aiWorkerAccess = true;
# Restricted sudo for ai-worker - security checks only
# Restricted sudo for ai-worker - security checks only (not for docker)
security.sudo.extraRules = [
{
users = [ "ai-worker" ];
@@ -69,15 +68,6 @@
command = "/run/current-system/sw/bin/sshd -T";
options = [ "NOPASSWD" ];
}
# Docker service checks
{
command = "/run/current-system/sw/bin/docker ps";
options = [ "NOPASSWD" ];
}
{
command = "/run/current-system/sw/bin/docker inspect *";
options = [ "NOPASSWD" ];
}
# Network diagnostics
{
command = "/run/current-system/sw/bin/ss -tlnp";