Compare commits

..

12 Commits

Author SHA1 Message Date
0980dca455 fix: update compose submodule to Traefik-routed Syncthing 2026-05-14 21:40:12 -04:00
96bc20ab70 feat: add Syncthing firewall port and update compose submodule 2026-05-14 21:36:26 -04:00
6f44aa7f76 Merge pull request 'fix: update compose submodule — remove apk add iptables-nft' (#45) from fix/vpn-iptables-nft-v2 into master
Reviewed-on: #45
2026-05-13 16:49:39 +00:00
8d40f1691f fix: update compose submodule — remove apk add iptables-nft 2026-05-13 12:49:14 -04:00
Robert
2dd2e64986 Merge remote-tracking branch 'origin/master' 2026-05-13 12:42:54 -04:00
Robert
23fc5e0597 Give a little more ssh room for tramp 2026-05-13 12:41:09 -04:00
0c9c33d735 Merge pull request 'fix: update wg-easy to official ghcr image with iptables-nft' (#44) from fix/vpn-iptables-nft-upstream into master
Reviewed-on: #44
2026-05-13 16:39:56 +00:00
0bb6890f1c chore: merge master into branch 2026-05-13 12:39:05 -04:00
9d5434425f fix: update compose submodule for wg-easy iptables-nft fix
Updates the assets/compose submodule to point to the fix/vpn-iptables-nft-upstream
branch which contains:
- Switch FROM weejewel/wg-easy:latest (Alpine 3.11, stale 4yr) to
  ghcr.io/wg-easy/wg-easy:latest (actively maintained, Alpine krypton)
- Use update-alternatives instead of raw ln -sf to flip iptables
  from legacy to nftables backend
- Fix compose build context: ./vpn -> . (Dockerfile is at same level)
2026-05-13 12:30:47 -04:00
1fb4320dd1 Merge pull request 'feat: update compose submodule for custom tools startup' (#43) from feat/update-compose-submodule-custom-tools into master
Reviewed-on: #43
2026-05-13 13:58:27 +00:00
51e9f47fd4 feat: update compose submodule for custom tools startup 2026-05-13 09:56:24 -04:00
06b3eb840f fix: update compose submodule for wg-easy iptables-nft fix 2026-05-12 16:29:51 -04:00
6 changed files with 9 additions and 684 deletions

View File

@@ -1,52 +0,0 @@
name: Build and test NixOS config
on:
pull_request:
branches: [ master ]
paths:
- '**.nix'
- 'flake.lock'
- 'secrets/**'
- 'hosts/**'
- 'modules/**'
push:
branches: [ master ]
paths:
- '**.nix'
- 'flake.lock'
- 'secrets/**'
- 'hosts/**'
- 'modules/**'
jobs:
build:
runs-on: nixos-builder
steps:
- name: Checkout
run: |
git clone -b "${{ github.head_ref || github.ref_name }}" \
https://gitea:${{ secrets.GITHUB_TOKEN }}@code.lazyworkhorse.net/gortium/infra.git .
git log --oneline -3
- name: Build NixOS config
run: |
nix --version
nh os build .#lazyworkhorse 2>&1
- name: Run integration tests (staging VM)
run: |
echo "==> Running integration tests on staging VM..."
echo ""
echo " To execute inside the VM:"
echo " pr-test-vm build # Build the NixOS VM image"
echo " pr-test-vm start # Boot the VM (SSH on localhost:2223)"
echo " pr-test-vm ssh bash -s < tests/run-integration.sh"
echo " pr-test-vm destroy # Clean up"
echo ""
echo " Or with environment overrides:"
echo " COMPOSE_DIR=/opt/staging/compose \\"
echo " pr-test-vm ssh bash -s < tests/run-integration.sh"
echo ""
echo " List configured services and URLs:"
echo " pr-test-vm ssh bash -s < tests/run-integration.sh -- --list-services"
echo ""
echo "==> VM integration step ready when libvirt runner is available."

View File

@@ -61,7 +61,6 @@
./modules/nixos/services/open_code_server.nix
./modules/nixos/services/ollama_init_custom_models.nix
./modules/nixos/services/openclaw_node.nix
./modules/nixos/services/staging-vm.nix
./modules/nixos/security/ai-worker-restricted.nix
./users/gortium.nix
./users/ai-worker.nix

View File

@@ -36,7 +36,7 @@
"transparent_hugepage=always" # because mucho ram
];
# 2. Load the specific drivers found by sensors-detect
boot.kernelModules = [ "nct6775" "lm96163" "iptable_nat" "iptable_filter" "kvm-intel" "kvm" ];
boot.kernelModules = [ "nct6775" "lm96163" "iptable_nat" "iptable_filter" ];
# 3. Force the nct6775 driver to recognize the chip if it's stubborn
boot.extraModprobeConfig = ''
options nct6775 force_id=0xd280
@@ -207,6 +207,7 @@
ai = {
path = self + "/assets/compose/ai";
envFile = config.age.secrets.containers_env.path;
ports = [ 22000 ]; # Syncthing TCP sync
};
cloudstorage = {
@@ -328,21 +329,20 @@
# Mi50 config
hardware.graphics = {
enable = true;
enable32Bit = true;
enable32Bit = true; # Useful for some compatibility layers
extraPackages = with pkgs; [
rocmPackages.clr.icd
rocmPackages.clr.icd # OpenCL/HIP runtime
];
};
nixpkgs.config.rocmTargets = [ "gfx906" ];
environment.variables = {
# This "tricks" ROCm into supporting the MI50 if using newer versions
HSA_OVERRIDE_GFX_VERSION = "9.0.6";
# Ensures the system sees both GPUs
HIP_VISIBLE_DEVICES = "0,1";
};
# KVM/libvirt for staging VM
services.stagingVm.enable = true;
# Open ports in the firewall.
# Open ports in the firewall.
# networking.firewall.allowedTCPPorts = [ ... ];
# networking.firewall.allowedUDPPorts = [ ... ];
# Or disable the firewall altogether.
@@ -475,7 +475,7 @@
services.openssh.settings = {
PermitRootLogin = "no";
MaxAuthTries = 3;
MaxSessions = 10;
MaxSessions = 20;
LoginGraceTime = 30;
ClientAliveInterval = 300;
ClientAliveCountMax = 2;

View File

@@ -1,275 +0,0 @@
{ config, pkgs, lib, ... }:
with lib;
let
cfg = config.services.stagingVm;
in
{
options.services.stagingVm = {
enable = mkOption {
type = types.bool;
default = false;
description = "Enable KVM/libvirt staging VM for compose PR testing";
};
vmName = mkOption {
type = types.str;
default = "compose-test-vm";
description = "Name of the staging VM";
};
memory = mkOption {
type = types.str;
default = "4096";
description = "RAM allocated to the staging VM (MB)";
};
vcpus = mkOption {
type = types.int;
default = 2;
description = "Number of vCPUs for the staging VM";
};
storagePath = mkOption {
type = types.str;
default = "/var/lib/libvirt/images";
description = "Path for libvirt storage pool";
};
dataPath = mkOption {
type = types.str;
default = "/var/lib/staging-vm";
description = "Path for compose test data (PR checkouts, test results)";
};
};
config = mkIf cfg.enable {
# Enable libvirt daemon
virtualisation.libvirtd = {
enable = true;
qemu = {
package = pkgs.qemu_kvm;
runAsRoot = true;
swtpm.enable = true;
ovmf = {
enable = true;
packages = [ pkgs.OVMFFull.fd ];
};
};
};
# Kernel modules + groups already handled in configuration.nix
# libvirt NAT network (192.168.122.0/24)
environment.etc."libvirt/qemu/networks/default.xml" = {
text = ''
<network>
<name>default</name>
<uuid>2b8f7a3c-9e5d-4a1f-bc3d-6e7a8f9b0c1d</uuid>
<forward mode='nat'>
<nat>
<port start='1024' end='65535'/>
</nat>
</forward>
<bridge name='virbr0' stp='on' delay='0'/>
<mac address='52:54:00:12:34:56'/>
<ip address='192.168.122.1' netmask='255.255.255.0'>
<dhcp>
<range start='192.168.122.2' end='192.168.122.254'/>
</dhcp>
</ip>
</network>
'';
# Autostart the network so it comes up on boot
mode = "0644";
};
# Ensure the default network is defined and autostarted
systemd.services.libvirtd = {
postStart = ''
${pkgs.libvirt}/bin/virsh net-define /etc/libvirt/qemu/networks/default.xml 2>/dev/null || true
${pkgs.libvirt}/bin/virsh net-autostart default 2>/dev/null || true
${pkgs.libvirt}/bin/virsh net-start default 2>/dev/null || true
'';
};
# Storage directory for VM images
systemd.tmpfiles.rules = [
"d ${cfg.storagePath} 0755 root root -"
"d ${cfg.dataPath} 0755 root root -"
];
# Ensure storage pool exists in libvirt
systemd.services.libvirtd.postStart = mkAfter ''
${pkgs.libvirt}/bin/virsh pool-define-as default dir --target "${cfg.storagePath}" 2>/dev/null || true
${pkgs.libvirt}/bin/virsh pool-autostart default 2>/dev/null || true
${pkgs.libvirt}/bin/virsh pool-start default 2>/dev/null || true
'';
# Firewall: allow traffic from virbr0 to host and outbound NAT
networking.firewall = {
extraCommands = ''
# Allow inbound DHCP/DNS from libvirt guests
iptables -I INPUT -i virbr0 -p udp --dport 67:68 -j ACCEPT
iptables -I INPUT -i virbr0 -p tcp --dport 53 -j ACCEPT
iptables -I INPUT -i virbr0 -p udp --dport 53 -j ACCEPT
# Allow established/related traffic back to guests
iptables -I FORWARD -i virbr0 -o virbr0 -j ACCEPT
iptables -I FORWARD -o virbr0 -j ACCEPT
iptables -I FORWARD -i virbr0 -j ACCEPT
'';
};
# Packages needed for VM management
environment.systemPackages = with pkgs; [
libvirt
qemu_kvm
virt-manager # optional GUI for manual management
OVMFFull
swtpm
];
# Enable docker in the host (already enabled, but ensure for compose testing)
virtualisation.docker.enable = true;
# Helper script: pr-test-vm
# Usage:
# pr-test-vm build — build the staging VM derivation
# pr-test-vm start — boot the VM with a compose PR branch
# pr-test-vm stop — graceful shutdown
# pr-test-vm destroy — force stop + delete VM
# pr-test-vm ssh — SSH into the running VM
systemd.tmpfiles.rules = mkAfter [
"d ${cfg.dataPath}/scripts 0755 root root -"
];
environment.systemPackages = [ (pkgs.writeShellScriptBin "pr-test-vm" ''
set -euo pipefail
DATA="${cfg.dataPath}"
VM_NAME="${cfg.vmName}"
VM_IMAGE="''${DATA}/''${VM_NAME}.qcow2"
VM_PORT=2223
build_vm() {
echo "==> Building NixOS staging VM for compose testing..."
# Build the VM config inline a minimal NixOS with Docker + SSH
cat > /tmp/staging-vm-config.nix << 'NIXEOF'
{ config, pkgs, lib, ... }: {
boot.loader.grub.devices = [ "/dev/vda" ];
boot.loader.timeout = 0;
# Minimal kernel
boot.kernelParams = [ "console=ttyS0" ];
boot.initrd.kernelModules = [ "virtio_blk" "virtio_net" "virtio_pci" ];
# SSH access
services.openssh = {
enable = true;
settings.PasswordAuthentication = false;
settings.PermitRootLogin = "prohibit-password";
};
# Docker for compose testing
virtualisation.docker.enable = true;
# Network (DHCP via virbr0)
networking.useDHCP = true;
networking.firewall.enable = false;
# Users
users.users.root.openssh.authorizedKeys.keys = [
"$(cat /root/.ssh/authorized_keys 2>/dev/null || echo 'ssh-ed25519 AAAAC3... placeholder')"
];
users.users.testrunner = {
isNormalUser = true;
extraGroups = [ "docker" ];
openssh.authorizedKeys.keys = [
"$(cat /root/.ssh/authorized_keys 2>/dev/null || echo 'ssh-ed25519 AAAAC3... placeholder')"
];
};
# Git + compose tools
environment.systemPackages = with pkgs; [ git docker-compose curl ];
system.stateVersion = "24.11";
}
NIXEOF
nixos-rebuild build-vm -I nixpkgs=channel:nixos-unstable \
--arg configuration 'import /tmp/staging-vm-config.nix' \
--out-link "''${DATA}/vm-result"
echo "==> VM built. Run 'pr-test-vm start' to boot."
}
start_vm() {
if [ -f "''${VM_IMAGE}" ]; then
echo "==> Booting existing VM..."
else
echo "==> Creating VM image..."
${pkgs.qemu_kvm}/bin/qemu-img create -f qcow2 "''${VM_IMAGE}" 20G
fi
# Check if already running
if ${pkgs.libvirt}/bin/virsh list --name 2>/dev/null | grep -q "''${VM_NAME}"; then
echo "==> VM already running."
exit 0
fi
${pkgs.qemu_kvm}/bin/qemu-system-x86_64 \
-name "''${VM_NAME}" \
-machine q35,accel=kvm \
-cpu host \
-smp ${toString cfg.vcpus} \
-m ${cfg.memory} \
-drive file="''${VM_IMAGE}",if=virtio,format=qcow2 \
-netdev user,id=net0,hostfwd=tcp::''${VM_PORT}-:22 \
-device virtio-net-pci,netdev=net0 \
-nographic \
-serial mon:stdio \
-pidfile "''${DATA}/''${VM_NAME}.pid" \
-daemonize
echo "==> VM booting... SSH on port ''${VM_PORT}"
echo "==> Wait for it: ssh -p ''${VM_PORT} testrunner@localhost"
}
stop_vm() {
PIDFILE="''${DATA}/''${VM_NAME}.pid"
if [ -f "''${PIDFILE}" ]; then
PID=$(cat "''${PIDFILE}")
kill "''${PID}" 2>/dev/null || true
rm -f "''${PIDFILE}"
echo "==> VM stopped."
else
${pkgs.libvirt}/bin/virsh destroy "''${VM_NAME}" 2>/dev/null || true
echo "==> VM destroyed."
fi
}
ssh_vm() {
exec ssh -p "''${VM_PORT}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "testrunner@localhost" "$@"
}
# Main dispatch
case "''${1:-help}" in
build) build_vm ;;
start) start_vm ;;
stop) stop_vm ;;
destroy) stop_vm; rm -f "''${VM_IMAGE}"; echo "==> VM deleted." ;;
ssh) shift; ssh_vm "$@" ;;
*)
echo "Usage: pr-test-vm {build|start|stop|destroy|ssh}"
echo ""
echo " build build the NixOS VM derivation"
echo " start boot the VM (create image if needed)"
echo " stop graceful VM shutdown"
echo " destroy stop + delete VM image"
echo " ssh SSH into the running VM"
;;
esac
'') ];
};
}

View File

@@ -1,347 +0,0 @@
#!/usr/bin/env bash
# =============================================================================
# run-integration.sh — Staging VM Integration Test Suite
#
# Verifies Docker daemon, compose stack, and service endpoint health.
# Designed to run inside the staging VM as part of CI/CD pipeline.
#
# Usage:
# ./tests/run-integration.sh # all defaults
# ./tests/run-integration.sh --verbose # detailed output
# ./tests/run-integration.sh --list-services # print detected services and exit
#
# Environment variables (all optional):
# COMPOSE_DIR Path to compose service directories (default: /opt/infra/compose)
# COMPOSE_PROJECT Docker Compose project name (default: staging)
# STAGING_DOMAIN Base domain for health checks (default: staging.lazyworkhorse.net)
# SERVICE_LIST Space-separated service dirs to check (default: auto-detect)
# HEALTH_URLS Space-separated URLs for health checks (default: auto-detect from SERVICE_LIST)
# HEALTH_TIMEOUT Curl timeout per check (seconds) (default: 5)
# HEALTH_RETRIES Number of retries per endpoint (default: 1)
# HEALTH_INTERVAL Seconds between retries (default: 2)
# =============================================================================
set -euo pipefail
# ---- Colors for readable output ----
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
BOLD='\033[1m'
NC='\033[0m' # No Color
# ---- Configuration (all env-overridable) ----
COMPOSE_DIR="${COMPOSE_DIR:-/opt/infra/compose}"
COMPOSE_PROJECT="${COMPOSE_PROJECT:-staging}"
STAGING_DOMAIN="${STAGING_DOMAIN:-staging.lazyworkhorse.net}"
HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-5}"
HEALTH_RETRIES="${HEALTH_RETRIES:-1}"
HEALTH_INTERVAL="${HEALTH_INTERVAL:-2}"
# Known compose service directories in order — override via SERVICE_LIST env var
DEFAULT_SERVICES=(
network
authentification
homepage
ai
cloudstorage
versioncontrol
backup
coms
finance
homeautomation
passwordmanager
)
# Map service directory -> default health check URL (relative to STAGING_DOMAIN)
# Override entirely via HEALTH_URLS env var.
declare -A DEFAULT_HEALTH_URLS
DEFAULT_HEALTH_URLS[network]="https://traefik.${STAGING_DOMAIN}/ping"
DEFAULT_HEALTH_URLS[authentification]="https://auth.${STAGING_DOMAIN}/api/verify"
DEFAULT_HEALTH_URLS[homepage]="https://${STAGING_DOMAIN}/"
DEFAULT_HEALTH_URLS[ai]="https://hermes.${STAGING_DOMAIN}/health"
DEFAULT_HEALTH_URLS[cloudstorage]="https://cloud.${STAGING_DOMAIN}/status.php"
DEFAULT_HEALTH_URLS[versioncontrol]="https://code.${STAGING_DOMAIN}/api/healthz"
# ---- Trackers ----
PASS_COUNT=0
FAIL_COUNT=0
WARN_COUNT=0
FAILURES=()
# ---- Helpers ----
log_info() { echo -e "${CYAN}[INFO]${NC} $*"; }
log_pass() { echo -e "${GREEN}[PASS]${NC} $*"; ((PASS_COUNT++)); }
log_fail() { echo -e "${RED}[FAIL]${NC} $*"; ((FAIL_COUNT++)); FAILURES+=("$*"); }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; ((WARN_COUNT++)); }
log_step() { echo -e "\n${BOLD}── $* ──${NC}"; }
log_raw() { echo -e " $*"; }
# Check if a command exists
require_cmd() {
if ! command -v "$1" &>/dev/null; then
log_fail "Required command not found: $1"
return 1
fi
}
# Retry a command with exponential-like backoff
retry() {
local cmd="$*"
local attempt=0
local max_attempts=$((HEALTH_RETRIES + 1))
local result
while [[ $attempt -lt $max_attempts ]]; do
if eval "$cmd" 2>/dev/null; then
return 0
fi
attempt=$((attempt + 1))
if [[ $attempt -lt $max_attempts ]]; then
sleep "$HEALTH_INTERVAL"
fi
done
return 1
}
# ---- Parse arguments ----
VERBOSE=false
LIST_SERVICES=false
POSITIONAL=()
while [[ $# -gt 0 ]]; do
case "$1" in
--verbose|-v) VERBOSE=true; shift ;;
--list-services) LIST_SERVICES=true; shift ;;
--) shift; POSITIONAL+=("$@"); break ;;
*) POSITIONAL+=("$1"); shift ;;
esac
done
set -- "${POSITIONAL[@]}"
# Resolve service list
if [[ -n "${SERVICE_LIST:-}" ]]; then
IFS=' ' read -ra SERVICES <<< "$SERVICE_LIST"
else
SERVICES=("${DEFAULT_SERVICES[@]}")
fi
# Resolve health URLs — default map with overrides from env
declare -A HEALTH_URLS
if [[ -n "${HEALTH_URLS:-}" ]]; then
# User-supplied mapping: "network=https://... authentification=https://..."
for pair in $HEALTH_URLS; do
key="${pair%%=*}"
val="${pair#*=}"
HEALTH_URLS["$key"]="$val"
done
else
for svc in "${SERVICES[@]}"; do
if [[ -n "${DEFAULT_HEALTH_URLS[$svc]:-}" ]]; then
HEALTH_URLS["$svc"]="${DEFAULT_HEALTH_URLS[$svc]}"
fi
done
fi
# --list-services mode (for CI integration)
if $LIST_SERVICES; then
echo "Configured services:"
for svc in "${SERVICES[@]}"; do
url="${HEALTH_URLS[$svc]:-no-health-check}"
echo " $svc -> $url"
done
exit 0
fi
# ---- Pre-flight ----
echo -e "${BOLD}============================================${NC}"
echo -e "${BOLD} Staging VM Integration Test Suite${NC}"
echo -e "${BOLD} $(date -u '+%Y-%m-%dT%H:%M:%SZ')${NC}"
echo -e "${BOLD}============================================${NC}"
# ---- Phase 1: Prerequisites ----
log_step "Phase 1: Prerequisites"
PREREQ_OK=true
for cmd in docker curl jq; do
if ! require_cmd "$cmd"; then
PREREQ_OK=false
fi
done
$PREREQ_OK && log_pass "All required commands available" || log_fail "Missing prerequisites"
# ---- Phase 2: Docker daemon ----
log_step "Phase 2: Docker Daemon"
if docker info --format '{{.ServerVersion}}' &>/dev/null; then
DOCKER_VERSION=$(docker info --format '{{.ServerVersion}}' 2>/dev/null)
log_pass "Docker daemon is running (version: $DOCKER_VERSION)"
if docker info --format '{{.Driver}}' 2>/dev/null | grep -qi "overlay"; then
log_pass "Storage driver: overlay"
else
log_warn "Non-overlay storage driver detected"
fi
else
log_fail "Docker daemon is NOT running or not accessible"
fi
# ---- Phase 3: Docker Compose stack ----
log_step "Phase 3: Compose Stack Status"
# Check if any compose files exist
COMPOSE_FILES=()
for svc in "${SERVICES[@]}"; do
cf="${COMPOSE_DIR}/${svc}/compose.yml"
if [[ -f "$cf" ]]; then
COMPOSE_FILES+=("$cf")
else
cf2="${COMPOSE_DIR}/${svc}/docker-compose.yml"
if [[ -f "$cf2" ]]; then
COMPOSE_FILES+=("$cf2")
else
log_warn "No compose file found for service '$svc' (expected: ${cf})"
fi
fi
done
if [[ ${#COMPOSE_FILES[@]} -eq 0 ]]; then
log_fail "No compose files found under COMPOSE_DIR=${COMPOSE_DIR}"
log_info "Skipping stack checks"
else
log_info "Found ${#COMPOSE_FILES[@]} compose file(s) in ${COMPOSE_DIR}"
# Build the compose file args
COMPOSE_CMD="docker compose -p ${COMPOSE_PROJECT}"
for cf in "${COMPOSE_FILES[@]}"; do
COMPOSE_CMD+=" -f ${cf}"
done
log_info "Project name: ${COMPOSE_PROJECT}"
# Check stack ps
if $VERBOSE; then
log_raw "--- docker compose ps output ---"
eval "$COMPOSE_CMD ps" 2>&1 | while IFS= read -r line; do log_raw "$line"; done
log_raw "--- end ---"
fi
# Get all services and their status
if STACK_STATUS=$(eval "$COMPOSE_CMD ps --format '{{.Name}}\t{{.Status}}'" 2>/dev/null); then
if [[ -z "$STACK_STATUS" ]]; then
log_warn "Stack exists but no running services — VM may be freshly provisioned"
else
ALL_RUNNING=true
RUNNING_COUNT=0
TOTAL_COUNT=0
while IFS=$'\t' read -r name status; do
TOTAL_COUNT=$((TOTAL_COUNT + 1))
status_lower=$(echo "$status" | tr '[:upper:]' '[:lower:]')
if echo "$status_lower" | grep -qE '^(up|running|healthy)'; then
RUNNING_COUNT=$((RUNNING_COUNT + 1))
$VERBOSE && log_pass " $name$status"
else
ALL_RUNNING=false
log_warn " $name$status (not healthy)"
fi
done <<< "$STACK_STATUS"
if [[ "$TOTAL_COUNT" -eq 0 ]]; then
log_fail "No services found in compose project"
elif $ALL_RUNNING && [[ "$TOTAL_COUNT" -eq "$RUNNING_COUNT" ]]; then
log_pass "All ${TOTAL_COUNT} service(s) running (${RUNNING_COUNT}/${TOTAL_COUNT})"
else
log_fail "${RUNNING_COUNT}/${TOTAL_COUNT} service(s) running — some services are down"
fi
fi
else
log_fail "Failed to query compose stack status"
fi
fi
# ---- Phase 4: Service health checks ----
log_step "Phase 4: Service Endpoint Health Checks"
ENDPOINT_CHECKS=0
ENDPOINT_PASS=0
for svc in "${SERVICES[@]}"; do
url="${HEALTH_URLS[$svc]:-}"
if [[ -z "$url" ]]; then
$VERBOSE && log_info "No health check URL for service '$svc' — skipping"
continue
fi
ENDPOINT_CHECKS=$((ENDPOINT_CHECKS + 1))
echo -ne " Checking ${svc} ... "
# Perform the HTTP health check with retries
if retry "curl -sf -o /dev/null -w '%{http_code}' --max-time ${HEALTH_TIMEOUT} '${url}' 2>/dev/null"; then
HTTP_CODE=$(curl -sf -o /dev/null -w '%{http_code}' --max-time "${HEALTH_TIMEOUT}" "${url}" 2>/dev/null || true)
ENDPOINT_PASS=$((ENDPOINT_PASS + 1))
echo -e "${GREEN}OK${NC} (HTTP ${HTTP_CODE})"
else
LAST_CODE=$(curl -s -o /dev/null -w '%{http_code}' --max-time "${HEALTH_TIMEOUT}" "${url}" 2>/dev/null || echo "000")
echo -e "${RED}FAIL${NC} (HTTP ${LAST_CODE})"
log_fail "Health check failed for ${svc} @ ${url}"
fi
done
if [[ $ENDPOINT_CHECKS -eq 0 ]]; then
log_warn "No health check URLs configured — skipping endpoint phase"
elif [[ $ENDPOINT_PASS -eq $ENDPOINT_CHECKS ]]; then
log_pass "All ${ENDPOINT_CHECKS} endpoint(s) healthy"
else
log_fail "${ENDPOINT_PASS}/${ENDPOINT_CHECKS} endpoint(s) healthy"
fi
# ---- Phase 5: Docker system sanity ----
log_step "Phase 5: Docker System Sanity"
# Check disk space for Docker
DOCKER_ROOT=$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || echo "/var/lib/docker")
log_info "Docker root: ${DOCKER_ROOT}"
if command -v df &>/dev/null && [[ -d "$DOCKER_ROOT" ]]; then
AVAIL_PCT=$(df -h "$DOCKER_ROOT" | awk 'NR==2 {print $5}' | tr -d '%')
if [[ -n "$AVAIL_PCT" ]]; then
if [[ "$AVAIL_PCT" -ge 90 ]]; then
log_warn "Docker storage is ${AVAIL_PCT}% full — consider cleanup"
else
log_pass "Docker storage at ${AVAIL_PCT}% — within limits"
fi
fi
fi
# Check for dangling images
DANGLING=$(docker images -f "dangling=true" -q 2>/dev/null | wc -l)
if [[ "$DANGLING" -gt 10 ]]; then
log_warn "${DANGLING} dangling images found — consider docker image prune"
fi
# ---- Summary ----
echo ""
echo -e "${BOLD}============================================${NC}"
echo -e "${BOLD} Test Summary${NC}"
echo -e "${BOLD} $(date -u '+%Y-%m-%dT%H:%M:%SZ')${NC}"
echo -e "${BOLD}============================================${NC}"
echo -e " ${GREEN}Passed:${NC} ${PASS_COUNT}"
echo -e " ${RED}Failed:${NC} ${FAIL_COUNT}"
echo -e " ${YELLOW}Warnings:${NC} ${WARN_COUNT}"
if [[ ${#FAILURES[@]} -gt 0 ]]; then
echo -e "\n${BOLD}Failed checks:${NC}"
for f in "${FAILURES[@]}"; do
echo -e " ${RED}${NC} $f"
done
fi
echo ""
if [[ $FAIL_COUNT -eq 0 ]]; then
echo -e "${GREEN}${BOLD}✓ All integration checks passed${NC}"
exit 0
else
echo -e "${RED}${BOLD}${FAIL_COUNT} integration check(s) failed${NC}"
exit 1
fi

View File

@@ -4,7 +4,7 @@
group = "ai-worker";
home = "/home/ai-worker";
createHome = true;
extraGroups = [ "docker" "libvirtd" ];
extraGroups = [ "docker" ];
shell = pkgs.bashInteractive;
openssh.authorizedKeys.keys = [
keys.users.ai-worker.main