Compare commits
6 Commits
kvm-pr
...
feat/nixos
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ef7f06166b | ||
|
|
e70516e78b | ||
|
|
ce3a327ff7 | ||
|
|
af38655170 | ||
| dcf5ac6b5e | |||
| d78de94f94 |
@@ -1,4 +1,4 @@
|
||||
name: Build and test NixOS config
|
||||
name: Build NixOS config
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
@@ -27,26 +27,7 @@ jobs:
|
||||
https://gitea:${{ secrets.GITHUB_TOKEN }}@code.lazyworkhorse.net/gortium/infra.git .
|
||||
git log --oneline -3
|
||||
|
||||
- name: Build NixOS config
|
||||
- name: Build NixOS config (lazyworkhorse)
|
||||
run: |
|
||||
nix --version
|
||||
nh os build .#lazyworkhorse 2>&1
|
||||
|
||||
- name: Run integration tests (staging VM)
|
||||
run: |
|
||||
echo "==> Running integration tests on staging VM..."
|
||||
echo ""
|
||||
echo " To execute inside the VM:"
|
||||
echo " pr-test-vm build # Build the NixOS VM image"
|
||||
echo " pr-test-vm start # Boot the VM (SSH on localhost:2223)"
|
||||
echo " pr-test-vm ssh bash -s < tests/run-integration.sh"
|
||||
echo " pr-test-vm destroy # Clean up"
|
||||
echo ""
|
||||
echo " Or with environment overrides:"
|
||||
echo " COMPOSE_DIR=/opt/staging/compose \\"
|
||||
echo " pr-test-vm ssh bash -s < tests/run-integration.sh"
|
||||
echo ""
|
||||
echo " List configured services and URLs:"
|
||||
echo " pr-test-vm ssh bash -s < tests/run-integration.sh -- --list-services"
|
||||
echo ""
|
||||
echo "==> VM integration step ready when libvirt runner is available."
|
||||
|
||||
Submodule assets/compose updated: f9fb28d560...6b82a26c25
@@ -1,106 +0,0 @@
|
||||
# ollama-gfx906/Dockerfile
|
||||
#
|
||||
# Custom ollama image with ROCm 6.1 + gfx906 (MI50) support.
|
||||
# The official ollama/rocm image ships ROCm 7.2 which dropped gfx906.
|
||||
# This uses v0.23.2's native CMake build system with AMDGPU_TARGETS including gfx906.
|
||||
#
|
||||
# Build: docker build -t ollama/ollama:rocm-gfx906 ai/ollama
|
||||
|
||||
FROM rocm/dev-ubuntu-22.04:6.1.2-complete AS builder
|
||||
|
||||
# Build dependencies (CMake, Ninja, Go)
|
||||
ARG CMAKEVERSION=3.31.2
|
||||
ARG NINJAVERSION=1.12.1
|
||||
ARG GOLANG_VERSION=1.22.0
|
||||
|
||||
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
curl git ccache build-essential pkg-config unzip \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install CMake from official binaries
|
||||
RUN curl -fsSL https://github.com/Kitware/CMake/releases/download/v${CMAKEVERSION}/cmake-${CMAKEVERSION}-linux-x86_64.tar.gz \
|
||||
| tar xz -C /usr/local --strip-components 1
|
||||
|
||||
# Install Ninja
|
||||
RUN curl -fsSL -o /tmp/ninja.zip \
|
||||
https://github.com/ninja-build/ninja/releases/download/v${NINJAVERSION}/ninja-linux.zip \
|
||||
&& unzip /tmp/ninja.zip -d /usr/local/bin && rm /tmp/ninja.zip
|
||||
|
||||
# Install Go
|
||||
RUN curl -fsSL https://go.dev/dl/go${GOLANG_VERSION}.linux-amd64.tar.gz \
|
||||
| tar xz -C /usr/local
|
||||
ENV PATH=/usr/local/go/bin:$PATH
|
||||
|
||||
ARG OLLAMA_VERSION=v0.23.2
|
||||
RUN git clone --depth 1 --branch ${OLLAMA_VERSION} https://github.com/ollama/ollama.git /build
|
||||
WORKDIR /build
|
||||
|
||||
# ROCm paths
|
||||
ENV HIP_PATH=/opt/rocm
|
||||
ENV ROCM_PATH=/opt/rocm
|
||||
ENV CMAKE_GENERATOR=Ninja
|
||||
ENV LDFLAGS=-s
|
||||
|
||||
# Step 1: Build CPU backends with GCC (no ROCm preset)
|
||||
# Pre-set CMAKE_HIP_COMPILER="" to prevent check_language(HIP) from
|
||||
# finding a HIP compiler (it searches /opt/rocm even without PATH).
|
||||
# Remove /opt/rocm from PATH to prevent find_program from finding hipcc.
|
||||
RUN mkdir -p build-cpu && \
|
||||
PATH=/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
|
||||
cmake -B build-cpu -DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_HIP_COMPILER="" \
|
||||
-DCMAKE_INSTALL_PREFIX=/build/dist && \
|
||||
cmake --build build-cpu --target ggml-cpu -- -l $(nproc) && \
|
||||
cmake --install build-cpu --component CPU --strip && \
|
||||
echo "=== CPU install ===" && \
|
||||
(find /build/dist/lib/ollama -type f -o -type l 2>&1 | head -20 || echo "empty")
|
||||
|
||||
# Step 2: Build HIP backend with ROCm preset + gfx906 target only
|
||||
# The ROCm 6 preset enables HIP language detection (enable_language(HIP))
|
||||
# which ensures GPU kernels are properly compiled for gfx906.
|
||||
# OLLAMA_RUNNER_DIR=rocm from the preset, so HIP goes to lib/ollama/rocm/
|
||||
# Need CMAKE_PREFIX_PATH so find_package(hip) finds hip-config.cmake
|
||||
# at /opt/rocm/lib/cmake/hip/hip-config.cmake.
|
||||
RUN mkdir -p build-hip && \
|
||||
cmake -B build-hip \
|
||||
--preset 'ROCm 6' \
|
||||
-DAMDGPU_TARGETS="gfx906:xnack-" \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_PREFIX_PATH="/opt/rocm" && \
|
||||
cmake --build build-hip --target ggml-hip -- -l $(nproc) && \
|
||||
cmake --install build-hip --component HIP --strip && \
|
||||
echo "=== HIP install ===" && \
|
||||
find /build/dist/lib/ollama -type f -o -type l | head -20
|
||||
|
||||
# Step 3: Build Go binary (GCC for CGo linking)
|
||||
ENV CGO_ENABLED=1
|
||||
RUN go build -trimpath -ldflags="-X=github.com/ollama/ollama/version.Version=${OLLAMA_VERSION}" -o /build/dist/ollama .
|
||||
|
||||
# ---------- Runtime image ----------
|
||||
FROM ubuntu:24.04
|
||||
|
||||
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
ca-certificates curl libstdc++6 libgomp1 libvulkan1 libopenblas0 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy ROCm 6.1 runtime libraries
|
||||
# These are needed at runtime by ggml-hip via LD_LIBRARY_PATH
|
||||
COPY --from=builder /opt/rocm/lib/ /opt/rocm/lib/
|
||||
COPY --from=builder /opt/rocm/share/ /opt/rocm/share/
|
||||
|
||||
# Copy ollama binary + all backends (CPU + HIP)
|
||||
# CPU install: /build/dist/lib/ollama/libggml-*.so
|
||||
# HIP install: /build/dist/lib/ollama/rocm/libggml-hip.so
|
||||
COPY --from=builder /build/dist/ollama /usr/bin/ollama
|
||||
COPY --from=builder /build/dist/lib/ollama/ /usr/lib/ollama/
|
||||
|
||||
RUN ldconfig
|
||||
|
||||
ENV LD_LIBRARY_PATH=/opt/rocm/lib:/usr/lib/ollama/rocm:/usr/lib/ollama
|
||||
ENV HSA_OVERRIDE_GFX_VERSION=9.0.6
|
||||
ENV HCC_AMDGPU_TARGET=gfx906
|
||||
ENV HSA_ENABLE_SDMA=0
|
||||
|
||||
EXPOSE 11434
|
||||
ENTRYPOINT ["/bin/ollama"]
|
||||
CMD ["serve"]
|
||||
@@ -61,7 +61,6 @@
|
||||
./modules/nixos/services/open_code_server.nix
|
||||
./modules/nixos/services/ollama_init_custom_models.nix
|
||||
./modules/nixos/services/openclaw_node.nix
|
||||
./modules/nixos/services/staging-vm.nix
|
||||
./modules/nixos/security/ai-worker-restricted.nix
|
||||
./users/gortium.nix
|
||||
./users/ai-worker.nix
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
"transparent_hugepage=always" # because mucho ram
|
||||
];
|
||||
# 2. Load the specific drivers found by sensors-detect
|
||||
boot.kernelModules = [ "nct6775" "lm96163" "iptable_nat" "iptable_filter" "kvm-intel" "kvm" ];
|
||||
boot.kernelModules = [ "nct6775" "lm96163" "iptable_nat" "iptable_filter" ];
|
||||
# 3. Force the nct6775 driver to recognize the chip if it's stubborn
|
||||
boot.extraModprobeConfig = ''
|
||||
options nct6775 force_id=0xd280
|
||||
@@ -191,6 +191,7 @@
|
||||
services.dockerStacks = {
|
||||
versioncontrol = {
|
||||
path = self + "/assets/compose/versioncontrol";
|
||||
envFile = config.age.secrets.containers_env.path;
|
||||
ports = [ 2222 ];
|
||||
};
|
||||
|
||||
@@ -328,21 +329,20 @@
|
||||
# Mi50 config
|
||||
hardware.graphics = {
|
||||
enable = true;
|
||||
enable32Bit = true;
|
||||
enable32Bit = true; # Useful for some compatibility layers
|
||||
extraPackages = with pkgs; [
|
||||
rocmPackages.clr.icd
|
||||
rocmPackages.clr.icd # OpenCL/HIP runtime
|
||||
];
|
||||
};
|
||||
nixpkgs.config.rocmTargets = [ "gfx906" ];
|
||||
environment.variables = {
|
||||
# This "tricks" ROCm into supporting the MI50 if using newer versions
|
||||
HSA_OVERRIDE_GFX_VERSION = "9.0.6";
|
||||
# Ensures the system sees both GPUs
|
||||
HIP_VISIBLE_DEVICES = "0,1";
|
||||
};
|
||||
|
||||
# KVM/libvirt for staging VM
|
||||
services.stagingVm.enable = true;
|
||||
|
||||
# Open ports in the firewall.
|
||||
# Open ports in the firewall.
|
||||
# networking.firewall.allowedTCPPorts = [ ... ];
|
||||
# networking.firewall.allowedUDPPorts = [ ... ];
|
||||
# Or disable the firewall altogether.
|
||||
|
||||
@@ -1,363 +0,0 @@
|
||||
{ config, pkgs, lib, ... }:
|
||||
|
||||
with lib;
|
||||
|
||||
let
|
||||
cfg = config.services.stagingVm;
|
||||
|
||||
# ── pr-test-vm helper script ──────────────────────────────────────────
|
||||
pr-test-vm = pkgs.writeShellScriptBin "pr-test-vm" ''
|
||||
set -euo pipefail
|
||||
|
||||
LIBVIRT_URI="qemu:///system"
|
||||
VM_DIR="${cfg.dataPath}"
|
||||
NETWORK="default"
|
||||
SCRIPT_NAME="$(basename "$0")"
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $SCRIPT_NAME <command> [options]
|
||||
|
||||
Commands:
|
||||
build <nixos-config> [--name <name>] Build VM image from a NixOS config
|
||||
start <vm-name> Start a VM
|
||||
stop <vm-name> Gracefully shut down a VM
|
||||
destroy <vm-name> Force-power-off and undefine a VM
|
||||
ssh [user@]<vm-name> SSH into a running VM
|
||||
console <vm-name> Connect to VM serial console
|
||||
list List all staging VMs
|
||||
status <vm-name> Show VM status
|
||||
|
||||
Examples:
|
||||
$SCRIPT_NAME build ./vm-config.nix --name my-test
|
||||
$SCRIPT_NAME start my-test
|
||||
$SCRIPT_NAME ssh root@my-test
|
||||
EOF
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Find the VM's IP address from the DHCP lease
|
||||
vm_ip() {
|
||||
local name="$1"
|
||||
local mac
|
||||
mac=$(${pkgs.libvirt}/bin/virsh -c "$LIBVIRT_URI" domiflist "$name" 2>/dev/null \
|
||||
| ${pkgs.gawk}/bin/awk 'NR>2 && $1 ~ /^vnet/ {print $NF; exit}')
|
||||
[ -z "$mac" ] && { echo "error: cannot find MAC for VM '$name'"; exit 1; }
|
||||
|
||||
local ip
|
||||
ip=$(${pkgs.libvirt}/bin/virsh -c "$LIBVIRT_URI" net-dhcp-leases "$NETWORK" 2>/dev/null \
|
||||
| ${pkgs.gawk}/bin/awk -v mac="$mac" '$0 ~ mac {gsub(/-.*/, "", $3); print $3; exit}')
|
||||
[ -z "$ip" ] && { echo "error: no DHCP lease found for VM '$name' (MAC: $mac)"; exit 1; }
|
||||
echo "$ip"
|
||||
}
|
||||
|
||||
case "''${1:-help}" in
|
||||
build)
|
||||
shift
|
||||
CONFIG="''${1:?Missing NixOS config path}"
|
||||
VM_NAME="''${2:-}"
|
||||
[ -f "$CONFIG" ] || { echo "error: config file not found: $CONFIG"; exit 1; }
|
||||
|
||||
# Extract name from --name flag or config basename
|
||||
if [ "''${2:-}" = "--name" ] && [ -n "''${3:-}" ]; then
|
||||
VM_NAME="$3"
|
||||
elif [ -z "$VM_NAME" ] || [ "''${VM_NAME#--}" != "$VM_NAME" ]; then
|
||||
VM_NAME="$(basename "$CONFIG" .nix)"
|
||||
fi
|
||||
|
||||
BUILD_DIR="$VM_DIR/$VM_NAME"
|
||||
echo "==> Building VM '$VM_NAME' from config: $CONFIG"
|
||||
mkdir -p "$BUILD_DIR"
|
||||
|
||||
# Build the NixOS VM derivation
|
||||
nix build --no-link -f "$CONFIG" vm 2>&1 || {
|
||||
echo "Trying flake build..."
|
||||
nix build "''${CONFIG%/.nix}#nixosConfigurations.$VM_NAME.config.system.build.vm" --no-link 2>&1 || {
|
||||
echo "error: failed to build VM (tried both import and flake)"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
echo "==> Build complete. Run 'pr-test-vm start $VM_NAME' to launch."
|
||||
;;
|
||||
|
||||
start)
|
||||
VM_NAME="''${1:?Missing VM name}"
|
||||
IMAGE="$VM_DIR/$VM_NAME/disk-image.qcow2"
|
||||
[ -f "$IMAGE" ] || { echo "error: no disk image found at $IMAGE. Build first."; exit 1; }
|
||||
|
||||
# Check if already running
|
||||
STATE=$(${pkgs.libvirt}/bin/virsh -c "$LIBVIRT_URI" domstate "$VM_NAME" 2>/dev/null || echo "undefined")
|
||||
if [ "$STATE" = "running" ]; then
|
||||
echo "VM '$VM_NAME' is already running."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "==> Starting VM '$VM_NAME'..."
|
||||
|
||||
# Undefine if defined but not running
|
||||
if [ "$STATE" != "undefined" ]; then
|
||||
${pkgs.libvirt}/bin/virsh -c "$LIBVIRT_URI" undefine "$VM_NAME" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Define and start with virt-install
|
||||
${pkgs.virt-manager}/bin/virt-install \
|
||||
--connect "$LIBVIRT_URI" \
|
||||
--name "$VM_NAME" \
|
||||
--memory "${toString cfg.memory}" \
|
||||
--vcpus "${toString cfg.vcpus}" \
|
||||
--disk "$IMAGE",bus=virtio \
|
||||
--import \
|
||||
--network network="$NETWORK",model=virtio \
|
||||
--graphics none \
|
||||
--console pty,target_type=virtio \
|
||||
--serial pty \
|
||||
--memballoon virtio \
|
||||
--rng /dev/urandom \
|
||||
--noautoconsole \
|
||||
--os-variant detect=on,name=generic
|
||||
|
||||
echo "==> VM '$VM_NAME' started. Get IP with: pr-test-vm status $VM_NAME"
|
||||
;;
|
||||
|
||||
stop)
|
||||
VM_NAME="''${1:?Missing VM name}"
|
||||
echo "==> Stoping VM '$VM_NAME'..."
|
||||
${pkgs.libvirt}/bin/virsh -c "$LIBVIRT_URI" shutdown "$VM_NAME" 2>/dev/null && {
|
||||
echo "Waiting for VM to shut down..."
|
||||
for i in $(seq 1 30); do
|
||||
STATE=$(${pkgs.libvirt}/bin/virsh -c "$LIBVIRT_URI" domstate "$VM_NAME" 2>/dev/null || echo "undefined")
|
||||
[ "$STATE" != "running" ] && { echo "VM stopped."; exit 0; }
|
||||
sleep 2
|
||||
done
|
||||
echo "warning: VM did not shut down gracefully, use 'destroy' for force"
|
||||
} || {
|
||||
echo "VM '$VM_NAME' not running or does not exist."
|
||||
}
|
||||
;;
|
||||
|
||||
destroy)
|
||||
VM_NAME="''${1:?Missing VM name}"
|
||||
echo "==> Destroying VM '$VM_NAME'..."
|
||||
${pkgs.libvirt}/bin/virsh -c "$LIBVIRT_URI" destroy "$VM_NAME" 2>/dev/null || true
|
||||
${pkgs.libvirt}/bin/virsh -c "$LIBVIRT_URI" undefine "$VM_NAME" 2>/dev/null || true
|
||||
echo "==> VM '$VM_NAME' destroyed and undefined."
|
||||
;;
|
||||
|
||||
ssh)
|
||||
TARGET="''${1:?Usage: $SCRIPT_NAME ssh [user@]<vm-name>}"
|
||||
# Split user@hostname if present
|
||||
if echo "$TARGET" | ${pkgs.gnugrep}/bin/grep -q '@'; then
|
||||
USER="''${TARGET%@*}"
|
||||
VM_NAME="''${TARGET#*@}"
|
||||
else
|
||||
VM_NAME="$TARGET"
|
||||
USER=""
|
||||
fi
|
||||
|
||||
IP=$(vm_ip "$VM_NAME") || exit 1
|
||||
if [ -n "$USER" ]; then
|
||||
exec ${pkgs.openssh}/bin/ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "''${USER}@''${IP}"
|
||||
else
|
||||
exec ${pkgs.openssh}/bin/ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$IP"
|
||||
fi
|
||||
;;
|
||||
|
||||
console)
|
||||
VM_NAME="''${1:?Missing VM name}"
|
||||
exec ${pkgs.libvirt}/bin/virsh -c "$LIBVIRT_URI" console "$VM_NAME"
|
||||
;;
|
||||
|
||||
list)
|
||||
echo "Staging VMs:"
|
||||
${pkgs.libvirt}/bin/virsh -c "$LIBVIRT_URI" list --all
|
||||
echo ""
|
||||
echo "Active networks:"
|
||||
${pkgs.libvirt}/bin/virsh -c "$LIBVIRT_URI" net-list
|
||||
echo ""
|
||||
echo "Storage pools:"
|
||||
${pkgs.libvirt}/bin/virsh -c "$LIBVIRT_URI" pool-list
|
||||
;;
|
||||
|
||||
status)
|
||||
VM_NAME="''${1:?Missing VM name}"
|
||||
echo "VM: $VM_NAME"
|
||||
STATE=$(${pkgs.libvirt}/bin/virsh -c "$LIBVIRT_URI" domstate "$VM_NAME" 2>/dev/null || echo "not found")
|
||||
echo "State: $STATE"
|
||||
if [ "$STATE" = "running" ]; then
|
||||
IP=$(vm_ip "$VM_NAME" 2>/dev/null || echo "N/A")
|
||||
echo "IP: $IP"
|
||||
${pkgs.libvirt}/bin/virsh -c "$LIBVIRT_URI" dommemstat "$VM_NAME" 2>/dev/null | head -3 || true
|
||||
fi
|
||||
;;
|
||||
|
||||
help|--help|-h)
|
||||
usage
|
||||
;;
|
||||
|
||||
*)
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
'';
|
||||
in
|
||||
{
|
||||
options.services.stagingVm = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = "Enable KVM/libvirt staging VM for compose PR testing";
|
||||
};
|
||||
|
||||
vmName = mkOption {
|
||||
type = types.str;
|
||||
default = "compose-test-vm";
|
||||
description = "Name of the staging VM";
|
||||
};
|
||||
|
||||
memory = mkOption {
|
||||
type = types.str;
|
||||
default = "4096";
|
||||
description = "RAM allocated to the staging VM (MB)";
|
||||
};
|
||||
|
||||
vcpus = mkOption {
|
||||
type = types.int;
|
||||
default = 2;
|
||||
description = "Number of vCPUs for the staging VM";
|
||||
};
|
||||
|
||||
storagePath = mkOption {
|
||||
type = types.str;
|
||||
default = "/var/lib/libvirt/images";
|
||||
description = "Path for libvirt storage pool";
|
||||
};
|
||||
|
||||
dataPath = mkOption {
|
||||
type = types.str;
|
||||
default = "/var/lib/staging-vm";
|
||||
description = "Path for compose test data (PR checkouts, test results)";
|
||||
};
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
# ── libvirtd with QEMU/KVM ──────────────────────────────────────────
|
||||
virtualisation.libvirtd = {
|
||||
enable = true;
|
||||
qemu = {
|
||||
package = pkgs.qemu_kvm;
|
||||
runAsRoot = true;
|
||||
swtpm.enable = true;
|
||||
ovmf = {
|
||||
enable = true;
|
||||
packages = [ pkgs.OVMF ];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
# ── System packages ─────────────────────────────────────────────────
|
||||
environment.systemPackages = with pkgs; [
|
||||
libvirt # virsh, virt-admin
|
||||
qemu_kvm # QEMU/KVM
|
||||
swtpm # Software TPM
|
||||
OVMF # UEFI firmware for VMs
|
||||
virt-manager # GUI + virt-install
|
||||
virt-viewer # SPICE/VNC viewer
|
||||
libguestfs # virt-customize, guestfish
|
||||
cdrtools # genisoimage for cloud-init ISOs
|
||||
jq # JSON parsing
|
||||
gawk # awk for DHCP lease parsing
|
||||
gnugrep # grep
|
||||
];
|
||||
|
||||
# ── User permissions ────────────────────────────────────────────────
|
||||
users.users.gortium.extraGroups = [ "libvirtd" ];
|
||||
|
||||
# ── Directories ─────────────────────────────────────────────────────
|
||||
systemd.tmpfiles.rules = [
|
||||
"d ${cfg.storagePath} 0755 root root -"
|
||||
"d ${cfg.dataPath} 0755 root root -"
|
||||
];
|
||||
|
||||
# ── Default NAT network (192.168.122.0/24) ──────────────────────────
|
||||
# Define the default libvirt NAT network using virsh postStart hook
|
||||
systemd.services.libvirtd = {
|
||||
postStart = ''
|
||||
set -e
|
||||
# Define the NAT network if it doesn't exist
|
||||
${pkgs.libvirt}/bin/virsh -c qemu:///system net-info default 2>/dev/null && {
|
||||
echo "Network 'default' already exists"
|
||||
} || {
|
||||
echo "Defining default NAT network (192.168.122.0/24)..."
|
||||
${pkgs.libvirt}/bin/virsh -c qemu:///system net-define /etc/libvirt/qemu/networks/default.xml
|
||||
}
|
||||
${pkgs.libvirt}/bin/virsh -c qemu:///system net-autostart default 2>/dev/null || true
|
||||
# Start the network if not active
|
||||
STATE=$(${pkgs.libvirt}/bin/virsh -c qemu:///system net-state default 2>/dev/null || echo "inactive")
|
||||
if [ "$STATE" != "active" ]; then
|
||||
${pkgs.libvirt}/bin/virsh -c qemu:///system net-start default 2>/dev/null || true
|
||||
fi
|
||||
echo "Default network ready."
|
||||
'';
|
||||
};
|
||||
|
||||
# Define the default network as an XML config file
|
||||
environment.etc."libvirt/qemu/networks/default.xml" = {
|
||||
text = ''
|
||||
<network>
|
||||
<name>default</name>
|
||||
<forward mode='nat'/>
|
||||
<bridge name='virbr0' stp='on' delay='0'/>
|
||||
<ip address='192.168.122.1' netmask='255.255.255.0'>
|
||||
<dhcp>
|
||||
<range start='192.168.122.2' end='192.168.122.254'/>
|
||||
</dhcp>
|
||||
</ip>
|
||||
</network>
|
||||
'';
|
||||
mode = "0644";
|
||||
};
|
||||
|
||||
# ── Storage pool ────────────────────────────────────────────────────
|
||||
systemd.services.libvirtd.postStart = mkAfter ''
|
||||
set -e
|
||||
${pkgs.libvirt}/bin/virsh -c qemu:///system pool-info default 2>/dev/null && {
|
||||
echo "Storage pool 'default' already exists"
|
||||
} || {
|
||||
echo "Defining storage pool at ${cfg.storagePath}..."
|
||||
${pkgs.libvirt}/bin/virsh -c qemu:///system pool-define-as \
|
||||
--name default --type dir --target "${cfg.storagePath}"
|
||||
}
|
||||
${pkgs.libvirt}/bin/virsh -c qemu:///system pool-autostart default 2>/dev/null || true
|
||||
STATE=$(${pkgs.libvirt}/bin/virsh -c qemu:///system pool-state default 2>/dev/null || echo "inactive")
|
||||
if [ "$STATE" != "running" ]; then
|
||||
${pkgs.libvirt}/bin/virsh -c qemu:///system pool-build default 2>/dev/null || true
|
||||
${pkgs.libvirt}/bin/virsh -c qemu:///system pool-start default 2>/dev/null || true
|
||||
fi
|
||||
echo "Storage pool ready."
|
||||
'';
|
||||
|
||||
# ── Firewall rules for libvirt guests ───────────────────────────────
|
||||
networking.firewall = {
|
||||
trustedInterfaces = [ "virbr0" ];
|
||||
|
||||
extraCommands = mkAfter ''
|
||||
# Allow DHCP (port 67/68) and DNS (port 53) to libvirt guests
|
||||
iptables -I INPUT -i virbr0 -p udp --dport 67:68 -j ACCEPT 2>/dev/null || true
|
||||
iptables -I INPUT -i virbr0 -p tcp --dport 53 -j ACCEPT 2>/dev/null || true
|
||||
iptables -I INPUT -i virbr0 -p udp --dport 53 -j ACCEPT 2>/dev/null || true
|
||||
|
||||
# Allow forwarding between the bridge and the outside world
|
||||
iptables -I FORWARD -i virbr0 -o virbr0 -j ACCEPT 2>/dev/null || true
|
||||
iptables -I FORWARD -o virbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || true
|
||||
iptables -I FORWARD -i virbr0 -j ACCEPT 2>/dev/null || true
|
||||
|
||||
# NAT for guest outbound traffic
|
||||
iptables -t nat -I POSTROUTING -s 192.168.122.0/24 -j MASQUERADE 2>/dev/null || true
|
||||
'';
|
||||
};
|
||||
|
||||
# ── pr-test-vm helper script ────────────────────────────────────────
|
||||
environment.systemPackages = [ pr-test-vm ];
|
||||
};
|
||||
}
|
||||
@@ -1,36 +1,36 @@
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IEdoTUQ4QSBWNEpt
|
||||
cGFNeVBBaDRqb3pLSEZGQW0wb3VmVnBoZCswUFkzbnBLUnJ0QTNrClRqVkk4RUVO
|
||||
d29KYjd5YUcwankvaTFmVHUxQVpDT2ZYWHRaY3JXTUtQMU0KLT4gKXBtQ3UsXi1n
|
||||
cmVhc2UgNnwxYCBXVyA/KCQmIHt9NAo3OTZVUHR2UXkvaEFwY0ZBdEJsaFpsbHJ0
|
||||
cklKcDVHcEdWMEdPSkpnN0FiRU43RW5hUWFMdjR3WFRRSFBLSGlmClM3cTNJWlNM
|
||||
TExkdHdXUHJISkNIaE1TTUxUc1NUWkV1a09HeFU3bVZwQXMKLS0tIGhOcXFTUElS
|
||||
azJJNnEreUhMWTJBaVZGSTJPRUFqQkVYS01KRENUVVpZSDgK8+8onFejroBo7MeO
|
||||
dW+so4lOsq4zJKn3f0cxmCFg1f0X8zt6h4Uc3A5Cvr1uU+6yw1FWmJ7xa3jJz3lO
|
||||
EEaKQJXYC+xIIKGcA7qILa0SFp4a/4OuYjcg27HrlPhg7u5wDhQrd0LdVEe1Xngp
|
||||
ZivX7P7HwIna3X8C+TL+K2v/AG2N/z86cdKfRvxyMKNbHhYw+CfHEnWgh8tJ++4h
|
||||
G9evNniuNqte6cQaRe7jODfPNW4FuY/Sb7barlJ/M9iAQdYAdyLAzU1LABeHeUfD
|
||||
wtHjxy9DUZ55Vg8bB8M2JJU9MkoRT4ewiVd9LeC1GWeVmKsm93wsmrov714i7U2j
|
||||
wHtDkjqEF2MmzuQc18sjNaAHiwz8j6o5xU2L/Q4+Q707yISWG7RGZYh389Cr1rnw
|
||||
siUq/Vunqw2wk13+J/4vu9nqt5mMktBaCtp+QiWIurjwB5LUAyChrSm+dg5lb0Mt
|
||||
UhSc0lq1+E3vxAXM2Hmk+vP86VD+6WJvAU82VFApF1s6zG2FU1/AcOVVf54nan/q
|
||||
f+rgSFfASHQCYSblUJHyEtwLNsWEmTGmOEn1buUKD/H0zatPQnc0rYpjlx2V0Sjd
|
||||
6yB5+wPrZ0AkN1pjcsPKOv8Kaog2DzqIjib+SaSTaRxWHQEb9uzvaReAcYI5HOpE
|
||||
gkC040HN33BItATbo4+hz70Im8Ni/VXD+g6yzM6Hj1hJL+PinTKeg5keQRFIZjMx
|
||||
grzievB2wVBBgLgN3qMdTFmpplaL7iL702JjXZUTTK9Izp+9wiCsV1fTa53FWDht
|
||||
ylFL5SWElqXjK+QBXxAe+Jk6VQov5HI21YDXL67S554ABeRok23wxrQ31TCI4xq9
|
||||
PQV7VtNRjyVud7S29m3OwpWOsgTZhn+JclHj2v4bNJzJkJnZRTmcvGPktzRI5+R4
|
||||
e5vxVhGnJDzI71txaHl8+xS1lu9VzCQUrxX6TXyTRV4KjIOz0g06JOBgmBRBvJca
|
||||
7MZbC65xpisl/gyLRbgkVga3t94dPV+dpZsn8eq6427IyRbKslJefatggR9//c6I
|
||||
5N5fl0fR3gJQMB+HRbipBH2YsdbdWJyb4Nn6STZxIfrqoG/xC6C1raF0xK7hUx6i
|
||||
4DUDSPohM8fOIswQPfE+FH3eygfzu/Ln5+ghsgHTEhgFvmgMvyxaAt6kHIzIUhMX
|
||||
M3dASr4VPDpIXuXsRWwYLEifhzxsuvwVxfwtsnCaR6XKijsYECWGDdYOWHdleeqx
|
||||
wDPhxEesfFVhKxhrKY9Ir8k9/FFBKQU/3GjW4+SMAg5Al1YEzxshP9vKuVcsei7W
|
||||
JDwAwotNXaCm6NBckiyZJE53ou6+gckPY7V9cOfnuH74Z9ywkFzB3HW3ZlonaGyM
|
||||
oGmLGcccavFtyhg5s/As4i6X8ARIpDiwe59Pn3GNXMctySqIrrr2ogUoXgrfFCie
|
||||
6GOTdeMW7GeOSdJUxCofghlspS/nq01Og77VI/beWYrIwLubSka6Zaltww9zgObk
|
||||
/FGEMgFkEpq7iyCvYSPA8F46pJKvnMP3S84AWCPmcTcHeg4lwGPvs6btexXBGdoz
|
||||
nkCyq7wdH5Nngm7jUbl88LtaLZPAQkuqXphBVTnrF9Ofbnb4iRZ2Op4xpx9rGyvx
|
||||
mO6UEhL6V1i2YZFNkNMg/W8aoMiUgBdqbkxaxblT9L0aNdlFU9+LbWYolURVEadd
|
||||
Qjv0Z1gMA+tsuBbVszwsMfneZ5+B9Q==
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IEdoTUQ4QSBMcVY2
|
||||
eWRnb05sMEZVZUpicGVYN2ZSUm1mUmVsdUsrSHR4c2I2SXVRWDF3CnlhUkxoeEx5
|
||||
M1NNcGR3bmx6ZW5RaHU3L2l4WEowdWYzQk0xa3E4ZUtwNkkKLT4gIi1ncmVhc2Ug
|
||||
SllwSzlIIEdAKltUKSBjSWpmCnljSXlZTXZBL2xuMEtma1NUNjdCM0dMNHJuVjFS
|
||||
enV5LzF1NlYzbFNURW1rTlI2aUxCRFFxK2ZJdktsTTU1ZSsKNUZiZmVQWQotLS0g
|
||||
bTVrZEdWMHFWK0Q5RzZUc09PYmlEQjNweXRxU2FETWNWTXRUVEFKUUFpdwqtUbmM
|
||||
kbnj4Q+9QlnJHuWBVu+BcWPl5HuiPUjrrxnWAuN6rFYd79H7qk9MagDB/2BAEk82
|
||||
iuQeqS0r8wAmp9bTzhbiMQEbtN96huSGA2aO9I/RoPU1jv1Upi8bNy+KX0jSsfV9
|
||||
+cGFm0JsBZ4o5acq4isanC+3YkNq+IHxyDqUdwWMIHiEkR26pwndyMAaJTCBAgTN
|
||||
kmdWMK4PxR36ElY+8J0tiOv6VmGS73rVS4R/2Rdc5ICx6QO6oQQqETKYxqcmr8eD
|
||||
8dgkZVpmBF/iuw7BYZ2U/p2PZSzgEVTVqfT3VO8WQ9ii2a4dw+2QEORgISbhxzoy
|
||||
WGB0q4X33QWEYSp0FNyAG4Kc9yTphUq+hMHfNOtz3XmTZLltcVfA6XQbKwB/nDbq
|
||||
G1gAfDwdEZ5OpN2IT0S6Zc2rKKeovqdFYWgqmDWiaBfqncl9qLH37KkpKqmUSQSe
|
||||
zyriQt8nZCzVrh4EKohjeLogVBsn0tPTYqiFnV+ZFK/kOWIYWduWDJcM3zwRVjx7
|
||||
Mr5l81gFv4WRHbnQB9eynGJLZYs1lI4/X9tKRUgUj0mN20Bt80NXGNPaJ3TAkrCO
|
||||
rX0tgjsX/Sc59JTHaMOT419+ob9UtDoS7mGlxswKfyhvjzluu+EdHd0GRJ5PUsSq
|
||||
8YlKYjRonBqhC0Ju9CRuZS0pk2bh72bG9z1Gb4LcJ0pLJ6lMfmF1j4zzfzsABe5G
|
||||
0bJF3ikzNxo6Wzsf3rk45/FvMhKjkI5O3Btnfp+Vkw+iRqDMh7wiD6cczNSp+Skd
|
||||
Et43NIobiLa6O95y2YEjqSkT5T0ug1nbLkygmxwffVn6RTWgScZSfBPvOKTINAVo
|
||||
J/MU2c0DaBm4glLfm4IWaJNcEmZn8+FWG6m42WEWTMEfSeAo5XXaEb0FsK0Yqd3+
|
||||
RlE/b6a/DdoNEAQOlEPSASsoQhTVvsiEwH4Pq5G0STHtSBBIT/xJow4pa/GOiQnn
|
||||
yAFva9F7KFYWA7bjbRbv+B21bvss0T+BK9HRF+bklSzjxBNfDEWXW0GhwIiahwXW
|
||||
Fxi3BUMZcfgoLg2NkhMb3irwJUoGqQFqn68e2jTJVyUATyVgGV5mzfjDBB8thcDt
|
||||
ehuIL/Y1bUVGWZteU/JAF7z+Fb1yBO2FJqpCKIcfd+JgkWVtQKaqYGjcuzbI7ce4
|
||||
fcrRDNCse2pBO1unE7HS160rK+dMWavlrsHHZKezsvNv7TwMj1SKjCKSVr8up7TC
|
||||
NLW0I6uXGEcUBj+RNqF6frdpw/Ve1WTAkIbznMV6kZ8cTfAGhOzJ0wxHMBSQka8X
|
||||
uMatPuGywu+dvjrJXTwD0gJD/Jrd88K685ahu86nSt/DYnxIYfQhwo/oZMR7E6kN
|
||||
4NgGtYjRU0asmio6sF8D1uA2YgmxNPRw2GwTqpS2XyYaEh3B6yjqa4pJ1vfo6t/g
|
||||
aPLhMuT4qt/eKMvSiR+sTaOMNkcLWmCpY762aYk4XUZFTYAAY5XsNSAU+Hs7o/Eu
|
||||
9NSMrIrpqLAcgzr/nZZTLqswbsYtdVl5WUd9uqdQe+AhTbrkcvHibUrVgW/XU+oq
|
||||
QdHXUyfn/IByp2uE7WZpfRVhwjXg/LQJh/ogksbzrh4/anOivku+n1ouciAAgnQn
|
||||
04i6taBu+393ysMC/sp7wZkp//yAZj2SNPOTzbakG8xWoVGzbqxLCIOIE7I97KBv
|
||||
G49jiEOS42vZZXSGLxQND+N0aOqosZfQ1WKpI9XjirB8qVOt/sr6uqEIx311V+BJ
|
||||
wrdoMa9hWmDFzf3+ThqfUuHOtxxJXReL7vC4J7K8iU6nCVIGJN7axifk
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
|
||||
@@ -1,347 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# run-integration.sh — Staging VM Integration Test Suite
|
||||
#
|
||||
# Verifies Docker daemon, compose stack, and service endpoint health.
|
||||
# Designed to run inside the staging VM as part of CI/CD pipeline.
|
||||
#
|
||||
# Usage:
|
||||
# ./tests/run-integration.sh # all defaults
|
||||
# ./tests/run-integration.sh --verbose # detailed output
|
||||
# ./tests/run-integration.sh --list-services # print detected services and exit
|
||||
#
|
||||
# Environment variables (all optional):
|
||||
# COMPOSE_DIR Path to compose service directories (default: /opt/infra/compose)
|
||||
# COMPOSE_PROJECT Docker Compose project name (default: staging)
|
||||
# STAGING_DOMAIN Base domain for health checks (default: staging.lazyworkhorse.net)
|
||||
# SERVICE_LIST Space-separated service dirs to check (default: auto-detect)
|
||||
# HEALTH_URLS Space-separated URLs for health checks (default: auto-detect from SERVICE_LIST)
|
||||
# HEALTH_TIMEOUT Curl timeout per check (seconds) (default: 5)
|
||||
# HEALTH_RETRIES Number of retries per endpoint (default: 1)
|
||||
# HEALTH_INTERVAL Seconds between retries (default: 2)
|
||||
# =============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ---- Colors for readable output ----
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
CYAN='\033[0;36m'
|
||||
BOLD='\033[1m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# ---- Configuration (all env-overridable) ----
|
||||
COMPOSE_DIR="${COMPOSE_DIR:-/opt/infra/compose}"
|
||||
COMPOSE_PROJECT="${COMPOSE_PROJECT:-staging}"
|
||||
STAGING_DOMAIN="${STAGING_DOMAIN:-staging.lazyworkhorse.net}"
|
||||
HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-5}"
|
||||
HEALTH_RETRIES="${HEALTH_RETRIES:-1}"
|
||||
HEALTH_INTERVAL="${HEALTH_INTERVAL:-2}"
|
||||
|
||||
# Known compose service directories in order — override via SERVICE_LIST env var
|
||||
DEFAULT_SERVICES=(
|
||||
network
|
||||
authentification
|
||||
homepage
|
||||
ai
|
||||
cloudstorage
|
||||
versioncontrol
|
||||
backup
|
||||
coms
|
||||
finance
|
||||
homeautomation
|
||||
passwordmanager
|
||||
)
|
||||
|
||||
# Map service directory -> default health check URL (relative to STAGING_DOMAIN)
|
||||
# Override entirely via HEALTH_URLS env var.
|
||||
declare -A DEFAULT_HEALTH_URLS
|
||||
DEFAULT_HEALTH_URLS[network]="https://traefik.${STAGING_DOMAIN}/ping"
|
||||
DEFAULT_HEALTH_URLS[authentification]="https://auth.${STAGING_DOMAIN}/api/verify"
|
||||
DEFAULT_HEALTH_URLS[homepage]="https://${STAGING_DOMAIN}/"
|
||||
DEFAULT_HEALTH_URLS[ai]="https://hermes.${STAGING_DOMAIN}/health"
|
||||
DEFAULT_HEALTH_URLS[cloudstorage]="https://cloud.${STAGING_DOMAIN}/status.php"
|
||||
DEFAULT_HEALTH_URLS[versioncontrol]="https://code.${STAGING_DOMAIN}/api/healthz"
|
||||
|
||||
# ---- Trackers ----
|
||||
PASS_COUNT=0
|
||||
FAIL_COUNT=0
|
||||
WARN_COUNT=0
|
||||
FAILURES=()
|
||||
|
||||
# ---- Helpers ----
|
||||
|
||||
log_info() { echo -e "${CYAN}[INFO]${NC} $*"; }
|
||||
log_pass() { echo -e "${GREEN}[PASS]${NC} $*"; ((PASS_COUNT++)); }
|
||||
log_fail() { echo -e "${RED}[FAIL]${NC} $*"; ((FAIL_COUNT++)); FAILURES+=("$*"); }
|
||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; ((WARN_COUNT++)); }
|
||||
log_step() { echo -e "\n${BOLD}── $* ──${NC}"; }
|
||||
log_raw() { echo -e " $*"; }
|
||||
|
||||
# Check if a command exists
|
||||
require_cmd() {
|
||||
if ! command -v "$1" &>/dev/null; then
|
||||
log_fail "Required command not found: $1"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Retry a command with exponential-like backoff
|
||||
retry() {
|
||||
local cmd="$*"
|
||||
local attempt=0
|
||||
local max_attempts=$((HEALTH_RETRIES + 1))
|
||||
local result
|
||||
|
||||
while [[ $attempt -lt $max_attempts ]]; do
|
||||
if eval "$cmd" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
if [[ $attempt -lt $max_attempts ]]; then
|
||||
sleep "$HEALTH_INTERVAL"
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# ---- Parse arguments ----
|
||||
VERBOSE=false
|
||||
LIST_SERVICES=false
|
||||
POSITIONAL=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--verbose|-v) VERBOSE=true; shift ;;
|
||||
--list-services) LIST_SERVICES=true; shift ;;
|
||||
--) shift; POSITIONAL+=("$@"); break ;;
|
||||
*) POSITIONAL+=("$1"); shift ;;
|
||||
esac
|
||||
done
|
||||
set -- "${POSITIONAL[@]}"
|
||||
|
||||
# Resolve service list
|
||||
if [[ -n "${SERVICE_LIST:-}" ]]; then
|
||||
IFS=' ' read -ra SERVICES <<< "$SERVICE_LIST"
|
||||
else
|
||||
SERVICES=("${DEFAULT_SERVICES[@]}")
|
||||
fi
|
||||
|
||||
# Resolve health URLs — default map with overrides from env
|
||||
declare -A HEALTH_URLS
|
||||
if [[ -n "${HEALTH_URLS:-}" ]]; then
|
||||
# User-supplied mapping: "network=https://... authentification=https://..."
|
||||
for pair in $HEALTH_URLS; do
|
||||
key="${pair%%=*}"
|
||||
val="${pair#*=}"
|
||||
HEALTH_URLS["$key"]="$val"
|
||||
done
|
||||
else
|
||||
for svc in "${SERVICES[@]}"; do
|
||||
if [[ -n "${DEFAULT_HEALTH_URLS[$svc]:-}" ]]; then
|
||||
HEALTH_URLS["$svc"]="${DEFAULT_HEALTH_URLS[$svc]}"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# --list-services mode (for CI integration)
|
||||
if $LIST_SERVICES; then
|
||||
echo "Configured services:"
|
||||
for svc in "${SERVICES[@]}"; do
|
||||
url="${HEALTH_URLS[$svc]:-no-health-check}"
|
||||
echo " $svc -> $url"
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ---- Pre-flight ----
|
||||
echo -e "${BOLD}============================================${NC}"
|
||||
echo -e "${BOLD} Staging VM Integration Test Suite${NC}"
|
||||
echo -e "${BOLD} $(date -u '+%Y-%m-%dT%H:%M:%SZ')${NC}"
|
||||
echo -e "${BOLD}============================================${NC}"
|
||||
|
||||
# ---- Phase 1: Prerequisites ----
|
||||
log_step "Phase 1: Prerequisites"
|
||||
|
||||
PREREQ_OK=true
|
||||
for cmd in docker curl jq; do
|
||||
if ! require_cmd "$cmd"; then
|
||||
PREREQ_OK=false
|
||||
fi
|
||||
done
|
||||
$PREREQ_OK && log_pass "All required commands available" || log_fail "Missing prerequisites"
|
||||
|
||||
# ---- Phase 2: Docker daemon ----
|
||||
log_step "Phase 2: Docker Daemon"
|
||||
|
||||
if docker info --format '{{.ServerVersion}}' &>/dev/null; then
|
||||
DOCKER_VERSION=$(docker info --format '{{.ServerVersion}}' 2>/dev/null)
|
||||
log_pass "Docker daemon is running (version: $DOCKER_VERSION)"
|
||||
|
||||
if docker info --format '{{.Driver}}' 2>/dev/null | grep -qi "overlay"; then
|
||||
log_pass "Storage driver: overlay"
|
||||
else
|
||||
log_warn "Non-overlay storage driver detected"
|
||||
fi
|
||||
else
|
||||
log_fail "Docker daemon is NOT running or not accessible"
|
||||
fi
|
||||
|
||||
# ---- Phase 3: Docker Compose stack ----
|
||||
log_step "Phase 3: Compose Stack Status"
|
||||
|
||||
# Check if any compose files exist
|
||||
COMPOSE_FILES=()
|
||||
for svc in "${SERVICES[@]}"; do
|
||||
cf="${COMPOSE_DIR}/${svc}/compose.yml"
|
||||
if [[ -f "$cf" ]]; then
|
||||
COMPOSE_FILES+=("$cf")
|
||||
else
|
||||
cf2="${COMPOSE_DIR}/${svc}/docker-compose.yml"
|
||||
if [[ -f "$cf2" ]]; then
|
||||
COMPOSE_FILES+=("$cf2")
|
||||
else
|
||||
log_warn "No compose file found for service '$svc' (expected: ${cf})"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ${#COMPOSE_FILES[@]} -eq 0 ]]; then
|
||||
log_fail "No compose files found under COMPOSE_DIR=${COMPOSE_DIR}"
|
||||
log_info "Skipping stack checks"
|
||||
else
|
||||
log_info "Found ${#COMPOSE_FILES[@]} compose file(s) in ${COMPOSE_DIR}"
|
||||
|
||||
# Build the compose file args
|
||||
COMPOSE_CMD="docker compose -p ${COMPOSE_PROJECT}"
|
||||
for cf in "${COMPOSE_FILES[@]}"; do
|
||||
COMPOSE_CMD+=" -f ${cf}"
|
||||
done
|
||||
|
||||
log_info "Project name: ${COMPOSE_PROJECT}"
|
||||
|
||||
# Check stack ps
|
||||
if $VERBOSE; then
|
||||
log_raw "--- docker compose ps output ---"
|
||||
eval "$COMPOSE_CMD ps" 2>&1 | while IFS= read -r line; do log_raw "$line"; done
|
||||
log_raw "--- end ---"
|
||||
fi
|
||||
|
||||
# Get all services and their status
|
||||
if STACK_STATUS=$(eval "$COMPOSE_CMD ps --format '{{.Name}}\t{{.Status}}'" 2>/dev/null); then
|
||||
if [[ -z "$STACK_STATUS" ]]; then
|
||||
log_warn "Stack exists but no running services — VM may be freshly provisioned"
|
||||
else
|
||||
ALL_RUNNING=true
|
||||
RUNNING_COUNT=0
|
||||
TOTAL_COUNT=0
|
||||
while IFS=$'\t' read -r name status; do
|
||||
TOTAL_COUNT=$((TOTAL_COUNT + 1))
|
||||
status_lower=$(echo "$status" | tr '[:upper:]' '[:lower:]')
|
||||
if echo "$status_lower" | grep -qE '^(up|running|healthy)'; then
|
||||
RUNNING_COUNT=$((RUNNING_COUNT + 1))
|
||||
$VERBOSE && log_pass " $name — $status"
|
||||
else
|
||||
ALL_RUNNING=false
|
||||
log_warn " $name — $status (not healthy)"
|
||||
fi
|
||||
done <<< "$STACK_STATUS"
|
||||
|
||||
if [[ "$TOTAL_COUNT" -eq 0 ]]; then
|
||||
log_fail "No services found in compose project"
|
||||
elif $ALL_RUNNING && [[ "$TOTAL_COUNT" -eq "$RUNNING_COUNT" ]]; then
|
||||
log_pass "All ${TOTAL_COUNT} service(s) running (${RUNNING_COUNT}/${TOTAL_COUNT})"
|
||||
else
|
||||
log_fail "${RUNNING_COUNT}/${TOTAL_COUNT} service(s) running — some services are down"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
log_fail "Failed to query compose stack status"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---- Phase 4: Service health checks ----
|
||||
log_step "Phase 4: Service Endpoint Health Checks"
|
||||
|
||||
ENDPOINT_CHECKS=0
|
||||
ENDPOINT_PASS=0
|
||||
|
||||
for svc in "${SERVICES[@]}"; do
|
||||
url="${HEALTH_URLS[$svc]:-}"
|
||||
if [[ -z "$url" ]]; then
|
||||
$VERBOSE && log_info "No health check URL for service '$svc' — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
ENDPOINT_CHECKS=$((ENDPOINT_CHECKS + 1))
|
||||
echo -ne " Checking ${svc} ... "
|
||||
|
||||
# Perform the HTTP health check with retries
|
||||
if retry "curl -sf -o /dev/null -w '%{http_code}' --max-time ${HEALTH_TIMEOUT} '${url}' 2>/dev/null"; then
|
||||
HTTP_CODE=$(curl -sf -o /dev/null -w '%{http_code}' --max-time "${HEALTH_TIMEOUT}" "${url}" 2>/dev/null || true)
|
||||
ENDPOINT_PASS=$((ENDPOINT_PASS + 1))
|
||||
echo -e "${GREEN}OK${NC} (HTTP ${HTTP_CODE})"
|
||||
else
|
||||
LAST_CODE=$(curl -s -o /dev/null -w '%{http_code}' --max-time "${HEALTH_TIMEOUT}" "${url}" 2>/dev/null || echo "000")
|
||||
echo -e "${RED}FAIL${NC} (HTTP ${LAST_CODE})"
|
||||
log_fail "Health check failed for ${svc} @ ${url}"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ $ENDPOINT_CHECKS -eq 0 ]]; then
|
||||
log_warn "No health check URLs configured — skipping endpoint phase"
|
||||
elif [[ $ENDPOINT_PASS -eq $ENDPOINT_CHECKS ]]; then
|
||||
log_pass "All ${ENDPOINT_CHECKS} endpoint(s) healthy"
|
||||
else
|
||||
log_fail "${ENDPOINT_PASS}/${ENDPOINT_CHECKS} endpoint(s) healthy"
|
||||
fi
|
||||
|
||||
# ---- Phase 5: Docker system sanity ----
|
||||
log_step "Phase 5: Docker System Sanity"
|
||||
|
||||
# Check disk space for Docker
|
||||
DOCKER_ROOT=$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || echo "/var/lib/docker")
|
||||
log_info "Docker root: ${DOCKER_ROOT}"
|
||||
|
||||
if command -v df &>/dev/null && [[ -d "$DOCKER_ROOT" ]]; then
|
||||
AVAIL_PCT=$(df -h "$DOCKER_ROOT" | awk 'NR==2 {print $5}' | tr -d '%')
|
||||
if [[ -n "$AVAIL_PCT" ]]; then
|
||||
if [[ "$AVAIL_PCT" -ge 90 ]]; then
|
||||
log_warn "Docker storage is ${AVAIL_PCT}% full — consider cleanup"
|
||||
else
|
||||
log_pass "Docker storage at ${AVAIL_PCT}% — within limits"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check for dangling images
|
||||
DANGLING=$(docker images -f "dangling=true" -q 2>/dev/null | wc -l)
|
||||
if [[ "$DANGLING" -gt 10 ]]; then
|
||||
log_warn "${DANGLING} dangling images found — consider docker image prune"
|
||||
fi
|
||||
|
||||
# ---- Summary ----
|
||||
echo ""
|
||||
echo -e "${BOLD}============================================${NC}"
|
||||
echo -e "${BOLD} Test Summary${NC}"
|
||||
echo -e "${BOLD} $(date -u '+%Y-%m-%dT%H:%M:%SZ')${NC}"
|
||||
echo -e "${BOLD}============================================${NC}"
|
||||
echo -e " ${GREEN}Passed:${NC} ${PASS_COUNT}"
|
||||
echo -e " ${RED}Failed:${NC} ${FAIL_COUNT}"
|
||||
echo -e " ${YELLOW}Warnings:${NC} ${WARN_COUNT}"
|
||||
|
||||
if [[ ${#FAILURES[@]} -gt 0 ]]; then
|
||||
echo -e "\n${BOLD}Failed checks:${NC}"
|
||||
for f in "${FAILURES[@]}"; do
|
||||
echo -e " ${RED}•${NC} $f"
|
||||
done
|
||||
fi
|
||||
|
||||
echo ""
|
||||
if [[ $FAIL_COUNT -eq 0 ]]; then
|
||||
echo -e "${GREEN}${BOLD}✓ All integration checks passed${NC}"
|
||||
exit 0
|
||||
else
|
||||
echo -e "${RED}${BOLD}✗ ${FAIL_COUNT} integration check(s) failed${NC}"
|
||||
exit 1
|
||||
fi
|
||||
@@ -4,7 +4,7 @@
|
||||
group = "ai-worker";
|
||||
home = "/home/ai-worker";
|
||||
createHome = true;
|
||||
extraGroups = [ "docker" "libvirtd" ];
|
||||
extraGroups = [ "docker" ];
|
||||
shell = pkgs.bashInteractive;
|
||||
openssh.authorizedKeys.keys = [
|
||||
keys.users.ai-worker.main
|
||||
|
||||
Reference in New Issue
Block a user