Compare commits

..

6 Commits

Author SHA1 Message Date
Robert
ef7f06166b Removed double file
Some checks failed
Build NixOS config / build (pull_request) Failing after 1s
2026-05-13 10:04:57 -04:00
Robert
e70516e78b Added access to the secret to version control 2026-05-13 10:03:14 -04:00
Robert
ce3a327ff7 The key changed apparently? it works with that one anyway 2026-05-13 10:01:58 -04:00
Robert
af38655170 Adding the runner key
Some checks failed
Build NixOS config / build (pull_request) Failing after 55s
2026-05-10 22:23:29 -04:00
dcf5ac6b5e fix: use nixos-builder label for NixOS CI 2026-05-10 22:15:53 -04:00
d78de94f94 feat: add NixOS build CI for infra PRs 2026-05-10 22:15:02 -04:00
9 changed files with 45 additions and 793 deletions

View File

@@ -1,4 +1,4 @@
name: Build and test NixOS config
name: Build NixOS config
on:
pull_request:
branches: [ master ]
@@ -27,26 +27,7 @@ jobs:
https://gitea:${{ secrets.GITHUB_TOKEN }}@code.lazyworkhorse.net/gortium/infra.git .
git log --oneline -3
- name: Build NixOS config
- name: Build NixOS config (lazyworkhorse)
run: |
nix --version
nh os build .#lazyworkhorse 2>&1
- name: Run integration tests (staging VM)
run: |
echo "==> Running integration tests on staging VM..."
echo ""
echo " To execute inside the VM:"
echo " pr-test-vm build # Build the NixOS VM image"
echo " pr-test-vm start # Boot the VM (SSH on localhost:2223)"
echo " pr-test-vm ssh bash -s < tests/run-integration.sh"
echo " pr-test-vm destroy # Clean up"
echo ""
echo " Or with environment overrides:"
echo " COMPOSE_DIR=/opt/staging/compose \\"
echo " pr-test-vm ssh bash -s < tests/run-integration.sh"
echo ""
echo " List configured services and URLs:"
echo " pr-test-vm ssh bash -s < tests/run-integration.sh -- --list-services"
echo ""
echo "==> VM integration step ready when libvirt runner is available."

View File

@@ -1,106 +0,0 @@
# ollama-gfx906/Dockerfile
#
# Custom ollama image with ROCm 6.1 + gfx906 (MI50) support.
# The official ollama/rocm image ships ROCm 7.2 which dropped gfx906.
# This uses v0.23.2's native CMake build system with AMDGPU_TARGETS including gfx906.
#
# Build: docker build -t ollama/ollama:rocm-gfx906 ai/ollama
FROM rocm/dev-ubuntu-22.04:6.1.2-complete AS builder
# Build dependencies (CMake, Ninja, Go)
ARG CMAKEVERSION=3.31.2
ARG NINJAVERSION=1.12.1
ARG GOLANG_VERSION=1.22.0
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
curl git ccache build-essential pkg-config unzip \
&& rm -rf /var/lib/apt/lists/*
# Install CMake from official binaries
RUN curl -fsSL https://github.com/Kitware/CMake/releases/download/v${CMAKEVERSION}/cmake-${CMAKEVERSION}-linux-x86_64.tar.gz \
| tar xz -C /usr/local --strip-components 1
# Install Ninja
RUN curl -fsSL -o /tmp/ninja.zip \
https://github.com/ninja-build/ninja/releases/download/v${NINJAVERSION}/ninja-linux.zip \
&& unzip /tmp/ninja.zip -d /usr/local/bin && rm /tmp/ninja.zip
# Install Go
RUN curl -fsSL https://go.dev/dl/go${GOLANG_VERSION}.linux-amd64.tar.gz \
| tar xz -C /usr/local
ENV PATH=/usr/local/go/bin:$PATH
ARG OLLAMA_VERSION=v0.23.2
RUN git clone --depth 1 --branch ${OLLAMA_VERSION} https://github.com/ollama/ollama.git /build
WORKDIR /build
# ROCm paths
ENV HIP_PATH=/opt/rocm
ENV ROCM_PATH=/opt/rocm
ENV CMAKE_GENERATOR=Ninja
ENV LDFLAGS=-s
# Step 1: Build CPU backends with GCC (no ROCm preset)
# Pre-set CMAKE_HIP_COMPILER="" to prevent check_language(HIP) from
# finding a HIP compiler (it searches /opt/rocm even without PATH).
# Remove /opt/rocm from PATH to prevent find_program from finding hipcc.
RUN mkdir -p build-cpu && \
PATH=/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
cmake -B build-cpu -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_HIP_COMPILER="" \
-DCMAKE_INSTALL_PREFIX=/build/dist && \
cmake --build build-cpu --target ggml-cpu -- -l $(nproc) && \
cmake --install build-cpu --component CPU --strip && \
echo "=== CPU install ===" && \
(find /build/dist/lib/ollama -type f -o -type l 2>&1 | head -20 || echo "empty")
# Step 2: Build HIP backend with ROCm preset + gfx906 target only
# The ROCm 6 preset enables HIP language detection (enable_language(HIP))
# which ensures GPU kernels are properly compiled for gfx906.
# OLLAMA_RUNNER_DIR=rocm from the preset, so HIP goes to lib/ollama/rocm/
# Need CMAKE_PREFIX_PATH so find_package(hip) finds hip-config.cmake
# at /opt/rocm/lib/cmake/hip/hip-config.cmake.
RUN mkdir -p build-hip && \
cmake -B build-hip \
--preset 'ROCm 6' \
-DAMDGPU_TARGETS="gfx906:xnack-" \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_PREFIX_PATH="/opt/rocm" && \
cmake --build build-hip --target ggml-hip -- -l $(nproc) && \
cmake --install build-hip --component HIP --strip && \
echo "=== HIP install ===" && \
find /build/dist/lib/ollama -type f -o -type l | head -20
# Step 3: Build Go binary (GCC for CGo linking)
ENV CGO_ENABLED=1
RUN go build -trimpath -ldflags="-X=github.com/ollama/ollama/version.Version=${OLLAMA_VERSION}" -o /build/dist/ollama .
# ---------- Runtime image ----------
FROM ubuntu:24.04
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
ca-certificates curl libstdc++6 libgomp1 libvulkan1 libopenblas0 \
&& rm -rf /var/lib/apt/lists/*
# Copy ROCm 6.1 runtime libraries
# These are needed at runtime by ggml-hip via LD_LIBRARY_PATH
COPY --from=builder /opt/rocm/lib/ /opt/rocm/lib/
COPY --from=builder /opt/rocm/share/ /opt/rocm/share/
# Copy ollama binary + all backends (CPU + HIP)
# CPU install: /build/dist/lib/ollama/libggml-*.so
# HIP install: /build/dist/lib/ollama/rocm/libggml-hip.so
COPY --from=builder /build/dist/ollama /usr/bin/ollama
COPY --from=builder /build/dist/lib/ollama/ /usr/lib/ollama/
RUN ldconfig
ENV LD_LIBRARY_PATH=/opt/rocm/lib:/usr/lib/ollama/rocm:/usr/lib/ollama
ENV HSA_OVERRIDE_GFX_VERSION=9.0.6
ENV HCC_AMDGPU_TARGET=gfx906
ENV HSA_ENABLE_SDMA=0
EXPOSE 11434
ENTRYPOINT ["/bin/ollama"]
CMD ["serve"]

View File

@@ -61,7 +61,6 @@
./modules/nixos/services/open_code_server.nix
./modules/nixos/services/ollama_init_custom_models.nix
./modules/nixos/services/openclaw_node.nix
./modules/nixos/services/staging-vm.nix
./modules/nixos/security/ai-worker-restricted.nix
./users/gortium.nix
./users/ai-worker.nix

View File

@@ -36,7 +36,7 @@
"transparent_hugepage=always" # because mucho ram
];
# 2. Load the specific drivers found by sensors-detect
boot.kernelModules = [ "nct6775" "lm96163" "iptable_nat" "iptable_filter" "kvm-intel" "kvm" ];
boot.kernelModules = [ "nct6775" "lm96163" "iptable_nat" "iptable_filter" ];
# 3. Force the nct6775 driver to recognize the chip if it's stubborn
boot.extraModprobeConfig = ''
options nct6775 force_id=0xd280
@@ -191,6 +191,7 @@
services.dockerStacks = {
versioncontrol = {
path = self + "/assets/compose/versioncontrol";
envFile = config.age.secrets.containers_env.path;
ports = [ 2222 ];
};
@@ -328,21 +329,20 @@
# Mi50 config
hardware.graphics = {
enable = true;
enable32Bit = true;
enable32Bit = true; # Useful for some compatibility layers
extraPackages = with pkgs; [
rocmPackages.clr.icd
rocmPackages.clr.icd # OpenCL/HIP runtime
];
};
nixpkgs.config.rocmTargets = [ "gfx906" ];
environment.variables = {
# This "tricks" ROCm into supporting the MI50 if using newer versions
HSA_OVERRIDE_GFX_VERSION = "9.0.6";
# Ensures the system sees both GPUs
HIP_VISIBLE_DEVICES = "0,1";
};
# KVM/libvirt for staging VM
services.stagingVm.enable = true;
# Open ports in the firewall.
# Open ports in the firewall.
# networking.firewall.allowedTCPPorts = [ ... ];
# networking.firewall.allowedUDPPorts = [ ... ];
# Or disable the firewall altogether.

View File

@@ -1,275 +0,0 @@
{ config, pkgs, lib, ... }:
with lib;
let
cfg = config.services.stagingVm;
in
{
options.services.stagingVm = {
enable = mkOption {
type = types.bool;
default = false;
description = "Enable KVM/libvirt staging VM for compose PR testing";
};
vmName = mkOption {
type = types.str;
default = "compose-test-vm";
description = "Name of the staging VM";
};
memory = mkOption {
type = types.str;
default = "4096";
description = "RAM allocated to the staging VM (MB)";
};
vcpus = mkOption {
type = types.int;
default = 2;
description = "Number of vCPUs for the staging VM";
};
storagePath = mkOption {
type = types.str;
default = "/var/lib/libvirt/images";
description = "Path for libvirt storage pool";
};
dataPath = mkOption {
type = types.str;
default = "/var/lib/staging-vm";
description = "Path for compose test data (PR checkouts, test results)";
};
};
config = mkIf cfg.enable {
# Enable libvirt daemon
virtualisation.libvirtd = {
enable = true;
qemu = {
package = pkgs.qemu_kvm;
runAsRoot = true;
swtpm.enable = true;
ovmf = {
enable = true;
packages = [ pkgs.OVMFFull.fd ];
};
};
};
# Kernel modules + groups already handled in configuration.nix
# libvirt NAT network (192.168.122.0/24)
environment.etc."libvirt/qemu/networks/default.xml" = {
text = ''
<network>
<name>default</name>
<uuid>2b8f7a3c-9e5d-4a1f-bc3d-6e7a8f9b0c1d</uuid>
<forward mode='nat'>
<nat>
<port start='1024' end='65535'/>
</nat>
</forward>
<bridge name='virbr0' stp='on' delay='0'/>
<mac address='52:54:00:12:34:56'/>
<ip address='192.168.122.1' netmask='255.255.255.0'>
<dhcp>
<range start='192.168.122.2' end='192.168.122.254'/>
</dhcp>
</ip>
</network>
'';
# Autostart the network so it comes up on boot
mode = "0644";
};
# Ensure the default network is defined and autostarted
systemd.services.libvirtd = {
postStart = ''
${pkgs.libvirt}/bin/virsh net-define /etc/libvirt/qemu/networks/default.xml 2>/dev/null || true
${pkgs.libvirt}/bin/virsh net-autostart default 2>/dev/null || true
${pkgs.libvirt}/bin/virsh net-start default 2>/dev/null || true
'';
};
# Storage directory for VM images
systemd.tmpfiles.rules = [
"d ${cfg.storagePath} 0755 root root -"
"d ${cfg.dataPath} 0755 root root -"
];
# Ensure storage pool exists in libvirt
systemd.services.libvirtd.postStart = mkAfter ''
${pkgs.libvirt}/bin/virsh pool-define-as default dir --target "${cfg.storagePath}" 2>/dev/null || true
${pkgs.libvirt}/bin/virsh pool-autostart default 2>/dev/null || true
${pkgs.libvirt}/bin/virsh pool-start default 2>/dev/null || true
'';
# Firewall: allow traffic from virbr0 to host and outbound NAT
networking.firewall = {
extraCommands = ''
# Allow inbound DHCP/DNS from libvirt guests
iptables -I INPUT -i virbr0 -p udp --dport 67:68 -j ACCEPT
iptables -I INPUT -i virbr0 -p tcp --dport 53 -j ACCEPT
iptables -I INPUT -i virbr0 -p udp --dport 53 -j ACCEPT
# Allow established/related traffic back to guests
iptables -I FORWARD -i virbr0 -o virbr0 -j ACCEPT
iptables -I FORWARD -o virbr0 -j ACCEPT
iptables -I FORWARD -i virbr0 -j ACCEPT
'';
};
# Packages needed for VM management
environment.systemPackages = with pkgs; [
libvirt
qemu_kvm
virt-manager # optional GUI for manual management
OVMFFull
swtpm
];
# Enable docker in the host (already enabled, but ensure for compose testing)
virtualisation.docker.enable = true;
# Helper script: pr-test-vm
# Usage:
# pr-test-vm build — build the staging VM derivation
# pr-test-vm start — boot the VM with a compose PR branch
# pr-test-vm stop — graceful shutdown
# pr-test-vm destroy — force stop + delete VM
# pr-test-vm ssh — SSH into the running VM
systemd.tmpfiles.rules = mkAfter [
"d ${cfg.dataPath}/scripts 0755 root root -"
];
environment.systemPackages = [ (pkgs.writeShellScriptBin "pr-test-vm" ''
set -euo pipefail
DATA="${cfg.dataPath}"
VM_NAME="${cfg.vmName}"
VM_IMAGE="''${DATA}/''${VM_NAME}.qcow2"
VM_PORT=2223
build_vm() {
echo "==> Building NixOS staging VM for compose testing..."
# Build the VM config inline a minimal NixOS with Docker + SSH
cat > /tmp/staging-vm-config.nix << 'NIXEOF'
{ config, pkgs, lib, ... }: {
boot.loader.grub.devices = [ "/dev/vda" ];
boot.loader.timeout = 0;
# Minimal kernel
boot.kernelParams = [ "console=ttyS0" ];
boot.initrd.kernelModules = [ "virtio_blk" "virtio_net" "virtio_pci" ];
# SSH access
services.openssh = {
enable = true;
settings.PasswordAuthentication = false;
settings.PermitRootLogin = "prohibit-password";
};
# Docker for compose testing
virtualisation.docker.enable = true;
# Network (DHCP via virbr0)
networking.useDHCP = true;
networking.firewall.enable = false;
# Users
users.users.root.openssh.authorizedKeys.keys = [
"$(cat /root/.ssh/authorized_keys 2>/dev/null || echo 'ssh-ed25519 AAAAC3... placeholder')"
];
users.users.testrunner = {
isNormalUser = true;
extraGroups = [ "docker" ];
openssh.authorizedKeys.keys = [
"$(cat /root/.ssh/authorized_keys 2>/dev/null || echo 'ssh-ed25519 AAAAC3... placeholder')"
];
};
# Git + compose tools
environment.systemPackages = with pkgs; [ git docker-compose curl ];
system.stateVersion = "24.11";
}
NIXEOF
nixos-rebuild build-vm -I nixpkgs=channel:nixos-unstable \
--arg configuration 'import /tmp/staging-vm-config.nix' \
--out-link "''${DATA}/vm-result"
echo "==> VM built. Run 'pr-test-vm start' to boot."
}
start_vm() {
if [ -f "''${VM_IMAGE}" ]; then
echo "==> Booting existing VM..."
else
echo "==> Creating VM image..."
${pkgs.qemu_kvm}/bin/qemu-img create -f qcow2 "''${VM_IMAGE}" 20G
fi
# Check if already running
if ${pkgs.libvirt}/bin/virsh list --name 2>/dev/null | grep -q "''${VM_NAME}"; then
echo "==> VM already running."
exit 0
fi
${pkgs.qemu_kvm}/bin/qemu-system-x86_64 \
-name "''${VM_NAME}" \
-machine q35,accel=kvm \
-cpu host \
-smp ${toString cfg.vcpus} \
-m ${cfg.memory} \
-drive file="''${VM_IMAGE}",if=virtio,format=qcow2 \
-netdev user,id=net0,hostfwd=tcp::''${VM_PORT}-:22 \
-device virtio-net-pci,netdev=net0 \
-nographic \
-serial mon:stdio \
-pidfile "''${DATA}/''${VM_NAME}.pid" \
-daemonize
echo "==> VM booting... SSH on port ''${VM_PORT}"
echo "==> Wait for it: ssh -p ''${VM_PORT} testrunner@localhost"
}
stop_vm() {
PIDFILE="''${DATA}/''${VM_NAME}.pid"
if [ -f "''${PIDFILE}" ]; then
PID=$(cat "''${PIDFILE}")
kill "''${PID}" 2>/dev/null || true
rm -f "''${PIDFILE}"
echo "==> VM stopped."
else
${pkgs.libvirt}/bin/virsh destroy "''${VM_NAME}" 2>/dev/null || true
echo "==> VM destroyed."
fi
}
ssh_vm() {
exec ssh -p "''${VM_PORT}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "testrunner@localhost" "$@"
}
# Main dispatch
case "''${1:-help}" in
build) build_vm ;;
start) start_vm ;;
stop) stop_vm ;;
destroy) stop_vm; rm -f "''${VM_IMAGE}"; echo "==> VM deleted." ;;
ssh) shift; ssh_vm "$@" ;;
*)
echo "Usage: pr-test-vm {build|start|stop|destroy|ssh}"
echo ""
echo " build build the NixOS VM derivation"
echo " start boot the VM (create image if needed)"
echo " stop graceful VM shutdown"
echo " destroy stop + delete VM image"
echo " ssh SSH into the running VM"
;;
esac
'') ];
};
}

View File

@@ -1,36 +1,36 @@
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IEdoTUQ4QSBWNEpt
cGFNeVBBaDRqb3pLSEZGQW0wb3VmVnBoZCswUFkzbnBLUnJ0QTNrClRqVkk4RUVO
d29KYjd5YUcwankvaTFmVHUxQVpDT2ZYWHRaY3JXTUtQMU0KLT4gKXBtQ3UsXi1n
cmVhc2UgNnwxYCBXVyA/KCQmIHt9NAo3OTZVUHR2UXkvaEFwY0ZBdEJsaFpsbHJ0
cklKcDVHcEdWMEdPSkpnN0FiRU43RW5hUWFMdjR3WFRRSFBLSGlmClM3cTNJWlNM
TExkdHdXUHJISkNIaE1TTUxUc1NUWkV1a09HeFU3bVZwQXMKLS0tIGhOcXFTUElS
azJJNnEreUhMWTJBaVZGSTJPRUFqQkVYS01KRENUVVpZSDgK8+8onFejroBo7MeO
dW+so4lOsq4zJKn3f0cxmCFg1f0X8zt6h4Uc3A5Cvr1uU+6yw1FWmJ7xa3jJz3lO
EEaKQJXYC+xIIKGcA7qILa0SFp4a/4OuYjcg27HrlPhg7u5wDhQrd0LdVEe1Xngp
ZivX7P7HwIna3X8C+TL+K2v/AG2N/z86cdKfRvxyMKNbHhYw+CfHEnWgh8tJ++4h
G9evNniuNqte6cQaRe7jODfPNW4FuY/Sb7barlJ/M9iAQdYAdyLAzU1LABeHeUfD
wtHjxy9DUZ55Vg8bB8M2JJU9MkoRT4ewiVd9LeC1GWeVmKsm93wsmrov714i7U2j
wHtDkjqEF2MmzuQc18sjNaAHiwz8j6o5xU2L/Q4+Q707yISWG7RGZYh389Cr1rnw
siUq/Vunqw2wk13+J/4vu9nqt5mMktBaCtp+QiWIurjwB5LUAyChrSm+dg5lb0Mt
UhSc0lq1+E3vxAXM2Hmk+vP86VD+6WJvAU82VFApF1s6zG2FU1/AcOVVf54nan/q
f+rgSFfASHQCYSblUJHyEtwLNsWEmTGmOEn1buUKD/H0zatPQnc0rYpjlx2V0Sjd
6yB5+wPrZ0AkN1pjcsPKOv8Kaog2DzqIjib+SaSTaRxWHQEb9uzvaReAcYI5HOpE
gkC040HN33BItATbo4+hz70Im8Ni/VXD+g6yzM6Hj1hJL+PinTKeg5keQRFIZjMx
grzievB2wVBBgLgN3qMdTFmpplaL7iL702JjXZUTTK9Izp+9wiCsV1fTa53FWDht
ylFL5SWElqXjK+QBXxAe+Jk6VQov5HI21YDXL67S554ABeRok23wxrQ31TCI4xq9
PQV7VtNRjyVud7S29m3OwpWOsgTZhn+JclHj2v4bNJzJkJnZRTmcvGPktzRI5+R4
e5vxVhGnJDzI71txaHl8+xS1lu9VzCQUrxX6TXyTRV4KjIOz0g06JOBgmBRBvJca
7MZbC65xpisl/gyLRbgkVga3t94dPV+dpZsn8eq6427IyRbKslJefatggR9//c6I
5N5fl0fR3gJQMB+HRbipBH2YsdbdWJyb4Nn6STZxIfrqoG/xC6C1raF0xK7hUx6i
4DUDSPohM8fOIswQPfE+FH3eygfzu/Ln5+ghsgHTEhgFvmgMvyxaAt6kHIzIUhMX
M3dASr4VPDpIXuXsRWwYLEifhzxsuvwVxfwtsnCaR6XKijsYECWGDdYOWHdleeqx
wDPhxEesfFVhKxhrKY9Ir8k9/FFBKQU/3GjW4+SMAg5Al1YEzxshP9vKuVcsei7W
JDwAwotNXaCm6NBckiyZJE53ou6+gckPY7V9cOfnuH74Z9ywkFzB3HW3ZlonaGyM
oGmLGcccavFtyhg5s/As4i6X8ARIpDiwe59Pn3GNXMctySqIrrr2ogUoXgrfFCie
6GOTdeMW7GeOSdJUxCofghlspS/nq01Og77VI/beWYrIwLubSka6Zaltww9zgObk
/FGEMgFkEpq7iyCvYSPA8F46pJKvnMP3S84AWCPmcTcHeg4lwGPvs6btexXBGdoz
nkCyq7wdH5Nngm7jUbl88LtaLZPAQkuqXphBVTnrF9Ofbnb4iRZ2Op4xpx9rGyvx
mO6UEhL6V1i2YZFNkNMg/W8aoMiUgBdqbkxaxblT9L0aNdlFU9+LbWYolURVEadd
Qjv0Z1gMA+tsuBbVszwsMfneZ5+B9Q==
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IEdoTUQ4QSBMcVY2
eWRnb05sMEZVZUpicGVYN2ZSUm1mUmVsdUsrSHR4c2I2SXVRWDF3CnlhUkxoeEx5
M1NNcGR3bmx6ZW5RaHU3L2l4WEowdWYzQk0xa3E4ZUtwNkkKLT4gIi1ncmVhc2Ug
SllwSzlIIEdAKltUKSBjSWpmCnljSXlZTXZBL2xuMEtma1NUNjdCM0dMNHJuVjFS
enV5LzF1NlYzbFNURW1rTlI2aUxCRFFxK2ZJdktsTTU1ZSsKNUZiZmVQWQotLS0g
bTVrZEdWMHFWK0Q5RzZUc09PYmlEQjNweXRxU2FETWNWTXRUVEFKUUFpdwqtUbmM
kbnj4Q+9QlnJHuWBVu+BcWPl5HuiPUjrrxnWAuN6rFYd79H7qk9MagDB/2BAEk82
iuQeqS0r8wAmp9bTzhbiMQEbtN96huSGA2aO9I/RoPU1jv1Upi8bNy+KX0jSsfV9
+cGFm0JsBZ4o5acq4isanC+3YkNq+IHxyDqUdwWMIHiEkR26pwndyMAaJTCBAgTN
kmdWMK4PxR36ElY+8J0tiOv6VmGS73rVS4R/2Rdc5ICx6QO6oQQqETKYxqcmr8eD
8dgkZVpmBF/iuw7BYZ2U/p2PZSzgEVTVqfT3VO8WQ9ii2a4dw+2QEORgISbhxzoy
WGB0q4X33QWEYSp0FNyAG4Kc9yTphUq+hMHfNOtz3XmTZLltcVfA6XQbKwB/nDbq
G1gAfDwdEZ5OpN2IT0S6Zc2rKKeovqdFYWgqmDWiaBfqncl9qLH37KkpKqmUSQSe
zyriQt8nZCzVrh4EKohjeLogVBsn0tPTYqiFnV+ZFK/kOWIYWduWDJcM3zwRVjx7
Mr5l81gFv4WRHbnQB9eynGJLZYs1lI4/X9tKRUgUj0mN20Bt80NXGNPaJ3TAkrCO
rX0tgjsX/Sc59JTHaMOT419+ob9UtDoS7mGlxswKfyhvjzluu+EdHd0GRJ5PUsSq
8YlKYjRonBqhC0Ju9CRuZS0pk2bh72bG9z1Gb4LcJ0pLJ6lMfmF1j4zzfzsABe5G
0bJF3ikzNxo6Wzsf3rk45/FvMhKjkI5O3Btnfp+Vkw+iRqDMh7wiD6cczNSp+Skd
Et43NIobiLa6O95y2YEjqSkT5T0ug1nbLkygmxwffVn6RTWgScZSfBPvOKTINAVo
J/MU2c0DaBm4glLfm4IWaJNcEmZn8+FWG6m42WEWTMEfSeAo5XXaEb0FsK0Yqd3+
RlE/b6a/DdoNEAQOlEPSASsoQhTVvsiEwH4Pq5G0STHtSBBIT/xJow4pa/GOiQnn
yAFva9F7KFYWA7bjbRbv+B21bvss0T+BK9HRF+bklSzjxBNfDEWXW0GhwIiahwXW
Fxi3BUMZcfgoLg2NkhMb3irwJUoGqQFqn68e2jTJVyUATyVgGV5mzfjDBB8thcDt
ehuIL/Y1bUVGWZteU/JAF7z+Fb1yBO2FJqpCKIcfd+JgkWVtQKaqYGjcuzbI7ce4
fcrRDNCse2pBO1unE7HS160rK+dMWavlrsHHZKezsvNv7TwMj1SKjCKSVr8up7TC
NLW0I6uXGEcUBj+RNqF6frdpw/Ve1WTAkIbznMV6kZ8cTfAGhOzJ0wxHMBSQka8X
uMatPuGywu+dvjrJXTwD0gJD/Jrd88K685ahu86nSt/DYnxIYfQhwo/oZMR7E6kN
4NgGtYjRU0asmio6sF8D1uA2YgmxNPRw2GwTqpS2XyYaEh3B6yjqa4pJ1vfo6t/g
aPLhMuT4qt/eKMvSiR+sTaOMNkcLWmCpY762aYk4XUZFTYAAY5XsNSAU+Hs7o/Eu
9NSMrIrpqLAcgzr/nZZTLqswbsYtdVl5WUd9uqdQe+AhTbrkcvHibUrVgW/XU+oq
QdHXUyfn/IByp2uE7WZpfRVhwjXg/LQJh/ogksbzrh4/anOivku+n1ouciAAgnQn
04i6taBu+393ysMC/sp7wZkp//yAZj2SNPOTzbakG8xWoVGzbqxLCIOIE7I97KBv
G49jiEOS42vZZXSGLxQND+N0aOqosZfQ1WKpI9XjirB8qVOt/sr6uqEIx311V+BJ
wrdoMa9hWmDFzf3+ThqfUuHOtxxJXReL7vC4J7K8iU6nCVIGJN7axifk
-----END AGE ENCRYPTED FILE-----

View File

@@ -1,347 +0,0 @@
#!/usr/bin/env bash
# =============================================================================
# run-integration.sh — Staging VM Integration Test Suite
#
# Verifies Docker daemon, compose stack, and service endpoint health.
# Designed to run inside the staging VM as part of CI/CD pipeline.
#
# Usage:
# ./tests/run-integration.sh # all defaults
# ./tests/run-integration.sh --verbose # detailed output
# ./tests/run-integration.sh --list-services # print detected services and exit
#
# Environment variables (all optional):
# COMPOSE_DIR Path to compose service directories (default: /opt/infra/compose)
# COMPOSE_PROJECT Docker Compose project name (default: staging)
# STAGING_DOMAIN Base domain for health checks (default: staging.lazyworkhorse.net)
# SERVICE_LIST Space-separated service dirs to check (default: auto-detect)
# HEALTH_URLS Space-separated URLs for health checks (default: auto-detect from SERVICE_LIST)
# HEALTH_TIMEOUT Curl timeout per check (seconds) (default: 5)
# HEALTH_RETRIES Number of retries per endpoint (default: 1)
# HEALTH_INTERVAL Seconds between retries (default: 2)
# =============================================================================
set -euo pipefail
# ---- Colors for readable output ----
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
BOLD='\033[1m'
NC='\033[0m' # No Color
# ---- Configuration (all env-overridable) ----
COMPOSE_DIR="${COMPOSE_DIR:-/opt/infra/compose}"
COMPOSE_PROJECT="${COMPOSE_PROJECT:-staging}"
STAGING_DOMAIN="${STAGING_DOMAIN:-staging.lazyworkhorse.net}"
HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-5}"
HEALTH_RETRIES="${HEALTH_RETRIES:-1}"
HEALTH_INTERVAL="${HEALTH_INTERVAL:-2}"
# Known compose service directories in order — override via SERVICE_LIST env var
DEFAULT_SERVICES=(
network
authentification
homepage
ai
cloudstorage
versioncontrol
backup
coms
finance
homeautomation
passwordmanager
)
# Map service directory -> default health check URL (relative to STAGING_DOMAIN)
# Override entirely via HEALTH_URLS env var.
declare -A DEFAULT_HEALTH_URLS
DEFAULT_HEALTH_URLS[network]="https://traefik.${STAGING_DOMAIN}/ping"
DEFAULT_HEALTH_URLS[authentification]="https://auth.${STAGING_DOMAIN}/api/verify"
DEFAULT_HEALTH_URLS[homepage]="https://${STAGING_DOMAIN}/"
DEFAULT_HEALTH_URLS[ai]="https://hermes.${STAGING_DOMAIN}/health"
DEFAULT_HEALTH_URLS[cloudstorage]="https://cloud.${STAGING_DOMAIN}/status.php"
DEFAULT_HEALTH_URLS[versioncontrol]="https://code.${STAGING_DOMAIN}/api/healthz"
# ---- Trackers ----
PASS_COUNT=0
FAIL_COUNT=0
WARN_COUNT=0
FAILURES=()
# ---- Helpers ----
log_info() { echo -e "${CYAN}[INFO]${NC} $*"; }
log_pass() { echo -e "${GREEN}[PASS]${NC} $*"; ((PASS_COUNT++)); }
log_fail() { echo -e "${RED}[FAIL]${NC} $*"; ((FAIL_COUNT++)); FAILURES+=("$*"); }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; ((WARN_COUNT++)); }
log_step() { echo -e "\n${BOLD}── $* ──${NC}"; }
log_raw() { echo -e " $*"; }
# Check if a command exists
require_cmd() {
if ! command -v "$1" &>/dev/null; then
log_fail "Required command not found: $1"
return 1
fi
}
# Retry a command with exponential-like backoff
retry() {
local cmd="$*"
local attempt=0
local max_attempts=$((HEALTH_RETRIES + 1))
local result
while [[ $attempt -lt $max_attempts ]]; do
if eval "$cmd" 2>/dev/null; then
return 0
fi
attempt=$((attempt + 1))
if [[ $attempt -lt $max_attempts ]]; then
sleep "$HEALTH_INTERVAL"
fi
done
return 1
}
# ---- Parse arguments ----
VERBOSE=false
LIST_SERVICES=false
POSITIONAL=()
while [[ $# -gt 0 ]]; do
case "$1" in
--verbose|-v) VERBOSE=true; shift ;;
--list-services) LIST_SERVICES=true; shift ;;
--) shift; POSITIONAL+=("$@"); break ;;
*) POSITIONAL+=("$1"); shift ;;
esac
done
set -- "${POSITIONAL[@]}"
# Resolve service list
if [[ -n "${SERVICE_LIST:-}" ]]; then
IFS=' ' read -ra SERVICES <<< "$SERVICE_LIST"
else
SERVICES=("${DEFAULT_SERVICES[@]}")
fi
# Resolve health URLs — default map with overrides from env
declare -A HEALTH_URLS
if [[ -n "${HEALTH_URLS:-}" ]]; then
# User-supplied mapping: "network=https://... authentification=https://..."
for pair in $HEALTH_URLS; do
key="${pair%%=*}"
val="${pair#*=}"
HEALTH_URLS["$key"]="$val"
done
else
for svc in "${SERVICES[@]}"; do
if [[ -n "${DEFAULT_HEALTH_URLS[$svc]:-}" ]]; then
HEALTH_URLS["$svc"]="${DEFAULT_HEALTH_URLS[$svc]}"
fi
done
fi
# --list-services mode (for CI integration)
if $LIST_SERVICES; then
echo "Configured services:"
for svc in "${SERVICES[@]}"; do
url="${HEALTH_URLS[$svc]:-no-health-check}"
echo " $svc -> $url"
done
exit 0
fi
# ---- Pre-flight ----
echo -e "${BOLD}============================================${NC}"
echo -e "${BOLD} Staging VM Integration Test Suite${NC}"
echo -e "${BOLD} $(date -u '+%Y-%m-%dT%H:%M:%SZ')${NC}"
echo -e "${BOLD}============================================${NC}"
# ---- Phase 1: Prerequisites ----
log_step "Phase 1: Prerequisites"
PREREQ_OK=true
for cmd in docker curl jq; do
if ! require_cmd "$cmd"; then
PREREQ_OK=false
fi
done
$PREREQ_OK && log_pass "All required commands available" || log_fail "Missing prerequisites"
# ---- Phase 2: Docker daemon ----
log_step "Phase 2: Docker Daemon"
if docker info --format '{{.ServerVersion}}' &>/dev/null; then
DOCKER_VERSION=$(docker info --format '{{.ServerVersion}}' 2>/dev/null)
log_pass "Docker daemon is running (version: $DOCKER_VERSION)"
if docker info --format '{{.Driver}}' 2>/dev/null | grep -qi "overlay"; then
log_pass "Storage driver: overlay"
else
log_warn "Non-overlay storage driver detected"
fi
else
log_fail "Docker daemon is NOT running or not accessible"
fi
# ---- Phase 3: Docker Compose stack ----
log_step "Phase 3: Compose Stack Status"
# Check if any compose files exist
COMPOSE_FILES=()
for svc in "${SERVICES[@]}"; do
cf="${COMPOSE_DIR}/${svc}/compose.yml"
if [[ -f "$cf" ]]; then
COMPOSE_FILES+=("$cf")
else
cf2="${COMPOSE_DIR}/${svc}/docker-compose.yml"
if [[ -f "$cf2" ]]; then
COMPOSE_FILES+=("$cf2")
else
log_warn "No compose file found for service '$svc' (expected: ${cf})"
fi
fi
done
if [[ ${#COMPOSE_FILES[@]} -eq 0 ]]; then
log_fail "No compose files found under COMPOSE_DIR=${COMPOSE_DIR}"
log_info "Skipping stack checks"
else
log_info "Found ${#COMPOSE_FILES[@]} compose file(s) in ${COMPOSE_DIR}"
# Build the compose file args
COMPOSE_CMD="docker compose -p ${COMPOSE_PROJECT}"
for cf in "${COMPOSE_FILES[@]}"; do
COMPOSE_CMD+=" -f ${cf}"
done
log_info "Project name: ${COMPOSE_PROJECT}"
# Check stack ps
if $VERBOSE; then
log_raw "--- docker compose ps output ---"
eval "$COMPOSE_CMD ps" 2>&1 | while IFS= read -r line; do log_raw "$line"; done
log_raw "--- end ---"
fi
# Get all services and their status
if STACK_STATUS=$(eval "$COMPOSE_CMD ps --format '{{.Name}}\t{{.Status}}'" 2>/dev/null); then
if [[ -z "$STACK_STATUS" ]]; then
log_warn "Stack exists but no running services — VM may be freshly provisioned"
else
ALL_RUNNING=true
RUNNING_COUNT=0
TOTAL_COUNT=0
while IFS=$'\t' read -r name status; do
TOTAL_COUNT=$((TOTAL_COUNT + 1))
status_lower=$(echo "$status" | tr '[:upper:]' '[:lower:]')
if echo "$status_lower" | grep -qE '^(up|running|healthy)'; then
RUNNING_COUNT=$((RUNNING_COUNT + 1))
$VERBOSE && log_pass " $name$status"
else
ALL_RUNNING=false
log_warn " $name$status (not healthy)"
fi
done <<< "$STACK_STATUS"
if [[ "$TOTAL_COUNT" -eq 0 ]]; then
log_fail "No services found in compose project"
elif $ALL_RUNNING && [[ "$TOTAL_COUNT" -eq "$RUNNING_COUNT" ]]; then
log_pass "All ${TOTAL_COUNT} service(s) running (${RUNNING_COUNT}/${TOTAL_COUNT})"
else
log_fail "${RUNNING_COUNT}/${TOTAL_COUNT} service(s) running — some services are down"
fi
fi
else
log_fail "Failed to query compose stack status"
fi
fi
# ---- Phase 4: Service health checks ----
log_step "Phase 4: Service Endpoint Health Checks"
ENDPOINT_CHECKS=0
ENDPOINT_PASS=0
for svc in "${SERVICES[@]}"; do
url="${HEALTH_URLS[$svc]:-}"
if [[ -z "$url" ]]; then
$VERBOSE && log_info "No health check URL for service '$svc' — skipping"
continue
fi
ENDPOINT_CHECKS=$((ENDPOINT_CHECKS + 1))
echo -ne " Checking ${svc} ... "
# Perform the HTTP health check with retries
if retry "curl -sf -o /dev/null -w '%{http_code}' --max-time ${HEALTH_TIMEOUT} '${url}' 2>/dev/null"; then
HTTP_CODE=$(curl -sf -o /dev/null -w '%{http_code}' --max-time "${HEALTH_TIMEOUT}" "${url}" 2>/dev/null || true)
ENDPOINT_PASS=$((ENDPOINT_PASS + 1))
echo -e "${GREEN}OK${NC} (HTTP ${HTTP_CODE})"
else
LAST_CODE=$(curl -s -o /dev/null -w '%{http_code}' --max-time "${HEALTH_TIMEOUT}" "${url}" 2>/dev/null || echo "000")
echo -e "${RED}FAIL${NC} (HTTP ${LAST_CODE})"
log_fail "Health check failed for ${svc} @ ${url}"
fi
done
if [[ $ENDPOINT_CHECKS -eq 0 ]]; then
log_warn "No health check URLs configured — skipping endpoint phase"
elif [[ $ENDPOINT_PASS -eq $ENDPOINT_CHECKS ]]; then
log_pass "All ${ENDPOINT_CHECKS} endpoint(s) healthy"
else
log_fail "${ENDPOINT_PASS}/${ENDPOINT_CHECKS} endpoint(s) healthy"
fi
# ---- Phase 5: Docker system sanity ----
log_step "Phase 5: Docker System Sanity"
# Check disk space for Docker
DOCKER_ROOT=$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || echo "/var/lib/docker")
log_info "Docker root: ${DOCKER_ROOT}"
if command -v df &>/dev/null && [[ -d "$DOCKER_ROOT" ]]; then
AVAIL_PCT=$(df -h "$DOCKER_ROOT" | awk 'NR==2 {print $5}' | tr -d '%')
if [[ -n "$AVAIL_PCT" ]]; then
if [[ "$AVAIL_PCT" -ge 90 ]]; then
log_warn "Docker storage is ${AVAIL_PCT}% full — consider cleanup"
else
log_pass "Docker storage at ${AVAIL_PCT}% — within limits"
fi
fi
fi
# Check for dangling images
DANGLING=$(docker images -f "dangling=true" -q 2>/dev/null | wc -l)
if [[ "$DANGLING" -gt 10 ]]; then
log_warn "${DANGLING} dangling images found — consider docker image prune"
fi
# ---- Summary ----
echo ""
echo -e "${BOLD}============================================${NC}"
echo -e "${BOLD} Test Summary${NC}"
echo -e "${BOLD} $(date -u '+%Y-%m-%dT%H:%M:%SZ')${NC}"
echo -e "${BOLD}============================================${NC}"
echo -e " ${GREEN}Passed:${NC} ${PASS_COUNT}"
echo -e " ${RED}Failed:${NC} ${FAIL_COUNT}"
echo -e " ${YELLOW}Warnings:${NC} ${WARN_COUNT}"
if [[ ${#FAILURES[@]} -gt 0 ]]; then
echo -e "\n${BOLD}Failed checks:${NC}"
for f in "${FAILURES[@]}"; do
echo -e " ${RED}${NC} $f"
done
fi
echo ""
if [[ $FAIL_COUNT -eq 0 ]]; then
echo -e "${GREEN}${BOLD}✓ All integration checks passed${NC}"
exit 0
else
echo -e "${RED}${BOLD}${FAIL_COUNT} integration check(s) failed${NC}"
exit 1
fi

View File

@@ -4,7 +4,7 @@
group = "ai-worker";
home = "/home/ai-worker";
createHome = true;
extraGroups = [ "docker" "libvirtd" ];
extraGroups = [ "docker" ];
shell = pkgs.bashInteractive;
openssh.authorizedKeys.keys = [
keys.users.ai-worker.main