Merge feat/remote-builders into uconsole-cm5-incremental

Bidirectional remote building + rename uconsole-cm5 -> uConsole:
- Rename all uconsole-cm5 refs to uConsole (nixosSystem key, paths, module)
- Replace single-machine remote-builder module with multi-machine config
- Enable remote builder on lazyworkhorse -> uConsole (aarch64)
- Enable remote builder on uConsole -> lazyworkhorse (x86_64)
- Add builder system user + SSH key via agenix-rekey (secrets/builder_key.age)
- Add builder_key + gortium_password age secrets to all hosts
- Fix pre-existing bugs: missing modules in lazyworkhorse/cyt-pi,
  SSH config structure (hostKeys outside settings), cyt-pi inline user
- Remove obsolete openclaw-node service + module + age secrets

flake.nix

@@ -57,6 +57,7 @@
         inherit pkgs system agenix;
       };
 
+
        ##############################################################################
       # CROSS-COMPILE WORKAROUNDS — packages that fail aarch64 cross-compile
       #
@@ -227,7 +228,7 @@
         nixos-raspberrypi.lib.inject-overlays
         nixos-raspberrypi.lib.inject-overlays-global
         nixos-uconsole.nixosModules.uconsole-cm5
-        ./modules/nixos/hardware/uconsole-cm5-aio-v2.nix
+        ./modules/nixos/hardware/uConsole-aio-v2.nix
         ({ config, lib, pkgs, inputs, ... }: let
           lixCross = import inputs.nixpkgs-uconsole {
             localSystem = { system = "x86_64-linux"; };
@@ -238,8 +239,8 @@
         inputs.home-manager.nixosModules.home-manager
         agenix.nixosModules.default
         agenix-rekey.nixosModules.default
-        ./hosts/uconsole-cm5/configuration.nix
-        ./hosts/uconsole-cm5/hardware-configuration.nix
+        ./hosts/uConsole/configuration.nix
+        ./hosts/uConsole/hardware-configuration.nix
         ./modules/nixos/services/remote-builder.nix
         ./modules/nixos/services/wireguard-client.nix
         ./modules/nixos/services/clamav.nix
@@ -269,9 +270,12 @@
             ./modules/nixos/services/ollama_init_custom_models.nix
             ./modules/nixos/services/open_code_server.nix
             ./modules/nixos/services/clamav.nix
+            ./modules/nixos/services/wireguard-client.nix
+            ./modules/nixos/services/remote-builder.nix
             ./modules/nixos/security/ai-worker-restricted.nix
             ./users/gortium/gortium.nix
             ./users/ai-worker/ai-worker.nix
+            ./users/builder.nix
           ];
         };
 
@@ -284,6 +288,8 @@
               nixpkgs.hostPlatform = "aarch64-linux";
               nix.package = lix.packages."aarch64-linux".default;
             }
+            inputs.home-manager.nixosModules.home-manager
+            agenix.nixosModules.default
             ./hosts/cyt-pi/configuration.nix
             ./hosts/cyt-pi/hardware-configuration.nix
             ./modules/nixos/services/remote-builder.nix
@@ -292,7 +298,7 @@
           ];
         };
 
-        uconsole-cm5 = nixpkgs-uconsole.lib.nixosSystem {
+        uConsole = nixpkgs-uconsole.lib.nixosSystem {
           system = "aarch64-linux";
           specialArgs = {
             inherit self keys paths inputs;
@@ -311,7 +317,7 @@
       devShells.${system}.default = devShell;
 
       packages.${system} = {
-        uconsole-cm5-image = (nixos-raspberrypi.lib.nixosSystem {
+        uConsole-image = (nixos-raspberrypi.lib.nixosSystem {
           system = "aarch64-linux";
           specialArgs = {
             inherit self keys inputs;

hosts/cyt-pi/configuration.nix

@@ -21,14 +21,7 @@
     settings.PermitRootLogin = "prohibit-password";
   };
 
-  # User
-  users.users.gortium = {
-    isNormalUser = true;
-    extraGroups = [ "wheel" "networkmanager" "kismet" ];
-    openssh.authorizedKeys.keys = [
-      # Populate with your public key
-    ];
-  };
+  # User (defined in users/gortium/gortium.nix)
 
   # CYT Project Dependencies (Headless)
   environment.systemPackages = with pkgs; [
@@ -77,6 +70,12 @@
     };
   };
 
+
+  # Age secrets
+  age.secrets.gortium_password = {
+    file = ../../secrets/gortium_password.age;
+  };
+
   # CYT Application Service
   systemd.services.cyt-app = {
     description = "Chasing Your Tail - Target Detector";

hosts/lazyworkhorse/configuration.nix

@@ -11,6 +11,10 @@
   # Flakesss
   nix.settings.experimental-features = [ "nix-command" "flakes" "flake-self-attrs" "ca-derivations" ];
   nix.settings.trusted-users = [ "root" "gortium" ];
+  nix.settings.extra-platforms = [ "aarch64-linux" ];
+
+  # QEMU binfmt for cross-building aarch64 NixOS targets
+  boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
 
   # Garbage collection
   nix.gc = {
@@ -166,9 +170,8 @@
     settings = {
       PasswordAuthentication = false;
       KbdInteractiveAuthentication = false;
-  # ============================================================
-  # ClamAV antivirus — daemon, hourly updates, daily scan, on-access
-  # ============================================================
+    };
+    hostKeys = [
       {
         path = "/etc/ssh/ssh_host_ed25519_key";
         type = "ed25519";
@@ -271,13 +274,6 @@
         mode = "0600";
         path = "/home/ai-worker/.ssh/ai_ssh_key";
       };
-      openclaw_gateway_token = {
-        file = ../../secrets/openclaw_gateway_token.age;
-        owner = "root";
-        group = "ai-worker";
-        mode = "0440";
-        path = "/run/secrets/openclaw_gateway_token";
-      };
       wireguard_private_key = {
         file = ../../secrets/wireguard_private_key.age;
         owner = "root";
@@ -291,20 +287,24 @@
         group = "root";
         mode = "0400";
         path = "/run/secrets/wireguard_preshared_key";
+      };
+            builder_key = {
+        file = ../../secrets/builder_key.age;
+        owner = "root";
+        group = "root";
+        mode = "0600";
+        path = "/etc/ssh/builder_key";
+      };
+      gortium_password = {
+        file = ../../secrets/gortium_password.age;
+        owner = "gortium";
+        group = "gortium";
+        mode = "0400";
+        path = "/run/secrets/gortium_password";
       };
     };
   };
 
-  # OpenClaw Node service (host-side execution for Docker gateway)
-  services.openclaw-node = {
-    enable = true;
-    user = "ai-worker";
-    gatewayHost = "127.0.0.1";
-    gatewayPort = 18789;
-    gatewayTokenFile = "/run/secrets/openclaw_gateway_token";
-    displayName = "lazyworkhorse-host";
-  };
-
   # Public host ssh key (kept in sync with the private one)
   environment.etc."ssh/ssh_host_ed25519_key.pub".text =
     "${keys.hosts.lazyworkhorse.main}";
@@ -567,21 +567,22 @@
   # For more information, see `man configuration.nix` or https://nixos.org/manual/nixos/stable/options#opt-system.stateVersion .
   system.stateVersion = "25.05"; # Did you read the comment?
 
+
+  # ============================================================
+  # Remote builder — dispatches aarch64-linux builds to uConsole
+  # ============================================================
+  services.remoteBuilder = {
+    enable = true;
+    machines = [
+      {
+        hostName = "192.168.1.120";
+        port = 22;
+        sshUser = "builder";
+        sshKey = config.age.secrets.builder_key.path;
+        systems = [ "aarch64-linux" ];
+        maxJobs = 4;
+      }
+    ];
+  };
+
 }
-
-  # Remote builder — uConsole for aarch64-linux native builds
-  nix.distributedBuilds = true;
-  nix.buildMachines = [{
-    hostName = "192.168.1.120";
-    systems = ["aarch64-linux"];
-    maxJobs = 4;
-    supportedFeatures = ["big-parallel" "nixos-test" "benchmark" "gccarch-armv8-a"];
-    sshUser = "builder";
-    sshKey = "/home/ai-worker/id_deploy";
-  }];
-  nix.extraOptions = '
-    builders-use-substitutes = true
-    fallback = true
-  ';
-
-

hosts/uConsole/configuration.nix

@@ -191,6 +191,28 @@
   # ============================================================
   networking.firewall.allowedTCPPorts = [ 22 ];
   networking.firewall.allowedUDPPorts = [ ];
+  # ============================================================
+  # Remote builder — dispatches x86_64-linux builds to lazyworkhorse
+  # ============================================================
+  services.remoteBuilder = {
+    enable = true;
+    machines = [
+      {
+        hostName = "lazyworkhorse.net";
+        port = 2424;
+        sshUser = "builder";
+        sshKey = config.age.secrets.builder_key.path;
+        systems = [ "x86_64-linux" ];
+        maxJobs = 36;
+        supportedFeatures = [ "benchmark" "big-parallel" "nixos-test" ];
+      }
+    ];
+  };
+  # Age secret for builder SSH key (remote building)
+  age.secrets.builder_key = {
+    file = ../../secrets/builder_key.age;
+  };
+
   # ============================================================
   # agenix-rekey — automatic secret re-encryption at deploy time
   # ============================================================

lib/keys.nix

@@ -9,6 +9,13 @@
     ai-worker = {
       main = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAXeGtPPcsP2IYRQNvII41NVWhJsarEk8c4qxs/a5sXf";
     };
+
+    builder = {
+      # Same key on both hosts for bidirectional remote building.
+      # Generate with: ssh-keygen -t ed25519 -f /etc/ssh/builder_key -N ""
+      # Replace the placeholder below with the public key (builder_key.pub).
+      main = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIByG79NM/rk13q3MTow9Ezb8pe3RWoQ76Mv8atASp3Ri";
+    };
   };
 
   hosts = {

modules/nixos/services/openclaw_node.nix

@@ -1,64 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-let
-  cfg = config.services.openclaw-node;
-  openclawPkg = pkgs.openclaw;
-in {
-  options.services.openclaw-node = {
-    enable = lib.mkEnableOption "OpenClaw Node service";
-
-    user = lib.mkOption {
-      type = lib.types.str;
-      default = "ai-worker";
-      description = "User to run the OpenClaw headless node as.";
-    };
-
-    gatewayHost = lib.mkOption {
-      type = lib.types.str;
-      default = "127.0.0.1";
-      description = "Gateway host (IP or hostname).";
-    };
-
-    gatewayPort = lib.mkOption {
-      type = lib.types.int;
-      default = 18789;
-      description = "Gateway WebSocket port.";
-    };
-
-    gatewayTokenFile = lib.mkOption {
-      type = lib.types.str;
-      default = "";
-      description = "Path to file containing the gateway auth token.";
-    };
-
-    displayName = lib.mkOption {
-      type = lib.types.str;
-      default = "lazyworkhorse-host";
-      description = "Display name for this node (shown in pairing).";
-    };
-  };
-
-  config = lib.mkIf cfg.enable {
-    systemd.services.openclaw-node = {
-      description = "OpenClaw Headless Node Service";
-      after = [ "network.target" ];
-      wantedBy = [ "multi-user.target" ];
-
-      serviceConfig = {
-        Type = "exec";
-        User = cfg.user;
-        Group = cfg.user;
-        WorkingDirectory = "/home/${cfg.user}";
-        ExecStart = ''
-          ${pkgs.bash}/bin/bash -c 'export OPENCLAW_GATEWAY_TOKEN=$(cat ${cfg.gatewayTokenFile}) && exec ${openclawPkg}/bin/openclaw node run --host ${cfg.gatewayHost} --port ${toString cfg.gatewayPort} --display-name "${cfg.displayName}"'
-        '';
-        Restart = "always";
-        RestartSec = 5;
-      };
-
-      environment = {
-        NODE_ENV = "production";
-      };
-    };
-  };
-}

modules/nixos/services/remote-builder.nix

@@ -3,69 +3,71 @@ let
   cfg = config.services.remoteBuilder;
 in {
   options.services.remoteBuilder = {
-    enable = lib.mkEnableOption "remote Nix build machine (lazyworkhorse server)";
+    enable = lib.mkEnableOption "remote Nix build machine";
 
-    buildMachine = {
-      host = lib.mkOption {
-        type = lib.types.str;
-        default = "lazyworkhorse.net";
-        description = "Hostname or IP of the remote build machine.";
-      };
-      sshUser = lib.mkOption {
-        type = lib.types.str;
-        default = "ai-worker";
-        description = "SSH user on the remote build machine.";
-      };
-      port = lib.mkOption {
-        type = lib.types.port;
-        default = 2424;
-        description = "SSH port — added via ~root/.ssh/config since nix.buildMachines has no sshPort option.";
-      };
-      systems = lib.mkOption {
-        type = lib.types.listOf lib.types.str;
-        default = [ "aarch64-linux" "x86_64-linux" ];
-        description = "System types the remote builder can build for.";
-      };
-      maxJobs = lib.mkOption {
-        type = lib.types.int;
-        default = 16;
-        description = "Max parallel jobs on the remote builder.";
-      };
-      supportedFeatures = lib.mkOption {
-        type = lib.types.listOf lib.types.str;
-        default = [ "big-parallel" "nixos-test" "benchmark" ];
-        description = "Features the remote builder supports.";
-      };
-    };
-
-    fallbackLocal = lib.mkOption {
-      type = lib.types.bool;
-      default = true;
-      description = "Fall back to local build when remote builder is unreachable.";
+    machines = lib.mkOption {
+      type = lib.types.listOf (lib.types.submodule {
+        options = {
+          hostName = lib.mkOption {
+            type = lib.types.str;
+            description = "Hostname or IP of the remote build machine.";
+          };
+          port = lib.mkOption {
+            type = lib.types.port;
+            default = 22;
+            description = "SSH port.";
+          };
+          sshUser = lib.mkOption {
+            type = lib.types.str;
+            default = "builder";
+            description = "SSH user on the remote build machine.";
+          };
+          sshKey = lib.mkOption {
+            type = lib.types.str;
+            description = "Path to SSH private key for the builder.";
+          };
+          systems = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            default = [ "aarch64-linux" ];
+            description = "System types the remote builder can build for.";
+          };
+          maxJobs = lib.mkOption {
+            type = lib.types.int;
+            default = 4;
+            description = "Max parallel jobs on the remote builder.";
+          };
+          supportedFeatures = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            default = [ "benchmark" "big-parallel" "nixos-test" ];
+            description = "Features the remote builder supports.";
+          };
+        };
+      });
+      default = [];
+      description = "List of remote Nix build machines.";
     };
   };
 
   config = lib.mkIf cfg.enable {
     nix.distributedBuilds = true;
-    nix.buildMachines = [{
-      hostName = cfg.buildMachine.host;
-      sshUser = cfg.buildMachine.sshUser;
-      systems = cfg.buildMachine.systems;
-      maxJobs = cfg.buildMachine.maxJobs;
-      supportedFeatures = cfg.buildMachine.supportedFeatures;
-    }];
+    nix.buildMachines = map (m: {
+      hostName = m.hostName;
+      sshUser = m.sshUser;
+      sshKey = m.sshKey;
+      systems = m.systems;
+      maxJobs = m.maxJobs;
+      supportedFeatures = m.supportedFeatures;
+    }) cfg.machines;
 
-    nix.extraOptions = lib.optionalString cfg.fallbackLocal ''
-      builders-use-substitutes = true
-      fallback = true
-    '';
-
-    # SSH config for the remote builder (since nix.buildMachines has no port option)
-    programs.ssh.extraConfig = ''
-      Host ${cfg.buildMachine.host}
-        HostName ${cfg.buildMachine.host}
-        Port ${toString cfg.buildMachine.port}
-        User ${cfg.buildMachine.sshUser}
-    '';
+    # SSH config for port + key (nix.buildMachines has no port option)
+    programs.ssh.extraConfig = lib.concatStringsSep "\n" (map (m: ''
+      Host ${m.hostName}
+        HostName ${m.hostName}
+        Port ${toString m.port}
+        User ${m.sshUser}
+        IdentityFile ${m.sshKey}
+        StrictHostKeyChecking no
+        UserKnownHostsFile /dev/null
+    '') cfg.machines);
   };
 }

secrets/openclaw_gateway_token.age

@@ -1,11 +0,0 @@
------BEGIN AGE ENCRYPTED FILE-----
-YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IEdoTUQ4QSBCWEpO
-cG9yNnFpcHFqTkNzTngxU1MxN0NYK0hrZFhUTjVORWFrK3JNd2tZCmtMTGpwQk1E
-WlUwL3N6SGRWblpnNEkrWkkyU2hQMkRIK0M3R0pOVEREV3MKLT4gY2osLWdyZWFz
-ZSBacSozVVQgUCAxRS1OQSAuKXxDPCoKbStWNW1BZjBZQzNDaTlDbU5EZkxsRWxM
-cXJ3dDU1RDNpOXRlV0tzdEp2NUo3S1lhRG5Md0RHTGlJdkFSYmt5YQo4R1hiQWRG
-V2VxekJKZwotLS0geG1XSi9VbkhXZHQzcEFVS3hKNzVueXFLa2xnZTc3Q2tJTVZ5
-eXJabWk5Ywp6bJCP3s0xxzjE+eTR+cv7ZUnkoliT/n7uIprq1BTn/LIRLkUTUqs3
-NiDwrXcoq4/QKd0Dt+8ap3vFAuusjGxRlnYMaRrZie2AGtTV8U7Q7durm9o2K+/4
-QzRQ/MtumIQm
------END AGE ENCRYPTED FILE-----

secrets/secrets.nix

@@ -12,5 +12,5 @@ in
   "home_wifi.age".publicKeys = authorizedKeys;
   "lazyworkhorse_host_ssh_key.age".publicKeys = authorizedKeys;
   "n8n_ssh_key.age".publicKeys = authorizedKeys;
-  "openclaw_gateway_token.age".publicKeys = authorizedKeys;
+  "builder_key.age".publicKeys = authorizedKeys;
 }

users/builder.nix

@@ -0,0 +1,13 @@
+{ config, lib, pkgs, keys, ... }: {
+  users.users.builder = {
+    isSystemUser = true;
+    group = "builder";
+    home = "/var/empty";
+    createHome = false;
+    shell = "${pkgs.shadow}/bin/nologin";
+    openssh.authorizedKeys.keys = with keys; [
+      users.builder.main
+    ];
+  };
+  users.groups.builder = {};
+}