From 1b27dadd6090e92ca4572614112e10dc0615c0b8 Mon Sep 17 00:00:00 2001 From: Alex Jackson Date: Sun, 6 Sep 2026 17:06:35 -0500 Subject: [PATCH 1/3] fix: preserve Tailscale policy rules across networkd restarts --- hosts/patroclus/hypervisor.nix | 10 ++ hosts/patroclus/tests/README.md | 29 ++++++ .../tests/networkd-routing-policy.nix | 97 +++++++++++++++++++ 3 files changed, 136 insertions(+) create mode 100644 hosts/patroclus/tests/README.md create mode 100644 hosts/patroclus/tests/networkd-routing-policy.nix diff --git a/hosts/patroclus/hypervisor.nix b/hosts/patroclus/hypervisor.nix index 643e410..210e677 100644 --- a/hosts/patroclus/hypervisor.nix +++ b/hosts/patroclus/hypervisor.nix @@ -1,3 +1,4 @@ +{ config, pkgs, ... }: let ethernetIface = "eno2"; # this is what the iface is named in the system bridgeIface = "br0"; # created by this file @@ -19,6 +20,10 @@ in networking.useNetworkd = true; systemd.network = { enable = true; + # Tailscale owns policy rules 5210-5270. networkd otherwise removes them + # when it starts/reconfigures links, which can interrupt tailnet routing + # and the host's MagicDNS path. Leave rules owned by other services alone. + config.networkConfig.ManageForeignRoutingPolicyRules = false; netdevs.${bridgeIface}.netdevConfig = { Name = bridgeIface; Kind = "bridge"; @@ -46,4 +51,9 @@ in }; }; }; + + system.build.networkdRoutingPolicyTest = import ./tests/networkd-routing-policy.nix { + inherit pkgs; + networkConfig = config.systemd.network.config.networkConfig; + }; } diff --git a/hosts/patroclus/tests/README.md b/hosts/patroclus/tests/README.md new file mode 100644 index 0000000..74f5330 --- /dev/null +++ b/hosts/patroclus/tests/README.md @@ -0,0 +1,29 @@ +# networkd / Tailscale routing regression + +Run from the repository root: + +```sh +nix build --no-link --print-build-logs \ + .#nixosConfigurations.patroclusStripped.config.system.build.networkdRoutingPolicyTest +``` + +This is a NixOS VM test, separate from the editor firewall's namespace test. +It requires a Linux builder with KVM/NixOS-test support. It boots an isolated +guest, runs the pinned systemd-networkd with the host's actual global settings, +and installs Tailscale-style IPv4 and IPv6 policy rules. + +It verifies that those rules survive: + +- a networkd restart; +- a managed-link reconfiguration; +- adding an editor bridge and restarting networkd. + +The negative control switches the guest to the previous `yes` default and +verifies that networkd really deletes the rules. Restoring the production +configuration must preserve them again. All restarts and configuration changes +occur in the disposable test VM, not on the machine running the test. + +This covers the routing-rule deletion observed during the September 6 deployment. +It does not emulate Tailscale's NextDNS-over-HTTPS connections or prove the entire +DNS interruption was caused solely by rule deletion. In particular, temporary +IPv6 address changes during activation remain a separate possible contributor. diff --git a/hosts/patroclus/tests/networkd-routing-policy.nix b/hosts/patroclus/tests/networkd-routing-policy.nix new file mode 100644 index 0000000..1af5f79 --- /dev/null +++ b/hosts/patroclus/tests/networkd-routing-policy.nix @@ -0,0 +1,97 @@ +# A real networkd restart test in a disposable NixOS VM. The production setting +# is passed in by hypervisor.nix; no network service on the test host is touched. +{ pkgs, networkConfig }: +pkgs.testers.runNixOSTest { + name = "patroclus-networkd-routing-policy"; + + nodes.machine = { + virtualisation.memorySize = 512; + networking.useNetworkd = true; + systemd.network = { + config.networkConfig = networkConfig; + netdevs."10-dummy0".netdevConfig = { + Name = "dummy0"; + Kind = "dummy"; + }; + networks."10-dummy0" = { + matchConfig.Name = "dummy0"; + address = [ "192.0.2.1/24" ]; + networkConfig.ConfigureWithoutCarrier = true; + linkConfig.RequiredForOnline = "no"; + }; + }; + }; + + testScript = '' + import json + + start_all() + machine.wait_for_unit("systemd-networkd.service") + + def wait_configured(interface): + machine.wait_until_succeeds( + f"networkctl status {interface} --no-pager | grep -F '(configured)'" + ) + + def add_rules(): + for family in ("-4", "-6"): + for rule in ( + "pref 5210 fwmark 0x80000/0xff0000 lookup main", + "pref 5230 fwmark 0x80000/0xff0000 lookup default", + "pref 5250 fwmark 0x80000/0xff0000 unreachable", + "pref 5270 lookup 52", + ): + machine.succeed(f"ip {family} rule add {rule}") + + def rules(): + return { + family: json.loads(machine.succeed(f"ip {family} -j rule show")) + for family in ("-4", "-6") + } + + wait_configured("dummy0") + add_rules() + expected = rules() + + with subtest("foreign IPv4 and IPv6 rules survive networkd restart"): + machine.succeed("systemctl restart systemd-networkd") + machine.wait_for_unit("systemd-networkd.service") + wait_configured("dummy0") + assert rules() == expected + + with subtest("foreign rules survive reconfiguration"): + machine.succeed("networkctl reconfigure dummy0") + wait_configured("dummy0") + assert rules() == expected + + with subtest("adding the editor bridge preserves foreign rules"): + machine.succeed("mkdir -p /run/systemd/network") + machine.succeed("printf '[NetDev]\\nName=agentbr0\\nKind=bridge\\n' > /run/systemd/network/30-agentbr0.netdev") + machine.succeed("printf '[Match]\\nName=agentbr0\\n[Network]\\nAddress=192.168.83.1/24\\nConfigureWithoutCarrier=yes\\nDHCP=no\\nIPv6AcceptRA=no\\nLinkLocalAddressing=no\\n' > /run/systemd/network/30-agentbr0.network") + machine.succeed("systemctl restart systemd-networkd") + machine.wait_for_unit("systemd-networkd.service") + wait_configured("agentbr0") + machine.succeed("ip -4 address show agentbr0 | grep -F 192.168.83.1/24") + assert rules() == expected + + with subtest("negative control reproduces deletion with the old default"): + # This makes the test sensitive to the original bug, rather than just + # checking that an option evaluates. It runs only inside this test VM. + machine.succeed("mkdir -p /run/systemd/networkd.conf.d") + machine.succeed("printf '[Network]\\nManageForeignRoutingPolicyRules=yes\\n' > /run/systemd/networkd.conf.d/99-control.conf") + machine.succeed("systemctl restart systemd-networkd") + machine.wait_for_unit("systemd-networkd.service") + wait_configured("dummy0") + for family in ("-4", "-6"): + machine.wait_until_succeeds(f"! ip {family} rule show | grep -E '^52(10|30|50|70):'") + assert rules() != expected + + with subtest("restoring production configuration preserves rules again"): + machine.succeed("rm /run/systemd/networkd.conf.d/99-control.conf") + add_rules() + machine.succeed("systemctl restart systemd-networkd") + machine.wait_for_unit("systemd-networkd.service") + wait_configured("dummy0") + assert rules() == expected + ''; +} From 88dd674f3fa26d4aab7260096030797db7d7f1a8 Mon Sep 17 00:00:00 2001 From: Alex Jackson Date: Sun, 6 Sep 2026 17:18:48 -0500 Subject: [PATCH 2/3] fix: bootstrap and administer the Grace editor guest --- components/website-editor/README.md | 53 ++++++++++++ components/website-editor/options.nix | 5 ++ components/website-editor/vm.nix | 118 +++++++++++++++++++++----- hosts/patroclus/configuration.nix | 8 +- 4 files changed, 164 insertions(+), 20 deletions(-) diff --git a/components/website-editor/README.md b/components/website-editor/README.md index 8eb6f46..5612126 100644 --- a/components/website-editor/README.md +++ b/components/website-editor/README.md @@ -16,6 +16,59 @@ Guest services start inside the VM, not on the host. - Guest: `192.168.83.2/24`, gateway `192.168.83.1` - Host bridge: `agentbr0`, containing only the `agent-grace` tap. +## Guest access and first boot + +From the homelab's `admin` account: + +```sh +ssh grace-editor +``` + +The host alias uses `~/.ssh/grace-editor`; its public key is declared in +`components.website-editor.authorizedKeys`. The private key stays on the host. +Guest SSH accepts key authentication from the bridge gateway only; port 22 is +not forwarded from the LAN. From another device, SSH into patroclus first. +The guest has passwordless sudo. Its own SSH host key persists on the home +volume, so rebuilding/rebooting it does not change the server identity. + +The private website repository requires a guest-specific GitHub deploy key. +Before the initial clone, create one **inside the guest**: + +```sh +install -d -m 700 ~/.ssh +ssh-keygen -t ed25519 -N '' -C grace-editor -f ~/.ssh/id_ed25519 +cat ~/.ssh/id_ed25519.pub +``` + +Add that public key to `ajaxbits/gracebobber` → Settings → Deploy keys, enabling +write access for publishing. Keep the private half inside the guest. The +bootstrap uses SSH transport and a pinned GitHub host key; it never receives +the administrator's broad GitHub token. Bootstrap initially fails until this +deploy key is provisioned; guest SSH remains available independently. + +Then run **inside the guest**: + +```sh +sudo systemctl restart grace-editor-bootstrap +sudo systemctl start opencode2-grace-editor grace-editor-preview +sudo systemctl status grace-editor-bootstrap opencode2-grace-editor grace-editor-preview +``` + +For bootstrap errors, use `sudo journalctl -u grace-editor-bootstrap -b` in +the guest. Its home filesystem is explicitly mounted before boot activation, +then tmpfiles sets the volume root's ownership to `agent:users`. Dependency +installation is stamped only after `npm ci` succeeds and repeated after a +lockfile/Node version change or incomplete install. + +OpenCode's generated server password is available to the guest administrator +in `sudo journalctl -u opencode2-grace-editor -b`. Use it to connect the web UI, +then connect GitHub Copilot there. That OAuth connection and the repository +deploy key serve separate purposes. Both OpenCode state and the checkout are +on `/home/agent` and survive a VM restart. + +`just dev` checks the preview over guest loopback, while reporting the LAN URL +to Grace. The agent is deliberately unable to connect to that host URL itself. + ## Network ownership This component does **not** enable the NixOS global nftables/NAT services or diff --git a/components/website-editor/options.nix b/components/website-editor/options.nix index 50b25b4..74acbf1 100644 --- a/components/website-editor/options.nix +++ b/components/website-editor/options.nix @@ -5,6 +5,11 @@ in { options.components.website-editor = { enable = mkEnableOption "Grace Bobber's isolated website-editor MicroVM"; + authorizedKeys = mkOption { + type = types.listOf types.str; + default = [ ]; + description = "Public SSH keys for the guest agent account. SSH is reachable from the host only, not forwarded to the LAN."; + }; lan = { interface = mkOption { type = types.strMatching "[a-zA-Z0-9_.-]+"; diff --git a/components/website-editor/vm.nix b/components/website-editor/vm.nix index 505bf0a..62bd49d 100644 --- a/components/website-editor/vm.nix +++ b/components/website-editor/vm.nix @@ -2,10 +2,43 @@ let cfg = config.components.website-editor; hostName = "grace-editor"; - repo = "https://github.com/ajaxbits/gracebobber.git"; + repo = "git@github.com:ajaxbits/gracebobber.git"; + tools = with pkgs; [ + inputs.llm-agents.packages.${pkgs.stdenv.hostPlatform.system}.opencode2 + bash + curl + exiftool + fd + git + gh + imagemagick + jujutsu + jq + just + nodejs_22 + openssh + poppler-utils + pkg-config + python3 + ripgrep + vips + ]; + globalOpenCodeConfig = pkgs.writeText "grace-editor-opencode.json" (builtins.toJSON { + "$schema" = "https://opencode.ai/config.json"; + update = "disable"; + warming = false; + }); in { config = lib.mkIf cfg.enable { + programs.ssh.extraConfig = '' + Host grace-editor + HostName ${cfg.vm.ip} + User agent + IdentityFile ~/.ssh/grace-editor + IdentitiesOnly yes + ''; + # Intentionally no autostart: run `systemctl start microvm@grace-editor` # when the editing environment is wanted. microvm.vms.${hostName} = { @@ -42,24 +75,50 @@ in home = "/home/agent"; createHome = true; extraGroups = [ "wheel" ]; + openssh.authorizedKeys.keys = cfg.authorizedKeys; }; security.sudo.wheelNeedsPassword = false; - environment.systemPackages = with pkgs; [ - inputs.llm-agents.packages.${pkgs.stdenv.hostPlatform.system}.opencode2 - bash - curl - exiftool - fd - git - imagemagick - jj - just - nodejs_22 - poppler-utils - pkg-config - ripgrep - vips + environment.systemPackages = tools; + nix.settings.experimental-features = [ "nix-command" "flakes" ]; + # GitHub's published Ed25519 host key (https://api.github.com/meta). + # First-boot cloning must not block on an interactive trust prompt. + programs.ssh.knownHosts."github.com".publicKey = + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl"; + + services.openssh = { + enable = true; + openFirewall = false; + settings = { + PasswordAuthentication = false; + KbdInteractiveAuthentication = false; + PermitRootLogin = "no"; + AllowUsers = [ "agent" ]; + }; + hostKeys = [ { + path = "/home/agent/.ssh-host-keys/ssh_host_ed25519_key"; + type = "ed25519"; + } ]; + }; + # Management is host-initiated; the host forwards only the web ports. + networking.firewall.extraCommands = '' + iptables -A nixos-fw -s ${cfg.vm.gateway}/32 -p tcp --dport 22 -j nixos-fw-accept + ''; + networking.firewall.extraStopCommands = '' + iptables -D nixos-fw -s ${cfg.vm.gateway}/32 -p tcp --dport 22 -j nixos-fw-accept 2>/dev/null || true + ''; + + # A newly formatted volume has a root-owned filesystem root. User + # creation alone doesn't chown an existing home/mount point. + fileSystems."/home/agent".neededForBoot = true; + systemd.tmpfiles.rules = [ + "d /home/agent 0700 agent users -" + "d /home/agent/.ssh-host-keys 0700 root root -" + "d /home/agent/.config 0700 agent users -" + "d /home/agent/.config/opencode 0700 agent users -" + # Update policy is global-only in V2. L (without +) preserves an + # existing guest configuration rather than overwriting user edits. + "L /home/agent/.config/opencode/opencode.json - - - - ${globalOpenCodeConfig}" ]; microvm = { @@ -96,22 +155,39 @@ in wantedBy = [ "multi-user.target" ]; after = [ "network-online.target" ]; wants = [ "network-online.target" ]; - path = with pkgs; [ git jj nodejs_22 ]; + path = tools; + environment.HOME = "/home/agent"; + unitConfig.RequiresMountsFor = "/home/agent"; serviceConfig = { Type = "oneshot"; + RemainAfterExit = true; User = "agent"; WorkingDirectory = "/home/agent"; + TimeoutStartSec = "15min"; }; script = '' + export GIT_SSH_COMMAND="ssh -o BatchMode=yes -o ConnectTimeout=15" if [ ! -d gracebobber/.git ]; then git clone ${repo} gracebobber fi cd gracebobber + if [ -z "$(jj config get user.name 2>/dev/null || true)" ]; then + jj config set --user user.name "Grace website editor" + fi + if [ -z "$(jj config get user.email 2>/dev/null || true)" ]; then + jj config set --user user.email "grace-editor@localhost" + fi if [ ! -d .jj ]; then jj git init --colocate + jj bookmark track main --remote=origin fi - if [ ! -d node_modules ]; then + # A failed npm ci may leave node_modules behind. Only record the + # lockfile and Node version once an installation actually succeeds. + signature="$(sha256sum package-lock.json | cut -d ' ' -f1):$(node --version)" + if [ ! -x node_modules/.bin/astro ] || + [ "$(cat node_modules/.grace-editor-deps 2>/dev/null || true)" != "$signature" ]; then npm ci + printf '%s\n' "$signature" > node_modules/.grace-editor-deps fi ''; }; @@ -120,12 +196,14 @@ in wantedBy = [ "multi-user.target" ]; after = [ "grace-editor-bootstrap.service" ]; requires = [ "grace-editor-bootstrap.service" ]; + path = tools ++ [ "/run/wrappers" "/run/current-system/sw" ]; serviceConfig = { User = "agent"; WorkingDirectory = "/home/agent/gracebobber"; Environment = [ "HOME=/home/agent" - "GRACE_EDITOR_PREVIEW_URL=http://172.22.0.10:${toString cfg.previewPort}" + "GRACE_EDITOR_PREVIEW_URL=http://${cfg.lan.hostIP}:${toString cfg.previewPort}" + "GRACE_EDITOR_PREVIEW_CHECK_URL=http://127.0.0.1:${toString cfg.previewPort}" ]; ExecStart = "${inputs.llm-agents.packages.${pkgs.stdenv.hostPlatform.system}.opencode2}/bin/opencode2 serve --hostname 0.0.0.0 --port ${toString cfg.editorPort}"; Restart = "on-failure"; @@ -137,6 +215,8 @@ in wantedBy = [ "multi-user.target" ]; after = [ "grace-editor-bootstrap.service" ]; requires = [ "grace-editor-bootstrap.service" ]; + path = tools; + environment.HOME = "/home/agent"; serviceConfig = { User = "agent"; WorkingDirectory = "/home/agent/gracebobber"; diff --git a/hosts/patroclus/configuration.nix b/hosts/patroclus/configuration.nix index 2482ed4..c4fe72d 100644 --- a/hosts/patroclus/configuration.nix +++ b/hosts/patroclus/configuration.nix @@ -54,7 +54,13 @@ in "github:ajaxbits/config#patroclus"; }; cloudflared.enable = true; - website-editor.enable = true; + website-editor = { + enable = true; + authorizedKeys = [ + # Private key stays on patroclus in ~admin/.ssh/grace-editor. + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICDgXkJ7RFsYogbUgTKwxRcGx70e5pULOzDFl8e5i9yp patroclus grace-editor administration" + ]; + }; ebooks.enable = false; mediacenter = { enable = true; From 5c334a667c2b04f0da003915641f52ecb060bcb3 Mon Sep 17 00:00:00 2001 From: Alex Jackson Date: Sun, 6 Sep 2026 17:21:43 -0500 Subject: [PATCH 3/3] docs: distinguish deploy keys from Actions API access --- components/website-editor/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/website-editor/README.md b/components/website-editor/README.md index 5612126..a990199 100644 --- a/components/website-editor/README.md +++ b/components/website-editor/README.md @@ -45,6 +45,9 @@ write access for publishing. Keep the private half inside the guest. The bootstrap uses SSH transport and a pinned GitHub host key; it never receives the administrator's broad GitHub token. Bootstrap initially fails until this deploy key is provisioned; guest SSH remains available independently. +Deploy keys authenticate Git operations only. Watching private Actions runs +with `gh` additionally requires a repository-scoped token with Actions read +access; Copilot authentication is not a substitute for that permission. Then run **inside the guest**: