From f89f07ae9844947611d32c9d371d6d85bc544ba0 Mon Sep 17 00:00:00 2001 From: Leynos Date: Fri, 3 Oct 2025 12:25:49 +0100 Subject: [PATCH 1/2] Enhance Vault bootstrap helper configuration --- docs/vault-appliance-module-design.md | 14 + docs/vault-applience-bootstrap-process.md | 87 ++++ scripts/__init__.py | 3 + scripts/bootstrap_vault_appliance.py | 419 ++++++++++++++++++ scripts/tests/_vendor/cmd_mox/__init__.py | 119 +++++ scripts/tests/conftest.py | 13 + .../tests/test_bootstrap_vault_appliance.py | 359 +++++++++++++++ 7 files changed, 1014 insertions(+) create mode 100644 docs/vault-applience-bootstrap-process.md create mode 100644 scripts/__init__.py create mode 100644 scripts/bootstrap_vault_appliance.py create mode 100644 scripts/tests/_vendor/cmd_mox/__init__.py create mode 100644 scripts/tests/conftest.py create mode 100644 scripts/tests/test_bootstrap_vault_appliance.py diff --git a/docs/vault-appliance-module-design.md b/docs/vault-appliance-module-design.md index 25c8aeb18..9b319d6e4 100644 --- a/docs/vault-appliance-module-design.md +++ b/docs/vault-appliance-module-design.md @@ -56,6 +56,20 @@ material required to bootstrap Vault in a deterministic, GitOps-friendly way. volumes, and load balancer to a DigitalOcean project via `digitalocean_project_resources`, keeping billing and governance aligned with wider platform conventions. +- **Bootstrap helper strategy (2024-12-06).** The Python bootstrapper resolves + the appliance IP via DigitalOcean tags before touching Vault. It records the + generated root token and unseal shares in DigitalOcean Secrets Manager using + a configurable prefix so repeated runs converge without manual data capture. + Subsequent invocations read the stored material, unseal Vault only when + necessary, and ensure the `secret/` KV v2 engine plus the `doks` AppRole exist + with deterministic TTLs. The helper writes the AppRole policy from a + temporary file to sidestep shell heredocs and stores the resulting role and + secret identifiers alongside the unseal keys for reuse by CI workflows. Tests + exercise happy and unhappy paths via `cmd-mox` backed command mocks to + validate DigitalOcean, Vault, and SSH interactions without hitting live APIs. + Operators can override the Vault address when routing through the managed + load balancer and supply a custom CA bundle so Vault CLI calls validate the + TLS chain without disabling verification. ## Future work diff --git a/docs/vault-applience-bootstrap-process.md b/docs/vault-applience-bootstrap-process.md new file mode 100644 index 000000000..d1f06a3c5 --- /dev/null +++ b/docs/vault-applience-bootstrap-process.md @@ -0,0 +1,87 @@ +# Vault appliance bootstrap process + +This guide explains how to initialise and maintain the DigitalOcean-hosted Vault +appliance with `scripts/bootstrap_vault_appliance.py`. The helper discovers the +appliance droplet, verifies Vault is running, initialises and unseals the +cluster when needed, enables the KV v2 engine, and provisions the AppRole used +by the DOKS deployment workflow. Re-running the script is safe: existing +configuration is reused whenever possible. + +## Prerequisites + +Before running the helper ensure the following tools are installed and +authenticated on the workstation or CI runner executing the script: + +- `doctl` logged in with access to the target DigitalOcean account. +- `vault` CLI able to reach the appliance network location. +- SSH access to the droplet using the user supplied via `--ssh-user` (defaults to + `root`). +- The Vault CA bundle and server credentials exported from the + `vault_appliance` OpenTofu module outputs. Save the CA certificate to disk so + it can be supplied through `--ca-cert-path`. + +The DigitalOcean Secrets Manager namespace identified by `--secret-prefix` must +exist. The script will create or update individual secrets inside that +namespace. + +## Running the script + +The helper may be launched directly thanks to the embedded [`uv`](https://github.com/astral-sh/uv) +metadata: + +```sh +./scripts/bootstrap_vault_appliance.py \ + --environment dev \ + --droplet-tag vault-dev \ + --secret-prefix dev-vault \ + --vault-address https://vault.dev.example:8200 \ + --ca-cert-path /secure/path/vault-ca.pem +``` + +Key options: + +- `--environment` and `--secret-prefix` scope the DigitalOcean Secrets Manager + entries used to persist unseal keys, the root token, and AppRole credentials. +- `--droplet-tag` locates the appliance droplet. The helper aborts if multiple + droplets share the tag to avoid operating on the wrong instance. +- `--vault-address` overrides the Vault API endpoint used by the CLI. When not + provided, the helper derives `https://:8200` automatically. +- `--ca-cert-path` writes the certificate bundle to `VAULT_CACERT` so the Vault + CLI trusts the appliance's TLS certificate. Omit the flag only when the CA is + already trusted by the host system. +- `--mount-path`, `--approle-name`, and `--policy-name` customise the KV engine + path and DOKS AppRole naming. + +All options accept the values documented in `scripts/bootstrap_vault_appliance.py +--help`. + +## Secrets management + +On the first run the helper: + +1. Initialises Vault with the requested number of key shares and threshold. +2. Stores each unseal share and the generated root token inside the secrets + manager under `-unseal-N` and `-root-token`. +3. Enables the KV v2 engine and writes the AppRole policy before generating a + role ID and secret ID. These are saved as `-role-id` and + `-secret-id`. + +Subsequent executions read the stored secrets, unseal Vault if necessary, and +update the AppRole credentials in-place. Failed operations (for example missing +unseal shares or an unhealthy systemd unit) raise descriptive errors without +modifying the appliance. + +## Troubleshooting + +- **`Vault remains sealed`** – verify that all unseal secrets exist in the + DigitalOcean Secrets Manager namespace. Missing shares cause the helper to + abort before any unseal attempts. +- **`certificate verify failed`** – ensure the CA bundle path supplied via + `--ca-cert-path` matches the file exported from the module outputs and that + the executing user can read it. +- **`No droplets tagged`** – confirm the OpenTofu module outputs include the tag + referenced by `--droplet-tag` and that the droplet is active in the target + account and region. + +For advanced usage, including integrating the helper in GitHub Actions, see the +architecture notes in `docs/cloud-native-ephemeral-previews.md`. diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 000000000..b737c1a9a --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1,3 @@ +"""Utility scripts package namespace for Wildside automation.""" + +from __future__ import annotations diff --git a/scripts/bootstrap_vault_appliance.py b/scripts/bootstrap_vault_appliance.py new file mode 100644 index 000000000..9a601bcff --- /dev/null +++ b/scripts/bootstrap_vault_appliance.py @@ -0,0 +1,419 @@ +#!/usr/bin/env -S uv run python +# /// script +# requires-python = ">=3.13" +# dependencies = ["plumbum"] +# /// + +"""Bootstrap the DigitalOcean-hosted Vault appliance. + +The helper discovers the droplet via tag lookup, verifies Vault is running, +initialises and unseals the cluster, enables the KV v2 secrets engine, and +provisions the DOKS AppRole while storing credentials in DigitalOcean Secrets +Manager. It is deliberately idempotent so reruns converge existing +deployments.""" + +from __future__ import annotations + +import argparse +import json +import sys +from dataclasses import dataclass +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Iterable, Sequence + +from plumbum import ProcessExecutionError, local + + +@dataclass(frozen=True) +class BootstrapOptions: + """Configuration derived from command line arguments.""" + + environment: str + droplet_tag: str + ssh_user: str + mount_path: str + approle_name: str + policy_name: str + secret_prefix: str + key_shares: int + key_threshold: int + vault_address: str | None = None + ca_cert_path: str | None = None + + +class CommandRunner: + """Thin wrapper around :mod:`plumbum` for deterministic command execution.""" + + def __init__(self, local_module=local): + self._local = local_module + + def run(self, command: str, *args: str, env: dict[str, str] | None = None) -> str: + cmd = self._local[command] + if env: + cmd = cmd.with_env(**env) + if args: + cmd = cmd[args] + return cmd() + + +class SecretStore: + """DigitalOcean Secrets Manager helper around ``doctl`` commands.""" + + def __init__(self, runner: CommandRunner, prefix: str): + self._runner = runner + self._prefix = prefix + + def _name(self, suffix: str) -> str: + return f"{self._prefix}-{suffix}" + + def get(self, suffix: str) -> str | None: + name = self._name(suffix) + try: + output = self._runner.run( + "doctl", + "secrets", + "manager", + "secrets", + "get", + name, + "--output", + "json", + ) + except ProcessExecutionError as exc: + if exc.retcode == 1: + return None + raise + payload = json.loads(output) + # ``doctl`` returns ``{"secret": {"value": "..."}}`` for ``--output json`` + try: + return payload["secret"]["value"] + except (KeyError, TypeError) as err: + raise RuntimeError(f"Unexpected secret payload for {name}") from err + + def put(self, suffix: str, value: str) -> None: + name = self._name(suffix) + try: + self._runner.run( + "doctl", + "secrets", + "manager", + "secrets", + "create", + name, + "--data", + value, + ) + except ProcessExecutionError as exc: + if exc.retcode != 10: + raise + # ``create`` exits 10 when the secret already exists. Update instead. + self._runner.run( + "doctl", + "secrets", + "manager", + "secrets", + "update", + name, + "--data", + value, + ) + + +def parse_args(argv: Sequence[str] | None = None) -> BootstrapOptions: + parser = argparse.ArgumentParser(description="Bootstrap the Vault appliance") + parser.add_argument( + "--environment", + required=True, + help="Logical environment identifier (for example 'dev').", + ) + parser.add_argument( + "--droplet-tag", + required=True, + help="DigitalOcean tag used to discover the Vault droplet.", + ) + parser.add_argument( + "--ssh-user", + default="root", + help="SSH user with access to the appliance (default: root).", + ) + parser.add_argument( + "--mount-path", + default="secret", + help="KV v2 mount point used for application secrets (default: secret).", + ) + parser.add_argument( + "--approle-name", + default="doks-deployer", + help="Name of the AppRole consumed by the DOKS workflow.", + ) + parser.add_argument( + "--policy-name", + default="doks-deployer", + help="Name of the policy bound to the DOKS AppRole.", + ) + parser.add_argument( + "--secret-prefix", + required=True, + help="Prefix for DigitalOcean Secrets that store unseal material.", + ) + parser.add_argument( + "--key-shares", + type=int, + default=5, + help="Number of unseal key shares to generate when initialising Vault.", + ) + parser.add_argument( + "--key-threshold", + type=int, + default=3, + help="Number of unseal shares required to unseal Vault.", + ) + parser.add_argument( + "--vault-address", + help=( + "Override the Vault API address. Defaults to the discovered droplet" + " IP on port 8200 over HTTPS." + ), + ) + parser.add_argument( + "--ca-cert-path", + help=( + "Optional path to the Vault certificate authority bundle. When set," + " exported as VAULT_CACERT for CLI calls." + ), + ) + args = parser.parse_args(argv) + return BootstrapOptions( + environment=args.environment, + droplet_tag=args.droplet_tag, + ssh_user=args.ssh_user, + mount_path=args.mount_path.rstrip("/"), + approle_name=args.approle_name, + policy_name=args.policy_name, + secret_prefix=args.secret_prefix, + key_shares=args.key_shares, + key_threshold=args.key_threshold, + vault_address=args.vault_address.rstrip("/") if args.vault_address else None, + ca_cert_path=args.ca_cert_path, + ) + + +def discover_droplet_ip(options: BootstrapOptions, runner: CommandRunner) -> str: + output = runner.run( + "doctl", + "compute", + "droplet", + "list", + "--tag-name", + options.droplet_tag, + "--format", + "PublicIPv4", + "--no-header", + ) + addresses = [line.strip() for line in output.splitlines() if line.strip()] + if not addresses: + raise RuntimeError( + f"No droplets tagged '{options.droplet_tag}' found in DigitalOcean." + ) + if len(addresses) > 1: + raise RuntimeError( + "Multiple droplets matched the Vault tag; aborting to avoid ambiguity." + ) + return addresses[0] + + +def verify_vault_service(ip: str, options: BootstrapOptions, runner: CommandRunner) -> None: + status = runner.run( + "ssh", + f"{options.ssh_user}@{ip}", + "sudo", + "systemctl", + "is-active", + "vault", + ).strip() + if status != "active": + raise RuntimeError(f"Vault systemd unit is not active (reported '{status}').") + + +def read_vault_status(runner: CommandRunner, env: dict[str, str]) -> dict: + output = runner.run("vault", "status", "-format=json", env=env) + return json.loads(output) + + +def initialise_vault( + options: BootstrapOptions, + runner: CommandRunner, + env: dict[str, str], + secrets: SecretStore, +) -> dict: + init_payload = runner.run( + "vault", + "operator", + "init", + "-key-shares", + str(options.key_shares), + "-key-threshold", + str(options.key_threshold), + "-format=json", + env=env, + ) + init_data = json.loads(init_payload) + unseal_keys: Iterable[str] = init_data.get("unseal_keys_b64", []) + root_token: str | None = init_data.get("root_token") + if not root_token: + raise RuntimeError("Vault did not return a root token during initialisation.") + for index, key in enumerate(unseal_keys, start=1): + secrets.put(f"unseal-{index}", key) + secrets.put("root-token", root_token) + return {"unseal_keys": list(unseal_keys), "root_token": root_token} + + +def load_unseal_keys(options: BootstrapOptions, secrets: SecretStore) -> list[str]: + keys: list[str] = [] + for index in range(1, options.key_shares + 1): + key = secrets.get(f"unseal-{index}") + if key: + keys.append(key) + return keys + + +def unseal_vault( + keys: Sequence[str], + options: BootstrapOptions, + runner: CommandRunner, + env: dict[str, str], +) -> None: + if len(keys) < options.key_threshold: + raise RuntimeError( + "Insufficient unseal keys available; aborting to keep Vault sealed." + ) + for key in keys[: options.key_threshold]: + runner.run("vault", "operator", "unseal", key, env=env) + + +def ensure_kv_mount(options: BootstrapOptions, runner: CommandRunner, env: dict[str, str]) -> None: + mounts = json.loads(runner.run("vault", "secrets", "list", "-format=json", env=env)) + mount_path = f"{options.mount_path}/" + current = mounts.get(mount_path) + if current and current.get("type") == "kv" and current.get("options", {}).get("version") == "2": + return + runner.run( + "vault", + "secrets", + "enable", + "-path", + options.mount_path, + "kv-v2", + env=env, + ) + + +def ensure_approle( + options: BootstrapOptions, + runner: CommandRunner, + env: dict[str, str], + secrets: SecretStore, +) -> None: + auth_methods = json.loads( + runner.run("vault", "auth", "list", "-format=json", env=env) + ) + if "approle/" not in auth_methods: + runner.run("vault", "auth", "enable", "approle", env=env) + + policy_hcl = ( + f'path "{options.mount_path}/data/*" {{\n' + " capabilities = [\"read\", \"list\"]\n" + "}\n" + ) + with TemporaryDirectory(prefix="vault-policy-") as tempdir: + policy_path = Path(tempdir, f"{options.policy_name}.hcl") + policy_path.write_text(policy_hcl, encoding="utf-8") + runner.run( + "vault", + "policy", + "write", + options.policy_name, + str(policy_path), + env=env, + ) + + runner.run( + "vault", + "write", + f"auth/approle/role/{options.approle_name}", + f"token_policies={options.policy_name}", + "secret_id_ttl=24h", + "token_ttl=1h", + "token_max_ttl=4h", + env=env, + ) + + role_id = runner.run( + "vault", + "read", + "-field=role_id", + f"auth/approle/role/{options.approle_name}/role-id", + env=env, + ).strip() + secret_id = runner.run( + "vault", + "write", + "-f", + "-field=secret_id", + f"auth/approle/role/{options.approle_name}/secret-id", + env=env, + ).strip() + + secrets.put("role-id", role_id) + secrets.put("secret-id", secret_id) + + +def bootstrap(options: BootstrapOptions, runner: CommandRunner | None = None) -> None: + command_runner = runner or CommandRunner() + secrets = SecretStore(command_runner, options.secret_prefix) + + ip_address = discover_droplet_ip(options, command_runner) + verify_vault_service(ip_address, options, command_runner) + + address = options.vault_address or f"https://{ip_address}:8200" + vault_env: dict[str, str] = {"VAULT_ADDR": address} + if options.ca_cert_path: + vault_env["VAULT_CACERT"] = options.ca_cert_path + status = read_vault_status(command_runner, vault_env) + + if not status.get("initialized", False): + init_data = initialise_vault(options, command_runner, vault_env, secrets) + unseal_vault(init_data["unseal_keys"], options, command_runner, vault_env) + vault_env["VAULT_TOKEN"] = init_data["root_token"] + else: + if status.get("sealed", False): + unseal_keys = load_unseal_keys(options, secrets) + unseal_vault(unseal_keys, options, command_runner, vault_env) + root_token = secrets.get("root-token") + if not root_token: + raise RuntimeError( + "Vault is initialised but no root token is stored; cannot proceed." + ) + vault_env["VAULT_TOKEN"] = root_token + + post_status = read_vault_status(command_runner, vault_env) + if post_status.get("sealed", False): + raise RuntimeError("Vault remains sealed after attempting to unseal it.") + + ensure_kv_mount(options, command_runner, vault_env) + ensure_approle(options, command_runner, vault_env, secrets) + + +def main(argv: Sequence[str] | None = None) -> int: + try: + options = parse_args(argv) + bootstrap(options) + except Exception as exc: # noqa: BLE001 - top-level guard + print(f"Error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/_vendor/cmd_mox/__init__.py b/scripts/tests/_vendor/cmd_mox/__init__.py new file mode 100644 index 000000000..e7097e667 --- /dev/null +++ b/scripts/tests/_vendor/cmd_mox/__init__.py @@ -0,0 +1,119 @@ +"""Minimal cmd-mox compatible helpers for command mocking in tests.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Dict, Iterable, List, Tuple + +from plumbum import ProcessExecutionError + + +@dataclass +class CommandCall: + args: Tuple[str, ...] + env: Dict[str, str] + + +@dataclass +class CommandResponse: + args: Tuple[str, ...] + stdout: str = "" + stderr: str = "" + exit_code: int = 0 + + +class MockCommandInvocation: + def __init__(self, command: "MockCommand", args: Iterable[str], env: Dict[str, str] | None = None): + self._command = command + self._args = tuple(args) + self._env = dict(env or {}) + + def __getitem__(self, more_args: Iterable[str] | str) -> "MockCommandInvocation": + if isinstance(more_args, tuple): + args = self._args + more_args + elif isinstance(more_args, list): + args = self._args + tuple(more_args) + else: + args = self._args + (more_args,) + return MockCommandInvocation(self._command, args, self._env) + + def with_env(self, **env: str) -> "MockCommandInvocation": + merged = dict(self._env) + merged.update(env) + return MockCommandInvocation(self._command, self._args, merged) + + def __call__(self) -> str: + return self._command.execute(self._args, self._env) + + +class MockCommand: + def __init__(self, name: str): + self.name = name + self.calls: List[CommandCall] = [] + self._queue: List[CommandResponse] = [] + + def queue(self, *args: str, stdout: str = "", stderr: str = "", exit_code: int = 0) -> None: + self._queue.append(CommandResponse(tuple(args), stdout, stderr, exit_code)) + + def execute(self, args: Tuple[str, ...], env: Dict[str, str]) -> str: + if not self._queue: + raise AssertionError(f"No queued responses remaining for command '{self.name}'.") + response = self._queue.pop(0) + if response.args and response.args != args: + raise AssertionError( + f"Command '{self.name}' expected args {response.args!r} but received {args!r}." + ) + self.calls.append(CommandCall(args=args, env=env)) + if response.exit_code: + raise ProcessExecutionError( + [self.name, *args], + response.exit_code, + response.stdout, + response.stderr, + ) + return response.stdout + + def __getitem__(self, args: Iterable[str] | str) -> MockCommandInvocation: + if isinstance(args, tuple): + values = args + elif isinstance(args, list): + values = tuple(args) + else: + values = (args,) + return MockCommandInvocation(self, values) + + def with_env(self, **env: str) -> MockCommandInvocation: + return MockCommandInvocation(self, tuple(), env) + + +class LocalProxy: + def __init__(self, registry: "CommandRegistry"): + self._registry = registry + + def __getitem__(self, name: str) -> MockCommand: + if name not in self._registry.commands: + raise KeyError(f"Command '{name}' not registered in CommandRegistry") + return self._registry.commands[name] + + +class CommandRegistry: + def __init__(self): + self.commands: Dict[str, MockCommand] = {} + self.local_proxy = LocalProxy(self) + + def create(self, name: str) -> MockCommand: + command = MockCommand(name) + self.commands[name] = command + return command + + def attach(self, module: Any) -> None: + module.local = self.local_proxy + + def detach(self, module: Any) -> None: + raise NotImplementedError("Detach is not supported by this stub.") + + +__all__ = [ + "CommandRegistry", + "MockCommand", +] diff --git a/scripts/tests/conftest.py b/scripts/tests/conftest.py new file mode 100644 index 000000000..2f983e7c6 --- /dev/null +++ b/scripts/tests/conftest.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +import sys +from pathlib import Path + + +def pytest_configure() -> None: + project_root = Path(__file__).resolve().parents[2] + if str(project_root) not in sys.path: + sys.path.insert(0, str(project_root)) + vendor_dir = Path(__file__).resolve().parent / "_vendor" + if str(vendor_dir) not in sys.path: + sys.path.insert(0, str(vendor_dir)) diff --git a/scripts/tests/test_bootstrap_vault_appliance.py b/scripts/tests/test_bootstrap_vault_appliance.py new file mode 100644 index 000000000..fab8e9608 --- /dev/null +++ b/scripts/tests/test_bootstrap_vault_appliance.py @@ -0,0 +1,359 @@ +from __future__ import annotations + +import json +from dataclasses import replace + +import pytest + +from cmd_mox import CommandRegistry + +from scripts.bootstrap_vault_appliance import BootstrapOptions, CommandRunner, bootstrap + + +def make_options() -> BootstrapOptions: + return BootstrapOptions( + environment="dev", + droplet_tag="vault-dev", + ssh_user="root", + mount_path="secret", + approle_name="doks", + policy_name="doks", + secret_prefix="dev-vault", + key_shares=5, + key_threshold=3, + vault_address="https://vault.dev.example:8200", + ) + + +def test_bootstrap_initialises_and_configures_vault(tmp_path) -> None: + registry = CommandRegistry() + doctl = registry.create("doctl") + vault = registry.create("vault") + ssh = registry.create("ssh") + runner = CommandRunner(local_module=registry.local_proxy) + + options = make_options() + + doctl.queue( + "compute", + "droplet", + "list", + "--tag-name", + options.droplet_tag, + "--format", + "PublicIPv4", + "--no-header", + stdout="203.0.113.10\n", + ) + ssh.queue( + f"{options.ssh_user}@203.0.113.10", + "sudo", + "systemctl", + "is-active", + "vault", + stdout="active\n", + ) + vault.queue( + "status", + "-format=json", + stdout=json.dumps({"initialized": False, "sealed": True}), + ) + init_payload = json.dumps( + { + "unseal_keys_b64": [ + "key-1", + "key-2", + "key-3", + "key-4", + "key-5", + ], + "root_token": "root-token", + } + ) + vault.queue( + "operator", + "init", + "-key-shares", + str(options.key_shares), + "-key-threshold", + str(options.key_threshold), + "-format=json", + stdout=init_payload, + ) + for index in range(1, options.key_shares + 1): + doctl.queue( + "secrets", + "manager", + "secrets", + "create", + f"{options.secret_prefix}-unseal-{index}", + "--data", + f"key-{index}", + ) + doctl.queue( + "secrets", + "manager", + "secrets", + "create", + f"{options.secret_prefix}-root-token", + "--data", + "root-token", + ) + for index in range(1, options.key_threshold + 1): + vault.queue("operator", "unseal", f"key-{index}") + vault.queue( + "status", + "-format=json", + stdout=json.dumps({"initialized": True, "sealed": False}), + ) + vault.queue( + "secrets", + "list", + "-format=json", + stdout=json.dumps({}), + ) + vault.queue( + "secrets", + "enable", + "-path", + options.mount_path, + "kv-v2", + ) + vault.queue( + "auth", + "list", + "-format=json", + stdout=json.dumps({}), + ) + vault.queue("auth", "enable", "approle") + vault.queue() + vault.queue( + "write", + f"auth/approle/role/{options.approle_name}", + f"token_policies={options.policy_name}", + "secret_id_ttl=24h", + "token_ttl=1h", + "token_max_ttl=4h", + ) + vault.queue( + "read", + "-field=role_id", + f"auth/approle/role/{options.approle_name}/role-id", + stdout="role-123\n", + ) + vault.queue( + "write", + "-f", + "-field=secret_id", + f"auth/approle/role/{options.approle_name}/secret-id", + stdout="secret-456\n", + ) + doctl.queue( + "secrets", + "manager", + "secrets", + "create", + f"{options.secret_prefix}-role-id", + "--data", + "role-123", + ) + doctl.queue( + "secrets", + "manager", + "secrets", + "create", + f"{options.secret_prefix}-secret-id", + "--data", + "secret-456", + ) + + bootstrap(options, runner=runner) + + policy_calls = [call for call in vault.calls if call.args[:2] == ("policy", "write")] + assert len(policy_calls) == 1 + assert policy_calls[0].env.get("VAULT_TOKEN") == "root-token" + assert policy_calls[0].env.get("VAULT_ADDR") == options.vault_address + + +def test_bootstrap_reuses_existing_configuration() -> None: + registry = CommandRegistry() + doctl = registry.create("doctl") + vault = registry.create("vault") + ssh = registry.create("ssh") + runner = CommandRunner(local_module=registry.local_proxy) + + options = replace(make_options(), ca_cert_path="/tmp/vault-ca.pem") + + doctl.queue( + "compute", + "droplet", + "list", + "--tag-name", + options.droplet_tag, + "--format", + "PublicIPv4", + "--no-header", + stdout="203.0.113.10\n", + ) + ssh.queue( + f"{options.ssh_user}@203.0.113.10", + "sudo", + "systemctl", + "is-active", + "vault", + stdout="active\n", + ) + vault.queue( + "status", + "-format=json", + stdout=json.dumps({"initialized": True, "sealed": False}), + ) + doctl.queue( + "secrets", + "manager", + "secrets", + "get", + f"{options.secret_prefix}-root-token", + "--output", + "json", + stdout=json.dumps({"secret": {"value": "root-token"}}), + ) + vault.queue( + "status", + "-format=json", + stdout=json.dumps({"initialized": True, "sealed": False}), + ) + vault.queue( + "secrets", + "list", + "-format=json", + stdout=json.dumps( + {"secret/": {"type": "kv", "options": {"version": "2"}}} + ), + ) + vault.queue( + "auth", + "list", + "-format=json", + stdout=json.dumps({"approle/": {}}), + ) + vault.queue() + vault.queue( + "write", + f"auth/approle/role/{options.approle_name}", + f"token_policies={options.policy_name}", + "secret_id_ttl=24h", + "token_ttl=1h", + "token_max_ttl=4h", + ) + vault.queue( + "read", + "-field=role_id", + f"auth/approle/role/{options.approle_name}/role-id", + stdout="role-abc\n", + ) + vault.queue( + "write", + "-f", + "-field=secret_id", + f"auth/approle/role/{options.approle_name}/secret-id", + stdout="secret-xyz\n", + ) + doctl.queue( + "secrets", + "manager", + "secrets", + "create", + f"{options.secret_prefix}-role-id", + "--data", + "role-abc", + exit_code=10, + ) + doctl.queue( + "secrets", + "manager", + "secrets", + "update", + f"{options.secret_prefix}-role-id", + "--data", + "role-abc", + ) + doctl.queue( + "secrets", + "manager", + "secrets", + "create", + f"{options.secret_prefix}-secret-id", + "--data", + "secret-xyz", + exit_code=10, + ) + doctl.queue( + "secrets", + "manager", + "secrets", + "update", + f"{options.secret_prefix}-secret-id", + "--data", + "secret-xyz", + ) + + bootstrap(options, runner=runner) + + init_calls = [call for call in vault.calls if call.args[:2] == ("operator", "init")] + assert not init_calls + first_call = vault.calls[0] + assert first_call.env.get("VAULT_ADDR") == options.vault_address + assert first_call.env.get("VAULT_CACERT") == "/tmp/vault-ca.pem" + + +def test_bootstrap_aborts_when_unseal_keys_missing() -> None: + registry = CommandRegistry() + doctl = registry.create("doctl") + vault = registry.create("vault") + ssh = registry.create("ssh") + runner = CommandRunner(local_module=registry.local_proxy) + + options = replace(make_options(), vault_address=None) + + doctl.queue( + "compute", + "droplet", + "list", + "--tag-name", + options.droplet_tag, + "--format", + "PublicIPv4", + "--no-header", + stdout="203.0.113.10\n", + ) + ssh.queue( + f"{options.ssh_user}@203.0.113.10", + "sudo", + "systemctl", + "is-active", + "vault", + stdout="active\n", + ) + vault.queue( + "status", + "-format=json", + stdout=json.dumps({"initialized": True, "sealed": True}), + ) + for index in range(1, options.key_shares + 1): + doctl.queue( + "secrets", + "manager", + "secrets", + "get", + f"{options.secret_prefix}-unseal-{index}", + "--output", + "json", + exit_code=1, + ) + + with pytest.raises(RuntimeError, match="Insufficient unseal keys"): + bootstrap(options, runner=runner) + + assert vault.calls[0].env.get("VAULT_ADDR") == "https://203.0.113.10:8200" + assert "VAULT_CACERT" not in vault.calls[0].env From 54130803549560b0648e891f716cef06991e5623 Mon Sep 17 00:00:00 2001 From: Leynos Date: Fri, 3 Oct 2025 14:56:40 +0100 Subject: [PATCH 2/2] Refactor bootstrap test setup --- .../tests/test_bootstrap_vault_appliance.py | 46 ++++++++++++++----- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/scripts/tests/test_bootstrap_vault_appliance.py b/scripts/tests/test_bootstrap_vault_appliance.py index fab8e9608..3a2ea5db1 100644 --- a/scripts/tests/test_bootstrap_vault_appliance.py +++ b/scripts/tests/test_bootstrap_vault_appliance.py @@ -5,7 +5,7 @@ import pytest -from cmd_mox import CommandRegistry +from cmd_mox import CommandRegistry, MockCommand from scripts.bootstrap_vault_appliance import BootstrapOptions, CommandRunner, bootstrap @@ -25,15 +25,9 @@ def make_options() -> BootstrapOptions: ) -def test_bootstrap_initialises_and_configures_vault(tmp_path) -> None: - registry = CommandRegistry() - doctl = registry.create("doctl") - vault = registry.create("vault") - ssh = registry.create("ssh") - runner = CommandRunner(local_module=registry.local_proxy) - - options = make_options() - +def setup_droplet_discovery_mock( + doctl: MockCommand, ssh: MockCommand, options: BootstrapOptions, ip: str +) -> None: doctl.queue( "compute", "droplet", @@ -43,16 +37,19 @@ def test_bootstrap_initialises_and_configures_vault(tmp_path) -> None: "--format", "PublicIPv4", "--no-header", - stdout="203.0.113.10\n", + stdout=f"{ip}\n", ) ssh.queue( - f"{options.ssh_user}@203.0.113.10", + f"{options.ssh_user}@{ip}", "sudo", "systemctl", "is-active", "vault", stdout="active\n", ) + + +def setup_vault_init_mock(vault: MockCommand, doctl: MockCommand, options: BootstrapOptions) -> None: vault.queue( "status", "-format=json", @@ -99,6 +96,9 @@ def test_bootstrap_initialises_and_configures_vault(tmp_path) -> None: "--data", "root-token", ) + + +def setup_vault_unseal_mock(vault: MockCommand, options: BootstrapOptions) -> None: for index in range(1, options.key_threshold + 1): vault.queue("operator", "unseal", f"key-{index}") vault.queue( @@ -106,6 +106,9 @@ def test_bootstrap_initialises_and_configures_vault(tmp_path) -> None: "-format=json", stdout=json.dumps({"initialized": True, "sealed": False}), ) + + +def setup_kv_mount_mock(vault: MockCommand, options: BootstrapOptions) -> None: vault.queue( "secrets", "list", @@ -119,6 +122,9 @@ def test_bootstrap_initialises_and_configures_vault(tmp_path) -> None: options.mount_path, "kv-v2", ) + + +def setup_approle_mock(vault: MockCommand, doctl: MockCommand, options: BootstrapOptions) -> None: vault.queue( "auth", "list", @@ -167,6 +173,22 @@ def test_bootstrap_initialises_and_configures_vault(tmp_path) -> None: "secret-456", ) + +def test_bootstrap_initialises_and_configures_vault(tmp_path) -> None: + registry = CommandRegistry() + doctl = registry.create("doctl") + vault = registry.create("vault") + ssh = registry.create("ssh") + runner = CommandRunner(local_module=registry.local_proxy) + + options = make_options() + + setup_droplet_discovery_mock(doctl, ssh, options, "203.0.113.10") + setup_vault_init_mock(vault, doctl, options) + setup_vault_unseal_mock(vault, options) + setup_kv_mount_mock(vault, options) + setup_approle_mock(vault, doctl, options) + bootstrap(options, runner=runner) policy_calls = [call for call in vault.calls if call.args[:2] == ("policy", "write")]