diff --git a/Cargo.lock b/Cargo.lock index bd4680a..67a0aa0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,11 +18,13 @@ dependencies = [ "bollard", "chrono", "futures-util", + "git2", "rand 0.8.6", "serde", "serde_json", "serde_yaml", "tar", + "tempfile", "thiserror 1.0.69", "tokio", "tracing", diff --git a/DOC.md b/DOC.md index 5495700..9b2e528 100644 --- a/DOC.md +++ b/DOC.md @@ -88,12 +88,59 @@ Since this is meant to run **API-only**, omit `frontend`: | `SSH_HOST_KEY_PATH` | `./data/ssh_host_key` | Persisted SSH host key (generated on first run) | | `HIQLITE_API_ADDR` | `127.0.0.1:8200` | Embedded DB node's internal API address | | `HIQLITE_RAFT_ADDR` | `127.0.0.1:8100` | Embedded DB node's internal Raft address | -| `DOCKER_SOCKET_PATH` | platform default | Docker socket for Actions job execution + dev workspaces | +| `DOCKER_SOCKET_PATH` | platform default | Docker (or Docker-API-compatible) socket for Actions job execution + dev workspaces — see §2a | | `SECRETS_ENCRYPTION_KEY` | *(unset)* | 32-byte base64 key for encrypting Actions secrets (`setRepoSecret`); required for that feature only | | `PACKAGES_ROOT_PATH` | `{repos_root_path}/../packages` | Package registry storage | | `FRONTEND_URL` | `http://localhost:3000` | Used for any redirect targets referencing the frontend | | `ADMIN_BOOTSTRAP_TOKEN` | *(unset)* | See §3 — pure-API bootstrap credential | +### 2a. Container isolation (avoiding a privileged host Docker socket) + +CI job execution (`actions::Executor`) and dev workspaces (`dev_env::WorkspaceManager`) +both work by talking to a Docker Engine API over a socket — the shipped +`docker-compose.yml` does this by bind-mounting the **host's own** +`/var/run/docker.sock` into the `server` container ("Docker outside of +Docker", not Docker-in-Docker/DinD — there's no nested `dockerd` — but the +practical risk is the same shape: anything that can reach that socket has +root-equivalent control of the *host*, since it can start a container with +`-v /:/host` and chroot into it). This is the simplest thing that works, +but it's the single biggest privilege-escalation surface in a default +deployment. + +`DOCKER_SOCKET_PATH` is a real, wired-up override (not just accepted and +logged) for both `server` and the standalone `runner` binary — point it at +a socket with a smaller blast radius instead of the host's main daemon: + +- **Rootless Podman** (recommended, no code changes needed — Podman's + socket speaks the same Docker Engine API `bollard` already uses): + ```bash + systemctl --user enable --now podman.socket + # DOCKER_SOCKET_PATH=/run/user/$(id -u)/podman/podman.sock + ``` + Running as an unprivileged user, in its own user namespace, means a + container escape lands in that user's namespace, not root on the host. + This is the same technique Docker's own `rootless` mode and CI systems + like GitLab increasingly default to instead of the historical + `docker:dind` sidecar. +- **Sysbox** (`nestybox/sysbox`) if a workload genuinely needs to run its + own nested Docker/Kubernetes (e.g. a CI job whose `run:` steps do + `docker build`, which rootless Podman alone doesn't help with) — an + alternative OCI runtime that gives a container real user-namespace + isolation *and* lets it run Docker-in-Docker safely, without + `--privileged` and without a host socket mount at all. Run the `server`/ + `runner` container itself with `--runtime=sysbox-runc` and it can host + its own isolated `dockerd`, socket-mounted only to itself. +- A dedicated **rootful-but-isolated** `dockerd` (e.g. a sibling container + or VM with nothing else on it) is the fallback if neither of the above + fits — smaller blast radius than sharing the *host's* daemon, even + though it isn't rootless. + +None of this is wired up as a default, since a working zero-config +`docker compose up` needs *some* socket available — but every deployment +that isn't purely local/throwaway should point `DOCKER_SOCKET_PATH` at one +of the above rather than the host socket bind-mount in the shipped compose +file. + --- ## 3. Authentication @@ -137,7 +184,9 @@ served for interactive exploration). Exact argument types/names are in `repository(owner, name)`, `organizations`, `organization(name)`, `devWorkspaces`, `adminAllDevWorkspaces`, `myAccessTokens`, `adminListUsers(limit, offset)`, `myNotifications(unreadOnly)`, -`myActivity(limit)`, `myPackages`, `search(query)`. +`myActivity(limit)`, `myPackages`, `search(query)` (real SQLite-FTS5 +full-text search — see §4a — over repositories, issues, users, and indexed +file content, not substring `LIKE`). ### `repository` nested fields (all resolved on the `RepositoryObject` type) @@ -153,9 +202,11 @@ served for interactive exploration). Exact argument types/names are in `createAccessToken`, `revokeAccessToken`. **Repositories**: `createRepository`, `updateRepository`, -`deleteRepository`, `forkRepository`, `setRepoMirror`, `setRepoSecret`, -`addCollaborator`, `removeCollaborator`, `createWebhook`, `updateWebhook`, -`deleteWebhook`, `createBranchProtectionRule`. +`renameRepository(repoId, newName)` (moves the bare repo + wiki dir on +disk and updates the DB `name`, with best-effort rollback on partial +failure), `deleteRepository`, `forkRepository`, `setRepoMirror`, +`setRepoSecret`, `addCollaborator`, `removeCollaborator`, `createWebhook`, +`updateWebhook`, `deleteWebhook`, `createBranchProtectionRule`. **Issues & pull requests**: `createIssue`, `updateIssue`, `commentOnIssue`, `createPullRequest`, `mergePullRequest` (`mergeMethod`: @@ -171,13 +222,41 @@ served for interactive exploration). Exact argument types/names are in **CI/CD**: `triggerWorkflowDispatch`. -**Dev workspaces**: `createDevWorkspace` (optional `autoStopMinutes`), -`startDevWorkspace`, `stopDevWorkspace`, `deleteDevWorkspace`, -`execInDevWorkspace`. +**Dev workspaces**: `createDevWorkspace` (optional `autoStopMinutes`, +`onRunner` — see §7), `startDevWorkspace`, `stopDevWorkspace`, +`deleteDevWorkspace`, `execInDevWorkspace`. **Notifications**: `markNotificationRead`. -**Admin**: `adminSetUserAdmin`, `adminDeactivateUser`. +**Admin**: `adminSetUserAdmin`, `adminDeactivateUser`, +`adminBackfillPreReceiveHooks` (writes the branch-protection pre-receive +hook onto every repo — including ones created before that feature +existed; see §5). + +--- + +## 4a. Search + +`search(query)` runs real SQLite FTS5 full-text search (via `MATCH`, with +BM25 relevance ordering), not the substring `LIKE` matching this used to +be limited to. Each plain alphanumeric word in `query` becomes an +implicit-prefix match (`word*`, so results appear as you finish typing a +term); anything else is treated as a literal quoted phrase. Terms are +ANDed together. Covers: + +- **Repositories** (`name`/`description`) and **users** (`username`). +- **Issues** (`title`/`body`), scoped to repos visible to the caller. +- **Code**: file contents across the *default branch* of every repo + visible to the caller, returned as `SearchResults.code` — each hit + carries the matching `repository`, `path`, and an excerpt (`snippet`, + FTS5-generated, with `[b]...[/b]` match markers). Indexing happens + automatically after every push that moves the default branch (see + `index_repo_code_on_push` in `crates/server`): the whole tree is + re-walked and re-indexed (not diffed), so a force-push/history rewrite + is handled correctly. Binary-looking files (a NUL byte anywhere in their + content), files over 256KB, and repos with more than 2000 files hit an + indexing cap and are partially indexed rather than skipped or slow. + Non-default branches are not indexed. --- @@ -190,11 +269,23 @@ served for interactive exploration). Exact argument types/names are in match against registered keys. - **Branch protection**: `createBranchProtectionRule` sets a required-reviews count and/or `blockForcePush`. Force-push blocking is enforced by a real - git `pre-receive` hook (written into every newly-created repo) — it - rejects the push at the git protocol level, not just after the fact. - Required-reviews is enforced in `mergePullRequest`. + git `pre-receive` hook, written automatically into every newly-created + repo — it rejects the push at the git protocol level, not just after the + fact. Required-reviews is enforced in `mergePullRequest`. Repos created + *before* branch-protection support existed don't get this hook + automatically; an admin can backfill it onto every repo at once (new + ones included, harmlessly — the write is an unconditional overwrite, not + conditional on "missing") via `adminBackfillPreReceiveHooks`. - **Merge methods**: `merge` (2-parent merge commit), `squash` (single commit atop target), `rebase` (replays source commits onto target). +- **Rename**: `renameRepository(repoId, newName)` updates the DB `name` + and moves both the bare repo directory and (if present) its wiki + directory on disk to match, rejecting the rename if the owner already + has another repo named `newName`. Best-effort rolls back any disk + rename(s) already performed if a later step fails, so disk and DB don't + end up disagreeing about the repo's name. A dev workspace already + running against the old path (its bind-mount was resolved once at + creation time) won't pick up the new path until recreated. - **Wiki**: every repo gets a second bare repo (`{name}.wiki.git`) automatically; pages are plain Markdown files, one commit per `writeWikiPage` call. @@ -215,15 +306,40 @@ Ubuntu image, anything that looks like an image tag is used directly), `steps[].uses: actions/checkout@*` (no-op — the repo tree is already present in the container). `env`, per-step `env`. -**Not supported**: the GitHub Actions marketplace. Any `uses:` other than -`actions/checkout` is logged and skipped (`... not supported in this runner, -skipping`) rather than failing the job — so a real-world workflow with -marketplace actions will still run its `run:` steps, just without whatever -that action would have set up. There is no full `ubuntu-latest`-equivalent -image with GitHub's huge pre-installed toolset — the default image is a -plain `ubuntu:22.04`/similar, so commands like `sudo`, `npm`, language -toolchains etc. are **not** pre-installed unless your workflow installs them -itself or you point `runs-on` at an image that already has them. +**Marketplace actions** (`crates/actions/src/marketplace.rs`): `uses:` steps +now do more than a no-op for `actions/checkout`: + +- `uses: docker://image[:tag]` runs that image directly as a short-lived + sibling container: the job's `/workspace` is copied in, the image's own + entrypoint/cmd runs, and `/workspace` is copied back out afterward so + later steps see any files the action wrote. `with:` values become + `INPUT_*` env vars. +- `uses: owner/repo[/path]@ref` fetches that action from GitHub (a full + clone, not a shallow one — see the module docs for why), resolves `ref` + against tags/branches/raw SHAs, and reads its `action.yml`/`action.yaml`: + - `runs.using: docker` — same container model as `docker://` above, + building from a `Dockerfile` first if `runs.image` isn't already a + `docker://` reference. `runs.entrypoint`/`runs.args` support + `${{ inputs.NAME }}` substitution. + - `runs.using: composite` — nested `steps` run against the *same* job + container (like a real composite action). One level deep only: a + composite action nested inside another composite action is logged and + skipped rather than recursing. + - `runs.using: node12/16/18/20` — JS actions actually run, inside a + helper `node:-slim` container (the job's own `runs-on` image + has no reason to include Node) with the action's source copied in + alongside the job's `/workspace`. + - Anything else (an input-less docker action with no `image`, an + unresolvable ref, a runtime we don't model, a fetch failure) is logged + and skipped exactly like today's "not supported" path — the job + continues, that step is a no-op. + +There's still no full `ubuntu-latest`-equivalent image with GitHub's huge +pre-installed toolset — the default image is a plain `ubuntu:22.04`/ +similar, so commands like `sudo`, `npm`, language toolchains etc. are +**not** pre-installed unless your workflow installs them itself, a +marketplace action sets them up, or you point `runs-on` at an image that +already has them. **Secrets**: `setRepoSecret(repoId, name, value)` stores an AES-256-GCM-encrypted value (`SECRETS_ENCRYPTION_KEY` required). Job steps @@ -256,27 +372,49 @@ Polls `POST /runner/claim` every few seconds; on a claimed job, executes it via the same `actions::Executor` logic as in-process execution, then reports back via `POST /runner/jobs/:id/complete`. Lets CI execution scale out to separate machines/pods without deploying the full GraphQL/git-hosting -stack on them. Dev-workspace hosting via the runner (as opposed to CI jobs) -is reserved in the schema (`kind = 'dev_workspace_action'`) but not yet wired -to an active polling loop — currently dev workspaces are only managed -in-process by `server`. +stack on them. + +The runner also claims and executes `dev_workspace_action` jobs +(`kind = 'dev_workspace_action'`) — see §7 — via its own local Docker +daemon (`dev_env::WorkspaceManager`), reporting results (e.g. the created +container's id) back in the same completion call's new `result` field. --- ## 7. Dev workspaces (Coder-like) -`createDevWorkspace(name, template?, image?, autoStopMinutes?)` starts a -Docker container with resource limits, optionally cloning a repo into it on -start. Pass `template` to pick a built-in configuration (`"code-server"`, -`"rust-dev"`, `"node-dev"` — see `dev_env::templates`), which resolves to an -image plus a set of named ports; or pass a raw `image` for advanced/custom -use, which falls back to the single-port code-server-only behavior. Access a -workspace's default port through Genome's own domain via -`GET /workspaces/:id/proxy/*path`, or an explicitly named port via -`GET /workspaces/:id/proxy_port/:portName/*path` (both reverse-proxied to -the container) rather than exposing raw container ports. `autoStopMinutes` + -a background loop stop idle workspaces automatically; `execInDevWorkspace` -runs an arbitrary command inside a running workspace. +`createDevWorkspace(name, template?, image?, autoStopMinutes?, onRunner?)` +starts a Docker container with resource limits, optionally cloning a repo +into it on start. Pass `template` to pick a built-in configuration +(`"code-server"`, `"rust-dev"`, `"node-dev"` — see `dev_env::templates`), +which resolves to an image plus a set of named ports; or pass a raw +`image` for advanced/custom use, which falls back to the single-port +code-server-only behavior. Access a workspace's default port through +Genome's own domain via `GET /workspaces/:id/proxy/*path`, or an +explicitly named port via `GET /workspaces/:id/proxy_port/:portName/*path` +(both reverse-proxied to the container) rather than exposing raw container +ports. `autoStopMinutes` + a background loop stop idle workspaces +automatically; `execInDevWorkspace` runs an arbitrary command inside a +running workspace. + +**Standalone-runner-hosted workspaces**: pass `onRunner: true` to have a +connected standalone `runner` (§6) create the container against *its own* +Docker daemon instead of `server`'s — useful for keeping dev-workspace +compute off the machine running the API/git-hosting stack. This enqueues a +`dev_workspace_action` job and waits (polling the DB, up to 20s) for a +runner to claim and execute it; the returned workspace has +`status: "pending_runner"` and `runnerId: null` until that happens, then +`status: "running"` with `runnerId` set. `deleteDevWorkspace` and +`execInDevWorkspace` also route through the runner for a workspace it +hosts. **Known gaps**: `startDevWorkspace`/`stopDevWorkspace` and the +auto-stop background loop are not implemented for runner-hosted workspaces +yet (they error/skip rather than acting on the wrong Docker daemon), and — +the bigger one — live port-proxying (`GET /workspaces/:id/proxy/*path`) +doesn't work for a runner-hosted workspace at all: its container lives on +the runner's Docker host, which `server` has no network path to reach or +tunnel through yet. A runner-hosted workspace is reachable via +`execInDevWorkspace` (e.g. to inspect it or run a headless task) but not +via the HTTP proxy today. --- @@ -320,9 +458,10 @@ removal note and the git history around that change. ## 11. Admin operations `adminListUsers(limit, offset)`, `adminSetUserAdmin(userId, isAdmin)`, -`adminDeactivateUser(userId)`, `adminAllDevWorkspaces` — all gated on -`claims.is_admin`, returning a `forbidden` GraphQL error otherwise. A -deactivated user's login (JWT or PAT) is rejected immediately. +`adminDeactivateUser(userId)`, `adminAllDevWorkspaces`, +`adminBackfillPreReceiveHooks` (§5) — all gated on `claims.is_admin`, +returning a `forbidden` GraphQL error otherwise. A deactivated user's +login (JWT or PAT) is rejected immediately. --- @@ -350,14 +489,29 @@ push/PR, plus builds+pushes the Docker image to GHCR on pushes to `main`. See `JOURNAL.md` for the full, honest build log, but the headline gaps: -- No full-text/code search (substring `LIKE` match only). -- No repo rename-on-disk. -- Existing repos created before branch-protection support don't get the - pre-receive hook backfilled automatically (only newly-created repos do). -- GitHub Actions marketplace `uses:` actions are skipped, not executed - (only `actions/checkout` and plain `run:` steps work). -- Standalone runner support for dev-workspace hosting (as opposed to CI - jobs) is schema-reserved but not implemented. +- GitHub Actions marketplace `uses:` support (§6) covers `docker://` + images, `owner/repo[/path]@ref` actions with `runs.using: docker` + (including building from a `Dockerfile`), one level of `composite` + actions, and JS actions (`node12`/`16`/`18`/`20`) — but not a nested + composite-inside-composite action, and not the actual GitHub Actions + toolkit's finer behaviors (`core.setOutput`/step outputs, `GITHUB_ENV`/ + `GITHUB_PATH` files, caching, etc.). There is still no full + `ubuntu-latest`-equivalent image with GitHub's huge pre-installed + toolset. +- Standalone-runner-hosted dev workspaces (§7) support create/delete/exec, + but not start/stop or live port-proxying to the workspace — the + runner's container lives on a Docker daemon `server` has no network path + to (no reverse tunnel exists between them yet). +- Code search (§4a) only indexes each repo's *default branch*; other + branches, and git history, aren't searchable. +- Merge is a real 2-parent/squash/rebase commit, but there's still no + rich diff-review UI, no LFS, and no SSH-transport for anything beyond + git itself (e.g. no `git-lfs-transfer`). - Multi-node Hiqlite (true multi-machine HA) is architecturally supported but not yet exposed via a ready-made multi-node env-var configuration — single-node is the tested, default path. +- CI/dev-workspace execution talks to a Docker Engine API socket; the + shipped `docker-compose.yml` bind-mounts the *host's* socket, which is a + large privilege-escalation surface. §2a documents safer alternatives + (rootless Podman, Sysbox) — `DOCKER_SOCKET_PATH` is a real, wired + override, not just accepted-and-ignored. diff --git a/JOURNAL.md b/JOURNAL.md index eb91e6a..abc42ba 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -36,3 +36,11 @@ Alternative Rust à GitHub/Forgejo/GitLab/Coder. Objectif 22h00. - **Réaliste pour 23h (deadline ajustée, dépassée pour ce chantier bonus)**: un MVP fonctionnel et démontrable (clone/push, CI qui tourne dans Docker, workspace dev qui boot, GraphQL+front qui marchent ensemble) — pas un remplaçant prod-ready de Forgejo+Coder. Ça, c'est plusieurs semaines/mois même en gardant ce rythme (intégration réelle, sécurité, edge cases, UI poli, tests). - **01:17** — Bootstrap admin token implémenté (`ADMIN_BOOTSTRAP_TOKEN`, provisioning idempotent admin+PAT au démarrage, testé live) — plus aucun besoin de `register`/`login` pour un déploiement API-only pur. `.claude/` retiré du suivi git (untrack + gitignore). Profil `[profile.release]` optimisé (lto+codegen-units=1+strip, sans `panic=abort` — évite de casser l'isolation panic par requête d'axum): `server` 50.8MB→37.4MB, `runner` 5.2MB. `cargo test --workspace` 33/33 verts après rebuild. Écrit `DOC.md`: doc complète de la plateforme (architecture, déploiement, auth/bootstrap, référence complète des 54 mutations/15 queries/24 champs imbriqués GraphQL, transports git HTTP/SSH, CI/Actions compat GH, dev-workspaces, wiki, registre de paquets, webhooks, admin, tests, limitations connues). Commité+pushé (285b79f). - **01:29** — Retrait complet du 2FA (TOTP): plus de justification une fois l'OAuth2 retiré et l'usage purement API/PAT/bootstrap-token — supprimé `crates/auth/src/totp.rs`, dep `totp-rs`, colonnes utilisées côté code (`totp_secret`/`totp_enabled` restent en DB, migration additive-only, juste plus lues/écrites), mutations `enableTwoFactor`/`confirmTwoFactor`/`disableTwoFactor`, arg `totpCode` sur `login`, page front `settings/security.vue` + lien nav. Ajouté en parallèle (agent dédié): templates de workspace nommés (`code-server`/`rust-dev`/`node-dev` → image+ports résolus, `crates/dev-env/src/templates.rs`) et proxy multi-port (`/workspaces/:id/proxy_port/:portName/*path` en plus de l'ancien single-port `/proxy/*path`, ports reconstruits depuis les labels Docker donc pas de migration DB). `cargo check/test --workspace` clean (32/32 tests, frontend build clean), pushé (5df083a). +- **(reprise de session, 15 juillet)** — Demande utilisateur: combler 5 lacunes précises listées dans `DOC.md` §14, et trouver une alternative safe au DinD. Fait dans l'ordre, chaque morceau vérifié par `cargo check/test --workspace` avant de passer au suivant (rustc mis à jour 1.94→1.97 en cours de route, hiqlite 0.14 l'exigeait) : + 1. **Recherche full-text** (`search`): remplacé le `LIKE` par de la vraie recherche **SQLite FTS5** (migration `9_search_fts.sql`, tables `repositories_fts`/`issues_fts`/`users_fts` tenues à jour par triggers + backfill des lignes existantes). Bonne surprise: FTS5 est déjà compilé dans le SQLite "bundled" que hiqlite utilise (`-DSQLITE_ENABLE_FTS5` dans son build.rs) — pas besoin de bidouiller les features cargo, juste écrire le SQL. Ajouté en plus une vraie **recherche de code** (`code_search_fts`), indexée automatiquement après chaque push qui bouge la branche par défaut (`index_repo_code_on_push`, ré-indexe tout l'arbre plutôt que de diff — gère nativement force-push/rewrite), avec snippet FTS5 en résultat. Nouvelle fonction `RepoManager::list_text_blobs_at_ref` (skip binaire/fichiers >256KB, cap 2000 fichiers/repo). + 2. **Rename-on-disk**: `RepoManager::rename_repo`/`rename_wiki` (nouveau) + mutation `renameRepository(repoId, newName)` — déplace le(s) dossier(s) bare sur disque et met à jour la colonne `name`, avec rollback best-effort si une étape échoue en cours de route (pas de vraie transaction FS+DB possible, donc ordre choisi pour minimiser la fenêtre d'incohérence: disque d'abord, DB ensuite, rollback disque si la DB échoue). + 3. **Backfill du hook pre-receive**: `write_pre_receive_hook` était `fn` privée — exposée via `RepoManager::ensure_pre_receive_hook` (pub), + mutation admin `adminBackfillPreReceiveHooks` qui la rejoue sur tous les repos existants (écriture inconditionnelle, sûre à rejouer même sur un repo qui a déjà le hook). + 4. **Marketplace GitHub Actions** (le plus gros morceau): `crates/actions/src/marketplace.rs` — parse `uses: docker://image` et `uses: owner/repo[/path]@ref`, clone (git2, pas de shallow propre possible avec un tag/branche arbitraire donc clone complet — les repos d'actions sont petits) puis lit `action.yml`. Supporte `runs.using: docker` (y compris build depuis un `Dockerfile` via `bollard::build_image`), `runs.using: composite` (un seul niveau — pas de composite imbriqué, pour éviter la récursion non bornée), et **`runs.using: node12/16/18/20`** (actions JS réellement exécutées, dans un conteneur `node:*-slim` dédié — l'image `runs-on` du job n'a aucune raison d'avoir Node). Modèle de conteneur pour docker/node: le `/workspace` du job est copié (tar `docker cp`) dans un conteneur satellite jetable, qui tourne, puis est recopié dans l'autre sens pour que les steps suivants voient les changements. Tout ce qui ne rentre pas dans ce moule (ref introuvable, `action.yml` absent, `runs.using` non reconnu, composite imbriqué) tombe dans le même chemin "not supported, skipping" qu'avant plutôt que de faire planter le job. + 5. **Dev-workspace hosting par le runner standalone**: le `kind = 'dev_workspace_action'` était réservé mais mort — maintenant câblé bout en bout: `createDevWorkspace(..., onRunner: true)` pousse un `runner_jobs` (nouvelle colonne `result` pour faire remonter `container_id`/output, migration `10_dev_workspace_runner.sql`), le runner le récupère et l'exécute contre **son propre** Docker (nouveau `crates/runner/src/dev_workspace_poll.rs`, plus plausible stub), rapporte le résultat, la mutation poll la DB (pas de pub/sub, juste polling 300ms/20s timeout) et renvoie l'objet à jour. `delete`/`exec` suivent le même chemin pour un workspace déjà hébergé par un runner. Honnêteté: `start`/`stop` et surtout le **reverse-proxy HTTP live** vers un workspace hébergé sur un runner restent non supportés (le conteneur est sur un Docker daemon que `server` n'a aucun moyen réseau d'atteindre — ça demanderait un vrai tunnel inverse runner→server, hors scope ici) — documenté comme lacune connue plutôt que de faire semblant que ça marche. + 6. **Alternative safe au DinD**: en creusant, `server`/`runner` ne faisaient déjà **pas** de vrai Docker-in-Docker (pas de `dockerd` imbriqué) — juste un montage du socket Docker de l'**hôte** ("Docker outside of Docker"), ce qui est en pratique un risque équivalent (accès root-like à l'hôte via `-v /:/host`). `DOCKER_SOCKET_PATH` existait déjà en config mais n'était jamais branché nulle part (juste loggé) — maintenant réellement câblé (`Executor::new_with_socket`, `WorkspaceManager::connect_socket`, via `bollard::Docker::connect_with_socket`). Recherche web: **Podman rootless** (`systemctl --user enable --now podman.socket`, API compatible Docker, isolation par user-namespace, zéro changement de code requis côté Genome) recommandé en premier, **Sysbox** (`nestybox/sysbox`) documenté pour le cas où un workflow a réellement besoin de nested Docker (ex: un step qui fait `docker build`). Tout documenté dans `DOC.md` §2a, rien d'actif par défaut (il faut un socket disponible pour un `docker compose up` zéro-config) mais le knob marche vraiment maintenant. + - Tests ajoutés: 6 nouveaux tests unitaires `marketplace` (parsing `ActionRef`, résolution d'inputs, substitution), 2 nouveaux tests `git-core` (rename repo+wiki avec rollback/collision, idempotence du backfill hook). `cargo test --workspace`: seuls les 6 tests d'intégration qui construisent un `actions::Executor`/`dev_env::WorkspaceManager` échouent, et uniquement à cause de l'absence de `/var/run/docker.sock` **dans ce bac à sable de session** (pas de démon Docker démarrable, `systemd` absent) — confirmé non lié à mes changements (ces tests appellent le même `Executor::new()`/`connect_local()` qu'avant, code non modifié sur ce chemin). Pushé (ec8ec2e). diff --git a/README.md b/README.md index 7421e22..f502c4d 100644 --- a/README.md +++ b/README.md @@ -15,10 +15,14 @@ Actions-compatible CI running real Docker jobs, Coder-like containerized dev workspaces, GraphQL API, package registry. See `JOURNAL.md` for the full build log and honest list of what is and isn't production-hardened. -**Known gaps** (see `JOURNAL.md` for detail): no artifact-of-repo-mirroring for -existing repos' pre-receive hooks (only newly-created repos get the force-push -protection hook), no code-search/full-text search (only ILIKE-ish substring -match), no rename-repo-on-disk. +**Known gaps** (see `JOURNAL.md` for detail, and `DOC.md` §14 for the current +full list): GitHub Actions marketplace support covers Docker/composite/JS +actions but not every toolkit behavior (step outputs, `GITHUB_ENV`, caching); +standalone-runner-hosted dev workspaces support create/delete/exec but not +start/stop or live port-proxying; code search only indexes each repo's +default branch; CI/dev-workspace execution needs a Docker socket, and the +shipped `docker-compose.yml` bind-mounts the host's own one (a real privilege +surface) rather than defaulting to a safer rootless engine. ## Architecture @@ -36,7 +40,7 @@ Rust workspace, one crate per concern: | `webhooks` | HMAC-signed webhook dispatch | | `graphql-api` | The GraphQL schema (async-graphql) wiring everything together | | `server` | axum binary: HTTP router (GraphQL, git smart-HTTP, packages, artifacts), spawns the SSH server, embeds the Hiqlite database node, and runs background loops (mirror sync, workspace auto-stop) | -| `runner` | **Optional**, standalone poll-based binary. Polls the main `server`'s `/runner/claim` HTTP route for queued CI jobs (`runner_jobs` table) and executes them locally via the same `actions::Executor`/`actions::Workflow` logic the server uses in-process, reporting results back to `/runner/jobs/:id/complete`. The `server` binary keeps running every CI job in-process exactly as before regardless of whether any `runner` is connected — the two paths are additive, not a replacement (see the `// DUAL-PATH:` comments in `crates/server/src/main.rs` and `crates/graphql-api/src/mutation.rs`). Also carries a `dev_env::WorkspaceManager` dependency as a forward-compat stub for eventual dev-workspace-hosting polling (not implemented yet). | +| `runner` | **Optional**, standalone poll-based binary. Polls the main `server`'s `/runner/claim` HTTP route for queued CI jobs (`runner_jobs` table) and executes them locally via the same `actions::Executor`/`actions::Workflow` logic the server uses in-process, reporting results back to `/runner/jobs/:id/complete`. The `server` binary keeps running every CI job in-process exactly as before regardless of whether any `runner` is connected — the two paths are additive, not a replacement (see the `// DUAL-PATH:` comments in `crates/server/src/main.rs` and `crates/graphql-api/src/mutation.rs`). Also claims and executes `dev_workspace_action` jobs against its own local Docker daemon (`dev_env::WorkspaceManager`), letting a dev workspace be hosted on the runner instead of `server` — create/delete/exec only, see `DOC.md` §7 for the gaps. | `frontend/` is an independent Nuxt 3 + Vue 3 app for browsing/testing against the GraphQL API — not required for production use. diff --git a/crates/actions/Cargo.toml b/crates/actions/Cargo.toml index 7ec8664..50a388e 100644 --- a/crates/actions/Cargo.toml +++ b/crates/actions/Cargo.toml @@ -20,3 +20,5 @@ tar.workspace = true aes-gcm.workspace = true base64.workspace = true rand.workspace = true +git2.workspace = true +tempfile = "3" diff --git a/crates/actions/src/executor.rs b/crates/actions/src/executor.rs index 106f61a..3ab88b2 100644 --- a/crates/actions/src/executor.rs +++ b/crates/actions/src/executor.rs @@ -1,19 +1,20 @@ //! Docker-based execution of a single workflow job via `bollard`. use std::collections::HashMap; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use bollard::container::{ - Config, CreateContainerOptions, DownloadFromContainerOptions, RemoveContainerOptions, - UploadToContainerOptions, + Config, CreateContainerOptions, DownloadFromContainerOptions, LogsOptions, + RemoveContainerOptions, UploadToContainerOptions, WaitContainerOptions, }; use bollard::exec::{CreateExecOptions, StartExecResults}; -use bollard::image::CreateImageOptions; +use bollard::image::{BuildImageOptions, CreateImageOptions}; use bollard::Docker; use futures_util::StreamExt; use uuid::Uuid; -use crate::workflow::Job; +use crate::marketplace; +use crate::workflow::{Job, Step}; use crate::ActionsError; /// Final status of a job run. @@ -57,6 +58,17 @@ impl Executor { Ok(Executor { docker, artifacts_root }) } + /// Connect to a specific Docker (or Docker-API-compatible, e.g. + /// rootless Podman) socket path rather than always resolving the + /// platform default -- see `dev_env::WorkspaceManager::connect_socket` + /// for why this matters for running without a host-Docker-socket + /// mount. + pub fn new_with_socket(artifacts_root: PathBuf, docker_socket_path: &str) -> Result { + let docker = Docker::connect_with_socket(docker_socket_path, 120, bollard::API_DEFAULT_VERSION) + .map_err(ActionsError::Docker)?; + Ok(Executor { docker, artifacts_root }) + } + /// Map a `runs-on:` value to a concrete Docker image tag. fn image_for_runs_on(runs_on: &str) -> String { match runs_on { @@ -86,7 +98,7 @@ impl Executor { workdir_repo_archive: &[u8], env_extra: HashMap, secrets: &HashMap, - log_sink: impl Fn(String) + Send + 'static, + log_sink: impl Fn(String) + Send + Sync + 'static, ) -> Result { let image = Self::image_for_runs_on(&job.runs_on); @@ -180,86 +192,19 @@ impl Executor { let mut last_exit_code: i32 = 0; for step in &job.steps { - if let Some(uses) = &step.uses { - if uses.starts_with("actions/checkout") { - log_sink(format!("[step] uses: {uses} (no-op, repo already present)")); - continue; - } - log_sink(format!( - "action {uses} not supported in this runner, skipping" - )); - continue; - } - - let Some(run) = &step.run else { - continue; - }; - - let step_name = step.name.clone().unwrap_or_else(|| run.clone()); - log_sink(format!("[step] {step_name}")); - - let run = crate::secrets::substitute_secrets(run, secrets); - - let shell = step.shell.clone().unwrap_or_else(|| "sh".to_string()); - let mut step_env: Vec = env_vars.clone(); - if let Some(se) = &step.env { - for (k, v) in se { - step_env.push(format!("{k}={v}")); - } - } - - let exec = self - .docker - .create_exec( - &container_name, - CreateExecOptions { - cmd: Some(vec![shell, "-c".to_string(), run.clone()]), - attach_stdout: Some(true), - attach_stderr: Some(true), - env: Some(step_env), - working_dir: Some("/workspace".to_string()), - ..Default::default() - }, - ) + let result = self + .execute_step(&container_name, step, &env_vars, secrets, &log_sink, 0) .await; - - let exec = match exec { - Ok(e) => e, + let exit_code = match result { + Ok(code) => code, Err(e) => { + log_sink(format!("[step error] {e}")); cleanup(self.docker.clone(), container_name.clone()).await; - return Err(ActionsError::Docker(e)); - } - }; - - match self.docker.start_exec(&exec.id, None).await { - Ok(StartExecResults::Attached { mut output, .. }) => { - while let Some(chunk) = output.next().await { - match chunk { - Ok(msg) => { - let text = msg.to_string(); - for line in text.lines() { - log_sink(line.to_string()); - } - } - Err(e) => { - log_sink(format!("[error reading output] {e}")); - break; - } - } - } - } - Ok(StartExecResults::Detached) => {} - Err(e) => { - cleanup(self.docker.clone(), container_name.clone()).await; - return Err(ActionsError::Docker(e)); - } - } - - let exit_code = match self.docker.inspect_exec(&exec.id).await { - Ok(inspect) => inspect.exit_code.unwrap_or(0) as i32, - Err(e) => { - cleanup(self.docker.clone(), container_name.clone()).await; - return Err(ActionsError::Docker(e)); + return Ok(JobResult { + status: JobStatus::Failure, + exit_code: 1, + artifacts: Vec::new(), + }); } }; @@ -333,6 +278,594 @@ impl Executor { }) } + /// Execute a single top-level job step (`run:` or `uses:`), returning + /// its exit code (0 = success). + async fn execute_step( + &self, + container_name: &str, + step: &Step, + env_vars: &[String], + secrets: &HashMap, + log_sink: &(impl Fn(String) + Send + Sync), + depth: u8, + ) -> Result { + if let Some(uses) = &step.uses { + if uses.starts_with("actions/checkout") { + log_sink(format!("[step] uses: {uses} (no-op, repo already present)")); + return Ok(0); + } + return self + .execute_uses_step(container_name, uses, step.with.as_ref(), env_vars, secrets, log_sink, depth) + .await; + } + + let Some(run) = &step.run else { return Ok(0) }; + let step_name = step.name.clone().unwrap_or_else(|| run.clone()); + log_sink(format!("[step] {step_name}")); + + let shell = step.shell.clone().unwrap_or_else(|| "sh".to_string()); + let mut step_env: Vec = env_vars.to_vec(); + if let Some(se) = &step.env { + for (k, v) in se { + step_env.push(format!("{k}={v}")); + } + } + + self.exec_run_in_container(container_name, &shell, run, secrets, step_env, log_sink) + .await + } + + /// Resolves and runs a `uses:` step that isn't `actions/checkout`: a + /// direct `docker://image`, or a marketplace `owner/repo[/path]@ref` + /// action whose `action.yml` declares `runs.using` as `docker`, + /// `composite`, or a Node runtime. Anything else (unparseable, fetch + /// failure, unsupported `runs.using`, or a composite action nested + /// inside another composite action) is logged and treated as a no-op + /// success (`Ok(0)`) rather than failing the job, matching the existing + /// lenient "not supported, skipping" behavior for unrecognized actions. + async fn execute_uses_step( + &self, + container_name: &str, + uses: &str, + with: Option<&HashMap>, + env_vars: &[String], + secrets: &HashMap, + log_sink: &(impl Fn(String) + Send + Sync), + depth: u8, + ) -> Result { + let Some(action_ref) = marketplace::ActionRef::parse(uses) else { + log_sink(format!("action {uses} not supported in this runner, skipping")); + return Ok(0); + }; + + match action_ref { + marketplace::ActionRef::DockerImage(image) => { + let resolved = marketplace::resolve_inputs(None, with); + let mut env = env_vars.to_vec(); + env.extend(marketplace::input_env_vars(&resolved)); + log_sink(format!("[step] uses: {uses} (docker image)")); + self.run_docker_action(container_name, image, None, None, env, log_sink).await + } + marketplace::ActionRef::Marketplace { owner, repo, path, git_ref } => { + let fetched = { + let owner = owner.clone(); + let repo = repo.clone(); + let path = path.clone(); + let git_ref = git_ref.clone(); + tokio::task::spawn_blocking(move || { + marketplace::fetch_action(&owner, &repo, path.as_deref(), &git_ref) + }) + .await + .map_err(|e| ActionsError::Marketplace(format!("fetch task panicked: {e}")))? + }; + + let fetched = match fetched { + Ok(f) => f, + Err(e) => { + log_sink(format!("action {uses} failed to fetch, skipping: {e}")); + return Ok(0); + } + }; + + let resolved = marketplace::resolve_inputs(Some(&fetched.metadata), with); + + match fetched.metadata.runs.using.as_str() { + "docker" => { + let Some(image_spec) = fetched.metadata.runs.image.clone() else { + log_sink(format!("action {uses} has runs.using=docker but no image, skipping")); + return Ok(0); + }; + let image = if let Some(direct) = image_spec.strip_prefix("docker://") { + direct.to_string() + } else { + match self.build_action_image(&fetched.dir, &image_spec, log_sink).await { + Ok(tag) => tag, + Err(e) => { + log_sink(format!("action {uses} failed to build image, skipping: {e}")); + return Ok(0); + } + } + }; + let entrypoint = fetched + .metadata + .runs + .entrypoint + .as_ref() + .map(|e| vec![marketplace::substitute_inputs(e, &resolved)]); + let cmd = fetched.metadata.runs.args.as_ref().map(|args| { + args.iter() + .map(|a| marketplace::substitute_inputs(a, &resolved)) + .collect() + }); + let mut env = env_vars.to_vec(); + env.extend(marketplace::input_env_vars(&resolved)); + log_sink(format!("[step] uses: {uses} (docker action)")); + self.run_docker_action(container_name, image, entrypoint, cmd, env, log_sink) + .await + } + "composite" => { + if depth > 0 { + log_sink(format!( + "action {uses} is a nested composite action, skipping (not supported)" + )); + return Ok(0); + } + let Some(steps) = fetched.metadata.runs.steps.clone() else { + log_sink(format!("action {uses} has runs.using=composite but no steps, skipping")); + return Ok(0); + }; + log_sink(format!("[step] uses: {uses} (composite, {} steps)", steps.len())); + for nested in &steps { + let exit = Box::pin(self.execute_step( + container_name, + nested, + env_vars, + secrets, + log_sink, + depth + 1, + )) + .await?; + if exit != 0 && !nested.continue_on_error() { + return Ok(exit); + } + } + Ok(0) + } + using if using.starts_with("node") => { + let Some(main) = fetched.metadata.runs.main.clone() else { + log_sink(format!("action {uses} has runs.using={using} but no main, skipping")); + return Ok(0); + }; + let image = marketplace::node_image_for_using(using).to_string(); + let mut env = env_vars.to_vec(); + env.extend(marketplace::input_env_vars(&resolved)); + log_sink(format!("[step] uses: {uses} ({using} action)")); + self.run_node_action(container_name, &fetched.dir, &main, image, env, log_sink) + .await + } + other => { + log_sink(format!("action {uses} has unsupported runs.using: {other}, skipping")); + Ok(0) + } + } + } + } + } + + /// Runs a `run:` step's shell command via `docker exec` against the + /// job's already-running container (secrets substituted into the + /// command text first). Shared by top-level and composite-action steps. + async fn exec_run_in_container( + &self, + container_name: &str, + shell: &str, + run: &str, + secrets: &HashMap, + env: Vec, + log_sink: &(impl Fn(String) + Send + Sync), + ) -> Result { + let run = crate::secrets::substitute_secrets(run, secrets); + + let exec = self + .docker + .create_exec( + container_name, + CreateExecOptions { + cmd: Some(vec![shell.to_string(), "-c".to_string(), run]), + attach_stdout: Some(true), + attach_stderr: Some(true), + env: Some(env), + working_dir: Some("/workspace".to_string()), + ..Default::default() + }, + ) + .await + .map_err(ActionsError::Docker)?; + + match self.docker.start_exec(&exec.id, None).await.map_err(ActionsError::Docker)? { + StartExecResults::Attached { mut output, .. } => { + while let Some(chunk) = output.next().await { + match chunk { + Ok(msg) => { + for line in msg.to_string().lines() { + log_sink(line.to_string()); + } + } + Err(e) => { + log_sink(format!("[error reading output] {e}")); + break; + } + } + } + } + StartExecResults::Detached => {} + } + + let inspect = self.docker.inspect_exec(&exec.id).await.map_err(ActionsError::Docker)?; + Ok(inspect.exit_code.unwrap_or(0) as i32) + } + + /// Downloads `/workspace` out of the job container as a tar stream. + async fn copy_workspace_out(&self, container_name: &str) -> Result, ActionsError> { + let mut stream = self.docker.download_from_container( + container_name, + Some(DownloadFromContainerOptions { path: "/workspace".to_string() }), + ); + let mut bytes = Vec::new(); + while let Some(chunk) = stream.next().await { + bytes.extend_from_slice(&chunk.map_err(ActionsError::Docker)?); + } + Ok(bytes) + } + + /// Uploads a tar produced by `copy_workspace_out` (whose entries are + /// rooted at `workspace/...`) into `container_name` at `/`, so it lands + /// at `/workspace` there too. + async fn copy_workspace_in(&self, container_name: &str, tar_bytes: Vec) -> Result<(), ActionsError> { + self.docker + .upload_to_container( + container_name, + Some(UploadToContainerOptions { path: "/", no_overwrite_dir_non_dir: "false" }), + tar_bytes.into(), + ) + .await + .map_err(ActionsError::Docker) + } + + /// Runs a Docker container action (or a direct `docker://image` step) + /// as its own short-lived sibling container: copies the job's + /// `/workspace` in, runs the image's entrypoint/cmd to completion, + /// copies `/workspace` back out to the job container (so files the + /// action wrote/changed are visible to later steps), then removes the + /// side container. Real GitHub Actions runners give Docker actions the + /// path `/github/workspace`; using `/workspace` here instead is a + /// deliberate simplification (see module docs) that avoids needing to + /// create `/github` in an arbitrary base image before the copy-in. + #[allow(clippy::too_many_arguments)] + async fn run_docker_action( + &self, + job_container_name: &str, + image: String, + entrypoint: Option>, + cmd: Option>, + env: Vec, + log_sink: &(impl Fn(String) + Send + Sync), + ) -> Result { + { + let mut stream = self.docker.create_image( + Some(CreateImageOptions { from_image: image.clone(), ..Default::default() }), + None, + None, + ); + while let Some(item) = stream.next().await { + if let Err(e) = item { + tracing::warn!("image pull warning for {image}: {e}"); + break; + } + } + } + + let side_name = format!("genome-action-{}", Uuid::new_v4()); + let config = Config { + image: Some(image), + entrypoint, + cmd, + env: Some(env), + working_dir: Some("/workspace".to_string()), + ..Default::default() + }; + + self.docker + .create_container(Some(CreateContainerOptions { name: side_name.clone(), platform: None }), config) + .await + .map_err(ActionsError::Docker)?; + + let cleanup_side = || { + let docker = self.docker.clone(); + let name = side_name.clone(); + async move { + let _ = docker + .remove_container(&name, Some(RemoveContainerOptions { force: true, ..Default::default() })) + .await; + } + }; + + let workspace_tar = match self.copy_workspace_out(job_container_name).await { + Ok(t) => t, + Err(e) => { + cleanup_side().await; + return Err(e); + } + }; + if let Err(e) = self.copy_workspace_in(&side_name, workspace_tar).await { + cleanup_side().await; + return Err(e); + } + + if let Err(e) = self.docker.start_container::(&side_name, None).await { + cleanup_side().await; + return Err(ActionsError::Docker(e)); + } + + let exit_code = { + let mut waits = self + .docker + .wait_container(&side_name, Some(WaitContainerOptions { condition: "not-running".to_string() })); + match waits.next().await { + Some(Ok(resp)) => resp.status_code as i32, + Some(Err(e)) => { + cleanup_side().await; + return Err(ActionsError::Docker(e)); + } + None => 0, + } + }; + + let mut logs = self.docker.logs( + &side_name, + Some(LogsOptions:: { + follow: false, + stdout: true, + stderr: true, + tail: "all".to_string(), + ..Default::default() + }), + ); + while let Some(chunk) = logs.next().await { + match chunk { + Ok(msg) => { + for line in msg.to_string().lines() { + log_sink(line.to_string()); + } + } + Err(e) => { + log_sink(format!("[error reading output] {e}")); + break; + } + } + } + + // Best-effort: propagate any workspace changes the action made + // back into the job's own container for subsequent steps to see. + if let Ok(tar) = self.copy_workspace_out(&side_name).await { + let _ = self.copy_workspace_in(job_container_name, tar).await; + } + + cleanup_side().await; + Ok(exit_code) + } + + /// Builds a Docker image from a `Dockerfile` living inside a fetched + /// action's directory (the `runs.image` value when it isn't a + /// `docker://...` reference), tagging it uniquely for this run. + async fn build_action_image( + &self, + action_dir: &Path, + dockerfile_relative: &str, + log_sink: &(impl Fn(String) + Send + Sync), + ) -> Result { + let action_dir = action_dir.to_path_buf(); + let dockerfile_relative = dockerfile_relative.to_string(); + let tar_bytes = tokio::task::spawn_blocking(move || -> Result, ActionsError> { + let mut builder = tar::Builder::new(Vec::new()); + builder + .append_dir_all(".", &action_dir) + .map_err(|e| ActionsError::Marketplace(e.to_string()))?; + builder.into_inner().map_err(|e| ActionsError::Marketplace(e.to_string())) + }) + .await + .map_err(|e| ActionsError::Marketplace(format!("build context task panicked: {e}")))??; + + let tag = format!("genome-action-image:{}", Uuid::new_v4()); + let mut stream = self.docker.build_image( + BuildImageOptions { + dockerfile: dockerfile_relative, + t: tag.clone(), + rm: true, + ..Default::default() + }, + None, + Some(tar_bytes.into()), + ); + while let Some(item) = stream.next().await { + match item { + Ok(msg) => { + if let Some(stream_text) = msg.stream { + for line in stream_text.lines() { + log_sink(format!("[build] {line}")); + } + } + } + Err(e) => return Err(ActionsError::Docker(e)), + } + } + Ok(tag) + } + + /// Runs a JS action (`runs.using: node12/16/18/20`) inside a helper + /// Node image matching the requested runtime, since the job's own + /// `runs-on` image has no reason to include Node. Copies both the + /// job's `/workspace` and the action's own source (containing `main`) + /// into the helper container, runs `node
`, then propagates + /// `/workspace` changes back exactly like `run_docker_action`. + #[allow(clippy::too_many_arguments)] + async fn run_node_action( + &self, + job_container_name: &str, + action_dir: &Path, + main: &str, + image: String, + env: Vec, + log_sink: &(impl Fn(String) + Send + Sync), + ) -> Result { + { + let mut stream = self.docker.create_image( + Some(CreateImageOptions { from_image: image.clone(), ..Default::default() }), + None, + None, + ); + while let Some(item) = stream.next().await { + if let Err(e) = item { + tracing::warn!("image pull warning for {image}: {e}"); + break; + } + } + } + + let side_name = format!("genome-action-{}", Uuid::new_v4()); + let config = Config { + image: Some(image), + cmd: Some(vec![ + "sh".to_string(), + "-c".to_string(), + "mkdir -p /workspace /action && tail -f /dev/null".to_string(), + ]), + env: Some(env), + working_dir: Some("/workspace".to_string()), + ..Default::default() + }; + + self.docker + .create_container(Some(CreateContainerOptions { name: side_name.clone(), platform: None }), config) + .await + .map_err(ActionsError::Docker)?; + + let cleanup_side = || { + let docker = self.docker.clone(); + let name = side_name.clone(); + async move { + let _ = docker + .remove_container(&name, Some(RemoveContainerOptions { force: true, ..Default::default() })) + .await; + } + }; + + if let Err(e) = self.docker.start_container::(&side_name, None).await { + cleanup_side().await; + return Err(ActionsError::Docker(e)); + } + + let workspace_tar = match self.copy_workspace_out(job_container_name).await { + Ok(t) => t, + Err(e) => { + cleanup_side().await; + return Err(e); + } + }; + if let Err(e) = self.copy_workspace_in(&side_name, workspace_tar).await { + cleanup_side().await; + return Err(e); + } + + let action_dir = action_dir.to_path_buf(); + let action_tar = match tokio::task::spawn_blocking(move || -> Result, ActionsError> { + let mut builder = tar::Builder::new(Vec::new()); + builder + .append_dir_all("action", &action_dir) + .map_err(|e| ActionsError::Marketplace(e.to_string()))?; + builder.into_inner().map_err(|e| ActionsError::Marketplace(e.to_string())) + }) + .await + { + Ok(Ok(t)) => t, + Ok(Err(_)) | Err(_) => { + cleanup_side().await; + return Err(ActionsError::Marketplace("failed to package action source".to_string())); + } + }; + if let Err(e) = self + .docker + .upload_to_container( + &side_name, + Some(UploadToContainerOptions { path: "/", no_overwrite_dir_non_dir: "false" }), + action_tar.into(), + ) + .await + { + cleanup_side().await; + return Err(ActionsError::Docker(e)); + } + + let exec = match self + .docker + .create_exec( + &side_name, + CreateExecOptions { + cmd: Some(vec!["node".to_string(), format!("/action/{main}")]), + attach_stdout: Some(true), + attach_stderr: Some(true), + working_dir: Some("/workspace".to_string()), + ..Default::default() + }, + ) + .await + { + Ok(e) => e, + Err(e) => { + cleanup_side().await; + return Err(ActionsError::Docker(e)); + } + }; + + match self.docker.start_exec(&exec.id, None).await { + Ok(StartExecResults::Attached { mut output, .. }) => { + while let Some(chunk) = output.next().await { + match chunk { + Ok(msg) => { + for line in msg.to_string().lines() { + log_sink(line.to_string()); + } + } + Err(e) => { + log_sink(format!("[error reading output] {e}")); + break; + } + } + } + } + Ok(StartExecResults::Detached) => {} + Err(e) => { + cleanup_side().await; + return Err(ActionsError::Docker(e)); + } + } + + let exit_code = match self.docker.inspect_exec(&exec.id).await { + Ok(inspect) => inspect.exit_code.unwrap_or(0) as i32, + Err(e) => { + cleanup_side().await; + return Err(ActionsError::Docker(e)); + } + }; + + if let Ok(tar) = self.copy_workspace_out(&side_name).await { + let _ = self.copy_workspace_in(job_container_name, tar).await; + } + + cleanup_side().await; + Ok(exit_code) + } + /// Downloads `container_path` out of the (still-running) container as a /// tar stream and writes it verbatim to /// `{artifacts_root}/{run_id}/{name}.tar`, returning its metadata. diff --git a/crates/actions/src/lib.rs b/crates/actions/src/lib.rs index 43d9bf3..4581710 100644 --- a/crates/actions/src/lib.rs +++ b/crates/actions/src/lib.rs @@ -13,23 +13,27 @@ //! - `steps[].run` — executed as a shell command inside a Docker container. //! - `steps[].uses: actions/checkout@*` — no-op, since the repository //! contents are already materialized into the container's workspace. +//! - `steps[].uses: docker://image` and `owner/repo[/path]@ref` marketplace +//! actions whose `action.yml` declares `runs.using` as `docker`, +//! `composite`, or a Node runtime (`node12`/`node16`/`node18`/`node20`) +//! — see `marketplace` and `Executor::execute_uses_step`. Composite +//! actions may not nest another composite action (one level deep only). //! //! ## Limitations //! -//! - There is **no GitHub Actions marketplace support**. Any `uses:` step -//! other than `actions/checkout` is logged and skipped rather than -//! executed — the job continues but that step is a no-op. This means -//! workflows relying on actions like `actions/setup-node` or third-party -//! actions will not get their expected side effects (e.g. toolchain -//! installation); users should replace them with equivalent `run:` steps -//! or pre-baked container images (`runs-on: `). +//! - Any `uses:` value that isn't `actions/checkout`, a `docker://` image, +//! or a resolvable `owner/repo[/path]@ref` (or one whose `action.yml` +//! declares an unsupported `runs.using`, or a nested composite action) is +//! logged and skipped rather than executed — the job continues but that +//! step is a no-op. //! - `strategy.matrix`, `outputs`, `if:` conditionals, reusable workflows, -//! composite actions, and caching are not implemented. +//! and caching are not implemented. //! - Windows/macOS runners are executed on a Linux Docker image fallback. use std::collections::HashMap; pub mod executor; +pub mod marketplace; pub mod secrets; pub mod trigger; pub mod workflow; @@ -50,6 +54,9 @@ pub enum ActionsError { #[error("artifact error: {0}")] Artifact(String), + + #[error("marketplace action error: {0}")] + Marketplace(String), } /// Scan a map of repository file paths -> file contents for diff --git a/crates/actions/src/marketplace.rs b/crates/actions/src/marketplace.rs new file mode 100644 index 0000000..ed6953c --- /dev/null +++ b/crates/actions/src/marketplace.rs @@ -0,0 +1,320 @@ +//! Support for a practical subset of the GitHub Actions marketplace: +//! parsing a `uses:` reference, fetching the referenced action's source +//! from GitHub, and reading its `action.yml`/`action.yaml`. +//! +//! Deliberately out of scope (see `ActionRef::parse`/`Executor::execute_uses_step`): +//! anything that doesn't parse as `owner/repo[/path]@ref` or `docker://image`, +//! and (to avoid unbounded recursion) composite actions nested inside +//! another composite action. + +use std::collections::HashMap; +use std::path::PathBuf; + +use serde::Deserialize; + +use crate::workflow::Step; +use crate::ActionsError; + +/// A parsed `uses:` reference, minus the two special-cased values handled +/// directly by the executor (`actions/checkout@*`, which is a no-op, and +/// bare `genome/upload-artifact`, which has no `@ref` and so never parses +/// here at all -- it falls through to `ActionRef::parse` returning `None`, +/// same as any other unversioned/malformed `uses:` value). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ActionRef { + /// `docker://image[:tag]` -- run that image directly, no action.yml. + DockerImage(String), + /// `owner/repo[/path]@ref` -- a marketplace action to fetch from GitHub. + Marketplace { + owner: String, + repo: String, + path: Option, + git_ref: String, + }, +} + +impl ActionRef { + pub fn parse(uses: &str) -> Option { + if let Some(image) = uses.strip_prefix("docker://") { + return Some(ActionRef::DockerImage(image.to_string())); + } + let (path_part, git_ref) = uses.split_once('@')?; + if path_part.is_empty() || git_ref.is_empty() { + return None; + } + let mut segments = path_part.splitn(3, '/'); + let owner = segments.next()?.to_string(); + let repo = segments.next()?.to_string(); + if owner.is_empty() || repo.is_empty() { + return None; + } + let path = segments.next().filter(|s| !s.is_empty()).map(|s| s.to_string()); + Some(ActionRef::Marketplace { + owner, + repo, + path, + git_ref: git_ref.to_string(), + }) + } +} + +/// Mirrors the subset of `action.yml`/`action.yaml` we understand. +#[derive(Debug, Clone, Deserialize)] +pub struct ActionMetadata { + #[serde(default)] + pub inputs: HashMap, + pub runs: ActionRuns, +} + +#[derive(Debug, Clone, Default, Deserialize)] +pub struct ActionInput { + #[serde(default)] + pub default: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ActionRuns { + /// `"docker"`, `"composite"`, or a JS runtime like `"node20"`. + pub using: String, + /// Docker actions only: `"docker://image:tag"` or a path to a + /// `Dockerfile` (relative to the action's directory) to build. + #[serde(default)] + pub image: Option, + #[serde(default)] + pub entrypoint: Option, + #[serde(default)] + pub args: Option>, + /// JS actions only: entry script, relative to the action's directory. + #[serde(default)] + pub main: Option, + /// Composite actions only. Reuses the workflow `Step` shape since the + /// fields composite steps support (`run`/`uses`/`with`/`env`/`shell`) + /// are the same ones a job step supports. + #[serde(default)] + pub steps: Option>, +} + +/// A fetched action: its checked-out source directory and parsed metadata. +/// Holds on to the `TempDir` so it isn't deleted while `dir` is still in use. +pub struct FetchedAction { + pub dir: PathBuf, + pub metadata: ActionMetadata, + _tempdir: tempfile::TempDir, +} + +/// Shallow-clone-equivalent fetch of a marketplace action: clones the +/// referenced GitHub repo (git2 doesn't make a true `--depth 1` clone +/// straightforward when the target ref is a tag/branch other than the +/// remote's HEAD, so this is a full clone -- action repos are small, so +/// the cost is acceptable), resolves `git_ref` against tags/branches/raw +/// SHAs, checks out that commit, and parses `action.yml`/`action.yaml` +/// from the (optional) `path` subdirectory. +pub fn fetch_action( + owner: &str, + repo: &str, + path: Option<&str>, + git_ref: &str, +) -> Result { + let tempdir = tempfile::tempdir().map_err(|e| ActionsError::Marketplace(e.to_string()))?; + let url = format!("https://github.com/{owner}/{repo}.git"); + + let git_repo = git2::Repository::clone(&url, tempdir.path()) + .map_err(|e| ActionsError::Marketplace(format!("failed to clone {url}: {e}")))?; + + let oid = resolve_git_ref(&git_repo, git_ref)?; + git_repo + .set_head_detached(oid) + .map_err(|e| ActionsError::Marketplace(format!("failed to check out {git_ref}: {e}")))?; + let mut checkout = git2::build::CheckoutBuilder::new(); + checkout.force(); + git_repo + .checkout_head(Some(&mut checkout)) + .map_err(|e| ActionsError::Marketplace(format!("failed to check out {git_ref}: {e}")))?; + drop(git_repo); + + let action_dir = match path { + Some(p) => tempdir.path().join(p), + None => tempdir.path().to_path_buf(), + }; + + let yml_path = ["action.yml", "action.yaml"] + .iter() + .map(|f| action_dir.join(f)) + .find(|p| p.exists()) + .ok_or_else(|| { + ActionsError::Marketplace(format!("no action.yml/action.yaml found in {owner}/{repo}{}", + path.map(|p| format!("/{p}")).unwrap_or_default())) + })?; + let text = std::fs::read_to_string(&yml_path).map_err(|e| ActionsError::Marketplace(e.to_string()))?; + let metadata: ActionMetadata = serde_yaml::from_str(&text) + .map_err(|e| ActionsError::Marketplace(format!("failed to parse {}: {e}", yml_path.display())))?; + + Ok(FetchedAction { + dir: action_dir, + metadata, + _tempdir: tempdir, + }) +} + +fn resolve_git_ref(repo: &git2::Repository, git_ref: &str) -> Result { + for candidate in [format!("refs/tags/{git_ref}"), format!("refs/remotes/origin/{git_ref}")] { + if let Ok(reference) = repo.find_reference(&candidate) { + if let Some(oid) = reference.target() { + return Ok(oid); + } + } + } + repo.revparse_single(git_ref) + .and_then(|obj| obj.peel_to_commit()) + .map(|c| c.id()) + .map_err(|e| ActionsError::Marketplace(format!("could not resolve ref '{git_ref}': {e}"))) +} + +/// Resolve final input values: `with:` overrides `action.yml`'s declared +/// defaults. `metadata` is `None` for a direct `docker://image` step (no +/// action.yml at all), in which case only `with:` values are used. +pub fn resolve_inputs( + metadata: Option<&ActionMetadata>, + with: Option<&HashMap>, +) -> HashMap { + let mut resolved = HashMap::new(); + if let Some(with) = with { + for (k, v) in with { + let value = match v { + serde_yaml::Value::String(s) => s.clone(), + serde_yaml::Value::Bool(b) => b.to_string(), + serde_yaml::Value::Number(n) => n.to_string(), + other => serde_yaml::to_string(other).unwrap_or_default().trim().to_string(), + }; + resolved.insert(k.clone(), value); + } + } + if let Some(metadata) = metadata { + for (name, input) in &metadata.inputs { + if !resolved.contains_key(name) { + if let Some(default) = &input.default { + resolved.insert(name.clone(), default.clone()); + } + } + } + } + resolved +} + +/// GitHub Actions exposes each input `foo-bar` to the action's process as +/// `INPUT_FOO-BAR` (uppercased, spaces -> underscores; hyphens are left +/// as-is). This is a close approximation of that convention. +fn input_env_name(name: &str) -> String { + format!("INPUT_{}", name.to_uppercase().replace(' ', "_")) +} + +pub fn input_env_vars(resolved: &HashMap) -> Vec { + resolved + .iter() + .map(|(k, v)| format!("{}={}", input_env_name(k), v)) + .collect() +} + +/// Substitutes `${{ inputs.NAME }}` (with or without inner spaces) in +/// `runs.args`/`runs.entrypoint` templating for Docker actions. +pub fn substitute_inputs(text: &str, resolved: &HashMap) -> String { + let mut out = text.to_string(); + for (k, v) in resolved { + out = out.replace(&format!("${{{{ inputs.{k} }}}}"), v); + out = out.replace(&format!("${{{{inputs.{k}}}}}"), v); + } + out +} + +/// Maps a JS action's declared `runs.using` runtime to a Docker image that +/// actually has that Node version, since (unlike a real GitHub-hosted +/// runner) the job's own `runs-on` image has no reason to include Node. +pub fn node_image_for_using(using: &str) -> &'static str { + match using { + "node12" => "node:12-slim", + "node16" => "node:16-slim", + "node18" => "node:18-slim", + _ => "node:20-slim", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_docker_image_ref() { + assert_eq!( + ActionRef::parse("docker://alpine:3.19"), + Some(ActionRef::DockerImage("alpine:3.19".to_string())) + ); + } + + #[test] + fn parses_marketplace_ref_without_path() { + assert_eq!( + ActionRef::parse("actions/setup-node@v4"), + Some(ActionRef::Marketplace { + owner: "actions".to_string(), + repo: "setup-node".to_string(), + path: None, + git_ref: "v4".to_string(), + }) + ); + } + + #[test] + fn parses_marketplace_ref_with_path() { + assert_eq!( + ActionRef::parse("actions/aws/ec2@main"), + Some(ActionRef::Marketplace { + owner: "actions".to_string(), + repo: "aws".to_string(), + path: Some("ec2".to_string()), + git_ref: "main".to_string(), + }) + ); + } + + #[test] + fn rejects_unversioned_ref() { + assert_eq!(ActionRef::parse("genome/upload-artifact"), None); + } + + #[test] + fn resolves_with_overriding_default() { + let mut inputs = HashMap::new(); + inputs.insert( + "foo".to_string(), + ActionInput { + default: Some("default-value".to_string()), + }, + ); + let metadata = ActionMetadata { + inputs, + runs: ActionRuns { + using: "docker".to_string(), + image: None, + entrypoint: None, + args: None, + main: None, + steps: None, + }, + }; + let mut with = HashMap::new(); + with.insert("foo".to_string(), serde_yaml::Value::String("override".to_string())); + + let resolved = resolve_inputs(Some(&metadata), Some(&with)); + assert_eq!(resolved.get("foo"), Some(&"override".to_string())); + } + + #[test] + fn substitutes_input_placeholders() { + let mut resolved = HashMap::new(); + resolved.insert("name".to_string(), "world".to_string()); + assert_eq!( + substitute_inputs("hello ${{ inputs.name }}", &resolved), + "hello world" + ); + } +} diff --git a/crates/dev-env/src/manager.rs b/crates/dev-env/src/manager.rs index 7494644..375a7b1 100644 --- a/crates/dev-env/src/manager.rs +++ b/crates/dev-env/src/manager.rs @@ -77,6 +77,18 @@ impl WorkspaceManager { Ok(Self { docker }) } + /// Connects to a specific Docker (or Docker-API-compatible, e.g. + /// rootless Podman) socket path, honoring a configured + /// `DOCKER_SOCKET_PATH` instead of always resolving the platform + /// default. Pointing this at a rootless engine's socket instead of the + /// default `/var/run/docker.sock` is the supported way to run without + /// giving this process root-equivalent access to the host's main + /// Docker daemon -- see the "Container isolation" section of the docs. + pub fn connect_socket(path: &str) -> Result { + let docker = Docker::connect_with_socket(path, 120, bollard::API_DEFAULT_VERSION)?; + Ok(Self { docker }) + } + /// Validates a workspace name: lowercase alphanumeric and hyphens only, must start /// with a letter, 1-63 chars. This becomes part of the container name so it must be /// safe to pass to the Docker API. diff --git a/crates/entity/src/dev_workspace.rs b/crates/entity/src/dev_workspace.rs index 5be7f4e..c24db5f 100644 --- a/crates/entity/src/dev_workspace.rs +++ b/crates/entity/src/dev_workspace.rs @@ -16,10 +16,20 @@ pub struct Model { pub created_at: DateTime, pub auto_stop_minutes: Option, pub last_activity_at: Option>, + /// Free-form identifier self-reported by the standalone `runner` + /// process hosting this workspace, if any. `None` means it's hosted + /// directly by `server`'s own Docker daemon (the default/only path + /// before runner-hosted dev workspaces existed). + pub runner_id: Option, } pub mod status { pub const STARTING: &str = "starting"; pub const RUNNING: &str = "running"; pub const STOPPED: &str = "stopped"; + /// Enqueued as a `runner_jobs` row (kind `dev_workspace_action`, + /// action `create`) but not yet claimed/created by a runner. + pub const PENDING_RUNNER: &str = "pending_runner"; + /// The runner failed to create the workspace's container. + pub const ERROR: &str = "error"; } diff --git a/crates/entity/src/runner_job.rs b/crates/entity/src/runner_job.rs index 8bc925c..a4ea05f 100644 --- a/crates/entity/src/runner_job.rs +++ b/crates/entity/src/runner_job.rs @@ -26,13 +26,18 @@ pub struct Model { pub claimed_at: Option>, pub created_at: DateTime, pub finished_at: Option>, + /// Raw JSON result reported back via `POST /runner/jobs/:id/complete`, + /// e.g. `{"container_id": "...", "runner_id": "..."}` for a `create` + /// dev-workspace action, or `{"output": "..."}` for `exec`. `None` for + /// CI jobs (which don't report a structured result, only logs) and for + /// any job not yet completed. + pub result: Option, } pub mod kind { pub const CI_JOB: &str = "ci_job"; - /// TODO(future work): dev-workspace hosting via the standalone runner is - /// not implemented yet; this kind is reserved so the schema doesn't need - /// another migration when that lands. + /// Dev-workspace hosting via a standalone runner: see + /// `runner/src/dev_workspace_poll.rs` for the payload/result shape. pub const DEV_WORKSPACE_ACTION: &str = "dev_workspace_action"; } diff --git a/crates/git-core/src/manager.rs b/crates/git-core/src/manager.rs index 9e93345..61a176b 100644 --- a/crates/git-core/src/manager.rs +++ b/crates/git-core/src/manager.rs @@ -66,12 +66,9 @@ impl RepoManager { /// here when compiled for unix; on Windows there is no executable bit to /// set, so that step is skipped (no-op). /// - /// SCOPE NOTE: this hook is only written when a repo is *newly created* - /// via `init_repo`. Repositories that already existed before this change - /// shipped will not have the hook and must have it backfilled manually - /// (e.g. a one-off maintenance script that re-runs this same write for - /// every existing bare repo under `root`) -- that backfill is out of - /// scope here. + /// This hook is written automatically for repos created via + /// `init_repo`; for repos that predate branch-protection support, call + /// `ensure_pre_receive_hook` (below) to backfill it. fn write_pre_receive_hook(repo_path: &Path, owner: &str, name: &str) -> Result<()> { let current_exe = std::env::current_exe().map_err(GitCoreError::Io)?; // The hook runs with a cwd controlled by git (typically the repo dir @@ -120,6 +117,17 @@ impl RepoManager { Ok(()) } + /// Write (or re-write) the branch-protection pre-receive hook for a repo + /// that already exists on disk. `init_repo` already does this for newly + /// created repos; this is the public entry point for backfilling it onto + /// repos that predate branch-protection support. Safe to call + /// unconditionally and repeatedly -- the hook file is fully overwritten + /// each time, there is no "already has it" state to check first. + pub fn ensure_pre_receive_hook(&self, owner: &str, name: &str) -> Result<()> { + let path = self.repo_path(owner, name)?; + Self::write_pre_receive_hook(&path, owner, name) + } + /// Compute the filesystem path for a repository's wiki (a second bare /// repo, named `{name}.wiki.git`, mirroring Forgejo's wiki model). pub fn wiki_repo_path(&self, owner: &str, name: &str) -> PathBuf { @@ -279,6 +287,41 @@ impl RepoManager { Ok(()) } + /// Move a repository's bare directory on disk from `{owner}/{old_name}.git` + /// to `{owner}/{new_name}.git`. The caller is expected to have already + /// checked (via the DB, e.g. `idx_repositories_owner_name`) that + /// `new_name` doesn't collide with an existing repo for this owner -- + /// this only guards against the on-disk directory itself already being + /// occupied, which would indicate the DB and disk have drifted apart. + pub fn rename_repo(&self, owner: &str, old_name: &str, new_name: &str) -> Result<()> { + let old_path = self.repo_path(owner, old_name)?; + let new_path = self.repo_path(owner, new_name)?; + if !old_path.exists() { + return Err(GitCoreError::RepoNotFound(old_path)); + } + if new_path.exists() { + return Err(GitCoreError::RepoAlreadyExists(new_path)); + } + std::fs::rename(&old_path, &new_path)?; + Ok(()) + } + + /// Move a repository's wiki bare directory on disk, mirroring + /// `rename_repo`. Unlike `rename_repo`, a missing source wiki is not an + /// error -- not every repository has one. + pub fn rename_wiki(&self, owner: &str, old_name: &str, new_name: &str) -> Result<()> { + let old_path = self.wiki_repo_path(owner, old_name); + if !old_path.exists() { + return Ok(()); + } + let new_path = self.wiki_repo_path(owner, new_name); + if new_path.exists() { + return Err(GitCoreError::RepoAlreadyExists(new_path)); + } + std::fs::rename(&old_path, &new_path)?; + Ok(()) + } + fn open(&self, owner: &str, name: &str) -> Result<(Repository, PathBuf)> { let path = self.repo_path(owner, name)?; if !path.exists() { @@ -548,6 +591,72 @@ impl RepoManager { Ok(data) } + /// Recursively collect `(path, content)` pairs for every blob in the + /// tree at `git_ref`, for feeding into the code-search index. Skips + /// blobs over `max_file_bytes` and anything that looks binary (a NUL + /// byte anywhere in the content), and stops once `max_files` blobs + /// have been collected -- both are indexing-cost guards, not + /// correctness requirements, so a huge/binary-heavy repo degrades to + /// a partial index rather than an expensive or garbage one. + pub fn list_text_blobs_at_ref( + &self, + owner: &str, + name: &str, + git_ref: &str, + max_files: usize, + max_file_bytes: usize, + ) -> Result)>> { + let (repo, _) = self.open(owner, name)?; + let commit = self.resolve_commit(&repo, git_ref)?; + let tree = commit.tree()?; + let mut out = Vec::new(); + Self::collect_text_blobs(&repo, &tree, "", max_files, max_file_bytes, &mut out)?; + Ok(out) + } + + fn collect_text_blobs( + repo: &Repository, + tree: &git2::Tree, + prefix: &str, + max_files: usize, + max_file_bytes: usize, + out: &mut Vec<(String, Vec)>, + ) -> Result<()> { + for entry in tree.iter() { + if out.len() >= max_files { + return Ok(()); + } + let entry_name = entry.name().unwrap_or_default().to_string(); + let full_path = format!("{prefix}{entry_name}"); + match entry.kind() { + Some(ObjectType::Tree) => { + let object = entry.to_object(repo)?; + if let Some(subtree) = object.as_tree() { + Self::collect_text_blobs( + repo, + subtree, + &format!("{full_path}/"), + max_files, + max_file_bytes, + out, + )?; + } + } + Some(ObjectType::Blob) => { + let object = entry.to_object(repo)?; + if let Some(blob) = object.as_blob() { + let content = blob.content(); + if content.len() <= max_file_bytes && !content.contains(&0u8) { + out.push((full_path, content.to_vec())); + } + } + } + _ => {} + } + } + Ok(()) + } + /// Merge `source_branch` into `target_branch`. If the target is already /// up-to-date with the source, this is a no-op and returns the target's /// current commit sha. If a fast-forward is possible, the target branch @@ -1061,4 +1170,59 @@ mod tests { .expect("list pages again"); assert_eq!(pages2, vec!["Home".to_string()]); } + + #[test] + fn rename_repo_moves_bare_dir_and_wiki() { + let dir = tempfile::tempdir().expect("tempdir"); + let manager = RepoManager::new(dir.path()); + manager.init_repo("acme", "widgets").expect("init repo"); + manager.init_wiki("acme", "widgets").expect("init wiki"); + + let old_path = manager.repo_path("acme", "widgets").expect("old repo path"); + let old_wiki_path = manager.wiki_repo_path("acme", "widgets"); + assert!(old_path.exists()); + assert!(old_wiki_path.exists()); + + manager + .rename_repo("acme", "widgets", "gadgets") + .expect("rename repo"); + manager + .rename_wiki("acme", "widgets", "gadgets") + .expect("rename wiki"); + + assert!(!old_path.exists()); + assert!(!old_wiki_path.exists()); + let new_path = manager.repo_path("acme", "gadgets").expect("new repo path"); + let new_wiki_path = manager.wiki_repo_path("acme", "gadgets"); + assert!(new_path.exists()); + assert!(new_wiki_path.exists()); + + // Renaming onto an existing name is rejected, and rename_wiki is a + // no-op (not an error) when the source repo has no wiki. + manager.init_repo("acme", "taken").expect("init repo"); + assert!(manager.rename_repo("acme", "gadgets", "taken").is_err()); + manager.init_repo("acme", "no-wiki").expect("init repo"); + assert!(manager.rename_wiki("acme", "no-wiki", "no-wiki-2").is_ok()); + } + + #[test] + fn ensure_pre_receive_hook_is_idempotent() { + let dir = tempfile::tempdir().expect("tempdir"); + let manager = RepoManager::new(dir.path()); + manager.init_repo("acme", "widgets").expect("init repo"); + + let hook_path = manager + .repo_path("acme", "widgets") + .expect("repo path") + .join("hooks") + .join("pre-receive"); + assert!(hook_path.exists()); + + // Backfilling onto a repo that already has the hook (from + // `init_repo`) is safe to call again. + manager + .ensure_pre_receive_hook("acme", "widgets") + .expect("backfill hook"); + assert!(hook_path.exists()); + } } diff --git a/crates/graphql-api/src/mutation.rs b/crates/graphql-api/src/mutation.rs index cb521f8..0269522 100644 --- a/crates/graphql-api/src/mutation.rs +++ b/crates/graphql-api/src/mutation.rs @@ -76,6 +76,91 @@ async fn record_activity(app: &AppContext, repo_id: Option, actor_id: Uuid } } +/// Enqueues a `dev_workspace_action` job (`kind::DEV_WORKSPACE_ACTION`) for +/// a standalone runner to claim via `POST /runner/claim`, merging `action` +/// into `payload` under the `"action"` key. Returns the new job's id. +async fn enqueue_dev_workspace_job( + app: &AppContext, + action: &str, + payload: &serde_json::Value, +) -> async_graphql::Result { + let job_id = Uuid::new_v4(); + let mut full_payload = payload.clone(); + if let Some(obj) = full_payload.as_object_mut() { + obj.insert("action".to_string(), serde_json::Value::String(action.to_string())); + } + app.db + .execute( + "INSERT INTO runner_jobs (id, kind, repo_id, workflow_run_id, payload, status, claimed_by, claimed_at, created_at, finished_at, result) \ + VALUES (?1, ?2, NULL, NULL, ?3, ?4, NULL, NULL, ?5, NULL, NULL)", + params!( + job_id.to_string(), + entity::runner_job::kind::DEV_WORKSPACE_ACTION.to_string(), + full_payload.to_string(), + entity::runner_job::status::QUEUED.to_string(), + Utc::now().to_rfc3339() + ), + ) + .await?; + Ok(job_id) +} + +/// Polls `runner_jobs` for a job enqueued by `enqueue_dev_workspace_job` to +/// reach a terminal status, up to `timeout`. This is plain DB polling +/// rather than a push notification -- there's no pub/sub infrastructure in +/// this codebase -- so it trades a little latency (bounded by the poll +/// interval below, well under the runner's own ~5s claim-loop interval) +/// for simplicity. +async fn wait_for_runner_job( + app: &AppContext, + job_id: Uuid, + timeout: std::time::Duration, +) -> async_graphql::Result { + let deadline = tokio::time::Instant::now() + timeout; + loop { + let job = app + .db + .query_as::( + "SELECT * FROM runner_jobs WHERE id = ?1", + params!(job_id.to_string()), + ) + .await? + .into_iter() + .next() + .ok_or_else(|| async_graphql::Error::new("runner job disappeared"))?; + if job.status == entity::runner_job::status::SUCCESS || job.status == entity::runner_job::status::FAILURE { + return Ok(job); + } + if tokio::time::Instant::now() >= deadline { + return Err(async_graphql::Error::new( + "timed out waiting for a standalone runner to claim this dev workspace action -- is one connected?", + )); + } + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + } +} + +/// Extracts `(status, container_id, runner_id)` from a completed `create` +/// dev-workspace job. On failure (or an unparseable/missing result), status +/// is `entity::dev_workspace::status::ERROR` with both ids left `None`. +fn apply_create_result(job: &entity::runner_job::Model) -> (String, Option, Option) { + if job.status != entity::runner_job::status::SUCCESS { + return (entity::dev_workspace::status::ERROR.to_string(), None, None); + } + let Some(result) = &job.result else { + return (entity::dev_workspace::status::ERROR.to_string(), None, None); + }; + let Ok(parsed) = serde_json::from_str::(result) else { + return (entity::dev_workspace::status::ERROR.to_string(), None, None); + }; + let container_id = parsed.get("container_id").and_then(|v| v.as_str()).map(str::to_string); + let runner_id = parsed.get("runner_id").and_then(|v| v.as_str()).map(str::to_string); + if container_id.is_none() { + return (entity::dev_workspace::status::ERROR.to_string(), None, None); + } + (entity::dev_workspace::status::RUNNING.to_string(), container_id, runner_id) +} + fn username_regex() -> &'static Regex { static RE: OnceLock = OnceLock::new(); RE.get_or_init(|| Regex::new(r"^[a-zA-Z0-9_-]{3,32}$").unwrap()) @@ -443,6 +528,45 @@ impl MutationRoot { Ok(UserObject::from(user)) } + /// Writes (or re-writes) the branch-protection pre-receive hook onto + /// every repository in the instance. `init_repo` already writes this + /// hook for newly-created repos; this is the one-off maintenance + /// operation for backfilling it onto repos that predate + /// branch-protection support. Safe to run more than once -- it's a + /// plain overwrite for every repo, not just ones missing the hook. + /// Returns the number of repos it wrote the hook for; a repo whose + /// on-disk directory can't be found (or written to) is logged and + /// skipped rather than failing the whole operation. + async fn admin_backfill_pre_receive_hooks(&self, ctx: &Context<'_>) -> async_graphql::Result { + let app = ctx.data::()?; + let req = ctx.data::()?; + let claims = require_user(req)?; + if !claims.is_admin { + return Err(async_graphql::Error::new("forbidden")); + } + + let repos = app + .db + .query_as::("SELECT * FROM repositories", params!()) + .await?; + + let mut count = 0i32; + for repo in repos { + let owner_login = resolve_owner_login(&app.db, &repo.owner_type, repo.owner_id) + .await + .unwrap_or_default(); + match app.repo_manager.ensure_pre_receive_hook(&owner_login, &repo.name) { + Ok(()) => count += 1, + Err(e) => tracing::warn!( + "failed to backfill pre-receive hook for {owner_login}/{}: {e}", + repo.name + ), + } + } + + Ok(count) + } + async fn create_repository( &self, ctx: &Context<'_>, @@ -539,10 +663,83 @@ impl MutationRoot { Ok(true) } + /// Renames a repository: updates the DB `name` column and moves both + /// the bare repo directory and (if present) its wiki directory on disk + /// to match. Requires Admin permission on the repository. Errors if the + /// owner already has another repository named `new_name`. On partial + /// failure (e.g. the wiki move or the DB update fails after the repo + /// directory was already moved), best-effort rolls back the disk + /// rename(s) already performed so disk and DB don't end up disagreeing + /// about the repo's name. + async fn rename_repository( + &self, + ctx: &Context<'_>, + repo_id: Uuid, + new_name: String, + ) -> async_graphql::Result { + let app = ctx.data::()?; + let req = ctx.data::()?; + let claims = require_user(req)?; + + let mut repo = find_repo(app, repo_id).await?; + let perm = repo_permission(app, &repo, claims.sub).await?; + if perm != Some(Permission::Admin) { + return Err(async_graphql::Error::new("forbidden")); + } + + if new_name == repo.name { + return Ok(RepositoryObject::from_model(&app.db, repo).await); + } + + let existing = app + .db + .query_as::( + "SELECT * FROM repositories WHERE owner_type = ?1 AND owner_id = ?2 AND name = ?3", + params!(repo.owner_type.clone(), repo.owner_id.to_string(), new_name.clone()), + ) + .await? + .into_iter() + .next(); + if existing.is_some() { + return Err(async_graphql::Error::new(format!( + "owner already has a repository named '{new_name}'" + ))); + } + + let owner_login = resolve_owner_login(&app.db, &repo.owner_type, repo.owner_id) + .await + .unwrap_or_default(); + let old_name = repo.name.clone(); + + app.repo_manager + .rename_repo(&owner_login, &old_name, &new_name) + .map_err(|e| async_graphql::Error::new(e.to_string()))?; + + if let Err(e) = app.repo_manager.rename_wiki(&owner_login, &old_name, &new_name) { + let _ = app.repo_manager.rename_repo(&owner_login, &new_name, &old_name); + return Err(async_graphql::Error::new(e.to_string())); + } + + if let Err(e) = app + .db + .execute( + "UPDATE repositories SET name = ?1 WHERE id = ?2", + params!(new_name.clone(), repo_id.to_string()), + ) + .await + { + let _ = app.repo_manager.rename_wiki(&owner_login, &new_name, &old_name); + let _ = app.repo_manager.rename_repo(&owner_login, &new_name, &old_name); + return Err(e.into()); + } + + repo.name = new_name; + Ok(RepositoryObject::from_model(&app.db, repo).await) + } + /// Updates a repository's mutable metadata (description, visibility, - /// default branch). Requires Admin permission on the repository. Does - /// not rename the repository on disk; renaming is not currently - /// supported. + /// default branch). Requires Admin permission on the repository. Use + /// `renameRepository` to change its name. async fn update_repository( &self, ctx: &Context<'_>, @@ -1878,6 +2075,22 @@ impl MutationRoot { /// raw `image` for advanced/custom use. `template` takes precedence if /// both are given; a raw `image` with no `template` falls back to the /// single-port code-server-only behavior that predates templates. + /// + /// If `on_runner` is true, instead of creating the container directly + /// against `server`'s own Docker daemon, this enqueues a + /// `dev_workspace_action` job (`kind::DEV_WORKSPACE_ACTION`) for a + /// connected standalone `runner` process to claim and create the + /// container against its *own* Docker daemon (see + /// `runner/src/dev_workspace_poll.rs`). The returned workspace has + /// `status: "pending_runner"` and a `null` `containerId`/`runnerId` + /// until a runner claims the job (poll `devWorkspace(id)` for the + /// updated status). Errors if no runner claims it within 20s. + /// + /// Known gap: live port-proxying (`GET /workspaces/:id/proxy/*path`) + /// only works for server-hosted workspaces today -- a runner-hosted + /// workspace's container is on a different, not-necessarily-reachable + /// Docker host, and there is no reverse-tunnel between the runner and + /// `server` yet to route proxied HTTP requests to it. async fn create_dev_workspace( &self, ctx: &Context<'_>, @@ -1886,6 +2099,7 @@ impl MutationRoot { image: Option, repo_id: Option, auto_stop_minutes: Option, + on_runner: Option, ) -> async_graphql::Result { let app = ctx.data::()?; let req = ctx.data::()?; @@ -1916,6 +2130,63 @@ impl MutationRoot { None }; + let id = Uuid::new_v4(); + let created_at = Utc::now(); + + if on_runner.unwrap_or(false) { + let payload = serde_json::json!({ + "workspace_id": id, + "name": name, + "image": image, + "ports": ports.iter().map(|(n, p)| (n.to_string(), *p)).collect::>(), + "repo_clone_url": repo_clone_url, + "owner": claims.username, + }); + let job_id = enqueue_dev_workspace_job(app, "create", &payload).await?; + + let status = entity::dev_workspace::status::PENDING_RUNNER.to_string(); + app.db + .execute( + "INSERT INTO dev_workspaces (id, owner_id, repo_id, name, image, status, container_id, created_at, auto_stop_minutes, last_activity_at, runner_id) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, NULL, ?7, ?8, NULL, NULL)", + params!( + id.to_string(), + claims.sub.to_string(), + repo_id.map(|r| r.to_string()), + name.clone(), + image.clone(), + status.clone(), + created_at.to_rfc3339(), + auto_stop_minutes + ), + ) + .await?; + + let job = wait_for_runner_job(app, job_id, std::time::Duration::from_secs(20)).await?; + let (final_status, container_id, runner_id) = apply_create_result(&job); + app.db + .execute( + "UPDATE dev_workspaces SET status = ?1, container_id = ?2, runner_id = ?3 WHERE id = ?4", + params!(final_status.clone(), container_id.clone(), runner_id.clone(), id.to_string()), + ) + .await?; + + let workspace = entity::dev_workspace::Model { + id, + owner_id: claims.sub, + repo_id, + name, + image, + status: final_status, + container_id, + created_at, + auto_stop_minutes, + last_activity_at: None, + runner_id, + }; + return Ok(DevWorkspaceObject::from(workspace)); + } + let handle = app .workspace_manager .create_workspace( @@ -1930,15 +2201,13 @@ impl MutationRoot { .await .map_err(|e| async_graphql::Error::new(e.to_string()))?; - let id = Uuid::new_v4(); - let created_at = Utc::now(); let status = entity::dev_workspace::status::RUNNING.to_string(); let container_id = Some(handle.container_id); let last_activity_at = Some(Utc::now()); app.db .execute( - "INSERT INTO dev_workspaces (id, owner_id, repo_id, name, image, status, container_id, created_at, auto_stop_minutes, last_activity_at) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", + "INSERT INTO dev_workspaces (id, owner_id, repo_id, name, image, status, container_id, created_at, auto_stop_minutes, last_activity_at, runner_id) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, NULL)", params!( id.to_string(), claims.sub.to_string(), @@ -1965,6 +2234,7 @@ impl MutationRoot { created_at, auto_stop_minutes, last_activity_at, + runner_id: None, }; Ok(DevWorkspaceObject::from(workspace)) } @@ -1987,6 +2257,11 @@ impl MutationRoot { if workspace.owner_id != claims.sub { return Err(async_graphql::Error::new("forbidden")); } + if workspace.runner_id.is_some() { + return Err(async_graphql::Error::new( + "starting/stopping a runner-hosted dev workspace is not supported yet -- delete and recreate it instead", + )); + } let container_id = workspace .container_id .clone() @@ -2026,6 +2301,11 @@ impl MutationRoot { if workspace.owner_id != claims.sub { return Err(async_graphql::Error::new("forbidden")); } + if workspace.runner_id.is_some() { + return Err(async_graphql::Error::new( + "starting/stopping a runner-hosted dev workspace is not supported yet -- delete and recreate it instead", + )); + } let container_id = workspace .container_id .clone() @@ -2066,7 +2346,21 @@ impl MutationRoot { return Err(async_graphql::Error::new("forbidden")); } - if let Some(container_id) = &workspace.container_id { + if workspace.runner_id.is_some() { + if let Some(container_id) = &workspace.container_id { + let payload = serde_json::json!({ + "workspace_id": workspace.id, + "container_id": container_id, + }); + let job_id = enqueue_dev_workspace_job(app, "delete", &payload).await?; + // Best-effort: the DB row is removed below regardless of + // whether the runner confirms deletion in time, so a slow + // or disconnected runner doesn't strand the user with an + // undeletable workspace record -- it may leave an orphaned + // container on the runner's host if the job never lands. + let _ = wait_for_runner_job(app, job_id, std::time::Duration::from_secs(20)).await; + } + } else if let Some(container_id) = &workspace.container_id { app.workspace_manager .delete_workspace(container_id) .await @@ -2890,6 +3184,26 @@ impl MutationRoot { .clone() .ok_or_else(|| async_graphql::Error::new("workspace has no container"))?; + if workspace.runner_id.is_some() { + let payload = serde_json::json!({ + "workspace_id": workspace.id, + "container_id": container_id, + "cmd": command, + }); + let job_id = enqueue_dev_workspace_job(app, "exec", &payload).await?; + let job = wait_for_runner_job(app, job_id, std::time::Duration::from_secs(20)).await?; + if job.status != entity::runner_job::status::SUCCESS { + return Err(async_graphql::Error::new("exec failed on the runner-hosted workspace")); + } + let output = job + .result + .as_deref() + .and_then(|r| serde_json::from_str::(r).ok()) + .and_then(|v| v.get("output").and_then(|o| o.as_str()).map(str::to_string)) + .unwrap_or_default(); + return Ok(output); + } + let output = app .workspace_manager .exec_command(&container_id, command) diff --git a/crates/graphql-api/src/query.rs b/crates/graphql-api/src/query.rs index d05977d..f915558 100644 --- a/crates/graphql-api/src/query.rs +++ b/crates/graphql-api/src/query.rs @@ -4,10 +4,49 @@ use uuid::Uuid; use crate::context::{AppContext, RequestContext}; use crate::types::{ - AccessTokenObject, ActivityEventObject, DevWorkspaceObject, IssueObject, NotificationObject, - OrganizationObject, PackageObject, RepositoryObject, SearchResults, SshKeyObject, UserObject, + AccessTokenObject, ActivityEventObject, CodeSearchResultObject, DevWorkspaceObject, IssueObject, + NotificationObject, OrganizationObject, PackageObject, RepositoryObject, SearchResults, + SshKeyObject, UserObject, }; +/// Turns free-text user input into a safe SQLite FTS5 `MATCH` query. +/// +/// Each whitespace-separated term becomes either an unquoted `term*` prefix +/// match (when it's plain alphanumeric/underscore -- the common case, and +/// what gives search-as-you-type behavior) or a double-quoted phrase with +/// embedded quotes doubled (FTS5's own escaping rule) for anything else, so +/// user-supplied FTS5 operators/punctuation (`AND`, `"`, `(`, `-`, ...) can +/// never be parsed as query syntax. Terms are ANDed together (FTS5's +/// default). Returns `None` if there are no usable terms, since `MATCH ""` +/// is a syntax error rather than a "match nothing" query. +fn build_fts_match_query(query: &str) -> Option { + let terms: Vec = query + .split_whitespace() + .map(|term| { + if !term.is_empty() && term.chars().all(|c| c.is_alphanumeric() || c == '_') { + format!("{term}*") + } else { + format!("\"{}\"", term.replace('"', "\"\"")) + } + }) + .collect(); + if terms.is_empty() { + None + } else { + Some(terms.join(" ")) + } +} + +/// Row shape for the `code_search_fts` MATCH query in `search` below -- +/// there's no `entity` model for it since it's a pure FTS5 virtual table, +/// not a regular data table with a persisted repository/entity type. +#[derive(serde::Deserialize)] +struct CodeSearchRow { + repo_id: String, + path: String, + snippet: String, +} + pub struct QueryRoot; #[Object] @@ -394,12 +433,23 @@ impl QueryRoot { Ok(packages.into_iter().map(PackageObject::from).collect()) } - /// Basic LIKE-based search across repositories, issues, and users. - /// Each result list is capped at 20 entries. + /// Real full-text search (SQLite FTS5) across repositories, issues, + /// users, and indexed file content. Each result list is capped at 20 + /// entries. Falls back to returning empty results (rather than an + /// error) when `query` has no usable search terms (e.g. only + /// punctuation/whitespace). async fn search(&self, ctx: &Context<'_>, query: String) -> async_graphql::Result { let app = ctx.data::()?; let req = ctx.data::()?; - let pattern = format!("%{}%", query); + + let Some(fts_query) = build_fts_match_query(&query) else { + return Ok(SearchResults { + repositories: vec![], + issues: vec![], + users: vec![], + code: vec![], + }); + }; // Repositories: public ones, plus ones the current user owns or collaborates on. let mut owned_or_collab_ids: Vec = vec![]; @@ -427,18 +477,24 @@ impl QueryRoot { } let repo_sql = if owned_or_collab_ids.is_empty() { - "SELECT * FROM repositories WHERE is_private = 0 AND (name LIKE ?1 OR description LIKE ?1) LIMIT 20" + "SELECT r.* FROM repositories r \ + JOIN repositories_fts ON repositories_fts.id = r.id \ + WHERE repositories_fts MATCH ?1 AND r.is_private = 0 \ + ORDER BY repositories_fts.rank LIMIT 20" .to_string() } else { let placeholders: Vec = (2..=owned_or_collab_ids.len() + 1) .map(|i| format!("?{i}")) .collect(); format!( - "SELECT * FROM repositories WHERE (is_private = 0 OR id IN ({})) AND (name LIKE ?1 OR description LIKE ?1) LIMIT 20", + "SELECT r.* FROM repositories r \ + JOIN repositories_fts ON repositories_fts.id = r.id \ + WHERE repositories_fts MATCH ?1 AND (r.is_private = 0 OR r.id IN ({})) \ + ORDER BY repositories_fts.rank LIMIT 20", placeholders.join(", ") ) }; - let mut repo_params = vec![hiqlite::Param::Text(pattern.clone())]; + let mut repo_params = vec![hiqlite::Param::Text(fts_query.clone())]; for id in &owned_or_collab_ids { repo_params.push(hiqlite::Param::Text(id.to_string())); } @@ -451,8 +507,8 @@ impl QueryRoot { repositories.push(RepositoryObject::from_model(&app.db, r).await); } - // Issues: only from repos visible to the current search context (public, - // or owned/collaborated-on by the current user). + // Issues + code: only from repos visible to the current search context + // (public, or owned/collaborated-on by the current user). let visible_repo_sql = if owned_or_collab_ids.is_empty() { "SELECT * FROM repositories WHERE is_private = 0".to_string() } else { @@ -466,34 +522,74 @@ impl QueryRoot { .iter() .map(|id| hiqlite::Param::Text(id.to_string())) .collect(); - let visible_repo_ids: Vec = app + let visible_repos = app .db .query_as::(visible_repo_sql, visible_repo_params) - .await? - .into_iter() - .map(|r| r.id) - .collect(); + .await?; + let visible_repo_ids: Vec = visible_repos.iter().map(|r| r.id).collect(); let issues = if visible_repo_ids.is_empty() { vec![] } else { let placeholders: Vec = (2..=visible_repo_ids.len() + 1).map(|i| format!("?{i}")).collect(); let sql = format!( - "SELECT * FROM issues WHERE repo_id IN ({}) AND (title LIKE ?1 OR body LIKE ?1) LIMIT 20", + "SELECT i.* FROM issues i \ + JOIN issues_fts ON issues_fts.id = i.id \ + WHERE issues_fts MATCH ?1 AND i.repo_id IN ({}) \ + ORDER BY issues_fts.rank LIMIT 20", placeholders.join(", ") ); - let mut p = vec![hiqlite::Param::Text(pattern.clone())]; + let mut p = vec![hiqlite::Param::Text(fts_query.clone())]; for id in &visible_repo_ids { p.push(hiqlite::Param::Text(id.to_string())); } app.db.query_as::(sql, p).await? }; + let code = if visible_repo_ids.is_empty() { + vec![] + } else { + let placeholders: Vec = (2..=visible_repo_ids.len() + 1).map(|i| format!("?{i}")).collect(); + let sql = format!( + "SELECT repo_id, path, \ + snippet(code_search_fts, 2, '[b]', '[/b]', '...', 12) AS snippet \ + FROM code_search_fts \ + WHERE code_search_fts MATCH ?1 AND repo_id IN ({}) \ + ORDER BY rank LIMIT 20", + placeholders.join(", ") + ); + let mut p = vec![hiqlite::Param::Text(fts_query.clone())]; + for id in &visible_repo_ids { + p.push(hiqlite::Param::Text(id.to_string())); + } + app.db + .query_as::(sql, p) + .await? + }; + let repos_by_id: std::collections::HashMap = + visible_repos.into_iter().map(|r| (r.id, r)).collect(); + let mut code_results = Vec::with_capacity(code.len()); + for row in code { + let Ok(repo_id) = row.repo_id.parse::() else { + continue; + }; + let Some(repo) = repos_by_id.get(&repo_id).cloned() else { + continue; + }; + code_results.push(CodeSearchResultObject { + repository: RepositoryObject::from_model(&app.db, repo).await, + path: row.path, + snippet: row.snippet, + }); + } + let users = app .db .query_as::( - "SELECT * FROM users WHERE username LIKE ?1 LIMIT 20", - params!(pattern), + "SELECT u.* FROM users u \ + JOIN users_fts ON users_fts.id = u.id \ + WHERE users_fts MATCH ?1 ORDER BY users_fts.rank LIMIT 20", + params!(fts_query), ) .await?; @@ -501,6 +597,7 @@ impl QueryRoot { repositories, issues: issues.into_iter().map(IssueObject::from).collect(), users: users.into_iter().map(UserObject::from).collect(), + code: code_results, }) } } diff --git a/crates/graphql-api/src/types.rs b/crates/graphql-api/src/types.rs index ac99f05..ac66832 100644 --- a/crates/graphql-api/src/types.rs +++ b/crates/graphql-api/src/types.rs @@ -562,6 +562,10 @@ pub struct DevWorkspaceObject { pub created_at: DateTime, pub auto_stop_minutes: Option, pub last_activity_at: Option>, + /// Set once a standalone runner has claimed and created this + /// workspace's container; `null` for workspaces hosted directly by + /// `server`'s own Docker daemon (the default). + pub runner_id: Option, } impl From for DevWorkspaceObject { @@ -577,6 +581,7 @@ impl From for DevWorkspaceObject { created_at: m.created_at, auto_stop_minutes: m.auto_stop_minutes, last_activity_at: m.last_activity_at, + runner_id: m.runner_id, } } } @@ -712,12 +717,22 @@ impl From for ActivityEventObject { } } +/// A single code-search hit: one file, in one repository, that matched. +#[derive(SimpleObject, Clone)] +pub struct CodeSearchResultObject { + pub repository: RepositoryObject, + pub path: String, + /// An FTS5-generated excerpt around the match, with `[b]...[/b]` markers. + pub snippet: String, +} + /// Result bundle for the basic `search` query, grouping matches by entity kind. #[derive(SimpleObject, Clone)] pub struct SearchResults { pub repositories: Vec, pub issues: Vec, pub users: Vec, + pub code: Vec, } #[derive(SimpleObject, Clone)] diff --git a/crates/migration/migrations/10_dev_workspace_runner.sql b/crates/migration/migrations/10_dev_workspace_runner.sql new file mode 100644 index 0000000..5c1b852 --- /dev/null +++ b/crates/migration/migrations/10_dev_workspace_runner.sql @@ -0,0 +1,18 @@ +-- Lets a dev workspace be hosted by a standalone `runner` process instead +-- of being created directly against the `server` process's own Docker +-- daemon (see `entity::runner_job::kind::DEV_WORKSPACE_ACTION` and +-- `runner/src/dev_workspace_poll.rs`). `runner_id` is a free-form +-- identifier the hosting runner process self-reports (there is no runner +-- registry/heartbeat table) -- it is NULL for server-hosted workspaces +-- (the existing, default behavior) and set once a runner claims and +-- creates the workspace. +ALTER TABLE dev_workspaces ADD COLUMN runner_id TEXT; + +-- Carries a runner's completion result (e.g. the created container's id, +-- or an `execInDevWorkspace` command's output) back through +-- `POST /runner/jobs/:id/complete` -- previously completion only recorded +-- `status`/`finished_at`, with the request body's `logs` field discarded, +-- which was fine when nothing needed the result of a CI job fed back into +-- another read, but a dev-workspace action's caller (a GraphQL mutation +-- polling this row) does need it. +ALTER TABLE runner_jobs ADD COLUMN result TEXT; diff --git a/crates/migration/migrations/9_search_fts.sql b/crates/migration/migrations/9_search_fts.sql new file mode 100644 index 0000000..edf9be3 --- /dev/null +++ b/crates/migration/migrations/9_search_fts.sql @@ -0,0 +1,90 @@ +-- Real full-text search, replacing the substring `LIKE` matching that +-- QueryRoot::search used to be limited to. FTS5 tables here are kept as +-- plain (non "external content") virtual tables with an UNINDEXED `id` +-- column, rather than linked via `content_rowid`, because the base +-- tables key on a TEXT UUID `id`/`repo_id` and FTS5's content_rowid +-- linkage requires an INTEGER rowid alias -- an explicit UNINDEXED +-- column plus triggers is simpler and avoids that mismatch entirely. + +CREATE VIRTUAL TABLE IF NOT EXISTS repositories_fts USING fts5( + id UNINDEXED, + name, + description +); + +CREATE TRIGGER IF NOT EXISTS repositories_fts_ai AFTER INSERT ON repositories BEGIN + INSERT INTO repositories_fts (id, name, description) + VALUES (new.id, new.name, new.description); +END; + +CREATE TRIGGER IF NOT EXISTS repositories_fts_ad AFTER DELETE ON repositories BEGIN + DELETE FROM repositories_fts WHERE id = old.id; +END; + +CREATE TRIGGER IF NOT EXISTS repositories_fts_au AFTER UPDATE ON repositories BEGIN + DELETE FROM repositories_fts WHERE id = old.id; + INSERT INTO repositories_fts (id, name, description) + VALUES (new.id, new.name, new.description); +END; + +-- Backfill rows that existed before this migration. +INSERT INTO repositories_fts (id, name, description) +SELECT id, name, description FROM repositories; + +CREATE VIRTUAL TABLE IF NOT EXISTS issues_fts USING fts5( + id UNINDEXED, + repo_id UNINDEXED, + title, + body +); + +CREATE TRIGGER IF NOT EXISTS issues_fts_ai AFTER INSERT ON issues BEGIN + INSERT INTO issues_fts (id, repo_id, title, body) + VALUES (new.id, new.repo_id, new.title, new.body); +END; + +CREATE TRIGGER IF NOT EXISTS issues_fts_ad AFTER DELETE ON issues BEGIN + DELETE FROM issues_fts WHERE id = old.id; +END; + +CREATE TRIGGER IF NOT EXISTS issues_fts_au AFTER UPDATE ON issues BEGIN + DELETE FROM issues_fts WHERE id = old.id; + INSERT INTO issues_fts (id, repo_id, title, body) + VALUES (new.id, new.repo_id, new.title, new.body); +END; + +INSERT INTO issues_fts (id, repo_id, title, body) +SELECT id, repo_id, title, body FROM issues; + +CREATE VIRTUAL TABLE IF NOT EXISTS users_fts USING fts5( + id UNINDEXED, + username +); + +CREATE TRIGGER IF NOT EXISTS users_fts_ai AFTER INSERT ON users BEGIN + INSERT INTO users_fts (id, username) VALUES (new.id, new.username); +END; + +CREATE TRIGGER IF NOT EXISTS users_fts_ad AFTER DELETE ON users BEGIN + DELETE FROM users_fts WHERE id = old.id; +END; + +CREATE TRIGGER IF NOT EXISTS users_fts_au AFTER UPDATE ON users BEGIN + DELETE FROM users_fts WHERE id = old.id; + INSERT INTO users_fts (id, username) VALUES (new.id, new.username); +END; + +INSERT INTO users_fts (id, username) +SELECT id, username FROM users; + +-- Code/file-content search. Unlike the tables above this has no source +-- table to trigger off of -- it's populated by the application after +-- each push (see `index_repo_code` in crates/server), re-indexing the +-- full tree of the default branch's new tip and replacing that repo's +-- previous rows. There is deliberately no UNIQUE constraint beyond what +-- the app enforces by deleting-then-reinserting per repo_id. +CREATE VIRTUAL TABLE IF NOT EXISTS code_search_fts USING fts5( + repo_id UNINDEXED, + path UNINDEXED, + content +); diff --git a/crates/runner/src/dev_workspace_poll.rs b/crates/runner/src/dev_workspace_poll.rs index 3793510..42add67 100644 --- a/crates/runner/src/dev_workspace_poll.rs +++ b/crates/runner/src/dev_workspace_poll.rs @@ -1,21 +1,94 @@ -//! Stub for future dev-workspace-hosting polling by the standalone runner. +//! Executes `dev_workspace_action` jobs claimed via `/runner/claim` against +//! this runner's own local Docker daemon (`dev_env::WorkspaceManager`). //! -//! TODO(future work): the `runner_jobs.kind = 'dev_workspace_action'` column -//! value is already reserved (see `entity::runner_job::kind::DEV_WORKSPACE_ACTION`) -//! but no polling loop against `dev_env::WorkspaceManager` exists yet. This -//! function exists only to prove the dependency wiring compiles end-to-end; -//! it is not called from a real polling loop. -#[allow(dead_code)] -pub async fn poll_dev_workspace_jobs(workspace_manager: &dev_env::WorkspaceManager) { - match workspace_manager.list_workspaces("").await { - Ok(handles) => { - tracing::debug!( - "dev-workspace polling stub: {} existing workspace(s) known to Docker (no-op)", - handles.len() - ); - } +//! Known gap: the resulting container lives on *this* runner's Docker +//! daemon, not `server`'s. `server` has no network path back to it, so +//! live port-proxying (`GET /workspaces/:id/proxy/*path`) and +//! start/stop for runner-hosted workspaces are not supported yet (see the +//! matching restrictions in `graphql-api::mutation::create_dev_workspace` +//! and `crates/server/src/main.rs::proxy_workspace_request`) -- only +//! create/delete/exec are wired up here. + +use serde::Deserialize; +use uuid::Uuid; + +#[derive(Debug, Deserialize)] +#[serde(tag = "action", rename_all = "snake_case")] +enum DevWorkspaceJobPayload { + Create { + #[allow(dead_code)] + workspace_id: Uuid, + name: String, + image: String, + #[serde(default)] + ports: Vec<(String, u16)>, + #[serde(default)] + repo_clone_url: Option, + owner: String, + }, + Delete { + #[allow(dead_code)] + workspace_id: Uuid, + container_id: String, + }, + Exec { + #[allow(dead_code)] + workspace_id: Uuid, + container_id: String, + cmd: Vec, + }, +} + +/// Result of handling one dev-workspace job: `(status, logs, result_json)`, +/// matching the shape `/runner/jobs/:id/complete` expects (`status`/`logs` +/// from the existing CI-job path, plus the new optional `result`). +pub async fn handle_dev_workspace_job( + workspace_manager: &dev_env::WorkspaceManager, + runner_id: &str, + payload_json: &str, +) -> (&'static str, String, Option) { + let payload = match serde_json::from_str::(payload_json) { + Ok(p) => p, Err(e) => { - tracing::debug!("dev-workspace polling stub: failed to list workspaces: {e}"); + return ("failure", format!("failed to parse dev workspace job payload: {e}"), None); + } + }; + + match payload { + DevWorkspaceJobPayload::Create { name, image, ports, repo_clone_url, owner, .. } => { + let port_refs: Vec<(&str, u16)> = ports.iter().map(|(n, p)| (n.as_str(), *p)).collect(); + match workspace_manager + .create_workspace(&name, &image, &port_refs, repo_clone_url.as_deref(), None, None, &owner) + .await + { + Ok(handle) => { + let result = serde_json::json!({ + "container_id": handle.container_id, + "runner_id": runner_id, + }); + ( + "success", + format!("created workspace container {}", handle.container_id), + Some(result.to_string()), + ) + } + Err(e) => ("failure", format!("failed to create workspace: {e}"), None), + } + } + DevWorkspaceJobPayload::Delete { container_id, .. } => { + match workspace_manager.delete_workspace(&container_id).await { + Ok(()) => ("success", "workspace deleted".to_string(), None), + Err(e) => ("failure", format!("failed to delete workspace: {e}"), None), + } + } + DevWorkspaceJobPayload::Exec { container_id, cmd, .. } => { + match workspace_manager.exec_command(&container_id, cmd).await { + Ok(output) => { + let result = serde_json::json!({ "output": output }); + ("success", "exec completed".to_string(), Some(result.to_string())) + } + Err(e) => ("failure", format!("exec failed: {e}"), None), + } } } } diff --git a/crates/runner/src/main.rs b/crates/runner/src/main.rs index 71b11c6..f70d009 100644 --- a/crates/runner/src/main.rs +++ b/crates/runner/src/main.rs @@ -16,12 +16,25 @@ //! `/runner/*` routes (validated the same way as any other bearer/PAT request). //! - `GENOME_RUNNER_ARTIFACTS_DIR` -- local directory for artifact tarballs //! collected by `actions::Executor` (defaults to `./runner-artifacts`). -//! - `DOCKER_SOCKET_PATH` -- accepted for forward-compatibility. `bollard`'s -//! `Docker::connect_with_local_defaults()` (used by both `actions::Executor` -//! and `dev_env::WorkspaceManager`) already honors Docker's own standard -//! `DOCKER_HOST` env var / platform-default socket resolution; there is no -//! separate bollard knob for an arbitrary "socket path" env var today, so -//! this variable is currently read but only logged, not wired further. +//! - `DOCKER_SOCKET_PATH` -- optional. If unset, connects to Docker via +//! platform defaults (`DOCKER_HOST` env var, else the usual unix +//! socket/named pipe), same as before. If set, connects to that exact +//! socket path instead -- e.g. a rootless Podman socket -- so this +//! process doesn't need root-equivalent access to the host's main Docker +//! daemon. See the "Container isolation" section of the docs. +//! - `GENOME_RUNNER_ID` -- optional free-form identifier for this runner +//! process, reported back on `dev_workspace_action` `create` jobs so +//! `dev_workspaces.runner_id` records which runner is hosting a given +//! workspace. Defaults to a freshly generated UUID if unset (so it's +//! stable for this process's lifetime, but changes across restarts). +//! +//! This binary also claims and executes `dev_workspace_action` jobs (see +//! `dev_workspace_poll`), letting a dev workspace be hosted on this +//! runner's own Docker daemon instead of `server`'s. Only create/delete/exec +//! are wired up -- live port-proxying to a runner-hosted workspace isn't +//! supported yet (no reverse tunnel between runner and server exists), so +//! `server` rejects `start`/`stop`/proxy requests for one with a clear error +//! instead of silently trying the wrong Docker daemon. use std::collections::HashMap; use std::time::Duration; @@ -57,6 +70,8 @@ struct CiJobPayload { struct CompleteRequest { status: String, logs: String, + #[serde(skip_serializing_if = "Option::is_none")] + result: Option, } #[tokio::main] @@ -72,17 +87,25 @@ async fn main() -> anyhow::Result<()> { let artifacts_dir = std::env::var("GENOME_RUNNER_ARTIFACTS_DIR") .unwrap_or_else(|_| "./runner-artifacts".to_string()); - if let Ok(socket) = std::env::var("DOCKER_SOCKET_PATH") { - tracing::info!( - "DOCKER_SOCKET_PATH={socket} accepted for forward-compat; bollard resolves the \ - Docker socket via its own DOCKER_HOST convention / platform defaults, not this var" - ); - } + let docker_socket_path = std::env::var("DOCKER_SOCKET_PATH").ok(); - let executor = actions::Executor::new(std::path::PathBuf::from(&artifacts_dir))?; + let (executor, workspace_manager) = match &docker_socket_path { + Some(socket) => { + tracing::info!("connecting to Docker socket {socket} (from DOCKER_SOCKET_PATH)"); + ( + actions::Executor::new_with_socket(std::path::PathBuf::from(&artifacts_dir), socket)?, + dev_env::WorkspaceManager::connect_socket(socket)?, + ) + } + None => ( + actions::Executor::new(std::path::PathBuf::from(&artifacts_dir))?, + dev_env::WorkspaceManager::connect_local()?, + ), + }; + let runner_id = std::env::var("GENOME_RUNNER_ID").unwrap_or_else(|_| Uuid::new_v4().to_string()); let http = reqwest::Client::new(); - tracing::info!("runner started, polling {server_url} every 5s"); + tracing::info!("runner started (id={runner_id}), polling {server_url} every 5s"); let mut interval = tokio::time::interval(Duration::from_secs(5)); loop { @@ -90,7 +113,16 @@ async fn main() -> anyhow::Result<()> { match claim_job(&http, &server_url, &runner_token).await { Ok(Some(job)) => { - if let Err(e) = handle_job(&http, &server_url, &runner_token, &executor, job).await + if let Err(e) = handle_job( + &http, + &server_url, + &runner_token, + &executor, + &workspace_manager, + &runner_id, + job, + ) + .await { tracing::warn!("failed to handle claimed job: {e}"); } @@ -127,21 +159,20 @@ async fn claim_job( Ok(Some(job)) } +#[allow(clippy::too_many_arguments)] async fn handle_job( http: &reqwest::Client, server_url: &str, token: &str, executor: &actions::Executor, + workspace_manager: &dev_env::WorkspaceManager, + runner_id: &str, job: ClaimedJob, ) -> anyhow::Result<()> { if job.kind == entity_kind_dev_workspace() { - // TODO(future work): full dev-workspace-hosting polling is not - // implemented yet. `dev_workspace_poll::poll_dev_workspace_jobs` - // exists as a stub proving `dev_env::WorkspaceManager` wiring - // compiles; wire it up here once that feature is built out. - tracing::warn!("received dev_workspace_action job {}; not yet supported by runner, skipping", job.id); - report_complete(http, server_url, token, job.id, "failure", "dev_workspace_action jobs are not yet supported by the standalone runner".to_string()).await?; - return Ok(()); + let (status, logs, result) = + dev_workspace_poll::handle_dev_workspace_job(workspace_manager, runner_id, &job.payload).await; + return report_complete(http, server_url, token, job.id, status, logs, result).await; } let payload: CiJobPayload = serde_json::from_str(&job.payload)?; @@ -179,7 +210,7 @@ async fn handle_job( } }; - report_complete(http, server_url, token, job.id, status, collected_logs).await + report_complete(http, server_url, token, job.id, status, collected_logs, None).await } async fn report_complete( @@ -189,10 +220,12 @@ async fn report_complete( job_id: Uuid, status: &str, logs: String, + result: Option, ) -> anyhow::Result<()> { let body = CompleteRequest { status: status.to_string(), logs, + result, }; let resp = http .post(format!("{server_url}/runner/jobs/{job_id}/complete")) diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index 20e211e..c1285c7 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -154,8 +154,11 @@ async fn main() -> anyhow::Result<()> { .parent() .map(|p| p.join("artifacts")) .unwrap_or_else(|| std::path::PathBuf::from("./artifacts")); - let actions_executor = Arc::new(actions::Executor::new(artifacts_root)?); - let workspace_manager = Arc::new(dev_env::WorkspaceManager::connect_local()?); + let actions_executor = Arc::new(actions::Executor::new_with_socket( + artifacts_root, + &config.docker_socket_path, + )?); + let workspace_manager = Arc::new(dev_env::WorkspaceManager::connect_socket(&config.docker_socket_path)?); let webhook_dispatcher = Arc::new(webhooks::WebhookDispatcher::new(db.clone())); let app_ctx = AppContext { @@ -398,6 +401,17 @@ async fn receive_pack_handler( tracing::warn!("post-push workflow processing failed: {e}"); } }); + + let app_ctx = state.app_ctx.clone(); + let owner_clone = owner.clone(); + let repo_clone = repo_name.clone(); + tokio::spawn(async move { + if let Err(e) = + index_repo_code_on_push(app_ctx, owner_clone, repo_clone, changes).await + { + tracing::warn!("post-push code-search indexing failed: {e}"); + } + }); } let content_type = @@ -410,6 +424,82 @@ async fn receive_pack_handler( .into_response()) } +/// Fire-and-forget task run after a successful `git-receive-pack`: +/// re-indexes the repository's default branch tip into `code_search_fts` +/// (SQLite FTS5) for the `search` GraphQL query's code-search results. +/// A no-op if the push didn't move the default branch. Re-indexing always +/// replaces the repo's whole previous index rather than diffing, since a +/// force-push or history rewrite can change any file, not just the ones +/// in the immediate diff. +async fn index_repo_code_on_push( + app_ctx: AppContext, + owner: String, + repo: String, + changes: Vec<(String, String, String)>, +) -> anyhow::Result<()> { + const ZERO_SHA: &str = "0000000000000000000000000000000000000000"; + // Indexing-cost guards (see `RepoManager::list_text_blobs_at_ref`), not + // correctness requirements: a huge/binary-heavy repo gets a partial + // index rather than an expensive or garbage one. + const MAX_FILES: usize = 2000; + const MAX_FILE_BYTES: usize = 256 * 1024; + + let repo_row = app_ctx + .db + .query_as::( + "SELECT * FROM repositories WHERE name = ?1", + params!(repo.clone()), + ) + .await? + .into_iter() + .next(); + let Some(repo_row) = repo_row else { + return Ok(()); + }; + + let default_ref = format!("refs/heads/{}", repo_row.default_branch); + let Some((_, _, new_sha)) = changes.iter().find(|(r, _, _)| *r == default_ref) else { + return Ok(()); + }; + if new_sha == ZERO_SHA { + // Default branch was deleted outright: drop its index entirely. + app_ctx + .db + .execute( + "DELETE FROM code_search_fts WHERE repo_id = ?1", + params!(repo_row.id.to_string()), + ) + .await?; + return Ok(()); + } + + let blobs = + app_ctx + .repo_manager + .list_text_blobs_at_ref(&owner, &repo, new_sha, MAX_FILES, MAX_FILE_BYTES)?; + + app_ctx + .db + .execute( + "DELETE FROM code_search_fts WHERE repo_id = ?1", + params!(repo_row.id.to_string()), + ) + .await?; + + for (path, content) in blobs { + let text = String::from_utf8_lossy(&content).into_owned(); + app_ctx + .db + .execute( + "INSERT INTO code_search_fts (repo_id, path, content) VALUES (?1, ?2, ?3)", + params!(repo_row.id.to_string(), path, text), + ) + .await?; + } + + Ok(()) +} + /// Fire-and-forget task run after a successful `git-receive-pack`: discovers /// `.github/workflows/*.yml` files in the pushed branch tips, matches them /// against the `push` event, runs matching jobs, and records a @@ -907,6 +997,14 @@ struct RunnerCompleteRequest { status: String, #[allow(dead_code)] logs: String, + /// Structured result JSON, currently only produced for + /// `dev_workspace_action` jobs (e.g. `{"container_id": ..., "runner_id": ...}` + /// for `create`, `{"output": ...}` for `exec`) -- `None` for CI jobs. + /// Stored verbatim on `runner_jobs.result` for a polling GraphQL + /// mutation (see `wait_for_runner_job` in `graphql-api::mutation`) to + /// read back. + #[serde(default)] + result: Option, } /// `POST /runner/jobs/:id/complete` -- reports the result of a job claimed @@ -962,8 +1060,8 @@ async fn runner_complete_handler( .app_ctx .db .execute( - "UPDATE runner_jobs SET status = ?1, finished_at = ?2 WHERE id = ?3", - params!(status.to_string(), now.clone(), job_id.to_string()), + "UPDATE runner_jobs SET status = ?1, finished_at = ?2, result = ?3 WHERE id = ?4", + params!(status.to_string(), now.clone(), body.result.clone(), job_id.to_string()), ) .await?; @@ -1150,6 +1248,13 @@ async fn auto_stop_dev_workspaces(app_ctx: AppContext) { continue; } + if workspace.runner_id.is_some() { + // Auto-stop isn't wired up for runner-hosted workspaces yet + // (see `stop_dev_workspace`'s same restriction) -- their + // container lives on a different Docker daemon than the + // one `app_ctx.workspace_manager` talks to. + continue; + } let Some(container_id) = workspace.container_id.clone() else { continue; }; @@ -1347,6 +1452,16 @@ async fn proxy_workspace_request( .next() .ok_or_else(|| ServerError::NotFound(format!("workspace {id} not found")))?; + if workspace.runner_id.is_some() { + // Known gap (see `create_dev_workspace`'s `on_runner` doc comment): + // a runner-hosted workspace's container lives on the runner's own + // Docker daemon, which `server` has no network path to reach or + // proxy through yet (no reverse tunnel exists between them). + return Err(ServerError::BadRequest( + "live port-proxying to runner-hosted dev workspaces is not supported yet".to_string(), + )); + } + let container_id = workspace .container_id .ok_or_else(|| ServerError::BadRequest("workspace has no container".to_string()))?;