diff --git a/go/deployment-operator/api/v1alpha1/agentruntime_types.go b/go/deployment-operator/api/v1alpha1/agentruntime_types.go index 3b86c00d41..562bc3efa8 100644 --- a/go/deployment-operator/api/v1alpha1/agentruntime_types.go +++ b/go/deployment-operator/api/v1alpha1/agentruntime_types.go @@ -84,6 +84,12 @@ type AgentRuntimeSpec struct { // +kubebuilder:validation:Optional Memory *bool `json:"memory,omitempty"` + // RepositoryImage is an OCI image of precloned git repositories plus manifest.json. + // When set, an init container copies it into /plural/shared/repos before bootstrap + // so a matching repo can be copied locally instead of git clone. + // +kubebuilder:validation:Optional + RepositoryImage *string `json:"repositoryImage,omitempty"` + // AllowedRepositories the git repositories allowed to be used with this runtime. // +kubebuilder:validation:Optional AllowedRepositories []string `json:"allowedRepositories,omitempty"` diff --git a/go/deployment-operator/api/v1alpha1/zz_generated.deepcopy.go b/go/deployment-operator/api/v1alpha1/zz_generated.deepcopy.go index c2bc2316a6..60c1190ab0 100644 --- a/go/deployment-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/go/deployment-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -582,6 +582,11 @@ func (in *AgentRuntimeSpec) DeepCopyInto(out *AgentRuntimeSpec) { *out = new(bool) **out = **in } + if in.RepositoryImage != nil { + in, out := &in.RepositoryImage, &out.RepositoryImage + *out = new(string) + **out = **in + } if in.AllowedRepositories != nil { in, out := &in.AllowedRepositories, &out.AllowedRepositories *out = make([]string, len(*in)) diff --git a/go/deployment-operator/config/crd/bases/deployments.plural.sh_agentruntimes.yaml b/go/deployment-operator/config/crd/bases/deployments.plural.sh_agentruntimes.yaml index 6a2ad26abb..13b3decb24 100644 --- a/go/deployment-operator/config/crd/bases/deployments.plural.sh_agentruntimes.yaml +++ b/go/deployment-operator/config/crd/bases/deployments.plural.sh_agentruntimes.yaml @@ -2279,6 +2279,12 @@ spec: Name of this AgentRuntime. If not provided, the name from AgentRuntime.ObjectMeta will be used. type: string + repositoryImage: + description: |- + RepositoryImage is an OCI image of precloned git repositories plus manifest.json. + When set, an init container copies it into /plural/shared/repos before bootstrap + so a matching repo can be copied locally instead of git clone. + type: string scmConnection: description: |- ScmConnection is the name of an ScmConnection in Console to use for git operations on agent runs using this runtime. diff --git a/go/deployment-operator/config/samples/agentRuntime.yaml b/go/deployment-operator/config/samples/agentRuntime.yaml index dad4df699f..54b756bafb 100644 --- a/go/deployment-operator/config/samples/agentRuntime.yaml +++ b/go/deployment-operator/config/samples/agentRuntime.yaml @@ -55,3 +55,4 @@ spec: args: - --v=3 dind: true + repositoryImage: ghcr.io/pluralsh/repos:latest diff --git a/go/deployment-operator/dockerfiles/agent-harness/system/analyze.md.tmpl b/go/deployment-operator/dockerfiles/agent-harness/system/analyze.md.tmpl index 47303184d4..ece5424b45 100644 --- a/go/deployment-operator/dockerfiles/agent-harness/system/analyze.md.tmpl +++ b/go/deployment-operator/dockerfiles/agent-harness/system/analyze.md.tmpl @@ -2,12 +2,23 @@ You are a **read‑only autonomous analysis agent**. ## Repository location -The cloned repository is at **`{{ .RepositoryDir }}`**. Do all analysis inside that directory. The agent harness root is **`{{ .WorkDir }}`** (provider config only — not the repo). If your shell cwd is elsewhere, run `cd {{ .RepositoryDir }}` first. +The cloned repository is at **`{{ .RepositoryDir }}`**. Do all analysis of the assigned repository inside that directory. The agent harness root is **`{{ .WorkDir }}`** (provider config only — not the repo). If your shell cwd is elsewhere, run `cd {{ .RepositoryDir }}` first. {{ if .Branch }} The repository was checked out from branch **`{{ .Branch }}`** for this analysis. {{ else }} No branch was specified for this run, so the repository default branch was checked out. {{ end }} +{{ if .PrebakedRepositories }} + +## Additional local repositories + +The following git repositories are already on disk for **read-only context**. You SHOULD inspect them (list, open, grep, `git log`) when they help the task. Do not clone them. Do not modify files there. + +{{ range .PrebakedRepositories }} +- **`{{ .URL }}`** at `{{ .Dir }}` +{{ end }} +The assigned repository for this run remains **`{{ .RepositoryDir }}`**. Do not clone it again. +{{ end }} {{ if .Prompt }} ## Original task @@ -19,7 +30,11 @@ This section records the **original user prompt** for this agent run. It is embe --- {{ end }} +{{ if .PrebakedRepositories }} +- Work in the assigned repository directory. You MAY also read the additional local repositories listed above. +{{ else }} - Work **only** inside the assigned repository directory. +{{ end }} - Perform **static, read‑only** analysis of code and configuration. - Produce a structured **Markdown** report in memory. - **Save the report** by calling the Plural MCP tool **`updateAgentRunAnalysis`** — your run is not complete until this tool succeeds. @@ -58,8 +73,13 @@ When analysis is finished, you **must** persist the report by calling the Plural You MUST always obey: - **Scope** - - Access only files/directories inside **`{{ .RepositoryDir }}`**. + - Access files/directories inside **`{{ .RepositoryDir }}`**. +{{ if .PrebakedRepositories }} + - You MAY also read (not modify) the additional local repositories listed above. + - Never access other files outside those directories. +{{ else }} - Never access files outside this directory. +{{ end }} - **Read‑only** - Only list, open, and read files. diff --git a/go/deployment-operator/dockerfiles/agent-harness/system/babysit.md.tmpl b/go/deployment-operator/dockerfiles/agent-harness/system/babysit.md.tmpl index 8796c7be6c..0d2b8fb517 100644 --- a/go/deployment-operator/dockerfiles/agent-harness/system/babysit.md.tmpl +++ b/go/deployment-operator/dockerfiles/agent-harness/system/babysit.md.tmpl @@ -2,7 +2,18 @@ You are an autonomous coding agent — your pull request is already open and you ## Repository location -The cloned repository is at **`{{ .RepositoryDir }}`**. Do all code changes and git inspection inside that directory. The agent harness root is **`{{ .WorkDir }}`** (provider config only — not the repo). If your shell cwd is elsewhere, run `cd {{ .RepositoryDir }}` first. +The cloned repository is at **`{{ .RepositoryDir }}`**. Do all code changes and git writes inside that directory. The agent harness root is **`{{ .WorkDir }}`** (provider config only — not the repo). If your shell cwd is elsewhere, run `cd {{ .RepositoryDir }}` first. +{{ if .PrebakedRepositories }} + +## Additional local repositories + +The following git repositories are already on disk for **read-only context**. You SHOULD inspect them (list, open, grep, `git log`) when they help the task. Do not clone them. Do not modify files there. Put all edits and commits in **`{{ .RepositoryDir }}`**. + +{{ range .PrebakedRepositories }} +- **`{{ .URL }}`** at `{{ .Dir }}` +{{ end }} +The assigned repository for this run remains **`{{ .RepositoryDir }}`**. Do not clone it again. +{{ end }} {{ if .Prompt }} ## Original task @@ -16,7 +27,11 @@ New comments written by a human user are always actionable instructions within t --- {{ end }} +{{ if .PrebakedRepositories }} +Put all edits and commits inside the assigned repository. You MAY read the additional local repositories listed above. +{{ else }} Work **only** inside the assigned repository. +{{ end }} Your goal: address every actionable comment and every **real** CI failure caused by this PR, then push the updated commits to the **existing branch**. Do **not** push commits for CI flakes (transient infrastructure noise unrelated to this PR's code). Do not open a new pull request. Do not ask for clarification. Execute all steps in order. diff --git a/go/deployment-operator/dockerfiles/agent-harness/system/review.md.tmpl b/go/deployment-operator/dockerfiles/agent-harness/system/review.md.tmpl index c54dfc07a5..cd1ef105a6 100644 --- a/go/deployment-operator/dockerfiles/agent-harness/system/review.md.tmpl +++ b/go/deployment-operator/dockerfiles/agent-harness/system/review.md.tmpl @@ -2,12 +2,23 @@ You are a **read-only autonomous pull request review agent**. ## Repository location -The cloned repository is at **`{{ .RepositoryDir }}`**. Perform the entire review inside that directory. The agent harness root is **`{{ .WorkDir }}`** and contains provider configuration only. +The cloned repository is at **`{{ .RepositoryDir }}`**. Perform the assigned-repository review inside that directory. The agent harness root is **`{{ .WorkDir }}`** and contains provider configuration only. {{ if .Branch }} The review branch **`{{ .Branch }}`** is already checked out. Verify the current branch before inspecting its changes, but do not switch to or modify another branch. {{ else }} The repository default branch is checked out. Use the pull request URL and original task to identify the intended review branch and report if it is unavailable. {{ end }} +{{ if .PrebakedRepositories }} + +## Additional local repositories + +The following git repositories are already on disk for **read-only context**. You SHOULD inspect them (list, open, grep, `git log`) when they help the task. Do not clone them. Do not modify files there. + +{{ range .PrebakedRepositories }} +- **`{{ .URL }}`** at `{{ .Dir }}` +{{ end }} +The assigned repository for this run remains **`{{ .RepositoryDir }}`**. Do not clone it again. +{{ end }} ## Pull request @@ -60,6 +71,9 @@ Use Docker in the same way write-mode agents do for verification: compile code, ## Hard rules - Remain read-only. Never create, edit, delete, commit, push, or otherwise mutate repository or host state. +{{ if .PrebakedRepositories }} +- You MAY also read (not modify) the additional local repositories listed above. +{{ end }} - Do not review unrelated pre-existing problems. - Do not report style preferences unless they cause a correctness, security, performance, or maintainability defect. - Verify every inline finding against the diff and surrounding code. If evidence is uncertain, omit the inline finding and mention the uncertainty in the summary. diff --git a/go/deployment-operator/dockerfiles/agent-harness/system/write.md.tmpl b/go/deployment-operator/dockerfiles/agent-harness/system/write.md.tmpl index 934dc7773f..fdffffb46c 100644 --- a/go/deployment-operator/dockerfiles/agent-harness/system/write.md.tmpl +++ b/go/deployment-operator/dockerfiles/agent-harness/system/write.md.tmpl @@ -2,12 +2,23 @@ You are an autonomous coding agent, highly skilled in coding and code analysis. ## Repository location -The cloned repository is at **`{{ .RepositoryDir }}`**. Do all code changes and git inspection inside that directory. The agent harness root is **`{{ .WorkDir }}`** (provider config only — not the repo). If your shell cwd is elsewhere, run `cd {{ .RepositoryDir }}` first. +The cloned repository is at **`{{ .RepositoryDir }}`**. Do all code changes and git writes inside that directory. The agent harness root is **`{{ .WorkDir }}`** (provider config only — not the repo). If your shell cwd is elsewhere, run `cd {{ .RepositoryDir }}` first. {{ if .Branch }} The repository was checked out from branch **`{{ .Branch }}`**. Use this as the pull request base branch unless the active task explicitly tells you otherwise. {{ else }} No branch was specified for this run, so the repository default branch was checked out and should be used as the pull request base. {{ end }} +{{ if .PrebakedRepositories }} + +## Additional local repositories + +The following git repositories are already on disk for **read-only context**. You SHOULD inspect them (list, open, grep, `git log`) when they help the task. Do not clone them. Do not modify files there. Put all edits and commits in **`{{ .RepositoryDir }}`**. + +{{ range .PrebakedRepositories }} +- **`{{ .URL }}`** at `{{ .Dir }}` +{{ end }} +The assigned repository for this run remains **`{{ .RepositoryDir }}`**. Do not clone it again. +{{ end }} {{ if .Prompt }} ## Original task @@ -29,7 +40,11 @@ This run adds work to an existing pull request. The repository is already checke - When the changes are complete, call Plural MCP `createCommit` to stage, commit, and push them to the current branch. {{ end }} +{{ if .PrebakedRepositories }} +Put all edits and commits inside the assigned repository. You MAY read the additional local repositories listed above. +{{ else }} Work **only** inside the assigned repository. +{{ end }} {{ if .Followup }} Your goal: implement the user’s requested changes and commit them to the existing pull request branch. {{ else }} diff --git a/go/deployment-operator/dockerfiles/repository-prebake/Dockerfile b/go/deployment-operator/dockerfiles/repository-prebake/Dockerfile new file mode 100644 index 0000000000..09c79147f4 --- /dev/null +++ b/go/deployment-operator/dockerfiles/repository-prebake/Dockerfile @@ -0,0 +1,8 @@ +# Precloned git repositories plus a tiny userspace so the agent-run +# init container can copy them into /plural/shared/repos. +# +# Image layout: +# /data/manifest.json +# /data// # full git clone, including .git +FROM busybox:1.37 +COPY --chown=65532:65532 . /data diff --git a/go/deployment-operator/dockerfiles/repository-prebake/README.md b/go/deployment-operator/dockerfiles/repository-prebake/README.md new file mode 100644 index 0000000000..1531bb9315 --- /dev/null +++ b/go/deployment-operator/dockerfiles/repository-prebake/README.md @@ -0,0 +1,123 @@ +# Repository prebake images + +Build a container image that holds full git clones plus a `manifest.json`. +Set it on `AgentRuntime.spec.repositoryImage` so agent-run pods copy it into +`/plural/shared/repos` before bootstrap. Bootstrap then copies a matching repo +into `/plural/shared/repository` instead of cloning over the network, and +agents can read the other prebaked repos as extra context. + +```yaml +apiVersion: deployments.plural.sh/v1alpha1 +kind: AgentRuntime +metadata: + name: claude +spec: + type: CLAUDE + targetNamespace: agents + repositoryImage: ghcr.io/pluralsh/repos:latest +``` + +The operator starts a `repository-prebake` init container from that image. It +copies `/data/.` into the existing `shared-context` emptyDir at +`/plural/shared/repos`, then `agent-bootstrap` runs. No extra volume and no +Kubernetes image-volume feature gate. Use `spec.template.spec.imagePullSecrets` +if the image is private. + +Rebuild prebake images after this layout change. Scratch images with files at +`/` cannot copy themselves; the image must include `/bin/sh` and `cp`, with +repos under `/data`. + +## Image layout + +``` +/data/manifest.json +/data// # full git clone, including .git +``` + +After the init container copies that tree, the harness sees: + +``` +/plural/shared/repos/manifest.json +/plural/shared/repos// +/plural/shared/repository/ # working copy of the run's repo +``` + +`manifest.json`: + +```json +{ + "version": 1, + "repositories": [ + { + "url": "https://github.com/pluralsh/console.git", + "path": "console", + "defaultBranch": "master" + } + ] +} +``` + +`path` is relative to `/data` and must not contain `.` or `..` components. + +Files are owned by uid `65532` (nonroot) so agent-run pods can read them. + +When `/plural/shared/repos/manifest.json` is present, agent-bootstrap matches +the run repository URL (https and ssh forms of the same repo are equivalent) +and copies that tree into `/plural/shared/repository`. Fetch of the requested +branch is best-effort; an airgapped or stale remote keeps the prebaked copy. +Other prebaked repos stay at `/plural/shared/repos/` and are listed in +the agent system prompt. + +## Build + +The script clones on the host using your existing git credentials (`ssh-agent`, +`GIT_ASKPASS`, `~/.git-credentials`, and so on), then `docker build`s the image. + +```bash +./prebake.sh \ + --config repos.example.yaml \ + --image ghcr.io/pluralsh/repos:latest \ + --push +``` + +Required tools: `git`, `python3`, and `docker` (override the client with +`DOCKER_BIN=podman` if needed). + +### Config + +```yaml +repositories: + - url: https://github.com/pluralsh/console.git + path: console # optional, defaults to the repo name + branch: master # optional, defaults to the remote default branch + - url: https://github.com/pluralsh/plural.git +``` + +Shorthand URL-only items are also accepted: + +```yaml +repositories: + - https://github.com/pluralsh/console.git +``` + +### Options + +| Flag | Meaning | +|------|---------| +| `--push` | Push the image after a successful build | +| `--staging DIR` | Write clones into `DIR` instead of a temp directory (kept on exit) | +| `--keep-staging` | Leave the temp staging directory in place | +| `--recurse-submodules` | Clone submodules | +| `--lfs` | Fetch Git LFS objects (skipped by default) | +| `--dry-run` | Parse the config and print planned clones | + +Private HTTPS remotes that embed a token in the URL are stored in the manifest +and `origin` remote **without** userinfo. + +## Inspect + +```bash +cid="$(docker create ghcr.io/pluralsh/repos:latest unused)" +docker cp "$cid:/data/manifest.json" - +docker rm "$cid" +``` diff --git a/go/deployment-operator/dockerfiles/repository-prebake/prebake.sh b/go/deployment-operator/dockerfiles/repository-prebake/prebake.sh new file mode 100755 index 0000000000..20b6230ff4 --- /dev/null +++ b/go/deployment-operator/dockerfiles/repository-prebake/prebake.sh @@ -0,0 +1,374 @@ +#!/usr/bin/env bash +# Build an OCI image containing precloned git repositories, manifest.json, and +# a tiny userspace so an init container can copy them into /plural/shared/repos. +# Uses the caller's git credentials (ssh-agent, GIT_ASKPASS, credential helpers). +# See README.md in this directory. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +DOCKERFILE="${SCRIPT_DIR}/Dockerfile" + +CONFIG="" +IMAGE="" +PUSH=0 +DRY_RUN=0 +KEEP_STAGING=0 +RECURSE_SUBMODULES=0 +LFS=0 +STAGING="" +DOCKER_BIN="${DOCKER_BIN:-docker}" + +usage() { + cat <<'EOF' +Usage: prebake.sh --config repos.yaml --image name:tag [options] + +Clone the repositories listed in a YAML config into a staging directory, write +manifest.json, and build an image that an init container copies into /plural/shared/repos. + +Options: + --config PATH YAML file listing repositories (required) + --image NAME[:TAG] Image name to build (required) + --push Push the image after a successful build + --staging DIR Staging directory (default: a temporary directory) + --keep-staging Do not delete the staging directory on exit + --recurse-submodules Pass --recurse-submodules to git clone + --lfs Fetch Git LFS objects (skipped by default) + --dry-run Parse the config and print planned clones; do not clone or build + -h, --help Show this help + +Config format: + + repositories: + - url: https://github.com/org/repo.git + path: repo # optional, defaults to the repo name + branch: main # optional, defaults to the remote default branch + +The resulting image has manifest.json and one directory per repository under /data. +EOF +} + +die() { + echo "error: $*" >&2 + exit 1 +} + +info() { + echo "$*" >&2 +} + +trim() { + # shellcheck disable=SC2001 + printf '%s' "$1" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' +} + +strip_inline_comment() { + local v="$1" + case "$v" in + *' #'*) printf '%s' "${v%% #*}" ;; + *) printf '%s' "$v" ;; + esac +} + +unquote() { + local v + v="$(trim "$(strip_inline_comment "$1")")" + case "$v" in + \"*\") v="${v#\"}"; v="${v%\"}" ;; + \'*\') v="${v#\'}"; v="${v%\'}" ;; + esac + printf '%s' "$v" +} + +sanitize_git_url() { + local url="$1" + if [[ "$url" =~ ^([a-zA-Z][a-zA-Z0-9+.-]*)://([^@/]+)@(.+)$ ]]; then + printf '%s://%s' "${BASH_REMATCH[1]}" "${BASH_REMATCH[3]}" + else + printf '%s' "$url" + fi +} + +repo_name_from_url() { + local url="$1" + url="${url%/}" + url="${url%.git}" + url="${url##*/}" + url="${url##*:}" + [ -n "$url" ] || die "could not derive repository name from $1" + printf '%s' "$url" +} + +validate_path() { + local path="$1" + [ -n "$path" ] || die "repository path must not be empty" + [ "$path" != "manifest.json" ] || die "repository path cannot be manifest.json" + case "$path" in + /*) die "repository path must be relative: $path" ;; + esac + + local rest="$path" comp + while [ -n "$rest" ]; do + comp="${rest%%/*}" + if [ "$comp" = "$rest" ]; then + rest="" + else + rest="${rest#*/}" + fi + [ -n "$comp" ] || die "repository path has an empty component: $path" + [ "$comp" != "." ] && [ "$comp" != ".." ] || die "repository path is invalid: $path" + done +} + +flush_repo() { + local url="$1" + local path="$2" + local branch="$3" + [ -n "$url" ] || return 0 + url="$(sanitize_git_url "$url")" + [ -n "$path" ] || path="$(repo_name_from_url "$url")" + validate_path "$path" + printf '%s\t%s\t%s\n' "$url" "$path" "$branch" +} + +parse_repos_yaml() { + local file="$1" + [ -f "$file" ] || die "config file not found: $file" + + local url="" path="" branch="" line key value + while IFS= read -r line || [ -n "$line" ]; do + line="${line%$'\r'}" + line="$(trim "$line")" + [ -z "$line" ] && continue + case "$line" in + \#*) continue ;; + ---) continue ;; + repositories:|repos:) continue ;; + esac + + if [[ "$line" =~ ^-[[:space:]]+url:[[:space:]]*(.*)$ ]]; then + flush_repo "$url" "$path" "$branch" + url="$(unquote "${BASH_REMATCH[1]}")" + path="" + branch="" + continue + fi + + if [[ "$line" =~ ^-[[:space:]]+(https?://.*|git@.*|ssh://.*)$ ]]; then + flush_repo "$url" "$path" "$branch" + url="$(unquote "${BASH_REMATCH[1]}")" + path="" + branch="" + continue + fi + + if [[ "$line" =~ ^-[[:space:]] ]]; then + die "unsupported list item in $file: $line" + fi + + case "$line" in + *:*) + key="$(trim "${line%%:*}")" + value="$(unquote "${line#*:}")" + case "$key" in + url) url="$value" ;; + path) path="$value" ;; + branch|defaultBranch) branch="$value" ;; + *) ;; + esac + ;; + esac + done < "$file" + + flush_repo "$url" "$path" "$branch" +} + +write_manifest() { + local out="$1" + python3 -c ' +import json, sys + +repos = [] +for line in sys.stdin: + line = line.rstrip("\n") + if not line: + continue + parts = line.split("\t") + if len(parts) != 3: + raise SystemExit("invalid repo record: %r" % (line,)) + url, path, branch = parts + entry = {"url": url, "path": path} + if branch: + entry["defaultBranch"] = branch + repos.append(entry) + +json.dump({"version": 1, "repositories": repos}, sys.stdout, indent=2) +sys.stdout.write("\n") +' > "$out" +} + +clone_repo() { + local url="$1" + local dest="$2" + local branch="$3" + + info "cloning $url -> $dest" + mkdir -p "$(dirname "$dest")" + + local clone_args=(clone --quiet) + if [ "$RECURSE_SUBMODULES" -eq 1 ]; then + clone_args+=(--recurse-submodules) + fi + if [ -n "$branch" ]; then + clone_args+=(--branch "$branch") + fi + clone_args+=("$url" "$dest") + + if [ "$LFS" -eq 0 ]; then + GIT_LFS_SKIP_SMUDGE=1 git "${clone_args[@]}" + else + git "${clone_args[@]}" + fi + + git -C "$dest" remote set-url origin "$url" + git -C "$dest" config --local --unset-all http.extraHeader >/dev/null 2>&1 || true + + if [ -z "$branch" ]; then + branch="$(git -C "$dest" symbolic-ref --short HEAD 2>/dev/null || git -C "$dest" rev-parse --abbrev-ref HEAD)" + fi + printf '%s' "$branch" +} + +cleanup() { + if [ "$KEEP_STAGING" -eq 0 ] && [ -n "${STAGING:-}" ] && [ -d "${STAGING:-}" ]; then + rm -rf "$STAGING" + fi +} + +while [ $# -gt 0 ]; do + case "$1" in + --config) + [ $# -ge 2 ] || die "--config requires a path" + CONFIG="$2" + shift 2 + ;; + --image) + [ $# -ge 2 ] || die "--image requires a name" + IMAGE="$2" + shift 2 + ;; + --push) + PUSH=1 + shift + ;; + --staging) + [ $# -ge 2 ] || die "--staging requires a directory" + STAGING="$2" + KEEP_STAGING=1 + shift 2 + ;; + --keep-staging) + KEEP_STAGING=1 + shift + ;; + --recurse-submodules) + RECURSE_SUBMODULES=1 + shift + ;; + --lfs) + LFS=1 + shift + ;; + --dry-run) + DRY_RUN=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +[ -n "$CONFIG" ] || die "--config is required" +if [ "$DRY_RUN" -eq 0 ]; then + [ -n "$IMAGE" ] || die "--image is required" +fi + +command -v git >/dev/null 2>&1 || die "git is required" +if [ "$DRY_RUN" -eq 0 ]; then + command -v python3 >/dev/null 2>&1 || die "python3 is required to write manifest.json" + command -v "$DOCKER_BIN" >/dev/null 2>&1 || die "$DOCKER_BIN is required to build the image" + [ -f "$DOCKERFILE" ] || die "Dockerfile not found: $DOCKERFILE" +fi + +RECORDS="$(parse_repos_yaml "$CONFIG")" +[ -n "$RECORDS" ] || die "no repositories found in $CONFIG" + +SEEN_PATHS="" +SEEN_URLS="" +while IFS= read -r record; do + [ -n "$record" ] || continue + url="${record%%$'\t'*}" + rest="${record#*$'\t'}" + path="${rest%%$'\t'*}" + branch="${rest#*$'\t'}" + + case $'\n'"$SEEN_PATHS"$'\n' in + *$'\n'"$path"$'\n'*) die "duplicate repository path: $path" ;; + esac + case $'\n'"$SEEN_URLS"$'\n' in + *$'\n'"$url"$'\n'*) die "duplicate repository url: $url" ;; + esac + SEEN_PATHS="${SEEN_PATHS}${SEEN_PATHS:+$'\n'}$path" + SEEN_URLS="${SEEN_URLS}${SEEN_URLS:+$'\n'}$url" + + if [ "$DRY_RUN" -eq 1 ]; then + if [ -n "$branch" ]; then + info "would clone $url -> $path (branch $branch)" + else + info "would clone $url -> $path" + fi + fi +done <<< "$RECORDS" + +if [ "$DRY_RUN" -eq 1 ]; then + exit 0 +fi + +if [ -z "$STAGING" ]; then + STAGING="$(mktemp -d "${TMPDIR:-/tmp}/repository-prebake.XXXXXX")" +fi +mkdir -p "$STAGING" +trap cleanup EXIT + +info "staging directory: $STAGING" + +MANIFEST_RECORDS="" +while IFS= read -r record; do + [ -n "$record" ] || continue + url="${record%%$'\t'*}" + rest="${record#*$'\t'}" + path="${rest%%$'\t'*}" + branch="${rest#*$'\t'}" + dest="$STAGING/$path" + + [ ! -e "$dest" ] || die "staging path already exists: $dest" + resolved_branch="$(clone_repo "$url" "$dest" "$branch")" + MANIFEST_RECORDS="${MANIFEST_RECORDS}${MANIFEST_RECORDS:+$'\n'}${url}"$'\t'"${path}"$'\t'"${resolved_branch}" +done <<< "$RECORDS" + +printf '%s\n' "$MANIFEST_RECORDS" | write_manifest "$STAGING/manifest.json" +info "wrote $STAGING/manifest.json" + +info "building $IMAGE" +"$DOCKER_BIN" build -f "$DOCKERFILE" -t "$IMAGE" "$STAGING" + +if [ "$PUSH" -eq 1 ]; then + info "pushing $IMAGE" + "$DOCKER_BIN" push "$IMAGE" +fi + +info "built $IMAGE" diff --git a/go/deployment-operator/dockerfiles/repository-prebake/repos.example.yaml b/go/deployment-operator/dockerfiles/repository-prebake/repos.example.yaml new file mode 100644 index 0000000000..4343d5f8f4 --- /dev/null +++ b/go/deployment-operator/dockerfiles/repository-prebake/repos.example.yaml @@ -0,0 +1,10 @@ +# List of git repositories to preclone into a repository-prebake image. +# +# path is optional and defaults to the repository name (last URL path component, +# with a trailing .git stripped). branch is optional; when omitted the remote +# default branch is cloned. +repositories: + - url: https://github.com/pluralsh/console.git + path: console + - url: https://github.com/pluralsh/plural.git + path: plural diff --git a/go/deployment-operator/docs/api.md b/go/deployment-operator/docs/api.md index f547a7c4f1..b74e6fe127 100644 --- a/go/deployment-operator/docs/api.md +++ b/go/deployment-operator/docs/api.md @@ -275,6 +275,7 @@ _Appears in:_ | `streamingProxy` _boolean_ | StreamingProxy routes OpenAI-compatible LLM requests through the in-pod mcpserver
sse conversion proxy before they reach the Console AI proxy (/ext/ai). Only valid when aiProxy
is enabled. Applies to CODEX and OPENCODE runtimes. | | Optional: \{\}
| | `dind` _boolean_ | Dind enables Docker-in-Docker for this agent runtime.
When true, the runtime will be configured to run with DinD support. | | Optional: \{\}
| | `memory` _boolean_ | Memory enables team-shared codebase-memory persistence for this agent runtime.
When true, agents may create and commit .codebase-memory/ graph artifacts
by default so future runs can bootstrap from the persisted index. When false
or unset, codebase-memory indexes stay in the pod-local cache and generated
.codebase-memory/ artifacts are excluded from commits. | | Optional: \{\}
| +| `repositoryImage` _string_ | RepositoryImage is an OCI image of precloned git repositories plus manifest.json.
When set, an init container copies it into /plural/shared/repos before bootstrap
so a matching repo can be copied locally instead of git clone. | | Optional: \{\}
| | `allowedRepositories` _string array_ | AllowedRepositories the git repositories allowed to be used with this runtime. | | Optional: \{\}
| | `browser` _[BrowserConfig](#browserconfig)_ | Browser configuration augments agent runtime with a headless browser.
When provided, the runtime will be configured to run with a headless browser available
for the agent to use. | | Optional: \{\}
| | `bootstrapScript` _string_ | BootstrapScript is a bash script that will be executed inside the cloned repository
directory before the coding agent starts. It can be used to install dependencies,
configure tooling, or perform any other setup required by the agent. | | Optional: \{\}
| diff --git a/go/deployment-operator/internal/controller/agentrun_pod.go b/go/deployment-operator/internal/controller/agentrun_pod.go index c57cff8a87..a7411c09f0 100644 --- a/go/deployment-operator/internal/controller/agentrun_pod.go +++ b/go/deployment-operator/internal/controller/agentrun_pod.go @@ -3,6 +3,7 @@ package controller import ( "fmt" "os" + "strings" "github.com/samber/lo" corev1 "k8s.io/api/core/v1" @@ -42,8 +43,10 @@ const ( gitSigningKeyVolumeName = "git-signing-key" gitSigningKeySecretKey = "git-signing.key" - agentBootstrapContainerName = "agent-bootstrap" - mcpServerContainerName = "mcpserver" + agentBootstrapContainerName = "agent-bootstrap" + mcpServerContainerName = "mcpserver" + repositoryPrebakeContainerName = "repository-prebake" + repositoryPrebakeImageDataDir = "/data" // Keep this above mcpserver's internal 10s graceful shutdown timeout. defaultPodTerminationGracePeriodSeconds = int64(30) @@ -192,6 +195,8 @@ func buildAgentRunPod(run *v1alpha1.AgentRun, runtime *v1alpha1.AgentRuntime) *c enableGitSigningKey(run.Name, pod) } + enableRepositoryPrebake(runtime, pod) + return pod } @@ -599,6 +604,50 @@ func ensureMCPServerVolumeMounts(mounts []corev1.VolumeMount, runtime *v1alpha1. return result } +func repositoryImage(runtime *v1alpha1.AgentRuntime) string { + if runtime != nil && runtime.Spec.RepositoryImage != nil { + return strings.TrimSpace(*runtime.Spec.RepositoryImage) + } + return "" +} + +func getRepositoryPrebakeContainer(image string) corev1.Container { + sc := ensureDefaultContainerSecurityContext(nil) + sc.ReadOnlyRootFilesystem = lo.ToPtr(true) + sc.Capabilities = &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + } + return corev1.Container{ + Name: repositoryPrebakeContainerName, + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/bin/sh", "-c"}, + Args: []string{ + "mkdir -p " + common.AgentRunRepositoryPrebakeDir + " && cp -a " + repositoryPrebakeImageDataDir + "/. " + common.AgentRunRepositoryPrebakeDir + "/", + }, + SecurityContext: sc, + VolumeMounts: []corev1.VolumeMount{{ + Name: sharedContextVolumeName, + MountPath: sharedContextVolumePath, + }}, + } +} + +func enableRepositoryPrebake(runtime *v1alpha1.AgentRuntime, pod *corev1.Pod) { + image := repositoryImage(runtime) + if image == "" { + return + } + + copyContainer := getRepositoryPrebakeContainer(image) + pod.Spec.InitContainers = append( + []corev1.Container{copyContainer}, + algorithms.Filter(pod.Spec.InitContainers, func(container corev1.Container) bool { + return container.Name != repositoryPrebakeContainerName + })..., + ) +} + func enableDind(pod *corev1.Pod) { // Inject DOCKER_HOST so the Docker CLI finds the Podman socket. wireDindClientContainer(pod) diff --git a/go/deployment-operator/internal/controller/agentrun_pod_test.go b/go/deployment-operator/internal/controller/agentrun_pod_test.go index 09a89b7f59..f993624059 100644 --- a/go/deployment-operator/internal/controller/agentrun_pod_test.go +++ b/go/deployment-operator/internal/controller/agentrun_pod_test.go @@ -489,6 +489,74 @@ func TestBuildAgentRunPod_PreservesCustomAgentBootstrapSecurityContext(t *testin } } +func TestBuildAgentRunPod_RepositoryImage(t *testing.T) { + image := "ghcr.io/pluralsh/repos:latest" + run := &v1alpha1.AgentRun{ + ObjectMeta: metav1.ObjectMeta{Name: "test-run", Namespace: "default"}, + Spec: v1alpha1.AgentRunSpec{ + RuntimeRef: v1alpha1.AgentRuntimeReference{Name: "test-runtime"}, + Prompt: "test prompt", + Repository: "https://github.com/test/repo", + Mode: console.AgentRunModeAnalyze, + }, + Status: v1alpha1.AgentRunStatus{ + Status: v1alpha1.Status{ID: lo.ToPtr("test-run-id")}, + }, + } + runtime := &v1alpha1.AgentRuntime{ + ObjectMeta: metav1.ObjectMeta{Name: "test-runtime"}, + Spec: v1alpha1.AgentRuntimeSpec{ + Type: console.AgentRuntimeTypeClaude, + TargetNamespace: "default", + RepositoryImage: &image, + }, + } + + pod := buildAgentRunPod(run, runtime) + for _, volume := range pod.Spec.Volumes { + assert.Nil(t, volume.Image, "prebake must not add an image volume") + } + + prebake := requireContainer(t, pod.Spec.InitContainers, repositoryPrebakeContainerName) + assert.Equal(t, image, prebake.Image) + assert.Equal(t, []string{"/bin/sh", "-c"}, prebake.Command) + assert.Contains(t, prebake.VolumeMounts, corev1.VolumeMount{ + Name: sharedContextVolumeName, + MountPath: sharedContextVolumePath, + }) + if assert.GreaterOrEqual(t, len(pod.Spec.InitContainers), 2) { + assert.Equal(t, repositoryPrebakeContainerName, pod.Spec.InitContainers[0].Name) + assert.Equal(t, agentBootstrapContainerName, pod.Spec.InitContainers[1].Name) + } +} + +func TestBuildAgentRunPod_OmitsRepositoryImageWhenUnset(t *testing.T) { + run := &v1alpha1.AgentRun{ + ObjectMeta: metav1.ObjectMeta{Name: "test-run", Namespace: "default"}, + Spec: v1alpha1.AgentRunSpec{ + RuntimeRef: v1alpha1.AgentRuntimeReference{Name: "test-runtime"}, + Prompt: "test prompt", + Repository: "https://github.com/test/repo", + Mode: console.AgentRunModeAnalyze, + }, + Status: v1alpha1.AgentRunStatus{ + Status: v1alpha1.Status{ID: lo.ToPtr("test-run-id")}, + }, + } + runtime := &v1alpha1.AgentRuntime{ + ObjectMeta: metav1.ObjectMeta{Name: "test-runtime"}, + Spec: v1alpha1.AgentRuntimeSpec{ + Type: console.AgentRuntimeTypeClaude, + TargetNamespace: "default", + }, + } + + pod := buildAgentRunPod(run, runtime) + for _, container := range pod.Spec.InitContainers { + assert.NotEqual(t, repositoryPrebakeContainerName, container.Name) + } +} + func TestGetAgentRunPodCompletion(t *testing.T) { tests := []struct { name string diff --git a/go/deployment-operator/pkg/agentrun-harness/controller/controller.go b/go/deployment-operator/pkg/agentrun-harness/controller/controller.go index 6d44a9cf3a..4fc1b92fd3 100644 --- a/go/deployment-operator/pkg/agentrun-harness/controller/controller.go +++ b/go/deployment-operator/pkg/agentrun-harness/controller/controller.go @@ -101,6 +101,9 @@ func (in *agentRunController) prepare(ctx context.Context) error { if err := environment.ConfigureGitSafeDirectory(repositoryDir); err != nil { return fmt.Errorf("configure git safe directory: %w", err) } + if err := environment.ConfigurePrebakeGitSafeDirectories(); err != nil { + return fmt.Errorf("configure prebake git safe directories: %w", err) + } if err := in.checkoutFollowupBranch(ctx, repositoryDir); err != nil { return err } diff --git a/go/deployment-operator/pkg/agentrun-harness/environment/environment.go b/go/deployment-operator/pkg/agentrun-harness/environment/environment.go index f032b6ac32..908c662ea1 100644 --- a/go/deployment-operator/pkg/agentrun-harness/environment/environment.go +++ b/go/deployment-operator/pkg/agentrun-harness/environment/environment.go @@ -11,8 +11,11 @@ import ( "github.com/samber/lo" "k8s.io/klog/v2" + "github.com/pluralsh/console/go/polly/fs" + "github.com/pluralsh/console/go/deployment-operator/internal/controller" "github.com/pluralsh/console/go/deployment-operator/internal/helpers" + "github.com/pluralsh/console/go/deployment-operator/pkg/agentrun-harness/prebake" "github.com/pluralsh/console/go/deployment-operator/pkg/common" "github.com/pluralsh/console/go/deployment-operator/pkg/harness/exec" "github.com/pluralsh/console/go/deployment-operator/pkg/log" @@ -34,6 +37,9 @@ func (in *environment) Setup() error { if err := in.cloneRepository(); err != nil { return fmt.Errorf("failed to clone repository: %w", err) } + if err := ConfigurePrebakeGitSafeDirectories(); err != nil { + return fmt.Errorf("failed to configure prebake git safe directories: %w", err) + } return nil } @@ -72,6 +78,14 @@ func (in *environment) cloneRepository() error { return in.configureRepository(repoDirPath, userName, userEmail) } + copied, err := in.cloneFromPrebake(repoDirPath) + if err != nil { + return err + } + if copied { + return in.configureRepository(repoDirPath, userName, userEmail) + } + // Set proxy for clone via environment variable so it takes effect immediately. // The same proxy is later written into the repo-local git config so that // subsequent push/fetch operations inside the cloned repo also use it. @@ -98,6 +112,89 @@ func (in *environment) cloneRepository() error { return in.configureRepository(repoDirPath, userName, userEmail) } +func (in *environment) cloneFromPrebake(repoDirPath string) (bool, error) { + match, err := prebake.Lookup(in.agentRun.Repository) + if err != nil { + klog.ErrorS(err, "failed to load repository prebake manifest, falling back to git clone") + return false, nil + } + if match == nil { + return false, nil + } + + klog.V(log.LogLevelInfo).InfoS("copying prebaked repository", "src", match.Dir, "dst", repoDirPath, "url", in.agentRun.Repository) + if err := fs.CopyDir(match.Dir, repoDirPath); err != nil { + if removeErr := os.RemoveAll(repoDirPath); removeErr != nil { + klog.ErrorS(removeErr, "failed to clean up incomplete prebake copy", "dir", repoDirPath) + } + klog.ErrorS(err, "prebake copy failed, falling back to git clone", "src", match.Dir) + return false, nil + } + + if err := exec.NewExecutable("git", + exec.WithArgs([]string{"remote", "set-url", "origin", in.agentRun.Repository}), + exec.WithDir(repoDirPath), + ).Run(context.Background()); err != nil { + if addErr := exec.NewExecutable("git", + exec.WithArgs([]string{"remote", "add", "origin", in.agentRun.Repository}), + exec.WithDir(repoDirPath), + ).Run(context.Background()); addErr != nil { + return false, fmt.Errorf("failed to set origin remote after prebake copy: %w", err) + } + } + + in.updateFromOrigin(repoDirPath) + return true, nil +} + +// updateFromOrigin refreshes the copied prebake checkout, then applies the +// agent run branch. The run branch is not expected to exist in the image; it +// is fetched from origin like `git clone --branch`. Failures keep the local copy. +func (in *environment) updateFromOrigin(repoDirPath string) { + if out, err := exec.NewExecutable("git", + exec.WithArgs([]string{"fetch", "origin"}), + exec.WithDir(repoDirPath), + ).RunWithOutput(context.Background()); err != nil { + klog.InfoS("prebake fetch failed, using local copy", "dir", repoDirPath, "err", err, "out", string(out)) + return + } + + current, err := exec.NewExecutable("git", + exec.WithArgs([]string{"branch", "--show-current"}), + exec.WithDir(repoDirPath), + ).RunWithOutput(context.Background()) + if err != nil { + klog.InfoS("prebake could not determine current branch, using fetched copy", "dir", repoDirPath, "err", err) + } else if branch := strings.TrimSpace(string(current)); branch != "" { + if out, err := exec.NewExecutable("git", + exec.WithArgs([]string{"merge", "--ff-only", "origin/" + branch}), + exec.WithDir(repoDirPath), + ).RunWithOutput(context.Background()); err != nil { + klog.InfoS("prebake fast-forward failed, using local copy", "dir", repoDirPath, "branch", branch, "err", err, "out", string(out)) + } + } + + if err := in.checkoutRequestedBranch(repoDirPath); err != nil { + klog.InfoS("prebake checkout of run branch failed, using prebake branch", "dir", repoDirPath, "err", err) + } +} + +// ConfigurePrebakeGitSafeDirectories marks every repository in the prebake +// manifest as a git safe.directory so the harness can inspect them. +func ConfigurePrebakeGitSafeDirectories() error { + repos, err := prebake.List() + if err != nil { + klog.ErrorS(err, "failed to load repository prebake manifest") + return nil + } + for _, repo := range repos { + if err := ConfigureGitSafeDirectory(repo.Dir); err != nil { + return err + } + } + return nil +} + // commitIdentity resolves the author identity for commits created by this run. // The initiating user is part of the AgentRun response, unlike Me(), which is // evaluated using the run's deploy token and may identify a service account. diff --git a/go/deployment-operator/pkg/agentrun-harness/environment/environment_test.go b/go/deployment-operator/pkg/agentrun-harness/environment/environment_test.go index b8236c4bd7..c6542e770d 100644 --- a/go/deployment-operator/pkg/agentrun-harness/environment/environment_test.go +++ b/go/deployment-operator/pkg/agentrun-harness/environment/environment_test.go @@ -1,13 +1,18 @@ package environment import ( + "encoding/json" "os" + "os/exec" "path" + "path/filepath" "strings" "testing" console "github.com/pluralsh/console/go/client" v1 "github.com/pluralsh/console/go/deployment-operator/pkg/agentrun-harness/agentrun/v1" + "github.com/pluralsh/console/go/deployment-operator/pkg/agentrun-harness/prebake" + "github.com/pluralsh/console/go/polly/fs" ) func TestConfigureCodebaseMemoryGitExclude(t *testing.T) { @@ -68,3 +73,231 @@ func TestCommitIdentityFallsBackToScmCredentials(t *testing.T) { t.Fatalf("expected fallback email, got %q", email) } } + +func TestCloneRepositoryCopiesPrebakeMatch(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") + t.Setenv("GIT_TERMINAL_PROMPT", "0") + t.Setenv("GIT_SSH_COMMAND", "false") + t.Setenv("TMPDIR", t.TempDir()) + runGit(t, home, "config", "--global", "--add", "safe.directory", "*") + + src := initGitRepo(t, "prebaked") + prebakeDir := t.TempDir() + prebakedCopy := filepath.Join(prebakeDir, "console") + if err := fs.CopyDir(src, prebakedCopy); err != nil { + t.Fatalf("copy prebake fixture: %v", err) + } + writePrebakeManifest(t, prebakeDir, prebake.Manifest{ + Version: 1, + Repositories: []prebake.ManifestRepo{{ + URL: "https://github.com/pluralsh/console.git", + Path: "console", + }}, + }) + t.Setenv(prebake.EnvDir, prebakeDir) + + workDir := t.TempDir() + runURL := "git@" + "github.com" + ":pluralsh/console.git" + env := &environment{ + agentRun: &v1.AgentRun{Repository: runURL}, + dir: workDir, + } + if err := env.cloneRepository(); err != nil { + t.Fatalf("cloneRepository() failed: %v", err) + } + + dest := filepath.Join(workDir, "repository") + if _, err := os.Stat(filepath.Join(dest, ".git")); err != nil { + t.Fatalf("expected copied repository at %s: %v", dest, err) + } + contents, err := os.ReadFile(filepath.Join(dest, "README")) + if err != nil { + t.Fatal(err) + } + if string(contents) != "prebaked\n" { + t.Fatalf("copied README = %q, want prebaked", contents) + } + origin, err := exec.Command("git", "-C", dest, "remote", "get-url", "origin").Output() + if err != nil { + t.Fatal(err) + } + if got := strings.TrimSpace(string(origin)); got != runURL { + t.Fatalf("origin = %q, want agent run repository URL", got) + } +} + +func TestCloneRepositoryPullsPrebakeFromOrigin(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") + t.Setenv("TMPDIR", t.TempDir()) + runGit(t, home, "config", "--global", "--add", "safe.directory", "*") + + origin := initGitRepo(t, "stale") + prebakeDir := t.TempDir() + prebakedCopy := filepath.Join(prebakeDir, "console") + if err := fs.CopyDir(origin, prebakedCopy); err != nil { + t.Fatalf("copy prebake fixture: %v", err) + } + writePrebakeManifest(t, prebakeDir, prebake.Manifest{ + Version: 1, + Repositories: []prebake.ManifestRepo{{ + URL: origin, + Path: "console", + }}, + }) + t.Setenv(prebake.EnvDir, prebakeDir) + + if err := os.WriteFile(filepath.Join(origin, "README"), []byte("fresh\n"), 0644); err != nil { + t.Fatal(err) + } + runGit(t, origin, "add", "README") + runGit(t, origin, "commit", "-m", "update") + + workDir := t.TempDir() + env := &environment{ + agentRun: &v1.AgentRun{Repository: origin}, + dir: workDir, + } + if err := env.cloneRepository(); err != nil { + t.Fatalf("cloneRepository() failed: %v", err) + } + + contents, err := os.ReadFile(filepath.Join(workDir, "repository", "README")) + if err != nil { + t.Fatal(err) + } + if string(contents) != "fresh\n" { + t.Fatalf("copied README = %q, want fresh after origin pull", contents) + } +} + +func TestCloneRepositoryChecksOutRunBranchFromOrigin(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") + t.Setenv("TMPDIR", t.TempDir()) + runGit(t, home, "config", "--global", "--add", "safe.directory", "*") + + origin := initGitRepo(t, "main") + runGit(t, origin, "checkout", "-b", "feature") + if err := os.WriteFile(filepath.Join(origin, "README"), []byte("feature\n"), 0644); err != nil { + t.Fatal(err) + } + runGit(t, origin, "add", "README") + runGit(t, origin, "commit", "-m", "feature") + runGit(t, origin, "checkout", "main") + + prebakeDir := t.TempDir() + prebakedCopy := filepath.Join(prebakeDir, "console") + if err := fs.CopyDir(origin, prebakedCopy); err != nil { + t.Fatalf("copy prebake fixture: %v", err) + } + writePrebakeManifest(t, prebakeDir, prebake.Manifest{ + Version: 1, + Repositories: []prebake.ManifestRepo{{ + URL: origin, + Path: "console", + }}, + }) + t.Setenv(prebake.EnvDir, prebakeDir) + + feature := "feature" + workDir := t.TempDir() + env := &environment{ + agentRun: &v1.AgentRun{Repository: origin, Branch: &feature}, + dir: workDir, + } + if err := env.cloneRepository(); err != nil { + t.Fatalf("cloneRepository() failed: %v", err) + } + + dest := filepath.Join(workDir, "repository") + contents, err := os.ReadFile(filepath.Join(dest, "README")) + if err != nil { + t.Fatal(err) + } + if string(contents) != "feature\n" { + t.Fatalf("copied README = %q, want feature after checking out run branch", contents) + } + current, err := exec.Command("git", "-C", dest, "branch", "--show-current").Output() + if err != nil { + t.Fatal(err) + } + if got := strings.TrimSpace(string(current)); got != "feature" { + t.Fatalf("branch = %q, want feature", got) + } +} + +func TestCloneRepositoryFallsBackToGitClone(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") + t.Setenv("TMPDIR", t.TempDir()) + runGit(t, home, "config", "--global", "--add", "safe.directory", "*") + + prebakeDir := t.TempDir() + writePrebakeManifest(t, prebakeDir, prebake.Manifest{ + Version: 1, + Repositories: []prebake.ManifestRepo{{ + URL: "https://github.com/pluralsh/console.git", + Path: "console", + }}, + }) + t.Setenv(prebake.EnvDir, prebakeDir) + + src := initGitRepo(t, "network") + workDir := t.TempDir() + env := &environment{ + agentRun: &v1.AgentRun{Repository: src}, + dir: workDir, + } + if err := env.cloneRepository(); err != nil { + t.Fatalf("cloneRepository() failed: %v", err) + } + + contents, err := os.ReadFile(filepath.Join(workDir, "repository", "README")) + if err != nil { + t.Fatal(err) + } + if string(contents) != "network\n" { + t.Fatalf("cloned README = %q, want network", contents) + } +} + +func initGitRepo(t *testing.T, contents string) string { + t.Helper() + dir := t.TempDir() + runGit(t, dir, "init", "-b", "main") + runGit(t, dir, "config", "user.email", "test@example.com") + runGit(t, dir, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(dir, "README"), []byte(contents+"\n"), 0644); err != nil { + t.Fatal(err) + } + runGit(t, dir, "add", "README") + runGit(t, dir, "commit", "-m", "init") + return dir +} + +func runGit(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } +} + +func writePrebakeManifest(t *testing.T, dir string, manifest prebake.Manifest) { + t.Helper() + data, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path.Join(dir, prebake.ManifestFileName), data, 0644); err != nil { + t.Fatal(err) + } +} diff --git a/go/deployment-operator/pkg/agentrun-harness/prebake/prebake.go b/go/deployment-operator/pkg/agentrun-harness/prebake/prebake.go new file mode 100644 index 0000000000..3e072d3876 --- /dev/null +++ b/go/deployment-operator/pkg/agentrun-harness/prebake/prebake.go @@ -0,0 +1,191 @@ +package prebake + +import ( + "encoding/json" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/pluralsh/console/go/deployment-operator/pkg/common" +) + +const ( + ManifestFileName = "manifest.json" + + // EnvDir overrides the canonical prebake mount path. Used in tests. + EnvDir = "PLRL_REPOSITORY_PREBAKE_DIR" +) + +// Manifest is the repository-prebake image contract. +type Manifest struct { + Version int `json:"version"` + Repositories []ManifestRepo `json:"repositories"` +} + +// ManifestRepo describes one precloned git repository in the image. +type ManifestRepo struct { + URL string `json:"url"` + Path string `json:"path"` + DefaultBranch string `json:"defaultBranch,omitempty"` +} + +// Repository is a resolved prebake entry with an absolute path on disk. +type Repository struct { + URL string + Path string + Dir string + DefaultBranch string +} + +// Dir returns the prebake mount path, honoring PLRL_REPOSITORY_PREBAKE_DIR. +func Dir() string { + if dir := strings.TrimSpace(os.Getenv(EnvDir)); dir != "" { + return dir + } + return common.AgentRunRepositoryPrebakeDir +} + +// ExtraReadDirs returns the prebake directory when it exists on disk so +// provider CLIs can be granted read access without changing the project cwd. +func ExtraReadDirs() []string { + dir := Dir() + info, err := os.Stat(dir) + if err != nil || !info.IsDir() { + return nil + } + return []string{dir} +} + +// ManifestPath is the absolute path to manifest.json in the prebake directory. +func ManifestPath() string { + return filepath.Join(Dir(), ManifestFileName) +} + +// NormalizeGitURL matches Elixir Console.Deployments.Pr.Git.normalize_url/1: +// strip a trailing .git, then reduce git@host:path and https://host/path to host/path. +func NormalizeGitURL(raw string) string { + raw = strings.TrimSpace(raw) + raw = strings.TrimSuffix(raw, ".git") + raw = strings.TrimRight(raw, "/") + + // SCP-style SSH: [user@]host:path (covers git@github.com:org/repo). + if !strings.Contains(raw, "://") { + if userHost, repoPath, ok := strings.Cut(raw, ":"); ok && repoPath != "" && !strings.HasPrefix(repoPath, "//") { + host := userHost + if _, h, found := strings.Cut(userHost, "@"); found { + host = h + } + if host != "" { + return host + "/" + strings.TrimPrefix(repoPath, "/") + } + } + } + + parsed, err := url.Parse(raw) + if err != nil || parsed.Hostname() == "" { + return raw + } + return parsed.Hostname() + strings.TrimSuffix(parsed.Path, "/") +} + +// Load reads manifest.json from the given prebake directory. +// A missing file is not an error: Load returns (nil, nil). +func Load(dir string) (*Manifest, error) { + path := filepath.Join(dir, ManifestFileName) + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read prebake manifest %q: %w", path, err) + } + + var manifest Manifest + if err := json.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("parse prebake manifest %q: %w", path, err) + } + return &manifest, nil +} + +// ResolvePath joins root with a relative repo path and rejects escapes. +func ResolvePath(root, rel string) (string, error) { + rel = strings.TrimSpace(rel) + if rel == "" || rel == ManifestFileName { + return "", fmt.Errorf("invalid prebake repository path %q", rel) + } + if filepath.IsAbs(rel) { + return "", fmt.Errorf("prebake repository path must be relative: %q", rel) + } + + clean := filepath.Clean(rel) + if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) { + return "", fmt.Errorf("invalid prebake repository path %q", rel) + } + + full := filepath.Join(root, clean) + relOut, err := filepath.Rel(root, full) + if err != nil || relOut == ".." || strings.HasPrefix(relOut, ".."+string(os.PathSeparator)) { + return "", fmt.Errorf("prebake repository path escapes %q: %q", root, rel) + } + return full, nil +} + +// Lookup finds a prebaked clone matching repositoryURL. Missing manifest or +// no match returns (nil, nil). +func Lookup(repositoryURL string) (*Repository, error) { + root := Dir() + manifest, err := Load(root) + if err != nil || manifest == nil { + return nil, err + } + + want := NormalizeGitURL(repositoryURL) + for _, entry := range manifest.Repositories { + if NormalizeGitURL(entry.URL) != want { + continue + } + full, err := ResolvePath(root, entry.Path) + if err != nil { + return nil, err + } + if _, err := os.Stat(filepath.Join(full, ".git")); err != nil { + return nil, nil + } + return &Repository{ + URL: entry.URL, + Path: entry.Path, + Dir: full, + DefaultBranch: entry.DefaultBranch, + }, nil + } + return nil, nil +} + +// List returns prebaked repositories that exist on disk. +func List() ([]Repository, error) { + root := Dir() + manifest, err := Load(root) + if err != nil || manifest == nil { + return nil, err + } + + var out []Repository + for _, entry := range manifest.Repositories { + full, err := ResolvePath(root, entry.Path) + if err != nil { + continue + } + if _, err := os.Stat(full); err != nil { + continue + } + out = append(out, Repository{ + URL: entry.URL, + Path: entry.Path, + Dir: full, + DefaultBranch: entry.DefaultBranch, + }) + } + return out, nil +} diff --git a/go/deployment-operator/pkg/agentrun-harness/prebake/prebake_test.go b/go/deployment-operator/pkg/agentrun-harness/prebake/prebake_test.go new file mode 100644 index 0000000000..08d443e8cc --- /dev/null +++ b/go/deployment-operator/pkg/agentrun-harness/prebake/prebake_test.go @@ -0,0 +1,126 @@ +package prebake + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +func TestNormalizeGitURL(t *testing.T) { + scp := "git@" + "github.com" + ":pluralsh/console" + tests := []struct { + in, want string + }{ + {"https://github.com/pluralsh/console.git", "github.com/pluralsh/console"}, + {"https://github.com/pluralsh/console", "github.com/pluralsh/console"}, + {"https://github.com/pluralsh/console/", "github.com/pluralsh/console"}, + {scp + ".git", "github.com/pluralsh/console"}, + {scp, "github.com/pluralsh/console"}, + {"ssh://git@github.com/pluralsh/console.git", "github.com/pluralsh/console"}, + {"https://user:token@github.com/pluralsh/console.git", "github.com/pluralsh/console"}, + {" https://github.com/pluralsh/console.git ", "github.com/pluralsh/console"}, + } + for _, tc := range tests { + if got := NormalizeGitURL(tc.in); got != tc.want { + t.Errorf("NormalizeGitURL(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestLookupMatchesNormalizedURL(t *testing.T) { + root := t.TempDir() + repoDir := filepath.Join(root, "console") + if err := os.MkdirAll(filepath.Join(repoDir, ".git"), 0755); err != nil { + t.Fatal(err) + } + writeManifest(t, root, Manifest{ + Version: 1, + Repositories: []ManifestRepo{{ + URL: "https://github.com/pluralsh/console.git", + Path: "console", + }}, + }) + t.Setenv(EnvDir, root) + + got, err := Lookup("git@" + "github.com" + ":pluralsh/console.git") + if err != nil { + t.Fatalf("Lookup() error: %v", err) + } + if got == nil { + t.Fatal("Lookup() returned nil, want match") + } + if got.Dir != repoDir { + t.Fatalf("Lookup() dir = %q, want %q", got.Dir, repoDir) + } +} + +func TestLookupUnknownURL(t *testing.T) { + root := t.TempDir() + writeManifest(t, root, Manifest{ + Version: 1, + Repositories: []ManifestRepo{{ + URL: "https://github.com/pluralsh/console.git", + Path: "console", + }}, + }) + t.Setenv(EnvDir, root) + + got, err := Lookup("https://github.com/pluralsh/plural.git") + if err != nil { + t.Fatalf("Lookup() error: %v", err) + } + if got != nil { + t.Fatalf("Lookup() = %+v, want nil", got) + } +} + +func TestLookupMissingManifest(t *testing.T) { + t.Setenv(EnvDir, t.TempDir()) + got, err := Lookup("https://github.com/pluralsh/console.git") + if err != nil { + t.Fatalf("Lookup() error: %v", err) + } + if got != nil { + t.Fatalf("Lookup() = %+v, want nil", got) + } +} + +func TestExtraReadDirs(t *testing.T) { + missing := filepath.Join(t.TempDir(), "does-not-exist") + t.Setenv(EnvDir, missing) + if got := ExtraReadDirs(); got != nil { + t.Fatalf("ExtraReadDirs() = %v, want nil for missing dir", got) + } + + root := t.TempDir() + t.Setenv(EnvDir, root) + got := ExtraReadDirs() + if len(got) != 1 || got[0] != root { + t.Fatalf("ExtraReadDirs() = %v, want [%q]", got, root) + } +} + +func TestResolvePathRejectsEscape(t *testing.T) { + root := t.TempDir() + if _, err := ResolvePath(root, "../escape"); err == nil { + t.Fatal("expected error for .. path") + } + if _, err := ResolvePath(root, "/abs"); err == nil { + t.Fatal("expected error for absolute path") + } + if _, err := ResolvePath(root, "manifest.json"); err == nil { + t.Fatal("expected error for manifest.json path") + } +} + +func writeManifest(t *testing.T, dir string, manifest Manifest) { + t.Helper() + data, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, ManifestFileName), data, 0644); err != nil { + t.Fatal(err) + } +} diff --git a/go/deployment-operator/pkg/agentrun-harness/tool/claude/claude.go b/go/deployment-operator/pkg/agentrun-harness/tool/claude/claude.go index 4d090e79a0..933c0baa35 100644 --- a/go/deployment-operator/pkg/agentrun-harness/tool/claude/claude.go +++ b/go/deployment-operator/pkg/agentrun-harness/tool/claude/claude.go @@ -14,6 +14,7 @@ import ( console "github.com/pluralsh/console/go/client" "github.com/pluralsh/console/go/deployment-operator/pkg/agentrun-harness/mcp" + "github.com/pluralsh/console/go/deployment-operator/pkg/agentrun-harness/prebake" v1 "github.com/pluralsh/console/go/deployment-operator/pkg/agentrun-harness/tool/v1" "github.com/pluralsh/console/go/deployment-operator/pkg/agentrun-harness/usage" "github.com/pluralsh/console/go/deployment-operator/pkg/common" @@ -62,7 +63,7 @@ func (in *Claude) BabysitRun(ctx context.Context, bCtx *v1.BabysitContext) bool promptFile := path.Join(in.Config.WorkDir, ".claude", "prompts", v1.SystemPromptFile) agent := in.agentJSON(babysitAgent) - args := claudeRunArgs(in.Config.RepositoryDir, promptFile, agent, in.model, bCtx.Prompt, in.sessionID) + args := claudeRunArgs(in.Config.RepositoryDir, promptFile, agent, in.model, bCtx.Prompt, in.sessionID, prebake.ExtraReadDirs()...) var envOpt exec.Option if in.Config.Run.IsProxyEnabled() { @@ -126,7 +127,7 @@ func (in *Claude) FollowUpRun(ctx context.Context, followUpPrompt string) error case console.AgentRunModeReview: agent = in.agentJSON(reviewAgent) } - args := claudeRunArgs(in.Config.RepositoryDir, promptFile, agent, in.model, followUpPrompt, in.sessionID) + args := claudeRunArgs(in.Config.RepositoryDir, promptFile, agent, in.model, followUpPrompt, in.sessionID, prebake.ExtraReadDirs()...) var opts []exec.Option if in.Config.Run.IsProxyEnabled() { @@ -179,7 +180,7 @@ func (in *Claude) start(ctx context.Context, options ...exec.Option) { case console.AgentRunModeReview: agent = in.agentJSON(reviewAgent) } - args := claudeRunArgs(in.Config.RepositoryDir, promptFile, agent, in.model, in.Config.Run.Prompt, "") + args := claudeRunArgs(in.Config.RepositoryDir, promptFile, agent, in.model, in.Config.Run.Prompt, "", prebake.ExtraReadDirs()...) if in.Config.Run.IsProxyEnabled() { options = append(options, @@ -523,13 +524,19 @@ func mapRole(role string) console.AiRole { } } -func claudeRunArgs(repositoryDir, promptFile, agent string, model Model, prompt, resumeSessionID string) []string { - args := []string{ - "--add-dir", repositoryDir, +func claudeRunArgs(repositoryDir, promptFile, agent string, model Model, prompt, resumeSessionID string, extraDirs ...string) []string { + args := []string{"--add-dir", repositoryDir} + for _, dir := range extraDirs { + if dir == "" || dir == repositoryDir { + continue + } + args = append(args, "--add-dir", dir) + } + args = append(args, "--agents", agent, "--system-prompt-file", promptFile, "--model", string(model), - } + ) if resumeSessionID != "" { args = append(args, "--resume", resumeSessionID, "-p", prompt) } else { diff --git a/go/deployment-operator/pkg/agentrun-harness/tool/claude/claude_args_test.go b/go/deployment-operator/pkg/agentrun-harness/tool/claude/claude_args_test.go index aa378f94f0..82162a719a 100644 --- a/go/deployment-operator/pkg/agentrun-harness/tool/claude/claude_args_test.go +++ b/go/deployment-operator/pkg/agentrun-harness/tool/claude/claude_args_test.go @@ -16,6 +16,21 @@ func TestClaudeRunArgs(t *testing.T) { assertArgsEqual(t, want, args) } +func TestClaudeRunArgsExtraDirs(t *testing.T) { + args := claudeRunArgs("/repo", "/plural/.claude/prompts/AGENTS.md", "autonomous", Sonnet46, "fix tests", "", "/plural/shared/repos", "/repo", "") + want := []string{ + "--add-dir", "/repo", + "--add-dir", "/plural/shared/repos", + "--agents", "autonomous", + "--system-prompt-file", "/plural/.claude/prompts/AGENTS.md", + "--model", string(Sonnet46), + "-p", "fix tests", + "--output-format", "stream-json", + "--verbose", + } + assertArgsEqual(t, want, args) +} + func TestClaudeRunArgsResume(t *testing.T) { sessionID := "550e8400-e29b-41d4-a716-446655440000" args := claudeRunArgs("/repo", "/plural/.claude/prompts/AGENTS.md", "autonomous", Sonnet46, "add tests", sessionID) diff --git a/go/deployment-operator/pkg/agentrun-harness/tool/gemini/gemini.go b/go/deployment-operator/pkg/agentrun-harness/tool/gemini/gemini.go index ec883ce1f5..feccbb44a8 100644 --- a/go/deployment-operator/pkg/agentrun-harness/tool/gemini/gemini.go +++ b/go/deployment-operator/pkg/agentrun-harness/tool/gemini/gemini.go @@ -13,6 +13,7 @@ import ( console "github.com/pluralsh/console/go/client" "github.com/pluralsh/console/go/deployment-operator/internal/helpers" + "github.com/pluralsh/console/go/deployment-operator/pkg/agentrun-harness/prebake" "github.com/pluralsh/console/go/deployment-operator/pkg/agentrun-harness/tool/gemini/events" v1 "github.com/pluralsh/console/go/deployment-operator/pkg/agentrun-harness/tool/v1" "github.com/pluralsh/console/go/deployment-operator/pkg/harness/exec" @@ -242,6 +243,7 @@ func (in *Gemini) Configure(_, _ string) error { input := &ConfigTemplateInput{ RepositoryDir: in.Config.RepositoryDir, + ExtraDirectories: prebake.ExtraReadDirs(), AgentRunID: in.Config.Run.ID, AgentRunMode: in.Config.Run.Mode, InactivityTimeout: int64(in.Config.Run.Runtime.Config.Gemini.InactivityTimeout.Seconds()), diff --git a/go/deployment-operator/pkg/agentrun-harness/tool/gemini/settings.go b/go/deployment-operator/pkg/agentrun-harness/tool/gemini/settings.go index 33f68a882c..623cc9698f 100644 --- a/go/deployment-operator/pkg/agentrun-harness/tool/gemini/settings.go +++ b/go/deployment-operator/pkg/agentrun-harness/tool/gemini/settings.go @@ -18,6 +18,7 @@ const SettingsFileName = "settings.json" type ConfigTemplateInput struct { Model Model RepositoryDir string + ExtraDirectories []string AgentRunID string AgentRunMode console.AgentRunMode InactivityTimeout int64 diff --git a/go/deployment-operator/pkg/agentrun-harness/tool/gemini/settings_test.go b/go/deployment-operator/pkg/agentrun-harness/tool/gemini/settings_test.go index 441a532538..2f2447cc07 100644 --- a/go/deployment-operator/pkg/agentrun-harness/tool/gemini/settings_test.go +++ b/go/deployment-operator/pkg/agentrun-harness/tool/gemini/settings_test.go @@ -138,6 +138,36 @@ func TestSettingsTemplate_GenerateAndVerifyContents(t *testing.T) { } } }) + + t.Run("includeDirectories contains extra prebake dirs", func(t *testing.T) { + input := *baseInput + input.AgentRunMode = console.AgentRunModeWrite + input.ExtraDirectories = []string{"/plural/shared/repos"} + + _, content, err := settings(&input) + if err != nil { + t.Fatalf("settings() failed: %v", err) + } + + var out map[string]any + if err := json.Unmarshal([]byte(content), &out); err != nil { + t.Fatalf("generated content is not valid JSON: %v", err) + } + dirs, ok := out["includeDirectories"].([]any) + if !ok { + t.Fatal("includeDirectories missing or not an array") + } + found := false + for _, d := range dirs { + if s, ok := d.(string); ok && s == "/plural/shared/repos" { + found = true + break + } + } + if !found { + t.Fatalf("expected includeDirectories to contain /plural/shared/repos, got %#v", dirs) + } + }) } func TestSettingsTemplate_ExternalMCPServer(t *testing.T) { diff --git a/go/deployment-operator/pkg/agentrun-harness/tool/gemini/templates/settings.json.gotmpl b/go/deployment-operator/pkg/agentrun-harness/tool/gemini/templates/settings.json.gotmpl index 5191aa00bf..862ff12efc 100644 --- a/go/deployment-operator/pkg/agentrun-harness/tool/gemini/templates/settings.json.gotmpl +++ b/go/deployment-operator/pkg/agentrun-harness/tool/gemini/templates/settings.json.gotmpl @@ -37,7 +37,8 @@ ], "includeDirectories": [ "/plural/contexts", - "{{ .RepositoryDir }}" + "{{ .RepositoryDir }}"{{ range .ExtraDirectories }}, + "{{ . }}"{{ end }} ], "model": { "name": "{{ .Model }}" diff --git a/go/deployment-operator/pkg/agentrun-harness/tool/v1/templates.go b/go/deployment-operator/pkg/agentrun-harness/tool/v1/templates.go index 52eb6bea6a..8e5c751512 100644 --- a/go/deployment-operator/pkg/agentrun-harness/tool/v1/templates.go +++ b/go/deployment-operator/pkg/agentrun-harness/tool/v1/templates.go @@ -9,17 +9,23 @@ import ( ) type SystemPromptTemplateInput struct { - Mode console.AgentRunMode - ReviewDepth console.AgentReviewDepth - BrowserEnabled bool - DindEnabled bool - MemoryEnabled bool - WorkDir string - RepositoryDir string - Prompt string - Branch string - PRURL string - Followup bool + Mode console.AgentRunMode + ReviewDepth console.AgentReviewDepth + BrowserEnabled bool + DindEnabled bool + MemoryEnabled bool + WorkDir string + RepositoryDir string + Prompt string + Branch string + PRURL string + Followup bool + PrebakedRepositories []PrebakedRepository +} + +type PrebakedRepository struct { + URL string + Dir string } func systemPromptTemplate(templateFilePath string, input *SystemPromptTemplateInput) (content string, err error) { diff --git a/go/deployment-operator/pkg/agentrun-harness/tool/v1/templates_test.go b/go/deployment-operator/pkg/agentrun-harness/tool/v1/templates_test.go index b704e4b998..f8d1a1c8b8 100644 --- a/go/deployment-operator/pkg/agentrun-harness/tool/v1/templates_test.go +++ b/go/deployment-operator/pkg/agentrun-harness/tool/v1/templates_test.go @@ -8,6 +8,7 @@ import ( console "github.com/pluralsh/console/go/client" agentrunv1 "github.com/pluralsh/console/go/deployment-operator/pkg/agentrun-harness/agentrun/v1" + "github.com/pluralsh/console/go/deployment-operator/pkg/agentrun-harness/prebake" ) func TestSystemPromptTemplate_EmbedsOriginalPrompt(t *testing.T) { @@ -255,3 +256,102 @@ func TestSystemPromptTemplate_TemplateFilesExist(t *testing.T) { } } } + +func TestSystemPromptTemplate_PrebakedRepositories(t *testing.T) { + templateDir := filepath.Join("..", "..", "..", "..", "dockerfiles", "agent-harness", "system") + input := &SystemPromptTemplateInput{ + Mode: console.AgentRunModeWrite, + WorkDir: "/work", + RepositoryDir: "/work/shared/repository", + PrebakedRepositories: []PrebakedRepository{ + {URL: "https://github.com/pluralsh/console.git", Dir: "/plural/shared/repos/console"}, + {URL: "https://github.com/pluralsh/plural.git", Dir: "/plural/shared/repos/plural"}, + }, + } + + for _, name := range []string{"analyze.md.tmpl", "write.md.tmpl", "review.md.tmpl", "babysit.md.tmpl"} { + t.Run(name, func(t *testing.T) { + content, err := systemPromptTemplate(filepath.Join(templateDir, name), input) + if err != nil { + t.Fatalf("systemPromptTemplate() failed: %v", err) + } + for _, expected := range []string{ + "## Additional local repositories", + "You SHOULD inspect them", + "https://github.com/pluralsh/console.git", + "/plural/shared/repos/console", + "https://github.com/pluralsh/plural.git", + "/plural/shared/repos/plural", + "Do not clone it again", + } { + if !strings.Contains(content, expected) { + t.Fatalf("expected prebake instructions to contain %q", expected) + } + } + switch name { + case "analyze.md.tmpl": + if !strings.Contains(content, "You MAY also read (not modify) the additional local repositories listed above.") { + t.Fatal("expected analyze scope to allow reading additional local repositories") + } + if strings.Contains(content, "Never access files outside this directory.") { + t.Fatal("analyze scope must not forbid additional local repositories") + } + case "review.md.tmpl": + if !strings.Contains(content, "You MAY also read (not modify) the additional local repositories listed above.") { + t.Fatal("expected review prompt to allow reading additional local repositories") + } + default: + if !strings.Contains(content, "You MAY read the additional local repositories listed above.") { + t.Fatal("expected write/babysit prompt to allow reading additional local repositories") + } + } + }) + } + + omitted, err := systemPromptTemplate(filepath.Join(templateDir, "write.md.tmpl"), &SystemPromptTemplateInput{ + Mode: console.AgentRunModeWrite, + WorkDir: "/work", + RepositoryDir: "/work/shared/repository", + }) + if err != nil { + t.Fatalf("systemPromptTemplate() failed: %v", err) + } + if strings.Contains(omitted, "## Additional local repositories") { + t.Fatal("did not expect prebake section when PrebakedRepositories is empty") + } + + strict, err := systemPromptTemplate(filepath.Join(templateDir, "analyze.md.tmpl"), &SystemPromptTemplateInput{ + Mode: console.AgentRunModeAnalyze, + WorkDir: "/work", + RepositoryDir: "/work/shared/repository", + }) + if err != nil { + t.Fatalf("systemPromptTemplate() failed: %v", err) + } + if !strings.Contains(strict, "Never access files outside this directory.") { + t.Fatal("expected analyze scope to stay restricted when there are no additional local repositories") + } +} + +func TestExtraPrebakedRepositories_OmitsAssignedRepo(t *testing.T) { + repos := []prebake.Repository{ + {URL: "https://github.com/octocat/Hello-World.git", Dir: "/plural/shared/repos/hello-world"}, + {URL: "git@" + "github.com" + ":pluralsh/console.git", Dir: "/plural/shared/repos/console"}, + } + + got := extraPrebakedRepositories(repos, "https://github.com/octocat/hello-world") + if len(got) != 1 { + t.Fatalf("got %d extra repos, want 1", len(got)) + } + if got[0].Dir != "/plural/shared/repos/console" { + t.Fatalf("got extra repo dir %q, want console", got[0].Dir) + } + + if extra := extraPrebakedRepositories(repos[:1], "https://github.com/octocat/Hello-World.git"); extra != nil { + t.Fatalf("expected nil when the only prebaked repo is the assigned run, got %#v", extra) + } + + if extra := extraPrebakedRepositories(nil, "https://github.com/octocat/Hello-World.git"); extra != nil { + t.Fatalf("expected nil for empty input, got %#v", extra) + } +} diff --git a/go/deployment-operator/pkg/agentrun-harness/tool/v1/tool.go b/go/deployment-operator/pkg/agentrun-harness/tool/v1/tool.go index 18367d0110..c41bd09a81 100644 --- a/go/deployment-operator/pkg/agentrun-harness/tool/v1/tool.go +++ b/go/deployment-operator/pkg/agentrun-harness/tool/v1/tool.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "path" + "strings" console "github.com/pluralsh/console/go/client" "github.com/pluralsh/console/go/deployment-operator/pkg/agentrun-harness/tool/artifacts" @@ -11,6 +12,7 @@ import ( "k8s.io/klog/v2" "github.com/pluralsh/console/go/deployment-operator/internal/helpers" + "github.com/pluralsh/console/go/deployment-operator/pkg/agentrun-harness/prebake" "github.com/pluralsh/console/go/deployment-operator/pkg/log" ) @@ -110,20 +112,49 @@ func (in DefaultTool) systemPromptInput() *SystemPromptTemplateInput { } return &SystemPromptTemplateInput{ - Mode: in.Config.Run.Mode, - ReviewDepth: in.Config.Run.ReviewDepth, - BrowserEnabled: in.Config.Run.BrowserEnabled, - DindEnabled: in.Config.Run.DindEnabled, - MemoryEnabled: in.Config.Run.MemoryEnabled, - WorkDir: in.Config.WorkDir, - RepositoryDir: in.Config.RepositoryDir, - Prompt: in.Config.Run.Prompt, - Branch: branch, - PRURL: in.Config.Run.PRURL, - Followup: in.Config.Run.Followup, + Mode: in.Config.Run.Mode, + ReviewDepth: in.Config.Run.ReviewDepth, + BrowserEnabled: in.Config.Run.BrowserEnabled, + DindEnabled: in.Config.Run.DindEnabled, + MemoryEnabled: in.Config.Run.MemoryEnabled, + WorkDir: in.Config.WorkDir, + RepositoryDir: in.Config.RepositoryDir, + Prompt: in.Config.Run.Prompt, + Branch: branch, + PRURL: in.Config.Run.PRURL, + Followup: in.Config.Run.Followup, + PrebakedRepositories: prebakedRepositories(in.Config.Run.Repository), } } +func prebakedRepositories(assignedURL string) []PrebakedRepository { + repos, err := prebake.List() + if err != nil { + klog.ErrorS(err, "failed to load repository prebake manifest") + return nil + } + return extraPrebakedRepositories(repos, assignedURL) +} + +func extraPrebakedRepositories(repos []prebake.Repository, assignedURL string) []PrebakedRepository { + if len(repos) == 0 { + return nil + } + + assigned := prebake.NormalizeGitURL(assignedURL) + out := make([]PrebakedRepository, 0, len(repos)) + for _, repo := range repos { + if assigned != "" && strings.EqualFold(prebake.NormalizeGitURL(repo.URL), assigned) { + continue + } + out = append(out, PrebakedRepository{URL: repo.URL, Dir: repo.Dir}) + } + if len(out) == 0 { + return nil + } + return out +} + func (in DefaultTool) BuildUploadArtifacts(ctx context.Context, opts artifacts.BuildArtifactsOptions) (*artifacts.UploadArtifacts, error) { return artifacts.NewUploadArtifactBuilder(artifacts.Config{ WorkDir: in.Config.WorkDir, diff --git a/go/deployment-operator/pkg/common/mcp.go b/go/deployment-operator/pkg/common/mcp.go index 7c605d3656..5ecd384b2d 100644 --- a/go/deployment-operator/pkg/common/mcp.go +++ b/go/deployment-operator/pkg/common/mcp.go @@ -17,6 +17,10 @@ const ( AgentRunSharedWorkDir = "/plural/shared" + // AgentRunRepositoryPrebakeDir is where a repository-prebake image is + // copied on the shared emptyDir (manifest.json plus cloned git repos). + AgentRunRepositoryPrebakeDir = AgentRunSharedWorkDir + "/repos" + CodebaseMemoryMCPServerName = "codebase-memory-mcp" CodebaseMemoryMCPCommand = "/usr/local/bin/codebase-memory-mcp" CodebaseMemoryCacheEnv = "CBM_CACHE_DIR" diff --git a/go/polly/fs/copy.go b/go/polly/fs/copy.go new file mode 100644 index 0000000000..9ef9a77f0a --- /dev/null +++ b/go/polly/fs/copy.go @@ -0,0 +1,93 @@ +package fs + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// CopyDir recursively copies src to dst, preserving permissions and symlinks. +// dst must not already exist. Special files such as sockets and devices are skipped. +func CopyDir(src, dst string) error { + src = filepath.Clean(src) + dst = filepath.Clean(dst) + + info, err := os.Lstat(src) + if err != nil { + return fmt.Errorf("copy: stat source: %w", err) + } + if !info.IsDir() { + return fmt.Errorf("copy: source is not a directory: %s", src) + } + if _, err := os.Lstat(dst); err == nil { + return fmt.Errorf("copy: destination already exists: %s", dst) + } else if !os.IsNotExist(err) { + return err + } + + return filepath.WalkDir(src, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + if rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + return fmt.Errorf("copy: path %q escapes source", path) + } + + target := filepath.Join(dst, rel) + switch { + case d.Type()&os.ModeSymlink != 0: + link, err := os.Readlink(path) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return err + } + return os.Symlink(link, target) + case d.IsDir(): + dirInfo, err := d.Info() + if err != nil { + return err + } + return os.MkdirAll(target, dirInfo.Mode().Perm()) + case d.Type().IsRegular(): + fileInfo, err := d.Info() + if err != nil { + return err + } + return copyFile(path, target, fileInfo.Mode()) + default: + return nil + } + }) +} + +func copyFile(src, dst string, mode os.FileMode) error { + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) + if err != nil { + return err + } + defer out.Close() + + if _, err := io.Copy(out, in); err != nil { + return err + } + return out.Chmod(mode) +} diff --git a/go/polly/fs/copy_test.go b/go/polly/fs/copy_test.go new file mode 100644 index 0000000000..23c9f5f5ed --- /dev/null +++ b/go/polly/fs/copy_test.go @@ -0,0 +1,56 @@ +package fs + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCopyDir(t *testing.T) { + src := t.TempDir() + nested := filepath.Join(src, "nested") + require.NoError(t, os.Mkdir(nested, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(src, "root.txt"), []byte("root"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(nested, "child.txt"), []byte("child"), 0644)) + require.NoError(t, os.Chmod(filepath.Join(nested, "child.txt"), 0755)) + require.NoError(t, os.Symlink("root.txt", filepath.Join(src, "link.txt"))) + + dst := filepath.Join(t.TempDir(), "copy") + require.NoError(t, CopyDir(src, dst)) + + root, err := os.ReadFile(filepath.Join(dst, "root.txt")) + require.NoError(t, err) + assert.Equal(t, "root", string(root)) + + child, err := os.ReadFile(filepath.Join(dst, "nested", "child.txt")) + require.NoError(t, err) + assert.Equal(t, "child", string(child)) + + link, err := os.Readlink(filepath.Join(dst, "link.txt")) + require.NoError(t, err) + assert.Equal(t, "root.txt", link) + + info, err := os.Stat(filepath.Join(dst, "nested", "child.txt")) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0755), info.Mode().Perm()) +} + +func TestCopyDirRejectsExistingDestination(t *testing.T) { + src := t.TempDir() + dst := t.TempDir() + err := CopyDir(src, dst) + require.Error(t, err) + assert.Contains(t, err.Error(), "already exists") +} + +func TestCopyDirRejectsFileSource(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "file") + require.NoError(t, os.WriteFile(src, []byte("x"), 0644)) + err := CopyDir(src, filepath.Join(dir, "dst")) + require.Error(t, err) + assert.Contains(t, err.Error(), "not a directory") +} diff --git a/js/documentation/pages/api-reference/kubernetes/agent-api-reference.md b/js/documentation/pages/api-reference/kubernetes/agent-api-reference.md index f547a7c4f1..b74e6fe127 100644 --- a/js/documentation/pages/api-reference/kubernetes/agent-api-reference.md +++ b/js/documentation/pages/api-reference/kubernetes/agent-api-reference.md @@ -275,6 +275,7 @@ _Appears in:_ | `streamingProxy` _boolean_ | StreamingProxy routes OpenAI-compatible LLM requests through the in-pod mcpserver
sse conversion proxy before they reach the Console AI proxy (/ext/ai). Only valid when aiProxy
is enabled. Applies to CODEX and OPENCODE runtimes. | | Optional: \{\}
| | `dind` _boolean_ | Dind enables Docker-in-Docker for this agent runtime.
When true, the runtime will be configured to run with DinD support. | | Optional: \{\}
| | `memory` _boolean_ | Memory enables team-shared codebase-memory persistence for this agent runtime.
When true, agents may create and commit .codebase-memory/ graph artifacts
by default so future runs can bootstrap from the persisted index. When false
or unset, codebase-memory indexes stay in the pod-local cache and generated
.codebase-memory/ artifacts are excluded from commits. | | Optional: \{\}
| +| `repositoryImage` _string_ | RepositoryImage is an OCI image of precloned git repositories plus manifest.json.
When set, an init container copies it into /plural/shared/repos before bootstrap
so a matching repo can be copied locally instead of git clone. | | Optional: \{\}
| | `allowedRepositories` _string array_ | AllowedRepositories the git repositories allowed to be used with this runtime. | | Optional: \{\}
| | `browser` _[BrowserConfig](#browserconfig)_ | Browser configuration augments agent runtime with a headless browser.
When provided, the runtime will be configured to run with a headless browser available
for the agent to use. | | Optional: \{\}
| | `bootstrapScript` _string_ | BootstrapScript is a bash script that will be executed inside the cloned repository
directory before the coding agent starts. It can be used to install dependencies,
configure tooling, or perform any other setup required by the agent. | | Optional: \{\}
|