diff --git a/README.md b/README.md index 52002d5..5d92091 100644 --- a/README.md +++ b/README.md @@ -355,6 +355,25 @@ loupectl repo list loupectl repo scan 1 # one-shot scan of repo id 1 ``` +Use `--scanner-config-file ` to attach a per-repo JSON +configuration at registration time. The file must contain a JSON object; +Loupe stores that object with the repo and passes it to each worker lease. +For the LLM scanner, supported fields are `max_concurrent_files`, +`max_file_bytes`, `per_request_timeout_seconds`, `include_extensions`, +`exclude_path_substrings`, and `extra_source_paths`. The last field adds +repository-relative files or directories to automatic source discovery. +Directories still use the extension allowlist; explicitly named files do +not, which makes it possible to include operational files such as +`Dockerfile`. Absolute paths, `..` traversal, missing paths, and symlink +targets outside the checked-out worktree are ignored. + +The scanner config is a stored snapshot; it does not follow later edits to +the local JSON file. Reload a changed profile without losing scan history: + +```bash +loupectl repo update --scanner-config-file +``` + Add `--verification-enabled` if this repo should route scan findings through verifier jobs before reporting. If the server-wide verification default is on, omit it to inherit the default, or pass @@ -473,6 +492,7 @@ the first one already covered. loupectl repo update --disable # pause scheduler loupectl repo update --enable loupectl repo update --interval 3600 # hourly +loupectl repo update --scanner-config-file profile.json loupectl repo update --verification-enabled # route via verify flow loupectl repo update --no-verification # skip verify; dispatch on insert loupectl repo update --require-approval # hold for human sign-off diff --git a/crates/loupe-cli/src/main.rs b/crates/loupe-cli/src/main.rs index 61a1308..90817bf 100644 --- a/crates/loupe-cli/src/main.rs +++ b/crates/loupe-cli/src/main.rs @@ -122,6 +122,10 @@ struct RepoUpdateArgs { /// `--disable` if you want to stop scheduled scans. #[arg(long)] interval: Option, + /// Replace the per-repo scanner settings with the JSON object in + /// this file. + #[arg(long, value_name = "PATH")] + scanner_config_file: Option, /// Route findings through the verify flow before dispatching. #[arg(long, conflicts_with = "no_verification")] verification_enabled: bool, @@ -151,6 +155,10 @@ struct RepoAddArgs { branch: Option, #[arg(long)] scan_interval_seconds: Option, + /// JSON file containing per-repo scanner settings. The parsed JSON + /// is stored with the repo and sent to workers with every lease. + #[arg(long, value_name = "PATH")] + scanner_config_file: Option, /// Owner of the tracker repo where findings get filed. Required /// unless `--no-reporting` is set. #[arg(long, required_unless_present = "no_reporting")] @@ -516,6 +524,12 @@ fn url(base: &reqwest::Url, path: &str) -> reqwest::Url { } async fn repo_add(client: &reqwest::Client, base: &reqwest::Url, a: RepoAddArgs) -> Result<()> { + let scanner_config = a + .scanner_config_file + .as_deref() + .map(load_scanner_config) + .transpose()? + .unwrap_or(serde_json::Value::Null); let require_approval = match (a.require_approval, a.no_require_approval) { (true, false) => Some(true), (false, true) => Some(false), @@ -543,7 +557,7 @@ async fn repo_add(client: &reqwest::Client, base: &reqwest::Url, a: RepoAddArgs) branch: a.branch, scan_interval_seconds: a.scan_interval_seconds, reporting, - scanner_config: serde_json::Value::Null, + scanner_config, verification_enabled, require_approval, }; @@ -557,6 +571,17 @@ async fn repo_add(client: &reqwest::Client, base: &reqwest::Url, a: RepoAddArgs) Ok(()) } +fn load_scanner_config(path: &Path) -> Result { + let raw = std::fs::read_to_string(path) + .with_context(|| format!("reading scanner config file {}", path.display()))?; + let value: serde_json::Value = serde_json::from_str(&raw) + .with_context(|| format!("parsing scanner config file {}", path.display()))?; + if !value.is_object() { + anyhow::bail!("scanner config file {} must contain a JSON object", path.display()); + } + Ok(value) +} + async fn repo_list( client: &reqwest::Client, base: &reqwest::Url, limit: Option, ) -> Result<()> { @@ -592,6 +617,7 @@ async fn repo_rm(client: &reqwest::Client, base: &reqwest::Url, id: i64) -> Resu async fn repo_update( client: &reqwest::Client, base: &reqwest::Url, a: RepoUpdateArgs, ) -> Result<()> { + let scanner_config = a.scanner_config_file.as_deref().map(load_scanner_config).transpose()?; let disabled = match (a.disable, a.enable) { (true, false) => Some(true), (false, true) => Some(false), @@ -612,6 +638,7 @@ async fn repo_update( disabled, scan_interval_seconds: a.interval, verification_enabled, + scanner_config, require_approval, inherit_require_approval: a.inherit_approval, }; @@ -1110,6 +1137,61 @@ mod tests { assert!(args.no_verification); } + #[test] + fn repo_add_accepts_a_scanner_config_file() { + let cli = Cli::try_parse_from([ + "loupectl", + "--server-url", + "https://loupe.example:8443", + "repo", + "add", + "--clone-url", + "https://github.com/acme/widget.git", + "--scanner-config-file", + "profiles/widget.json", + "--no-reporting", + ]) + .unwrap(); + let Cmd::Repo(RepoCmd::Add(args)) = cli.cmd else { + panic!("expected repo add command"); + }; + assert_eq!(args.scanner_config_file, Some(PathBuf::from("profiles/widget.json"))); + } + + #[test] + fn scanner_config_file_must_contain_a_json_object() { + let suffix = + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos(); + let path = std::env::temp_dir() + .join(format!("loupectl-scanner-config-{}-{suffix}.json", std::process::id())); + std::fs::write(&path, "[]\n").unwrap(); + + let error = load_scanner_config(&path).unwrap_err(); + std::fs::remove_file(&path).unwrap(); + + assert!(error.to_string().contains("must contain a JSON object")); + } + + #[test] + fn repo_update_accepts_a_scanner_config_file() { + let cli = Cli::try_parse_from([ + "loupectl", + "--server-url", + "https://loupe.example:8443", + "repo", + "update", + "7", + "--scanner-config-file", + "profiles/widget.json", + ]) + .unwrap(); + let Cmd::Repo(RepoCmd::Update(args)) = cli.cmd else { + panic!("expected repo update command"); + }; + assert_eq!(args.id, 7); + assert_eq!(args.scanner_config_file, Some(PathBuf::from("profiles/widget.json"))); + } + #[test] fn repo_set_github_reporting_parses_explicit_pat() { let cli = Cli::try_parse_from([ diff --git a/crates/loupe-proto/src/registry.rs b/crates/loupe-proto/src/registry.rs index a8afaa0..16a23f0 100644 --- a/crates/loupe-proto/src/registry.rs +++ b/crates/loupe-proto/src/registry.rs @@ -101,7 +101,7 @@ pub struct SetRepoGithubReportingRequest { /// `disabled_at = now`; `disabled = Some(false)` clears it. The repo's /// reporting destination, clone URL, and PAT cannot be patched: those /// are register-time inputs, and changing them would silently affect -/// where new findings get filed. Re-register the repo for that. +/// where new findings get filed. Re-register the repo for those fields. /// /// `require_approval` is tri-state on the wire: omitted = leave the /// existing per-repo override alone; `Some(true)` / `Some(false)` = @@ -118,6 +118,8 @@ pub struct UpdateRepoRequest { #[serde(default, skip_serializing_if = "Option::is_none")] pub verification_enabled: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub scanner_config: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub require_approval: Option, #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub inherit_require_approval: bool, @@ -269,6 +271,19 @@ mod tests { assert_eq!(back.verification_enabled, None); } + #[test] + fn update_repo_request_round_trips_scanner_config() { + let req = UpdateRepoRequest { + protocol_version: PROTOCOL_VERSION, + scanner_config: Some(json!({"extra_source_paths": ["Dockerfile"]})), + ..UpdateRepoRequest::default() + }; + let serialized = serde_json::to_string(&req).unwrap(); + let round_trip: UpdateRepoRequest = serde_json::from_str(&serialized).unwrap(); + assert_eq!(round_trip, req); + assert!(serialized.contains("extra_source_paths")); + } + fn summary_with_reporting(reporting: Option) -> RepoSummary { RepoSummary { id: 1, diff --git a/crates/loupe-server/src/routes/repos.rs b/crates/loupe-server/src/routes/repos.rs index 02b96c1..121d01b 100644 --- a/crates/loupe-server/src/routes/repos.rs +++ b/crates/loupe-server/src/routes/repos.rs @@ -141,7 +141,7 @@ pub async fn list( } /// `PATCH /v1/repos/:id` — admin only. Toggles `disabled`, swaps the -/// scan interval, or flips the verification flag. Each field is +/// scan interval, scanner configuration, or flips the verification flag. Each field is /// independently optional; absent fields are left alone. The clone URL /// and reporting destination are intentionally not patchable — those /// would silently change where new findings get filed, so re-register @@ -166,6 +166,7 @@ pub async fn update( disabled: req.disabled, scan_interval_seconds: req.scan_interval_seconds.map(|v| v as i64), verification_enabled: req.verification_enabled, + scanner_config: req.scanner_config, require_approval: if req.inherit_require_approval { Some(None) } else { diff --git a/crates/loupe-server/tests/repos.rs b/crates/loupe-server/tests/repos.rs index f222e3b..8e1ecb9 100644 --- a/crates/loupe-server/tests/repos.rs +++ b/crates/loupe-server/tests/repos.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use loupe_core::ReportingDestination; use loupe_proto::{ ListReposResponse, RegisterRepoRequest, ReportingSetup, ReportingSummary, RotateRepoPatRequest, - SetRepoGithubReportingRequest, PROTOCOL_VERSION, + SetRepoGithubReportingRequest, UpdateRepoRequest, PROTOCOL_VERSION, }; use loupe_server::init::run_init; use loupe_server::{serve, AppState, Config}; @@ -319,6 +319,43 @@ async fn repo_registration_inherits_verification_default_unless_pinned() { f.handle.shutdown().await; } +#[tokio::test] +async fn admin_can_update_a_repo_scanner_config() { + let f = bring_up().await; + let admin = admin_client(&f.ca_cert_pem, &f.admin_cert_pem, &f.admin_key_pem, f.addr); + let repo_id = create_repo(&admin, ReportingSetup::Manual).await; + let scanner_config = serde_json::json!({ + "include_extensions": ["ts", "tsx"], + "extra_source_paths": ["Dockerfile"] + }); + let req = UpdateRepoRequest { + protocol_version: PROTOCOL_VERSION, + scanner_config: Some(scanner_config.clone()), + ..UpdateRepoRequest::default() + }; + + let resp = admin + .patch(format!("https://loupe-server/v1/repos/{repo_id}")) + .json(&req) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 204, "update scanner config: {}", resp.status()); + + let stored: String = + f.db.with_conn(|connection| { + Ok(connection.query_row( + "SELECT scanner_config FROM registered_repos WHERE id = ?1", + [repo_id], + |row| row.get(0), + )?) + }) + .unwrap(); + assert_eq!(serde_json::from_str::(&stored).unwrap(), scanner_config); + + f.handle.shutdown().await; +} + #[tokio::test] async fn admin_can_rotate_a_repo_github_pat() { let f = bring_up().await; diff --git a/crates/loupe-storage/src/repos.rs b/crates/loupe-storage/src/repos.rs index 0b0446c..0e9dc86 100644 --- a/crates/loupe-storage/src/repos.rs +++ b/crates/loupe-storage/src/repos.rs @@ -137,6 +137,7 @@ pub struct RepoUpdate { pub disabled: Option, pub scan_interval_seconds: Option, pub verification_enabled: Option, + pub scanner_config: Option, pub require_approval: Option>, } @@ -144,6 +145,7 @@ pub fn update(conn: &Connection, id: i64, patch: &RepoUpdate, now: i64) -> rusql if patch.disabled.is_none() && patch.scan_interval_seconds.is_none() && patch.verification_enabled.is_none() + && patch.scanner_config.is_none() && patch.require_approval.is_none() { return Ok(get(conn, id)?.is_some()); @@ -166,6 +168,10 @@ pub fn update(conn: &Connection, id: i64, patch: &RepoUpdate, now: i64) -> rusql sets.push("verification_enabled = ?"); binds.push((v as i64).into()); } + if let Some(config) = &patch.scanner_config { + sets.push("scanner_config = ?"); + binds.push(serde_json::to_string(config).expect("JSON values are serialisable").into()); + } if let Some(ra) = patch.require_approval { match ra { None => sets.push("require_approval = NULL"), @@ -390,6 +396,9 @@ mod tests { disabled: Some(false), scan_interval_seconds: Some(7200), verification_enabled: Some(true), + scanner_config: Some(serde_json::json!({ + "extra_source_paths": ["Dockerfile"] + })), require_approval: Some(Some(true)), }, 456, @@ -401,6 +410,7 @@ mod tests { assert_eq!(row.disabled_at, None); assert_eq!(row.scan_interval_seconds, Some(7200)); assert!(row.verification_enabled); + assert_eq!(row.scanner_config, serde_json::json!({"extra_source_paths": ["Dockerfile"]})); assert_eq!(row.require_approval, Some(true)); // Clearing require_approval drops back to "inherit". diff --git a/crates/loupe-worker/src/config.rs b/crates/loupe-worker/src/config.rs index 1c16a82..07074d4 100644 --- a/crates/loupe-worker/src/config.rs +++ b/crates/loupe-worker/src/config.rs @@ -463,6 +463,7 @@ impl Default for WorkerConfig { per_request_timeout: DEFAULT_REQUEST_TIMEOUT, include_extensions: scanner_defaults.include_extensions, exclude_path_substrings: scanner_defaults.exclude_path_substrings, + extra_source_paths: scanner_defaults.extra_source_paths, }, bkb: BkbConfig { api_url: DEFAULT_BKB_API_URL.to_owned() }, } diff --git a/crates/loupe-worker/src/source_discovery.rs b/crates/loupe-worker/src/source_discovery.rs index 425d0d6..d2d830e 100644 --- a/crates/loupe-worker/src/source_discovery.rs +++ b/crates/loupe-worker/src/source_discovery.rs @@ -33,6 +33,12 @@ pub struct ScannerConfig { /// output directories. Legacy custom strings still match as path /// substrings, except `/name` matches an exact component. pub exclude_path_substrings: Vec, + /// Additional repository-relative files or directories to scan. + /// Directories still honour `include_extensions`; explicitly named + /// files are accepted regardless of extension so operators can add + /// files such as `Dockerfile` and `Caddyfile`. Paths are constrained + /// to the checked-out worktree. + pub extra_source_paths: Vec, } impl Default for ScannerConfig { @@ -43,6 +49,7 @@ impl Default for ScannerConfig { per_request_timeout: DEFAULT_REQUEST_TIMEOUT, include_extensions: default_extensions(), exclude_path_substrings: default_excludes(), + extra_source_paths: Vec::new(), } } } @@ -65,6 +72,7 @@ pub struct ScannerConfigPatch { pub per_request_timeout_seconds: Option, pub include_extensions: Option>, pub exclude_path_substrings: Option>, + pub extra_source_paths: Option>, } impl ScannerConfig { @@ -84,6 +92,9 @@ impl ScannerConfig { if let Some(v) = p.exclude_path_substrings { self.exclude_path_substrings = v; } + if let Some(v) = p.extra_source_paths { + self.extra_source_paths = v; + } } } @@ -165,12 +176,15 @@ pub(crate) fn walk_source_files(workdir: &Path, cfg: &ScannerConfig) -> Vec Vec, files: Vec, + explicit_files: Vec, } fn discover_roots(workdir: &Path, cfg: &ScannerConfig) -> DiscoveryRoots { @@ -192,13 +207,52 @@ fn discover_roots(workdir: &Path, cfg: &ScannerConfig) -> DiscoveryRoots { add_cargo_roots(workdir, cfg, &mut roots); add_marker_roots(workdir, cfg, &mut roots); add_dotnet_roots(workdir, cfg, &mut roots); + add_configured_paths(workdir, cfg, &mut roots); roots.roots.sort(); roots.roots.dedup(); roots.files.sort(); roots.files.dedup(); + roots.explicit_files.sort(); + roots.explicit_files.dedup(); roots } +fn add_configured_paths(workdir: &Path, cfg: &ScannerConfig, roots: &mut DiscoveryRoots) { + let Ok(canonical_workdir) = std::fs::canonicalize(workdir) else { + tracing::warn!(path = %workdir.display(), "cannot resolve scanner worktree"); + return; + }; + + for configured in &cfg.extra_source_paths { + let relative = Path::new(configured); + if configured.trim().is_empty() + || relative.is_absolute() + || !relative + .components() + .all(|component| matches!(component, std::path::Component::Normal(_))) + { + tracing::warn!(path = %configured, "ignoring invalid extra_source_paths entry"); + continue; + } + + let candidate = workdir.join(relative); + let Ok(canonical_candidate) = std::fs::canonicalize(&candidate) else { + tracing::warn!(path = %configured, "extra_source_paths entry does not exist"); + continue; + }; + if !canonical_candidate.starts_with(&canonical_workdir) { + tracing::warn!(path = %configured, "extra_source_paths entry escapes the worktree"); + continue; + } + + if candidate.is_dir() { + roots.roots.push(candidate); + } else if candidate.is_file() { + roots.explicit_files.push(candidate); + } + } +} + fn add_cargo_roots(workdir: &Path, cfg: &ScannerConfig, roots: &mut DiscoveryRoots) { let workspace_exclude = parse_workspace(&workdir.join("Cargo.toml")) .map(|workspace| workspace.exclude) @@ -375,6 +429,10 @@ fn collect_file(path: &Path, cfg: &ScannerConfig, out: &mut Vec) { if !has_allowed_extension(path, &cfg.include_extensions) { return; } + collect_explicit_file(path, cfg, out); +} + +fn collect_explicit_file(path: &Path, cfg: &ScannerConfig, out: &mut Vec) { if is_excluded_path(path, &cfg.exclude_path_substrings) { return; } @@ -510,11 +568,53 @@ mod tests { fn patch_overrides_only_the_fields_present() { let mut cfg = ScannerConfig::default(); let original_excludes = cfg.exclude_path_substrings.clone(); - let patch: ScannerConfigPatch = - serde_json::from_str(r#"{"include_extensions":["c","h"]}"#).unwrap(); + let patch: ScannerConfigPatch = serde_json::from_str( + r#"{"include_extensions":["c","h"],"extra_source_paths":["ops/Dockerfile"]}"#, + ) + .unwrap(); cfg.apply_patch(patch); assert_eq!(cfg.include_extensions, vec!["c".to_owned(), "h".to_owned()]); assert_eq!(cfg.exclude_path_substrings, original_excludes); + assert_eq!(cfg.extra_source_paths, vec!["ops/Dockerfile".to_owned()]); + } + + #[test] + fn configured_paths_augment_project_roots_and_allow_extensionless_files() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join("package.json"), "{}\n").unwrap(); + std::fs::create_dir_all(tmp.path().join("src")).unwrap(); + std::fs::write(tmp.path().join("src/index.ts"), "// app\n").unwrap(); + std::fs::create_dir_all(tmp.path().join("scripts")).unwrap(); + std::fs::write(tmp.path().join("scripts/devnet.mjs"), "// operator script\n").unwrap(); + std::fs::write(tmp.path().join("Dockerfile"), "FROM scratch\n").unwrap(); + + let cfg = ScannerConfig { + extra_source_paths: vec!["scripts".into(), "Dockerfile".into()], + ..ScannerConfig::default() + }; + let names = rel_names(tmp.path(), walk_source_files(tmp.path(), &cfg)); + + for expected in ["src/index.ts", "scripts/devnet.mjs", "Dockerfile"] { + assert!(names.iter().any(|name| name == expected), "missing {expected} in {names:?}"); + } + } + + #[test] + fn configured_paths_cannot_escape_the_worktree() { + let tmp = tempfile::tempdir().unwrap(); + let repo = tmp.path().join("repo"); + std::fs::create_dir_all(repo.join("src")).unwrap(); + std::fs::write(repo.join("package.json"), "{}\n").unwrap(); + std::fs::write(repo.join("src/index.ts"), "// app\n").unwrap(); + std::fs::write(tmp.path().join("outside.ts"), "// must not scan\n").unwrap(); + + let cfg = ScannerConfig { + extra_source_paths: vec!["../outside.ts".into()], + ..ScannerConfig::default() + }; + let names = rel_names(&repo, walk_source_files(&repo, &cfg)); + + assert_eq!(names, vec!["src/index.ts"]); } #[test]