Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,25 @@ loupectl repo list
loupectl repo scan 1 # one-shot scan of repo id 1
```

Use `--scanner-config-file <path.json>` to attach a per-repo JSON
configuration at registration time. The file must contain a JSON object;
Loupe stores that object with the repo and passes it to each worker lease.
For the LLM scanner, supported fields are `max_concurrent_files`,
`max_file_bytes`, `per_request_timeout_seconds`, `include_extensions`,
`exclude_path_substrings`, and `extra_source_paths`. The last field adds
repository-relative files or directories to automatic source discovery.
Directories still use the extension allowlist; explicitly named files do
not, which makes it possible to include operational files such as
`Dockerfile`. Absolute paths, `..` traversal, missing paths, and symlink
targets outside the checked-out worktree are ignored.

The scanner config is a stored snapshot; it does not follow later edits to
the local JSON file. Reload a changed profile without losing scan history:

```bash
loupectl repo update <repo-id> --scanner-config-file <path.json>
```

Add `--verification-enabled` if this repo should route scan findings
through verifier jobs before reporting. If the server-wide verification
default is on, omit it to inherit the default, or pass
Expand Down Expand Up @@ -473,6 +492,7 @@ the first one already covered.
loupectl repo update <id> --disable # pause scheduler
loupectl repo update <id> --enable
loupectl repo update <id> --interval 3600 # hourly
loupectl repo update <id> --scanner-config-file profile.json
loupectl repo update <id> --verification-enabled # route via verify flow
loupectl repo update <id> --no-verification # skip verify; dispatch on insert
loupectl repo update <id> --require-approval # hold for human sign-off
Expand Down
84 changes: 83 additions & 1 deletion crates/loupe-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,10 @@ struct RepoUpdateArgs {
/// `--disable` if you want to stop scheduled scans.
#[arg(long)]
interval: Option<u64>,
/// Replace the per-repo scanner settings with the JSON object in
/// this file.
#[arg(long, value_name = "PATH")]
scanner_config_file: Option<PathBuf>,
/// Route findings through the verify flow before dispatching.
#[arg(long, conflicts_with = "no_verification")]
verification_enabled: bool,
Expand Down Expand Up @@ -151,6 +155,10 @@ struct RepoAddArgs {
branch: Option<String>,
#[arg(long)]
scan_interval_seconds: Option<u64>,
/// JSON file containing per-repo scanner settings. The parsed JSON
/// is stored with the repo and sent to workers with every lease.
#[arg(long, value_name = "PATH")]
scanner_config_file: Option<PathBuf>,
/// Owner of the tracker repo where findings get filed. Required
/// unless `--no-reporting` is set.
#[arg(long, required_unless_present = "no_reporting")]
Expand Down Expand Up @@ -516,6 +524,12 @@ fn url(base: &reqwest::Url, path: &str) -> reqwest::Url {
}

async fn repo_add(client: &reqwest::Client, base: &reqwest::Url, a: RepoAddArgs) -> Result<()> {
let scanner_config = a
.scanner_config_file
.as_deref()
.map(load_scanner_config)
.transpose()?
.unwrap_or(serde_json::Value::Null);
let require_approval = match (a.require_approval, a.no_require_approval) {
(true, false) => Some(true),
(false, true) => Some(false),
Expand Down Expand Up @@ -543,7 +557,7 @@ async fn repo_add(client: &reqwest::Client, base: &reqwest::Url, a: RepoAddArgs)
branch: a.branch,
scan_interval_seconds: a.scan_interval_seconds,
reporting,
scanner_config: serde_json::Value::Null,
scanner_config,
verification_enabled,
require_approval,
};
Expand All @@ -557,6 +571,17 @@ async fn repo_add(client: &reqwest::Client, base: &reqwest::Url, a: RepoAddArgs)
Ok(())
}

fn load_scanner_config(path: &Path) -> Result<serde_json::Value> {
let raw = std::fs::read_to_string(path)
.with_context(|| format!("reading scanner config file {}", path.display()))?;
let value: serde_json::Value = serde_json::from_str(&raw)
.with_context(|| format!("parsing scanner config file {}", path.display()))?;
if !value.is_object() {
anyhow::bail!("scanner config file {} must contain a JSON object", path.display());
}
Ok(value)
}

async fn repo_list(
client: &reqwest::Client, base: &reqwest::Url, limit: Option<i64>,
) -> Result<()> {
Expand Down Expand Up @@ -592,6 +617,7 @@ async fn repo_rm(client: &reqwest::Client, base: &reqwest::Url, id: i64) -> Resu
async fn repo_update(
client: &reqwest::Client, base: &reqwest::Url, a: RepoUpdateArgs,
) -> Result<()> {
let scanner_config = a.scanner_config_file.as_deref().map(load_scanner_config).transpose()?;
let disabled = match (a.disable, a.enable) {
(true, false) => Some(true),
(false, true) => Some(false),
Expand All @@ -612,6 +638,7 @@ async fn repo_update(
disabled,
scan_interval_seconds: a.interval,
verification_enabled,
scanner_config,
require_approval,
inherit_require_approval: a.inherit_approval,
};
Expand Down Expand Up @@ -1110,6 +1137,61 @@ mod tests {
assert!(args.no_verification);
}

#[test]
fn repo_add_accepts_a_scanner_config_file() {
let cli = Cli::try_parse_from([
"loupectl",
"--server-url",
"https://loupe.example:8443",
"repo",
"add",
"--clone-url",
"https://github.com/acme/widget.git",
"--scanner-config-file",
"profiles/widget.json",
"--no-reporting",
])
.unwrap();
let Cmd::Repo(RepoCmd::Add(args)) = cli.cmd else {
panic!("expected repo add command");
};
assert_eq!(args.scanner_config_file, Some(PathBuf::from("profiles/widget.json")));
}

#[test]
fn scanner_config_file_must_contain_a_json_object() {
let suffix =
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos();
let path = std::env::temp_dir()
.join(format!("loupectl-scanner-config-{}-{suffix}.json", std::process::id()));
std::fs::write(&path, "[]\n").unwrap();

let error = load_scanner_config(&path).unwrap_err();
std::fs::remove_file(&path).unwrap();

assert!(error.to_string().contains("must contain a JSON object"));
}

#[test]
fn repo_update_accepts_a_scanner_config_file() {
let cli = Cli::try_parse_from([
"loupectl",
"--server-url",
"https://loupe.example:8443",
"repo",
"update",
"7",
"--scanner-config-file",
"profiles/widget.json",
])
.unwrap();
let Cmd::Repo(RepoCmd::Update(args)) = cli.cmd else {
panic!("expected repo update command");
};
assert_eq!(args.id, 7);
assert_eq!(args.scanner_config_file, Some(PathBuf::from("profiles/widget.json")));
}

#[test]
fn repo_set_github_reporting_parses_explicit_pat() {
let cli = Cli::try_parse_from([
Expand Down
17 changes: 16 additions & 1 deletion crates/loupe-proto/src/registry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ pub struct SetRepoGithubReportingRequest {
/// `disabled_at = now`; `disabled = Some(false)` clears it. The repo's
/// reporting destination, clone URL, and PAT cannot be patched: those
/// are register-time inputs, and changing them would silently affect
/// where new findings get filed. Re-register the repo for that.
/// where new findings get filed. Re-register the repo for those fields.
///
/// `require_approval` is tri-state on the wire: omitted = leave the
/// existing per-repo override alone; `Some(true)` / `Some(false)` =
Expand All @@ -118,6 +118,8 @@ pub struct UpdateRepoRequest {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub verification_enabled: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub scanner_config: Option<serde_json::Value>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub require_approval: Option<bool>,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub inherit_require_approval: bool,
Expand Down Expand Up @@ -269,6 +271,19 @@ mod tests {
assert_eq!(back.verification_enabled, None);
}

#[test]
fn update_repo_request_round_trips_scanner_config() {
let req = UpdateRepoRequest {
protocol_version: PROTOCOL_VERSION,
scanner_config: Some(json!({"extra_source_paths": ["Dockerfile"]})),
..UpdateRepoRequest::default()
};
let serialized = serde_json::to_string(&req).unwrap();
let round_trip: UpdateRepoRequest = serde_json::from_str(&serialized).unwrap();
assert_eq!(round_trip, req);
assert!(serialized.contains("extra_source_paths"));
}

fn summary_with_reporting(reporting: Option<ReportingSummary>) -> RepoSummary {
RepoSummary {
id: 1,
Expand Down
3 changes: 2 additions & 1 deletion crates/loupe-server/src/routes/repos.rs
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ pub async fn list(
}

/// `PATCH /v1/repos/:id` — admin only. Toggles `disabled`, swaps the
/// scan interval, or flips the verification flag. Each field is
/// scan interval, scanner configuration, or flips the verification flag. Each field is
/// independently optional; absent fields are left alone. The clone URL
/// and reporting destination are intentionally not patchable — those
/// would silently change where new findings get filed, so re-register
Expand All @@ -166,6 +166,7 @@ pub async fn update(
disabled: req.disabled,
scan_interval_seconds: req.scan_interval_seconds.map(|v| v as i64),
verification_enabled: req.verification_enabled,
scanner_config: req.scanner_config,
require_approval: if req.inherit_require_approval {
Some(None)
} else {
Expand Down
39 changes: 38 additions & 1 deletion crates/loupe-server/tests/repos.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use std::sync::Arc;
use loupe_core::ReportingDestination;
use loupe_proto::{
ListReposResponse, RegisterRepoRequest, ReportingSetup, ReportingSummary, RotateRepoPatRequest,
SetRepoGithubReportingRequest, PROTOCOL_VERSION,
SetRepoGithubReportingRequest, UpdateRepoRequest, PROTOCOL_VERSION,
};
use loupe_server::init::run_init;
use loupe_server::{serve, AppState, Config};
Expand Down Expand Up @@ -319,6 +319,43 @@ async fn repo_registration_inherits_verification_default_unless_pinned() {
f.handle.shutdown().await;
}

#[tokio::test]
async fn admin_can_update_a_repo_scanner_config() {
let f = bring_up().await;
let admin = admin_client(&f.ca_cert_pem, &f.admin_cert_pem, &f.admin_key_pem, f.addr);
let repo_id = create_repo(&admin, ReportingSetup::Manual).await;
let scanner_config = serde_json::json!({
"include_extensions": ["ts", "tsx"],
"extra_source_paths": ["Dockerfile"]
});
let req = UpdateRepoRequest {
protocol_version: PROTOCOL_VERSION,
scanner_config: Some(scanner_config.clone()),
..UpdateRepoRequest::default()
};

let resp = admin
.patch(format!("https://loupe-server/v1/repos/{repo_id}"))
.json(&req)
.send()
.await
.unwrap();
assert_eq!(resp.status(), 204, "update scanner config: {}", resp.status());

let stored: String =
f.db.with_conn(|connection| {
Ok(connection.query_row(
"SELECT scanner_config FROM registered_repos WHERE id = ?1",
[repo_id],
|row| row.get(0),
)?)
})
.unwrap();
assert_eq!(serde_json::from_str::<serde_json::Value>(&stored).unwrap(), scanner_config);

f.handle.shutdown().await;
}

#[tokio::test]
async fn admin_can_rotate_a_repo_github_pat() {
let f = bring_up().await;
Expand Down
10 changes: 10 additions & 0 deletions crates/loupe-storage/src/repos.rs
Original file line number Diff line number Diff line change
Expand Up @@ -137,13 +137,15 @@ pub struct RepoUpdate {
pub disabled: Option<bool>,
pub scan_interval_seconds: Option<i64>,
pub verification_enabled: Option<bool>,
pub scanner_config: Option<serde_json::Value>,
pub require_approval: Option<Option<bool>>,
}

pub fn update(conn: &Connection, id: i64, patch: &RepoUpdate, now: i64) -> rusqlite::Result<bool> {
if patch.disabled.is_none()
&& patch.scan_interval_seconds.is_none()
&& patch.verification_enabled.is_none()
&& patch.scanner_config.is_none()
&& patch.require_approval.is_none()
{
return Ok(get(conn, id)?.is_some());
Expand All @@ -166,6 +168,10 @@ pub fn update(conn: &Connection, id: i64, patch: &RepoUpdate, now: i64) -> rusql
sets.push("verification_enabled = ?");
binds.push((v as i64).into());
}
if let Some(config) = &patch.scanner_config {
sets.push("scanner_config = ?");
binds.push(serde_json::to_string(config).expect("JSON values are serialisable").into());
}
if let Some(ra) = patch.require_approval {
match ra {
None => sets.push("require_approval = NULL"),
Expand Down Expand Up @@ -390,6 +396,9 @@ mod tests {
disabled: Some(false),
scan_interval_seconds: Some(7200),
verification_enabled: Some(true),
scanner_config: Some(serde_json::json!({
"extra_source_paths": ["Dockerfile"]
})),
require_approval: Some(Some(true)),
},
456,
Expand All @@ -401,6 +410,7 @@ mod tests {
assert_eq!(row.disabled_at, None);
assert_eq!(row.scan_interval_seconds, Some(7200));
assert!(row.verification_enabled);
assert_eq!(row.scanner_config, serde_json::json!({"extra_source_paths": ["Dockerfile"]}));
assert_eq!(row.require_approval, Some(true));

// Clearing require_approval drops back to "inherit".
Expand Down
1 change: 1 addition & 0 deletions crates/loupe-worker/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,7 @@ impl Default for WorkerConfig {
per_request_timeout: DEFAULT_REQUEST_TIMEOUT,
include_extensions: scanner_defaults.include_extensions,
exclude_path_substrings: scanner_defaults.exclude_path_substrings,
extra_source_paths: scanner_defaults.extra_source_paths,
},
bkb: BkbConfig { api_url: DEFAULT_BKB_API_URL.to_owned() },
}
Expand Down
Loading