Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -211,13 +211,24 @@ Access Logs use the shared table pagination controls with 15 rows by default and
15, 25, 50 and 100. Filtering and paging run in the controller. Changing filters or page size,
or choosing Refresh, starts again on the first page.

The searchable host dropdown includes configured domains and hosts found in the retained
log scan. The status dropdown lists status codes found in that scan. General search matches
host, method, or path and applies after a short typing pause; Apply filters also applies the
current host and status selection immediately.

Each result has a short-lived snapshot so new requests and log rotation cannot move rows
between pages. Snapshots expire after two minutes and may be evicted sooner under load; the
UI announces a reset and returns to the first page. Refresh loads the latest logs. The controller
keeps at most eight snapshots within a 16 MiB cache. They are temporary, are lost on restart,
and are excluded from backups. The existing bounded log scan and visible truncation notice
still apply; pagination does not promise access to logs beyond that retained window.

Access logs render a country flag when the stored entry supplies a country code.
The web service and UI do not perform GeoIP lookups. Unknown countries have no flag.
Country flags in access logs and language settings use
[country-flag-icons](https://github.com/catamphetamine/country-flag-icons) under the MIT license.
Language flags do not depend on a GeoIP database.

### Certificates requested from proxy hosts

Create and Edit offer a new ACME certificate using the host's domains. The host row also
Expand Down
3 changes: 3 additions & 0 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

21 changes: 20 additions & 1 deletion controller/src/runtime/access_logs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ pub(crate) const MAX_SNAPSHOTS: usize = 8;
pub(crate) const MAX_SNAPSHOT_BYTES: usize = 16 * 1024 * 1024;
pub(crate) const MAX_SINGLE_SNAPSHOT_BYTES: usize = MAX_LOG_BYTES;
const MAX_SNAPSHOT_ID: usize = 64;
const MAX_AVAILABLE_HOSTS: usize = MAX_RECORDS;
const MAX_PATH: usize = 2048;
const MAX_HOST: usize = 253;
const MAX_METHOD: usize = 32;
Expand Down Expand Up @@ -139,6 +140,10 @@ pub(crate) struct AccessLogResponse {
pub limit: usize,
pub offset: usize,
pub total: usize,
#[serde(rename = "availableHosts")]
pub available_hosts: Vec<String>,
#[serde(rename = "availableStatuses")]
pub available_statuses: Vec<u16>,
#[serde(rename = "hasMore")]
pub has_more: bool,
pub truncated: bool,
Expand All @@ -165,6 +170,15 @@ pub(crate) struct AccessLogEntry {
pub protocol: String,
}

fn bounded_available_hosts(hosts: impl IntoIterator<Item = String>) -> Vec<String> {
let mut bounded = hosts
.into_iter()
.take(MAX_AVAILABLE_HOSTS)
.collect::<Vec<_>>();
bounded.shrink_to_fit();
bounded
}

#[derive(Clone, Debug, PartialEq, Eq)]
pub(super) struct SnapshotQuery {
host: Option<String>,
Expand Down Expand Up @@ -269,6 +283,11 @@ fn matches_query(entry: &AccessLogEntry, query: &ValidatedAccessLogQuery) -> boo
.is_none_or(|host| entry.host.eq_ignore_ascii_case(host))
&& query.status.is_none_or(|status| entry.status == status)
&& query.search.as_ref().is_none_or(|search| {
let search = search.trim();
if search.is_empty() {
return true;
}
let search = search.to_ascii_lowercase();
[
entry.host.as_str(),
entry.method.as_str(),
Expand All @@ -278,7 +297,7 @@ fn matches_query(entry: &AccessLogEntry, query: &ValidatedAccessLogQuery) -> boo
entry.upstream.as_deref().unwrap_or(""),
]
.into_iter()
.any(|field| field.contains(search))
.any(|field| field.to_ascii_lowercase().contains(&search))
})
}

Expand Down
84 changes: 80 additions & 4 deletions controller/src/runtime/access_logs/cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ struct CachedSnapshot {
entries: Vec<AccessLogEntry>,
total: usize,
truncated: bool,
available_hosts: Vec<String>,
available_statuses: Vec<u16>,
expires_at: Instant,
expires_at_utc: String,
bytes: usize,
Expand Down Expand Up @@ -88,6 +90,8 @@ impl SnapshotCache {
let bytes = snapshot_memory_bytes(
capture_query,
&capture.entries,
&capture.available_hosts,
&capture.available_statuses,
snapshot_id.capacity(),
expires_at_utc.capacity(),
);
Expand All @@ -111,6 +115,8 @@ impl SnapshotCache {
total: capture.entries.len(),
entries: capture.entries,
truncated: capture.truncated,
available_hosts: capture.available_hosts,
available_statuses: capture.available_statuses,
expires_at,
expires_at_utc,
bytes,
Expand Down Expand Up @@ -183,6 +189,8 @@ fn page_response(
limit: query.limit,
offset: query.offset,
total: snapshot.total,
available_hosts: snapshot.available_hosts.clone(),
available_statuses: snapshot.available_statuses.clone(),
has_more,
truncated: snapshot.truncated,
snapshot: snapshot_id.to_owned(),
Expand All @@ -194,6 +202,8 @@ fn page_response(
fn snapshot_memory_bytes(
query: &ValidatedAccessLogQuery,
entries: &Vec<AccessLogEntry>,
available_hosts: &[String],
available_statuses: &[u16],
snapshot_id_capacity: usize,
expires_at_capacity: usize,
) -> usize {
Expand All @@ -202,6 +212,10 @@ fn snapshot_memory_bytes(
.saturating_add(snapshot_id_capacity)
.saturating_add(expires_at_capacity)
.saturating_add(query_memory_bytes(query))
.saturating_add(available_filters_memory_bytes(
available_hosts,
available_statuses,
))
.saturating_add(
entries
.capacity()
Expand Down Expand Up @@ -239,9 +253,15 @@ pub(super) fn cap_snapshot_entries(
entries: Vec<AccessLogEntry>,
query: &ValidatedAccessLogQuery,
mut truncated: bool,
available_hosts: &[String],
available_statuses: &[u16],
) -> (Vec<AccessLogEntry>, bool) {
let base = SNAPSHOT_FIXED_BYTES
.saturating_add(query_memory_bytes(query))
.saturating_add(available_filters_memory_bytes(
available_hosts,
available_statuses,
))
.saturating_add(SNAPSHOT_ID_BYTES * 2)
.saturating_add(64);
let mut used = base;
Expand All @@ -261,7 +281,8 @@ pub(super) fn cap_snapshot_entries(
let mut bounded = Vec::with_capacity(keep);
bounded.extend(entries.into_iter().take(keep));
bounded.shrink_to_fit();
while snapshot_memory_bytes(query, &bounded, 64, 64) > MAX_SINGLE_SNAPSHOT_BYTES
while snapshot_memory_bytes(query, &bounded, available_hosts, available_statuses, 64, 64)
> MAX_SINGLE_SNAPSHOT_BYTES
&& !bounded.is_empty()
{
let next_len = bounded.len().saturating_sub(1);
Expand All @@ -271,6 +292,14 @@ pub(super) fn cap_snapshot_entries(
(bounded, truncated)
}

fn available_filters_memory_bytes(hosts: &[String], statuses: &[u16]) -> usize {
hosts
.len()
.saturating_mul(size_of::<String>())
.saturating_add(hosts.iter().map(String::capacity).sum::<usize>())
.saturating_add(statuses.len().saturating_mul(size_of::<u16>()))
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down Expand Up @@ -303,11 +332,56 @@ mod tests {
bytes: 0,
protocol: "HTTP/2".into(),
});
assert!(snapshot_memory_bytes(&query, &entries, 64, 64) > MAX_SINGLE_SNAPSHOT_BYTES);
let (bounded, truncated) = cap_snapshot_entries(entries, &query, false);
assert!(
snapshot_memory_bytes(&query, &entries, &[], &[], 64, 64) > MAX_SINGLE_SNAPSHOT_BYTES
);
let (bounded, truncated) = cap_snapshot_entries(entries, &query, false, &[], &[]);
assert!(!truncated);
assert_eq!(bounded.len(), 1);
assert!(snapshot_memory_bytes(&query, &bounded, 64, 64) < 4096);
assert!(snapshot_memory_bytes(&query, &bounded, &[], &[], 64, 64) < 4096);
}

#[test]
fn metadata_is_reserved_before_truncating_snapshot_entries() {
let query = query();
let hosts = (0..1000)
.map(|index| format!("host-{index}.{}.example", "a".repeat(200)))
.collect::<Vec<_>>();
let statuses = vec![200];
let entry = AccessLogEntry {
timestamp: "2026-09-13T00:00:00Z".into(),
host: "app.example".into(),
method: "GET".into(),
path: format!("/{}", "a".repeat(2047)),
status: 200,
duration_ms: 1,
client_ip: "192.0.2.1".into(),
upstream: None,
bytes: 0,
protocol: "HTTP/2".into(),
};
let (entries, truncated) =
cap_snapshot_entries(vec![entry; 2000], &query, false, &hosts, &statuses);
assert!(truncated);
assert!(!entries.is_empty());
let cache = SnapshotCache::new();
let response = cache
.insert(
&query,
&query,
CapturedLogs {
entries,
truncated,
available_hosts: hosts,
available_statuses: statuses,
},
false,
Instant::now(),
)
.unwrap();
assert_eq!(response.available_hosts.len(), 1000);
assert!(response.truncated);
assert!(cache.state.lock().unwrap().bytes <= MAX_SINGLE_SNAPSHOT_BYTES);
}

#[test]
Expand All @@ -323,6 +397,8 @@ mod tests {
CapturedLogs {
entries,
truncated: false,
available_hosts: Vec::new(),
available_statuses: Vec::new(),
},
false,
Instant::now(),
Expand Down
28 changes: 26 additions & 2 deletions controller/src/runtime/access_logs/reader.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use std::{
cmp::{Ordering, Reverse},
collections::BTreeSet,
fs::{self, File},
io::{ErrorKind, Read, Seek, SeekFrom},
path::Path,
Expand Down Expand Up @@ -28,6 +29,8 @@ pub(crate) enum ReadError {
pub(crate) struct CapturedLogs {
pub(crate) entries: Vec<AccessLogEntry>,
pub(crate) truncated: bool,
pub(crate) available_hosts: Vec<String>,
pub(crate) available_statuses: Vec<u16>,
}

struct IndexedEntry {
Expand Down Expand Up @@ -91,6 +94,8 @@ pub(crate) fn read_blocking(
limit: query.limit,
offset: query.offset,
total,
available_hosts: capture.available_hosts,
available_statuses: capture.available_statuses,
has_more: query.offset.saturating_add(page_len) < total,
truncated: capture.truncated,
snapshot: String::new(),
Expand All @@ -103,6 +108,8 @@ fn empty_capture() -> CapturedLogs {
CapturedLogs {
entries: Vec::new(),
truncated: false,
available_hosts: Vec::new(),
available_statuses: Vec::new(),
}
}

Expand Down Expand Up @@ -153,6 +160,8 @@ fn read_snapshot_blocking(
let mut scanned = 0usize;
let mut truncated = inventory_truncated || file_limit_truncated;
let mut matches = Vec::new();
let mut available_hosts = BTreeSet::new();
let mut available_statuses = BTreeSet::new();
for (_, name) in files {
if remaining == 0 || scanned >= MAX_RECORDS {
truncated = true;
Expand Down Expand Up @@ -188,6 +197,8 @@ fn read_snapshot_blocking(
let Some(entry) = parse_entry(&value, cutoff) else {
continue;
};
available_hosts.insert(entry.host.clone());
available_statuses.insert(entry.status);
if matches_query(&entry, query) {
matches.push(IndexedEntry {
timestamp_nanos: OffsetDateTime::parse(&entry.timestamp, &Rfc3339)
Expand All @@ -202,8 +213,21 @@ fn read_snapshot_blocking(
}
matches.sort_by(compare_indexed_entries);
let matches = matches.into_iter().map(|indexed| indexed.entry).collect();
let (entries, truncated) = cap_snapshot_entries(matches, query, truncated);
Ok(CapturedLogs { entries, truncated })
let available_hosts = super::bounded_available_hosts(available_hosts);
let available_statuses = available_statuses.into_iter().collect::<Vec<_>>();
let (entries, truncated) = cap_snapshot_entries(
matches,
query,
truncated,
&available_hosts,
&available_statuses,
);
Ok(CapturedLogs {
entries,
truncated,
available_hosts,
available_statuses,
})
}

fn compare_indexed_entries(left: &IndexedEntry, right: &IndexedEntry) -> Ordering {
Expand Down
32 changes: 32 additions & 0 deletions controller/src/runtime/access_logs/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,32 @@ fn filters_and_pagination_report_filtered_indexes() {
assert_eq!(response.entries[0].path, "/two");
}

#[test]
fn search_matches_partial_host_without_returning_unrelated_rows() {
let state = TempState::new(true);
let now = OffsetDateTime::now_utc().unix_timestamp() as f64;
state.write(
"access.log",
&format!(
"{}\n{}\n",
line(now, "ts-laser.example", "/laser", 200, "laser"),
line(now, "other.example", "/other", 200, "other")
),
);
let mut filtered = query(20, 0);
filtered.search = Some("ts-laser".into());

let response = read_blocking(state.path(), &filtered).unwrap();

assert_eq!(response.total, 1);
assert_eq!(response.entries[0].host, "ts-laser.example");
assert_eq!(
response.available_hosts,
vec!["other.example", "ts-laser.example"]
);
assert_eq!(response.available_statuses, vec![200]);
}

#[test]
fn malformed_and_partial_lines_are_ignored() {
let state = TempState::new(true);
Expand Down Expand Up @@ -336,6 +362,8 @@ fn snapshot_cache_expiry_is_fixed_and_evicts_lru_entries() {
CapturedLogs {
entries: Vec::new(),
truncated: false,
available_hosts: Vec::new(),
available_statuses: Vec::new(),
},
false,
now,
Expand Down Expand Up @@ -374,6 +402,8 @@ fn snapshot_cache_expiry_is_fixed_and_evicts_lru_entries() {
CapturedLogs {
entries: Vec::new(),
truncated: false,
available_hosts: Vec::new(),
available_statuses: Vec::new(),
},
false,
now,
Expand All @@ -392,6 +422,8 @@ fn snapshot_cache_expiry_is_fixed_and_evicts_lru_entries() {
CapturedLogs {
entries: Vec::new(),
truncated: false,
available_hosts: Vec::new(),
available_statuses: Vec::new(),
},
false,
now,
Expand Down
Loading
Loading