diff --git a/.dumplingconf.example b/.dumplingconf.example index 777aa4d..c8497f1 100644 --- a/.dumplingconf.example +++ b/.dumplingconf.example @@ -126,9 +126,19 @@ delete = [ # not_like / not_ilike — negation of like / ilike (prefer over regex look-around) # regex / iregex — Rust regex crate (not PCRE); look-around / backrefs rejected at load # not_regex / not_iregex — negation of regex / iregex -# lt / lte / gt / gte — numeric compare (values parsed as f64) +# lt / lte / gt / gte — numeric compare (values parsed as f64); +# set format = "datetime" for ISO/Postgres timestamps, +# or format = "date" for calendar dates (YYYY-MM-DD). +# Also valid on eq / neq. Unparseable cells fail closed. # is_null / not_null — NULL check (no value needed) +# Example: keep ~18 months of orders by timestamp (plus NULL created): +# [row_filters."public.listing_order"] +# retain = [ +# { column = "created", op = "gte", value = "2025-02-14", format = "datetime" }, +# { column = "created", op = "is_null" }, +# ] + # --------------------------------------------------------------------------- # [[column_cases]] — conditional per-column strategy overrides # diff --git a/AGENTS.md b/AGENTS.md index 8cee123..71aa178 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -192,6 +192,8 @@ Two code paths: JSON path traversal is supported: `payload.profile.tier` (dot) and `payload__profile__tier` (Django double-underscore). Array elements are traversed by evaluating each item's fields, so list-of-dicts structures work naturally. +Comparison operators (`lt` / `lte` / `gt` / `gte`, and optionally `eq` / `neq`) accept `format = "datetime"` or `format = "date"` to compare ISO-8601 / Postgres timestamp text (or calendar dates) instead of numbers. Thresholds are validated at config load; unparseable cell values fail closed (do not match). + ### Residual PII Scanning (`scan.rs`) `ScanningWriter` wraps the output `Write` stream, intercepting bytes and passing them to `OutputScanner`. The scanner applies regex detectors line-by-line for: diff --git a/CHANGELOG.md b/CHANGELOG.md index addcd04..215f252 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht ## [Unreleased] +### Added + +- **Datetime / date predicate comparisons**: `lt` / `lte` / `gt` / `gte` / `eq` / `neq` accept optional `format = "datetime"` (ISO-8601 / Postgres timestamp text as instants) or `format = "date"` (calendar dates). Thresholds are validated at config load; unparseable cell values fail closed ([#87](https://github.com/ababic/dumpling/issues/87)). + ## [0.8.0] - 2026-08-14 ### Added diff --git a/README.md b/README.md index 4348372..653dba9 100644 --- a/README.md +++ b/README.md @@ -328,9 +328,11 @@ Supported predicate operators: | `not_like` / `not_ilike` | Negation of `like` / `ilike` (prefer these over negative lookahead) | | `regex` / `iregex` | [Rust `regex`](https://docs.rs/regex/) crate (`iregex` is case-insensitive). **Not PCRE** — look-around (`(?!…)`, `(?=…)`, etc.), backreferences, and possessive quantifiers are unsupported and **rejected at config load** (fail closed). | | `not_regex` / `not_iregex` | Negation of `regex` / `iregex` | -| `lt` / `lte` / `gt` / `gte` | Numeric compare (values parsed as numbers) | +| `lt` / `lte` / `gt` / `gte` | Numeric compare by default (values parsed as numbers). Set `format = "datetime"` to compare ISO-8601 / Postgres timestamp text as instants, or `format = "date"` for calendar dates (`YYYY-MM-DD`). Unparseable cell values fail closed (do not match). | | `is_null` / `not_null` | No value needed | +`format = "datetime"` / `"date"` is also valid on `eq` / `neq` (instant or date equality). Thresholds are validated at config load; cell values that do not parse do not match. + Invalid or unsupported `regex` / `iregex` / `not_regex` / `not_iregex` patterns fail config load (and `dumpling lint-policy`) instead of silently matching nothing. For “scrub unless staff domain” policies, prefer a default scrub plus a `keep` case (see Conditional cases cookbook above), or scrub via `not_ilike` / `not_iregex` cases without a default rule: @@ -376,6 +378,18 @@ delete = [ ] ``` +Retain a recent window on timestamp columns without year-prefix `like` workarounds: + +```toml +[row_filters."public.listing_order"] +retain = [ + { column = "created", op = "gte", value = "2025-02-14", format = "datetime" }, + { column = "created", op = "is_null" }, +] +``` + +`format = "datetime"` accepts common `pg_dump` shapes (`2025-03-14 12:34:56.789+00`, `T` separators, `Z`, short offsets). Date-only thresholds are treated as midnight UTC. Use `format = "date"` to compare calendar dates only. + Row filtering works for both `INSERT ... VALUES (...)` and `COPY ... FROM stdin` rows. --- diff --git a/docs/src/configuration.md b/docs/src/configuration.md index c02841e..cacb0dd 100644 --- a/docs/src/configuration.md +++ b/docs/src/configuration.md @@ -379,9 +379,11 @@ salt = "${file:/run/secrets/dumpling_hmac_key}" | `not_like` / `not_ilike` | Negation of `like` / `ilike` (prefer these over negative lookahead) | | `regex` / `iregex` | [Rust `regex`](https://docs.rs/regex/) crate (`iregex` is case-insensitive). **Not PCRE** — look-around, backreferences, and possessive quantifiers are unsupported and **rejected at config load** (fail closed). | | `not_regex` / `not_iregex` | Negation of `regex` / `iregex` | -| `lt` / `lte` / `gt` / `gte` | Numeric compare (values parsed as numbers) | +| `lt` / `lte` / `gt` / `gte` | Numeric compare by default. With `format = "datetime"`, compare ISO-8601 / Postgres timestamp text as instants; with `format = "date"`, compare calendar dates. Unparseable cells fail closed. | | `is_null` / `not_null` | No value needed | +`format = "datetime"` / `"date"` also applies to `eq` / `neq`. Threshold strings are validated at config load. + Invalid or unsupported regex patterns fail config load and surface as `invalid-regex-predicate` in `dumpling lint-policy`. For “scrub unless allowlisted domain” policies, prefer a default scrub plus a `keep` case, or positive `not_ilike` / `not_iregex` cases — see the README *Conditional per-column cases* and *Row filtering* sections. Nested JSON targeting is supported in predicate `column` values via either: diff --git a/src/filter.rs b/src/filter.rs index 72a8cb9..87fd3d9 100644 --- a/src/filter.rs +++ b/src/filter.rs @@ -3,6 +3,7 @@ use crate::settings::{ ResolvedConfig, When, }; use crate::transform::{apply_anonymizer, AnonymizerRegistry, Replacement}; +use chrono::{DateTime, NaiveDate, NaiveDateTime, Utc}; use std::collections::HashMap; use std::sync::Mutex; @@ -57,6 +58,12 @@ fn predicate_matches(pred: &Predicate, columns: &[String], cells: &[Option<&str> let case_insensitive = pred .case_insensitive .unwrap_or(matches!(op, "ilike" | "not_ilike")); + let value_format = pred + .format + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_ascii_lowercase); // Fetch value(s) match op { "eq" | "neq" | "like" | "ilike" | "not_like" | "not_ilike" | "lt" | "lte" | "gt" @@ -66,12 +73,32 @@ fn predicate_matches(pred: &Predicate, columns: &[String], cells: &[Option<&str> None => return false, }; match op { - "eq" => targets - .iter() - .any(|cell| cmp_eq(cell.as_deref(), v, case_insensitive)), - "neq" => !targets - .iter() - .any(|cell| cmp_eq(cell.as_deref(), v, case_insensitive)), + "eq" => { + if let Some(fmt) = value_format.as_deref() { + targets.iter().any(|cell| { + cmp_order_with_format(cell.as_deref(), v, fmt) + .map(|o| o == 0) + .unwrap_or(false) + }) + } else { + targets + .iter() + .any(|cell| cmp_eq(cell.as_deref(), v, case_insensitive)) + } + } + "neq" => { + if let Some(fmt) = value_format.as_deref() { + !targets.iter().any(|cell| { + cmp_order_with_format(cell.as_deref(), v, fmt) + .map(|o| o == 0) + .unwrap_or(false) + }) + } else { + !targets + .iter() + .any(|cell| cmp_eq(cell.as_deref(), v, case_insensitive)) + } + } "like" | "ilike" | "not_like" | "not_ilike" => { let matched = targets .iter() @@ -96,26 +123,25 @@ fn predicate_matches(pred: &Predicate, columns: &[String], cells: &[Option<&str> !matched } } - "lt" => targets.iter().any(|cell| { - cmp_order(cell.as_deref(), v) - .map(|o| o < 0) - .unwrap_or(false) - }), - "lte" => targets.iter().any(|cell| { - cmp_order(cell.as_deref(), v) - .map(|o| o <= 0) - .unwrap_or(false) - }), - "gt" => targets.iter().any(|cell| { - cmp_order(cell.as_deref(), v) - .map(|o| o > 0) - .unwrap_or(false) - }), - "gte" => targets.iter().any(|cell| { - cmp_order(cell.as_deref(), v) - .map(|o| o >= 0) - .unwrap_or(false) - }), + "lt" | "lte" | "gt" | "gte" => { + let order = |cell: Option<&str>| -> Option { + match value_format.as_deref() { + Some(fmt) => cmp_order_with_format(cell, v, fmt), + None => cmp_order_numeric(cell, v), + } + }; + targets.iter().any(|cell| { + order(cell.as_deref()) + .map(|o| match op { + "lt" => o < 0, + "lte" => o <= 0, + "gt" => o > 0, + "gte" => o >= 0, + _ => false, + }) + .unwrap_or(false) + }) + } _ => false, } } @@ -535,7 +561,7 @@ fn parse_f64(s: &str) -> Option { s.parse::().ok() } -fn cmp_order(cell: Option<&str>, rhs: &serde_json::Value) -> Option { +fn cmp_order_numeric(cell: Option<&str>, rhs: &serde_json::Value) -> Option { let rv = match rhs { serde_json::Value::Number(n) => n.as_f64(), serde_json::Value::String(s) => parse_f64(s), @@ -551,6 +577,144 @@ fn cmp_order(cell: Option<&str>, rhs: &serde_json::Value) -> Option { } } +fn cmp_order_with_format(cell: Option<&str>, rhs: &serde_json::Value, format: &str) -> Option { + let rhs_raw = value_to_string(rhs)?; + match format { + "datetime" => { + let left = parse_predicate_datetime(cell?)?; + let right = parse_predicate_datetime(&rhs_raw)?; + Some(left.cmp(&right) as i32) + } + "date" => { + let left = parse_predicate_date(cell?)?; + let right = parse_predicate_date(&rhs_raw)?; + Some(left.cmp(&right) as i32) + } + _ => None, + } +} + +/// Parse a calendar date from dump text or a config threshold (`YYYY-MM-DD` or a longer timestamp). +pub(crate) fn parse_predicate_date(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + if let Ok(d) = NaiveDate::parse_from_str(trimmed, "%Y-%m-%d") { + return Some(d); + } + if trimmed.len() >= 10 { + if let Ok(d) = NaiveDate::parse_from_str(&trimmed[..10], "%Y-%m-%d") { + return Some(d); + } + } + parse_predicate_datetime(trimmed).map(|dt| dt.date_naive()) +} + +/// Parse ISO-8601 / Postgres timestamp text into a UTC instant. +/// +/// Accepts common `pg_dump` shapes (`YYYY-MM-DD[ T]HH:MM:SS[.fff][+HH[:MM]|Z]`), +/// date-only values (midnight UTC), and naive datetimes (interpreted as UTC). +pub(crate) fn parse_predicate_datetime(raw: &str) -> Option> { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + let normalized = normalize_timestamp_text(trimmed); + + if let Ok(d) = NaiveDate::parse_from_str(&normalized, "%Y-%m-%d") { + return d + .and_hms_opt(0, 0, 0) + .map(|ndt| DateTime::::from_naive_utc_and_offset(ndt, Utc)); + } + + let with_offset = [ + "%Y-%m-%d %H:%M:%S%.f%:z", + "%Y-%m-%d %H:%M:%S%:z", + "%Y-%m-%dT%H:%M:%S%.f%:z", + "%Y-%m-%dT%H:%M:%S%:z", + "%Y-%m-%d %H:%M%:z", + "%Y-%m-%dT%H:%M%:z", + ]; + for fmt in &with_offset { + if let Ok(dt) = DateTime::parse_from_str(&normalized, fmt) { + return Some(dt.with_timezone(&Utc)); + } + } + + if let Ok(dt) = DateTime::parse_from_rfc3339(&normalized) { + return Some(dt.with_timezone(&Utc)); + } + + let naive = [ + "%Y-%m-%d %H:%M:%S%.f", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%dT%H:%M:%S%.f", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%d %H:%M", + "%Y-%m-%dT%H:%M", + ]; + for fmt in &naive { + if let Ok(ndt) = NaiveDateTime::parse_from_str(&normalized, fmt) { + return Some(DateTime::::from_naive_utc_and_offset(ndt, Utc)); + } + } + None +} + +/// Normalize Postgres-ish offsets (`+00`, `-0500`, `Z`) into forms chrono accepts (`+00:00`). +fn normalize_timestamp_text(input: &str) -> String { + let mut s = input.trim().to_string(); + // Date-only `YYYY-MM-DD` must not be rewritten: the trailing `-DD` looks like an offset. + if NaiveDate::parse_from_str(&s, "%Y-%m-%d").is_ok() { + return s; + } + if s.ends_with('Z') || s.ends_with('z') { + s.pop(); + s.push_str("+00:00"); + return s; + } + + // Only rewrite offsets on values that include a date/time separator (space or `T`), + // so calendar dates and other hyphenated text are left alone. + if !(s.contains('T') || s.contains(' ')) { + return s; + } + + let bytes = s.as_bytes(); + let mut i = bytes.len(); + while i > 0 && (bytes[i - 1].is_ascii_digit() || bytes[i - 1] == b':') { + i -= 1; + } + if i == 0 || (bytes[i - 1] != b'+' && bytes[i - 1] != b'-') { + return s; + } + let sign_idx = i - 1; + // Require a datetime body before the offset (digit immediately before sign). + if sign_idx == 0 || !bytes[sign_idx - 1].is_ascii_digit() { + return s; + } + let offset = &s[sign_idx..]; + let sign = &offset[..1]; + let rest = &offset[1..]; + let rewritten = match rest { + // +00 / -05 + hh if hh.len() == 2 && hh.chars().all(|c| c.is_ascii_digit()) => { + Some(format!("{sign}{hh}:00")) + } + // +0000 / -0500 + hhmm if hhmm.len() == 4 && hhmm.chars().all(|c| c.is_ascii_digit()) => { + Some(format!("{sign}{}:{}", &hhmm[..2], &hhmm[2..])) + } + // already +00:00 + _ => None, + }; + if let Some(off) = rewritten { + s.replace_range(sign_idx.., &off); + } + s +} + // Simple global regex cache lazy_static::lazy_static! { static ref REGEX_CACHE: Mutex> = Mutex::new(HashMap::new()); @@ -595,6 +759,7 @@ mod tests { value: Some(serde_json::json!(".*@myco\\.com$")), values: None, case_insensitive: None, + format: None, }, Predicate { column: "email".to_string(), @@ -602,6 +767,7 @@ mod tests { value: Some(serde_json::json!(".*@myco\\.com$")), values: None, case_insensitive: None, + format: None, }, ], delete: vec![Predicate { @@ -610,6 +776,7 @@ mod tests { value: Some(serde_json::json!(".*@example\\.com$")), values: None, case_insensitive: None, + format: None, }], }, ); @@ -666,6 +833,7 @@ mod tests { value: Some(serde_json::json!("%@wearecrew.com")), values: None, case_insensitive: None, + format: None, }, Predicate { column: "email".to_string(), @@ -673,6 +841,7 @@ mod tests { value: Some(serde_json::json!(".*@reskinned\\.clothing$")), values: None, case_insensitive: None, + format: None, }, ], delete: vec![], @@ -726,6 +895,7 @@ mod tests { value: Some(serde_json::json!("%@wearecrew.com")), values: None, case_insensitive: None, + format: None, }], }, ); @@ -771,6 +941,7 @@ mod tests { value: Some(serde_json::json!("gold")), values: None, case_insensitive: None, + format: None, }], delete: vec![], }, @@ -817,6 +988,7 @@ mod tests { value: Some(serde_json::json!("primary")), values: None, case_insensitive: None, + format: None, }], delete: vec![], }, @@ -1070,4 +1242,193 @@ mod tests { assert_eq!(v2["meta"], serde_json::json!({})); assert_eq!(v2["items"], serde_json::json!([])); } + + #[test] + fn datetime_format_gte_retains_recent_rows() { + let cfg = ResolvedConfig { + salt: None, + rules: HashMap::new(), + row_filters: { + let mut m = HashMap::new(); + m.insert( + "public.listing_order".to_string(), + RowFilterSet { + retain: vec![ + Predicate { + column: "created".to_string(), + op: "gte".to_string(), + value: Some(serde_json::json!("2025-02-14")), + values: None, + case_insensitive: None, + format: Some("datetime".to_string()), + }, + Predicate { + column: "created".to_string(), + op: "is_null".to_string(), + value: None, + values: None, + case_insensitive: None, + format: None, + }, + ], + delete: vec![], + }, + ); + m + }, + column_cases: HashMap::new(), + sensitive_columns: HashMap::new(), + output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, + source_path: None, + }; + let cols = vec!["id".to_string(), "created".to_string()]; + assert!(should_keep_row( + &cfg, + Some("public"), + "listing_order", + &cols, + &[Some("1"), Some("2025-03-14 12:34:56.789+00")] + )); + assert!(should_keep_row( + &cfg, + Some("public"), + "listing_order", + &cols, + &[Some("2"), Some("2025-02-14T00:00:00Z")] + )); + assert!(!should_keep_row( + &cfg, + Some("public"), + "listing_order", + &cols, + &[Some("3"), Some("2025-02-13 23:59:59+00")] + )); + assert!(should_keep_row( + &cfg, + Some("public"), + "listing_order", + &cols, + &[Some("4"), None] + )); + assert!(!should_keep_row( + &cfg, + Some("public"), + "listing_order", + &cols, + &[Some("5"), Some("not-a-timestamp")] + )); + } + + #[test] + fn date_format_compares_calendar_dates_only() { + let cfg = ResolvedConfig { + salt: None, + rules: HashMap::new(), + row_filters: { + let mut m = HashMap::new(); + m.insert( + "public.events".to_string(), + RowFilterSet { + retain: vec![Predicate { + column: "created".to_string(), + op: "gte".to_string(), + value: Some(serde_json::json!("2025-02-14")), + values: None, + case_insensitive: None, + format: Some("date".to_string()), + }], + delete: vec![], + }, + ); + m + }, + column_cases: HashMap::new(), + sensitive_columns: HashMap::new(), + output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, + source_path: None, + }; + let cols = vec!["created".to_string()]; + assert!(should_keep_row( + &cfg, + Some("public"), + "events", + &cols, + &[Some("2025-02-14 00:00:00-05")] + )); + assert!(!should_keep_row( + &cfg, + Some("public"), + "events", + &cols, + &[Some("2025-02-13")] + )); + } + + #[test] + fn datetime_eq_matches_equivalent_instant_forms() { + let cfg = ResolvedConfig { + salt: None, + rules: HashMap::new(), + row_filters: { + let mut m = HashMap::new(); + m.insert( + "public.events".to_string(), + RowFilterSet { + retain: vec![Predicate { + column: "created".to_string(), + op: "eq".to_string(), + value: Some(serde_json::json!("2025-03-14T12:34:56Z")), + values: None, + case_insensitive: None, + format: Some("datetime".to_string()), + }], + delete: vec![], + }, + ); + m + }, + column_cases: HashMap::new(), + sensitive_columns: HashMap::new(), + output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, + source_path: None, + }; + let cols = vec!["created".to_string()]; + assert!(should_keep_row( + &cfg, + Some("public"), + "events", + &cols, + &[Some("2025-03-14 12:34:56+00")] + )); + assert!(!should_keep_row( + &cfg, + Some("public"), + "events", + &cols, + &[Some("2025-03-14 12:34:57+00")] + )); + } + + #[test] + fn parse_predicate_datetime_accepts_common_pg_dump_shapes() { + let a = parse_predicate_datetime("2025-03-14 12:34:56.789+00").unwrap(); + let b = parse_predicate_datetime("2025-03-14T12:34:56.789Z").unwrap(); + let c = parse_predicate_datetime("2025-03-14 12:34:56.789+00:00").unwrap(); + assert_eq!(a, b); + assert_eq!(a, c); + assert!(parse_predicate_datetime("2025-02-14").is_some()); + assert!(parse_predicate_datetime("bogus").is_none()); + assert_eq!( + parse_predicate_date("2025-03-14 12:34:56+00") + .unwrap() + .to_string(), + "2025-03-14" + ); + } } diff --git a/src/lint.rs b/src/lint.rs index d6f6cd3..ce81a9e 100644 --- a/src/lint.rs +++ b/src/lint.rs @@ -657,6 +657,7 @@ mod tests { value: Some(serde_json::json!("^(?!andy).*")), values: None, case_insensitive: None, + format: None, }], delete: vec![], }, @@ -682,6 +683,7 @@ mod tests { value: Some(serde_json::json!("^(?!staff).*")), values: None, case_insensitive: None, + format: None, }], delete: vec![], }, diff --git a/src/settings.rs b/src/settings.rs index a12371c..0eac0f2 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -732,6 +732,58 @@ fn validate_predicate(pred: &Predicate, path: &str) -> anyhow::Result<()> { KNOWN_PREDICATE_OPS.join(", ") ); } + let format = pred + .format + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_ascii_lowercase); + if let Some(ref fmt) = format { + const TEMPORAL_OPS: &[&str] = &["eq", "neq", "lt", "lte", "gt", "gte"]; + if !TEMPORAL_OPS.contains(&op.as_str()) { + anyhow::bail!( + "{}.format is only valid with operators eq, neq, lt, lte, gt, gte (got '{}')", + path, + pred.op + ); + } + if fmt != "datetime" && fmt != "date" { + anyhow::bail!( + "{}.format has unknown value '{}'; expected one of: datetime, date", + path, + pred.format.as_deref().unwrap_or("") + ); + } + let Some(value) = pred.value.as_ref() else { + anyhow::bail!( + "{}.value is required when format = '{}' for operator '{}'", + path, + fmt, + op + ); + }; + let Some(raw) = predicate_value_as_pattern_string(value) else { + anyhow::bail!( + "{}.value must be a string, number, or boolean when format = '{}'", + path, + fmt + ); + }; + let parsed_ok = match fmt.as_str() { + "datetime" => crate::filter::parse_predicate_datetime(&raw).is_some(), + "date" => crate::filter::parse_predicate_date(&raw).is_some(), + _ => false, + }; + if !parsed_ok { + anyhow::bail!( + "{}.value '{}' is not a valid {} threshold for format = '{}'", + path, + raw, + fmt, + fmt + ); + } + } match op.as_str() { "regex" | "iregex" | "not_regex" | "not_iregex" => { let Some(value) = pred.value.as_ref() else { @@ -1308,6 +1360,9 @@ pub struct Predicate { /// crate (not PCRE). Look-around, backreferences, and possessive quantifiers are unsupported /// and rejected at config load. Prefer `not_like` / `not_ilike` / `not_regex` / `not_iregex` /// for “match unless …” policies instead of negative lookahead. + /// + /// For `lt` / `lte` / `gt` / `gte` / `eq` / `neq`, set `format = "datetime"` or + /// `format = "date"` to compare ISO-8601 / Postgres timestamp text instead of numbers. pub op: String, /// Single value for eq/neq/like/ilike/not_like/not_ilike/regex/iregex/not_regex/not_iregex/lt/lte/gt/gte #[serde(default)] @@ -1318,6 +1373,12 @@ pub struct Predicate { /// Case-insensitive match for eq/neq/like/not_like (overridden by ilike/not_ilike/iregex/not_iregex) #[serde(default)] pub case_insensitive: Option, + /// Value interpretation for comparison operators: omit for numeric `lt`/`lte`/`gt`/`gte`; + /// `"datetime"` parses ISO-8601 / Postgres timestamp text as instants; `"date"` compares + /// calendar dates (`YYYY-MM-DD`). Only valid with `eq` / `neq` / `lt` / `lte` / `gt` / `gte`. + /// Unparseable cell values fail closed (predicate does not match). + #[serde(default)] + pub format: Option, } /// Lookup row filter set by schema-qualified or unqualified table name @@ -2168,4 +2229,60 @@ retain = [ assert_eq!(set.retain[1].op, "not_ilike"); let _ = fs::remove_file(path); } + + #[test] + fn datetime_format_predicate_loads_and_rejects_bad_thresholds() { + let ok = write_temp_config( + r#" +[row_filters."public.listing_order"] +retain = [ + { column = "created", op = "gte", value = "2025-02-14", format = "datetime" }, + { column = "created", op = "is_null" }, +] +"#, + ); + let cfg = load_config(Some(&ok), false).expect("datetime format predicate should load"); + let set = cfg + .row_filters + .get("public.listing_order") + .expect("filters"); + assert_eq!(set.retain[0].format.as_deref(), Some("datetime")); + let _ = fs::remove_file(ok); + + let bad_value = write_temp_config( + r#" +[row_filters."public.listing_order"] +retain = [{ column = "created", op = "gte", value = "last-year", format = "datetime" }] +"#, + ); + let err = load_config(Some(&bad_value), false) + .expect_err("unparseable datetime threshold should fail"); + let msg = format!("{:#}", err); + assert!(msg.contains("not a valid datetime threshold")); + let _ = fs::remove_file(bad_value); + + let bad_op = write_temp_config( + r#" +[row_filters."public.listing_order"] +retain = [{ column = "created", op = "like", value = "2025%", format = "datetime" }] +"#, + ); + let err = + load_config(Some(&bad_op), false).expect_err("format with like should fail validation"); + let msg = format!("{:#}", err); + assert!(msg.contains("format is only valid with operators")); + let _ = fs::remove_file(bad_op); + + let bad_format = write_temp_config( + r#" +[row_filters."public.listing_order"] +retain = [{ column = "created", op = "gte", value = "2025-02-14", format = "epoch" }] +"#, + ); + let err = load_config(Some(&bad_format), false) + .expect_err("unknown format should fail validation"); + let msg = format!("{:#}", err); + assert!(msg.contains("expected one of: datetime, date")); + let _ = fs::remove_file(bad_format); + } } diff --git a/src/sql.rs b/src/sql.rs index a5e0b1a..f93c8e3 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -2726,6 +2726,7 @@ mod tests { value: Some(serde_json::json!(".*@myco\\.com$")), values: None, case_insensitive: None, + format: None, }], delete: vec![crate::settings::Predicate { column: "email".to_string(), @@ -2733,6 +2734,7 @@ mod tests { value: Some(serde_json::json!(".*@example\\.com$")), values: None, case_insensitive: None, + format: None, }], }, ); @@ -2784,6 +2786,7 @@ COPY public.events (id, email, the_date) FROM stdin; value: Some(serde_json::json!("%@example.com")), values: None, case_insensitive: None, + format: None, }], all: vec![], }, @@ -2889,6 +2892,7 @@ COPY public.users (id, email) FROM stdin; value: Some(serde_json::json!("true")), values: None, case_insensitive: None, + format: None, }], all: vec![], }, @@ -2923,6 +2927,7 @@ COPY public.users (id, email) FROM stdin; serde_json::json!("GB"), ]), case_insensitive: None, + format: None, }], all: vec![], }, @@ -3021,6 +3026,7 @@ INSERT INTO public.users (id, email, country, is_admin) VALUES value: Some(serde_json::json!("%@wearecrew.com")), values: None, case_insensitive: None, + format: None, }, crate::settings::Predicate { column: "email".into(), @@ -3028,6 +3034,7 @@ INSERT INTO public.users (id, email, country, is_admin) VALUES value: Some(serde_json::json!("%@reskinned.clothing")), values: None, case_insensitive: None, + format: None, }, ], all: vec![], @@ -3160,6 +3167,7 @@ COPY public.users (id, email) FROM stdin; value: Some(serde_json::json!("%@wearecrew.com")), values: None, case_insensitive: None, + format: None, }, crate::settings::Predicate { column: "email".into(), @@ -3167,6 +3175,7 @@ COPY public.users (id, email) FROM stdin; value: Some(serde_json::json!("%@reskinned.clothing")), values: None, case_insensitive: None, + format: None, }, ], }, @@ -3758,6 +3767,7 @@ INSERT INTO public.users (id, email, first_name, password, dob, notes) VALUES value: Some(serde_json::json!("gold")), values: None, case_insensitive: None, + format: None, }], delete: vec![crate::settings::Predicate { column: "payload__events__kind".to_string(), @@ -3765,6 +3775,7 @@ INSERT INTO public.users (id, email, first_name, password, dob, notes) VALUES value: Some(serde_json::json!("drop")), values: None, case_insensitive: None, + format: None, }], }, );