From 60bf722eb81a00697aa24c16de4f5ae41e142165 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 12 Oct 2026 13:16:56 +0800 Subject: [PATCH 1/6] feat(core): stage A04 history projections and bounded finance export Add source-only A04 read projection helpers, synthetic test fixtures, PostgreSQL snapshot test definitions and allowlisted JSON output. Pending S11 module wiring, dependencies, real source integration, authorization, complete finance export and executable test evidence. --- services/core/src/finance_export.rs | 194 ++++++++++++++ services/core/src/history_read.rs | 211 +++++++++++++++ .../history_read/fixtures/projection-v1.json | 249 ++++++++++++++++++ services/core/src/history_read/snapshot.rs | 39 +++ services/core/src/history_read/usage.rs | 101 +++++++ services/core/tests/finance_export.rs | 151 +++++++++++ services/core/tests/history_read.rs | 235 +++++++++++++++++ 7 files changed, 1180 insertions(+) create mode 100644 services/core/src/finance_export.rs create mode 100644 services/core/src/history_read.rs create mode 100644 services/core/src/history_read/fixtures/projection-v1.json create mode 100644 services/core/src/history_read/snapshot.rs create mode 100644 services/core/src/history_read/usage.rs create mode 100644 services/core/tests/finance_export.rs create mode 100644 services/core/tests/history_read.rs diff --git a/services/core/src/finance_export.rs b/services/core/src/finance_export.rs new file mode 100644 index 000000000..b681e140e --- /dev/null +++ b/services/core/src/finance_export.rs @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! Allowlisted usage JSON for the finance ZIP. This is not an HTTP endpoint or +//! a complete finance bundle. The caller must authorize and select rows under +//! the same read snapshot; no raw storage object implements Serialize here. +use crate::history_read::{LogQuotaInput, QuotaMoneyProjection, project_log}; +use serde::Serialize; +use std::io::{self, Write}; + +pub const MAX_EXPORT_ROWS: usize = 200_000; +pub const MAX_EXPORT_BYTES: usize = 64 * 1024 * 1024; +pub const MAX_EXPORT_ROW_BYTES: usize = 32 * 1024; +const MAX_TEXT_BYTES: usize = 1024; + +/// Deliberately no Serialize/Debug: `other` can contain private provider data. +pub struct UsageExportInput<'a> { + pub log_id: i64, + pub user_id: i64, + pub created_at: i64, + pub log_type: i32, + pub username: &'a str, + pub token_name: &'a str, + pub model_name: &'a str, + pub quota: Option, + pub prompt_tokens: i64, + pub completion_tokens: i64, + pub use_time: i64, + pub is_stream: bool, + pub channel_id: i64, + pub group: &'a str, + pub other: &'a str, +} + +#[derive(Serialize)] +struct UsageExportRow<'a> { + #[serde(flatten)] + money: QuotaMoneyProjection, + log_id: i64, + user_id: i64, + created_at: i64, + #[serde(rename = "type")] + log_type: i32, + username: &'a str, + token_name: &'a str, + model_name: &'a str, + // Preserve NULL as NULL. Never manufacture a zero stored quota. + quota: Option, + prompt_tokens: i64, + completion_tokens: i64, + use_time: i64, + is_stream: bool, + channel_id: i64, + group: &'a str, +} + +impl<'a> From<&UsageExportInput<'a>> for UsageExportRow<'a> { + fn from(row: &UsageExportInput<'a>) -> Self { + Self { + money: project_log(LogQuotaInput { quota: row.quota, log_type: row.log_type, other: row.other }), + log_id: row.log_id, + user_id: row.user_id, + created_at: row.created_at, + log_type: row.log_type, + username: row.username, + token_name: row.token_name, + model_name: row.model_name, + quota: row.quota, + prompt_tokens: row.prompt_tokens, + completion_tokens: row.completion_tokens, + use_time: row.use_time, + is_stream: row.is_stream, + channel_id: row.channel_id, + group: row.group, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExportError { + InvalidLimits, + InvalidRow, + RowLimit, + ByteLimit, + OutOfOrder, + Encoding, + Io, + Aborted, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ExportReceipt { + pub rows: usize, + pub bytes: usize, +} + +/// Bounded UTF-8 JSON array writer. Input is pull-driven, so no producer task or +/// unbounded queue is needed. Use an owned 0600 spool file in integration, not +/// a live network writer while holding a database transaction. +/// +/// An error poisons the writer: finish cannot turn an incomplete export into a +/// valid-looking complete array. Delete only the caller's own failed spool. +pub struct UsageJsonWriter { + writer: W, + max_rows: usize, + max_bytes: usize, + rows: usize, + bytes: usize, + last_order: Option<(i64, i64)>, + failed: bool, +} + +impl UsageJsonWriter { + pub fn new(mut writer: W, max_rows: usize, max_bytes: usize) -> Result { + if max_rows == 0 || max_rows > MAX_EXPORT_ROWS || !(2..=MAX_EXPORT_BYTES).contains(&max_bytes) { + return Err(ExportError::InvalidLimits); + } + writer.write_all(b"[").map_err(|_| ExportError::Io)?; + Ok(Self { writer, max_rows, max_bytes, rows: 0, bytes: 1, last_order: None, failed: false }) + } + + pub fn push(&mut self, row: &UsageExportInput<'_>) -> Result<(), ExportError> { + if self.failed { + return Err(ExportError::Aborted); + } + let result = self.push_checked(row); + if result.is_err() { + self.failed = true; + } + result + } + + fn push_checked(&mut self, row: &UsageExportInput<'_>) -> Result<(), ExportError> { + if self.rows == self.max_rows { + return Err(ExportError::RowLimit); + } + if row.log_id <= 0 || row.user_id <= 0 || row.created_at < 0 + || ![1, 2, 3, 4, 6].contains(&row.log_type) + || [row.username, row.token_name, row.model_name, row.group].into_iter().any(|value| value.len() > MAX_TEXT_BYTES) + { + return Err(ExportError::InvalidRow); + } + let order = (row.created_at, row.log_id); + if self.last_order.is_some_and(|last| order <= last) { + return Err(ExportError::OutOfOrder); + } + let mut encoded = BoundedRow { bytes: Vec::new() }; + serde_json::to_writer(&mut encoded, &UsageExportRow::from(row)) + .map_err(|_| ExportError::Encoding)?; + let separator = usize::from(self.rows > 0); + let required = self.bytes.checked_add(separator) + .and_then(|value| value.checked_add(encoded.bytes.len())) + .and_then(|value| value.checked_add(1)) // Reserve the closing bracket. + .ok_or(ExportError::ByteLimit)?; + if required > self.max_bytes { + return Err(ExportError::ByteLimit); + } + if separator != 0 { + self.writer.write_all(b",").map_err(|_| ExportError::Io)?; + } + self.writer.write_all(&encoded.bytes).map_err(|_| ExportError::Io)?; + self.bytes = required - 1; + self.rows += 1; + self.last_order = Some(order); + Ok(()) + } + + /// The outer ZIP manifest must separately mark a deliberate source row + /// truncation after fetching a cap+1 sentinel in the same snapshot. + pub fn finish(mut self) -> Result<(W, ExportReceipt), ExportError> { + if self.failed { + return Err(ExportError::Aborted); + } + self.writer.write_all(b"]").map_err(|_| ExportError::Io)?; + self.writer.flush().map_err(|_| ExportError::Io)?; + Ok((self.writer, ExportReceipt { rows: self.rows, bytes: self.bytes + 1 })) + } +} + +struct BoundedRow { + bytes: Vec, +} + +impl Write for BoundedRow { + fn write(&mut self, data: &[u8]) -> io::Result { + if data.len() > MAX_EXPORT_ROW_BYTES.saturating_sub(self.bytes.len()) { + return Err(io::Error::other("finance export row limit")); + } + self.bytes.extend_from_slice(data); + Ok(data.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} diff --git a/services/core/src/history_read.rs b/services/core/src/history_read.rs new file mode 100644 index 000000000..f61978851 --- /dev/null +++ b/services/core/src/history_read.rs @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! Bounded, read-only display projections. These functions neither authenticate +//! a caller nor establish the provenance of an imported row. A storage adapter +//! must verify A01/A03 scope and source evidence before calling them. +//! +//! Keep legacy quota integers unchanged. Only consumption logs carry the +//! captured credits-per-USD meaning used here; refunds and aggregate buckets do +//! not acquire that meaning from a similar field name. +use num_rational::BigRational; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use std::collections::BTreeMap; + +pub mod snapshot; +pub mod usage; + +pub const MAX_RECORDS: usize = 10_000; +pub const MAX_BASES: usize = 64; +pub const MAX_METADATA_BYTES: usize = 64 * 1024; +pub const MAX_CREDIT_BASIS: i64 = 9_007_199_254_740_991; +pub const LOG_TYPE_CONSUME: i32 = 2; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct QuotaMoneyProjection { + #[serde(skip_serializing_if = "Option::is_none")] + pub quota_usd: Option, + pub quota_money_available: bool, + pub quota_money_basis: &'static str, +} + +impl QuotaMoneyProjection { + pub fn unknown(basis: &'static str) -> Self { + Self { + quota_usd: None, + quota_money_available: false, + quota_money_basis: basis, + } + } + + fn known(amount: &BigRational, basis: &'static str) -> Self { + Self { + quota_usd: Some(decimal_usd(amount)), + quota_money_available: true, + quota_money_basis: basis, + } + } +} + +/// A borrowed projection input, not a request body or an authorization token. +/// `None` preserves a stored NULL instead of silently converting it to zero. +#[derive(Clone, Copy)] +pub struct LogQuotaInput<'a> { + pub quota: Option, + pub log_type: i32, + pub other: &'a str, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct QuotaStatisticProjection { + #[serde(flatten)] + pub money: QuotaMoneyProjection, + pub unpriced_quota_records: usize, + pub quota_projection_truncated: bool, +} + +// Borrow the original number token: a JSON f64 would silently round a value +// such as 9007199254740991.1 into an apparently valid integer anchor. +#[derive(Deserialize)] +struct CapturedAnchor<'a> { + #[serde(borrow)] + pricing_credits_per_usd: &'a RawValue, +} + +pub fn captured_credits_per_usd(other: &str) -> Option { + // A derived struct can also accept a JSON sequence. The source contract + // requires an object, as does main's map decoder. + if other.len() > MAX_METADATA_BYTES || !other.trim_start().starts_with('{') { + return None; + } + let object: CapturedAnchor<'_> = serde_json::from_str(other).ok()?; + positive_integer_token(object.pricing_credits_per_usd.get()) +} + +fn positive_integer_token(raw: &str) -> Option { + if raw.is_empty() || raw.len() > 64 || !raw.as_bytes()[0].is_ascii_digit() { + return None; + } + let (mantissa, exponent) = match raw.find(['e', 'E']) { + Some(at) => (&raw[..at], raw[at + 1..].parse::().ok()?), + None => (raw, 0), + }; + if !(-18..=18).contains(&exponent) { + return None; + } + let (whole, fraction) = mantissa.split_once('.').unwrap_or((mantissa, "")); + // The enclosing RawValue has already checked JSON number syntax. Check + // digits here as well so this helper never relies on floating-point parsing. + if !whole.bytes().chain(fraction.bytes()).all(|b| b.is_ascii_digit()) { + return None; + } + let mut digits = format!("{whole}{fraction}"); + let shift = exponent - i32::try_from(fraction.len()).ok()?; + if shift < 0 { + let remove = usize::try_from(-shift).ok()?; + if remove > digits.len() || !digits[digits.len() - remove..].bytes().all(|b| b == b'0') { + return None; + } + digits.truncate(digits.len() - remove); + } + let digits = digits.trim_start_matches('0'); + let append = usize::try_from(shift.max(0)).ok()?; + if digits.is_empty() || digits.len().checked_add(append)? > 16 { + return None; + } + let value = digits.parse::().ok()?.checked_mul(10_i64.pow(append as u32))?; + (value > 0 && value <= MAX_CREDIT_BASIS).then_some(value) +} + +/// Round exactly once, to 18 decimal places, half away from zero. All callers +/// supply nonnegative consumption amounts. No float is used at any point. +fn decimal_usd(amount: &BigRational) -> String { + let scaled = amount * BigRational::from_integer(1_000_000_000_000_000_000_u64.into()); + let mut units = scaled.to_integer(); + if scaled - BigRational::from_integer(units.clone()) >= BigRational::new(1.into(), 2.into()) { + units += 1; + } + let mut decimal = format!("{:0>19}", units.to_string()); + decimal.insert(decimal.len() - 18, '.'); + decimal.trim_end_matches('0').trim_end_matches('.').to_owned() +} + +pub fn project_log(input: LogQuotaInput<'_>) -> QuotaMoneyProjection { + let Some(quota) = input.quota else { + return QuotaMoneyProjection::unknown("invalid_stored_quota"); + }; + if input.log_type != LOG_TYPE_CONSUME { + return QuotaMoneyProjection::unknown("not_consumption"); + } + let Some(basis) = captured_credits_per_usd(input.other).filter(|_| quota >= 0) else { + return QuotaMoneyProjection::unknown("missing_credit_basis"); + }; + QuotaMoneyProjection::known( + &BigRational::new(quota.into(), basis.into()), + "captured_credits_per_usd", + ) +} + +/// Project a complete, authorized consumption selection. The repository must +/// use the same filters and snapshot for the raw sum, count, page and this scan. +/// Do not pass only a page and describe its sum as the all-history statistic. +/// Non-consumption rows in this input are invalid, not silently excluded. +pub fn project_consumption_statistic<'a>( + rows: impl IntoIterator>, + source_truncated: bool, +) -> QuotaStatisticProjection { + let mut sums = BTreeMap::::new(); + let mut count = 0; + let mut unpriced = 0; + let limited = |unknown| QuotaStatisticProjection { + money: QuotaMoneyProjection::unknown("projection_limit"), + unpriced_quota_records: unknown, + quota_projection_truncated: true, + }; + for input in rows { + count += 1; + if count > MAX_RECORDS { + return limited(unpriced); + } + let quota = input.quota.filter(|value| *value >= 0); + let basis = captured_credits_per_usd(input.other); + let (Some(quota), Some(basis)) = (quota, basis) else { + unpriced += 1; + continue; + }; + if input.log_type != LOG_TYPE_CONSUME { + unpriced += 1; + continue; + } + if !sums.contains_key(&basis) && sums.len() == MAX_BASES { + return limited(unpriced); + } + let sum = sums.entry(basis).or_default(); + let Some(next) = sum.checked_add(i128::from(quota)) else { + return limited(unpriced); + }; + *sum = next; + } + if source_truncated { + return limited(unpriced); + } + let money = if unpriced > 0 { + QuotaMoneyProjection::unknown("aggregate_missing_credit_basis") + } else { + let total = sums.into_iter().fold(BigRational::from_integer(0.into()), |total, (basis, quota)| { + total + BigRational::new(quota.into(), basis.into()) + }); + QuotaMoneyProjection::known( + &total, + if count == 0 { "empty_result" } else { "captured_credits_per_usd" }, + ) + }; + QuotaStatisticProjection { + money, + unpriced_quota_records: unpriced, + quota_projection_truncated: false, + } +} + +pub fn project_bucket_without_components() -> QuotaMoneyProjection { + QuotaMoneyProjection::unknown("bucket_without_credit_basis") +} diff --git a/services/core/src/history_read/fixtures/projection-v1.json b/services/core/src/history_read/fixtures/projection-v1.json new file mode 100644 index 000000000..47c48e836 --- /dev/null +++ b/services/core/src/history_read/fixtures/projection-v1.json @@ -0,0 +1,249 @@ +{ + "contract": "a04/projection-v1", + "status": "synthetic fixtures; Rust execution blocked", + "main_sha": "53bf14fca577c61e75de2b3e1f1edf93e4e2fd18", + "log_cases": [ + { + "name": "known", + "quota": 500000, + "log_type": 2, + "other": "{\"pricing_credits_per_usd\":500000}", + "expected": { + "quota_usd": "1", + "quota_money_available": true, + "quota_money_basis": "captured_credits_per_usd" + } + }, + { + "name": "round_up_once", + "quota": 2, + "log_type": 2, + "other": "{\"pricing_credits_per_usd\":3}", + "expected": { + "quota_usd": "0.666666666666666667", + "quota_money_available": true, + "quota_money_basis": "captured_credits_per_usd" + } + }, + { + "name": "smallest_usd_basis", + "quota": 1, + "log_type": 2, + "other": "{\"pricing_credits_per_usd\":9007199254740991}", + "expected": { + "quota_usd": "0.000000000000000111", + "quota_money_available": true, + "quota_money_basis": "captured_credits_per_usd" + } + }, + { + "name": "exact_exponent", + "quota": 5, + "log_type": 2, + "other": "{\"pricing_credits_per_usd\":5e5}", + "expected": { + "quota_usd": "0.00001", + "quota_money_available": true, + "quota_money_basis": "captured_credits_per_usd" + } + }, + { + "name": "known_zero", + "quota": 0, + "log_type": 2, + "other": "{\"pricing_credits_per_usd\":500000}", + "expected": { + "quota_usd": "0", + "quota_money_available": true, + "quota_money_basis": "captured_credits_per_usd" + } + }, + { + "name": "missing_basis", + "quota": 500000, + "log_type": 2, + "other": "{}", + "expected": { + "quota_money_available": false, + "quota_money_basis": "missing_credit_basis" + } + }, + { + "name": "zero_without_basis", + "quota": 0, + "log_type": 2, + "other": "{}", + "expected": { + "quota_money_available": false, + "quota_money_basis": "missing_credit_basis" + } + }, + { + "name": "negative_consumption", + "quota": -1, + "log_type": 2, + "other": "{\"pricing_credits_per_usd\":1}", + "expected": { + "quota_money_available": false, + "quota_money_basis": "missing_credit_basis" + } + }, + { + "name": "null_quota", + "quota": null, + "log_type": 2, + "other": "{\"pricing_credits_per_usd\":1}", + "expected": { + "quota_money_available": false, + "quota_money_basis": "invalid_stored_quota" + } + }, + { + "name": "refund", + "quota": -50, + "log_type": 6, + "other": "{\"pricing_credits_per_usd\":1}", + "expected": { + "quota_money_available": false, + "quota_money_basis": "not_consumption" + } + }, + { + "name": "metadata_not_json", + "quota": 1, + "log_type": 2, + "other": "not json", + "expected": { + "quota_money_available": false, + "quota_money_basis": "missing_credit_basis" + } + }, + { + "name": "string_basis", + "quota": 1, + "log_type": 2, + "other": "{\"pricing_credits_per_usd\":\"1\"}", + "expected": { + "quota_money_available": false, + "quota_money_basis": "missing_credit_basis" + } + }, + { + "name": "fraction_near_safe_max", + "quota": 1, + "log_type": 2, + "other": "{\"pricing_credits_per_usd\":9007199254740991.1}", + "expected": { + "quota_money_available": false, + "quota_money_basis": "missing_credit_basis" + } + }, + { + "name": "out_of_range", + "quota": 1, + "log_type": 2, + "other": "{\"pricing_credits_per_usd\":9007199254740992}", + "expected": { + "quota_money_available": false, + "quota_money_basis": "missing_credit_basis" + } + }, + { + "name": "duplicate_basis", + "quota": 1, + "log_type": 2, + "other": "{\"pricing_credits_per_usd\":1,\"pricing_credits_per_usd\":2}", + "expected": { + "quota_money_available": false, + "quota_money_basis": "missing_credit_basis" + } + } + ], + "statistic_cases": [ + { + "name": "empty_result", + "rows": [], + "source_truncated": false, + "expected": { + "quota_usd": "0", + "quota_money_available": true, + "quota_money_basis": "empty_result", + "unpriced_quota_records": 0, + "quota_projection_truncated": false + } + }, + { + "name": "different_bases", + "rows": [ + { + "quota": 1, + "log_type": 2, + "other": "{\"pricing_credits_per_usd\":3}" + }, + { + "quota": 1, + "log_type": 2, + "other": "{\"pricing_credits_per_usd\":6}" + } + ], + "source_truncated": false, + "expected": { + "quota_usd": "0.5", + "quota_money_available": true, + "quota_money_basis": "captured_credits_per_usd", + "unpriced_quota_records": 0, + "quota_projection_truncated": false + } + }, + { + "name": "known_unknown_mixed", + "rows": [ + { + "quota": 1, + "log_type": 2, + "other": "{\"pricing_credits_per_usd\":1}" + }, + { + "quota": 1, + "log_type": 2, + "other": "{}" + } + ], + "source_truncated": false, + "expected": { + "quota_money_available": false, + "quota_money_basis": "aggregate_missing_credit_basis", + "unpriced_quota_records": 1, + "quota_projection_truncated": false + } + }, + { + "name": "truncated_empty_prefix", + "rows": [], + "source_truncated": true, + "expected": { + "quota_money_available": false, + "quota_money_basis": "projection_limit", + "unpriced_quota_records": 0, + "quota_projection_truncated": true + } + }, + { + "name": "truncated_unknown_prefix", + "rows": [ + { + "quota": 1, + "log_type": 2, + "other": "{}" + } + ], + "source_truncated": true, + "expected": { + "quota_money_available": false, + "quota_money_basis": "projection_limit", + "unpriced_quota_records": 1, + "quota_projection_truncated": true + } + } + ] +} diff --git a/services/core/src/history_read/snapshot.rs b/services/core/src/history_read/snapshot.rs new file mode 100644 index 000000000..8c03ea536 --- /dev/null +++ b/services/core/src/history_read/snapshot.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! A transaction boundary, not an identity or storage-schema adapter. +use sqlx::{PgConnection, PgPool, Postgres, Transaction}; + +/// Authentication reads, row filters, counts, pages and projections must all +/// execute through this one connection. Do not hold it while awaiting a client +/// download: spool a bounded export, close the snapshot, then send its bytes. +pub struct ReadSnapshot<'a> { + transaction: Transaction<'a, Postgres>, +} + +impl<'a> ReadSnapshot<'a> { + pub async fn begin(pool: &'a PgPool) -> Result { + let mut transaction = pool.begin().await?; + sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY") + .execute(&mut *transaction) + .await?; + sqlx::query("SET LOCAL statement_timeout = '5s'") + .execute(&mut *transaction) + .await?; + sqlx::query("SET LOCAL idle_in_transaction_session_timeout = '5s'") + .execute(&mut *transaction) + .await?; + // PostgreSQL 18 is the task's isolated database target. This also bounds + // a succession of quick queries that never reaches statement_timeout. + sqlx::query("SET LOCAL transaction_timeout = '30s'") + .execute(&mut *transaction) + .await?; + Ok(Self { transaction }) + } + + pub fn connection(&mut self) -> &mut PgConnection { + &mut self.transaction + } + + pub async fn finish(self) -> Result<(), sqlx::Error> { + self.transaction.rollback().await + } +} diff --git a/services/core/src/history_read/usage.rs b/services/core/src/history_read/usage.rs new file mode 100644 index 000000000..366f4fa63 --- /dev/null +++ b/services/core/src/history_read/usage.rs @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! The arithmetic part of main's UsageProjector, deliberately separate from +//! per-log USD conversion. This does not validate migration records or refunds. +//! A03 must supply a complete, verified baseline and reversal set before use. +use num_rational::BigRational; +use serde::Serialize; + +const MAX_QUOTA: i64 = 9_007_199_254_740_991; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum UsageRounding { + HalfAwayFromZero, + TowardZero, +} + +/// Arithmetic inputs only. Do not derive Deserialize or use this as evidence +/// that a client-selected divisor, historical owner or refund is authorized. +pub struct UsageArithmeticInput<'a> { + pub current_used: i64, + pub historical_raw: i64, + pub reversed_raw: i64, + pub reversed_normalized: i64, + pub divisor: &'a str, + pub rounding: UsageRounding, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct UsageProjection { + pub normalized_used_quota: i64, + pub historical_raw_quota: i64, + pub historical_normalized_quota: i64, + pub post_migration_quota: i64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum UsageError { + InvalidBasis, + InvalidCounter, + UnprovenReduction, + OutOfRange, +} + +fn parse_divisor(text: &str) -> Option { + // Bound allocation before parsing; main permits decimal divisors with up + // to 18 fractional digits. Neither scientific notation nor floats apply. + if text.is_empty() || text.len() > 64 { + return None; + } + let (whole, fraction) = text.split_once('.').unwrap_or((text, "")); + if whole.is_empty() + || !whole.bytes().all(|b| b.is_ascii_digit()) + || !fraction.bytes().all(|b| b.is_ascii_digit()) + || fraction.len() > 18 + || (text.contains('.') && fraction.is_empty()) + { + return None; + } + let denominator = 10_i64.pow(fraction.len() as u32); + let numerator = format!("{whole}{fraction}").parse().ok()?; + let divisor = BigRational::new(numerator, denominator.into()); + (divisor > BigRational::from_integer(1.into())).then_some(divisor) +} + +pub fn project_migrated_usage(input: &UsageArithmeticInput<'_>) -> Result { + for value in [input.current_used, input.historical_raw, input.reversed_raw, input.reversed_normalized] { + if !(0..=MAX_QUOTA).contains(&value) { + return Err(UsageError::InvalidCounter); + } + } + if input.reversed_raw > input.historical_raw { + return Err(UsageError::InvalidBasis); + } + let divisor = parse_divisor(input.divisor).ok_or(UsageError::InvalidBasis)?; + let scaled = BigRational::from_integer(input.historical_raw.into()) / divisor; + let mut normalized = scaled.to_integer(); + if input.rounding == UsageRounding::HalfAwayFromZero + && scaled - BigRational::from_integer(normalized.clone()) >= BigRational::new(1.into(), 2.into()) + { + normalized += 1; + } + // Conversion happens only after exact rounding. The divisor is > 1 and + // the raw baseline is bounded, so no valid result needs a wider DTO. + let normalized: i64 = normalized.to_string().parse().map_err(|_| UsageError::OutOfRange)?; + let historical = normalized.checked_sub(input.reversed_normalized).ok_or(UsageError::OutOfRange)?; + let delta = input.current_used.checked_sub(input.historical_raw) + .and_then(|value| value.checked_add(input.reversed_raw)) + .ok_or(UsageError::OutOfRange)?; + if delta < 0 { + return Err(UsageError::UnprovenReduction); + } + let total = historical.checked_add(delta).ok_or(UsageError::OutOfRange)?; + if historical < 0 || !(0..=MAX_QUOTA).contains(&total) { + return Err(UsageError::OutOfRange); + } + Ok(UsageProjection { + normalized_used_quota: total, + historical_raw_quota: input.historical_raw - input.reversed_raw, + historical_normalized_quota: historical, + post_migration_quota: delta, + }) +} diff --git a/services/core/tests/finance_export.rs b/services/core/tests/finance_export.rs new file mode 100644 index 000000000..539794715 --- /dev/null +++ b/services/core/tests/finance_export.rs @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +use lmm_core::finance_export::{ + ExportError, UsageExportInput, UsageJsonWriter, MAX_EXPORT_BYTES, MAX_EXPORT_ROWS, +}; +use serde_json::Value; +use std::io::{self, Write}; + +fn row(id: i64, other: &str) -> UsageExportInput<'_> { + UsageExportInput { + log_id: id, user_id: 7, created_at: 1700000000, log_type: 2, + username: "synthetic-user", token_name: "synthetic-token", model_name: "local-model", + quota: Some(500000), prompt_tokens: 5, completion_tokens: 2, use_time: 1, + is_stream: false, channel_id: 9, group: "default", other, + } +} + +#[test] +fn export_has_an_explicit_field_allowlist() { + let input = row(1, r#"{"pricing_credits_per_usd":500000,"api_key":"SYNTHETIC_SECRET","admin_info":{"user_id":8},"provider_payload":"SYNTHETIC_PROVIDER_DATA"}"#); + let mut writer = UsageJsonWriter::new(Vec::new(), 10, 10000).unwrap(); + writer.push(&input).unwrap(); + let (bytes, receipt) = writer.finish().unwrap(); + let text = String::from_utf8(bytes).unwrap(); + assert!(!text.contains("SYNTHETIC_SECRET")); + assert!(!text.contains("SYNTHETIC_PROVIDER_DATA")); + let decoded: Value = serde_json::from_str(&text).unwrap(); + let mut fields: Vec<_> = decoded[0].as_object().unwrap().keys().map(String::as_str).collect(); + fields.sort_unstable(); + let mut expected = vec!["log_id", "user_id", "created_at", "type", "username", "token_name", "model_name", "quota", "prompt_tokens", "completion_tokens", "use_time", "is_stream", "channel_id", "group", "quota_usd", "quota_money_available", "quota_money_basis"]; + expected.sort_unstable(); + assert_eq!(fields, expected); + assert_eq!(decoded[0]["quota"], 500000); + assert_eq!(decoded[0]["quota_usd"], "1"); + assert_eq!(receipt.rows, 1); + assert_eq!(receipt.bytes, text.len()); +} + +#[test] +fn export_null_and_refund_are_not_repriced_as_consumption() { + let mut null_row = row(1, r#"{"pricing_credits_per_usd":500000}"#); + null_row.quota = None; + let mut refund = row(2, r#"{"pricing_credits_per_usd":500000}"#); + refund.quota = Some(-50); + refund.log_type = 6; + let mut writer = UsageJsonWriter::new(Vec::new(), 2, 10000).unwrap(); + writer.push(&null_row).unwrap(); + writer.push(&refund).unwrap(); + let (bytes, _) = writer.finish().unwrap(); + let decoded: Value = serde_json::from_slice(&bytes).unwrap(); + assert!(decoded[0]["quota"].is_null()); + assert_eq!(decoded[0]["quota_money_basis"], "invalid_stored_quota"); + assert_eq!(decoded[1]["quota"], -50); + assert_eq!(decoded[1]["quota_money_basis"], "not_consumption"); + assert!(decoded[1].get("quota_usd").is_none()); +} + +#[test] +fn export_round_trips_unicode_quotes_controls_and_formula_text_as_json_strings() { + let mut input = row(1, "{}"); + input.username = "测试\n\"quote\"\u{0000}"; + input.token_name = "=1+1"; + input.model_name = "+cmd|'/C synthetic'!A0"; + let mut writer = UsageJsonWriter::new(Vec::new(), 1, 10000).unwrap(); + writer.push(&input).unwrap(); + let (bytes, _) = writer.finish().unwrap(); + let decoded: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(decoded[0]["username"], input.username); + assert_eq!(decoded[0]["token_name"], "=1+1"); + assert_eq!(decoded[0]["model_name"], input.model_name); + // This is JSON in a ZIP, not CSV, an XLSX cell or executable JavaScript. + // A later CSV adapter needs its own formula-injection protection. +} + +#[test] +fn export_enforces_row_order_and_duplicate_rejection() { + let mut writer = UsageJsonWriter::new(Vec::new(), 3, 10000).unwrap(); + writer.push(&row(2, "{}")).unwrap(); + assert_eq!(writer.push(&row(1, "{}")), Err(ExportError::OutOfOrder)); + assert!(matches!(writer.finish(), Err(ExportError::Aborted))); + let mut writer = UsageJsonWriter::new(Vec::new(), 3, 10000).unwrap(); + writer.push(&row(1, "{}")).unwrap(); + assert_eq!(writer.push(&row(1, "{}")), Err(ExportError::OutOfOrder)); +} + +#[test] +fn export_limits_cannot_be_raised_past_the_server_ceiling() { + for (rows, bytes) in [(0, 2), (1, 1), (MAX_EXPORT_ROWS + 1, 100), (1, MAX_EXPORT_BYTES + 1)] { + assert!(matches!(UsageJsonWriter::new(Vec::new(), rows, bytes), Err(ExportError::InvalidLimits))); + } +} + +#[test] +fn export_row_limit_poisoning_does_not_claim_a_complete_partial_file() { + let mut writer = UsageJsonWriter::new(Vec::new(), 1, 10000).unwrap(); + writer.push(&row(1, "{}")).unwrap(); + assert_eq!(writer.push(&row(2, "{}")), Err(ExportError::RowLimit)); + assert_eq!(writer.push(&row(3, "{}")), Err(ExportError::Aborted)); + assert!(matches!(writer.finish(), Err(ExportError::Aborted))); +} + +#[test] +fn export_byte_limit_reserves_the_closing_bracket() { + let mut sample = UsageJsonWriter::new(Vec::new(), 1, 10000).unwrap(); + sample.push(&row(1, "{}")).unwrap(); + let (bytes, _) = sample.finish().unwrap(); + let mut exact = UsageJsonWriter::new(Vec::new(), 1, bytes.len()).unwrap(); + exact.push(&row(1, "{}")).unwrap(); + assert_eq!(exact.finish().unwrap().0.len(), bytes.len()); + let mut short = UsageJsonWriter::new(Vec::new(), 1, bytes.len() - 1).unwrap(); + assert_eq!(short.push(&row(1, "{}")), Err(ExportError::ByteLimit)); + assert!(matches!(short.finish(), Err(ExportError::Aborted))); +} + +#[test] +fn export_rejects_unbounded_text_before_encoding() { + let long = "x".repeat(1025); + let mut input = row(1, "{}"); + input.model_name = &long; + let mut writer = UsageJsonWriter::new(Vec::new(), 1, 10000).unwrap(); + assert_eq!(writer.push(&input), Err(ExportError::InvalidRow)); +} + +#[test] +fn export_empty_result_is_a_real_empty_json_array() { + let writer = UsageJsonWriter::new(Vec::new(), 1, 2).unwrap(); + let (bytes, receipt) = writer.finish().unwrap(); + assert_eq!(bytes, b"[]"); + assert_eq!(receipt.rows, 0); + assert_eq!(receipt.bytes, 2); +} + +struct BrokenSink { remaining: usize } +impl Write for BrokenSink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if self.remaining == 0 { + return Err(io::Error::new(io::ErrorKind::BrokenPipe, "synthetic disconnect")); + } + let written = self.remaining.min(bytes.len()); + self.remaining -= written; + Ok(written) + } + fn flush(&mut self) -> io::Result<()> { Ok(()) } +} + +#[test] +fn export_writer_failure_cannot_be_retried_as_success() { + let mut writer = UsageJsonWriter::new(BrokenSink { remaining: 10 }, 2, 10000).unwrap(); + assert_eq!(writer.push(&row(1, "{}")), Err(ExportError::Io)); + assert_eq!(writer.push(&row(2, "{}")), Err(ExportError::Aborted)); + assert!(matches!(writer.finish(), Err(ExportError::Aborted))); +} diff --git a/services/core/tests/history_read.rs b/services/core/tests/history_read.rs new file mode 100644 index 000000000..366c56498 --- /dev/null +++ b/services/core/tests/history_read.rs @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! Projection tests are synthetic. The PG tests use SQLx-created disposable +//! databases; run them only under A09's exclusive synthetic resource lease. +use lmm_core::history_read::{ + LogQuotaInput, MAX_BASES, MAX_CREDIT_BASIS, MAX_METADATA_BYTES, MAX_RECORDS, + captured_credits_per_usd, project_bucket_without_components, + project_consumption_statistic, project_log, + snapshot::ReadSnapshot, + usage::{UsageArithmeticInput, UsageError, UsageRounding, project_migrated_usage}, +}; +use serde_json::Value; +use sqlx::PgPool; + +fn consumption(quota: i64, other: &str) -> LogQuotaInput<'_> { + LogQuotaInput { quota: Some(quota), log_type: 2, other } +} + +#[test] +fn projection_accepts_exact_integer_json_forms() { + for raw in ["500000", "5e5", "500000.0", "500000000e-3", "0.0000000000005e18"] { + let other = format!(r#"{{"pricing_credits_per_usd":{raw}}}"#); + assert_eq!(captured_credits_per_usd(&other), Some(500000), "{raw}"); + } + assert_eq!(captured_credits_per_usd(&format!(r#"{{"pricing_credits_per_usd":{MAX_CREDIT_BASIS}}}"#)), Some(MAX_CREDIT_BASIS)); +} + +#[test] +fn projection_rejects_unbounded_ambiguous_or_inexact_anchors() { + for raw in ["0", "-1", "true", "null", "[]", "{}", r#""500000""#, "1.5", "9007199254740991.1", "9007199254740992", "1e19", "1e-19", "1e999999999999999999999", "0.000000000000000001"] { + assert_eq!(captured_credits_per_usd(&format!(r#"{{"pricing_credits_per_usd":{raw}}}"#)), None, "{raw}"); + } + for other in ["{}", "[]", "[500000]", "null", "not JSON", r#"{"pricing_credits_per_usd":1,"pricing_credits_per_usd":2}"#, r#"{"pricing_credits_per_usd":1} {}"#] { + assert_eq!(captured_credits_per_usd(other), None, "{other}"); + } + let large = format!(r#"{{"pricing_credits_per_usd":1,"padding":"{}"}}"#, "x".repeat(MAX_METADATA_BYTES)); + assert_eq!(captured_credits_per_usd(&large), None); +} + +#[test] +fn projection_preserves_raw_quota_and_unknown_semantics() { + let input = consumption(1, r#"{"pricing_credits_per_usd":3}"#); + assert_eq!(project_log(input).quota_usd.as_deref(), Some("0.333333333333333333")); + assert_eq!(input.quota, Some(1)); + assert_eq!(project_log(consumption(0, "{}")).quota_usd, None); + assert_eq!(project_log(consumption(-1, r#"{"pricing_credits_per_usd":3}"#)).quota_money_basis, "missing_credit_basis"); + assert_eq!(project_log(LogQuotaInput { quota: None, ..input }).quota_money_basis, "invalid_stored_quota"); + for log_type in [0, 1, 3, 4, 5, 6, 7] { + assert_eq!(project_log(LogQuotaInput { log_type, ..input }).quota_money_basis, "not_consumption"); + } + assert_eq!(project_bucket_without_components().quota_money_basis, "bucket_without_credit_basis"); +} + +#[test] +fn projection_rounds_only_once_after_adding_different_bases() { + let result = project_consumption_statistic([ + consumption(1, r#"{"pricing_credits_per_usd":3}"#), + consumption(1, r#"{"pricing_credits_per_usd":6}"#), + ], false); + assert_eq!(result.money.quota_usd.as_deref(), Some("0.5")); + let result = project_consumption_statistic([ + consumption(1, r#"{"pricing_credits_per_usd":3}"#); 3 + ], false); + assert_eq!(result.money.quota_usd.as_deref(), Some("1")); +} + +#[test] +fn projection_rounds_an_exact_mixed_basis_half_tie_away_from_zero() { + let rows = [ + consumption(1, r#"{"pricing_credits_per_usd":524288}"#), + consumption(1, r#"{"pricing_credits_per_usd":3814697265625}"#), + ]; + assert_eq!(project_consumption_statistic(rows, false).money.quota_usd.as_deref(), Some("0.000001907348894957")); + assert_eq!(project_consumption_statistic(rows.into_iter().rev(), false).money.quota_usd.as_deref(), Some("0.000001907348894957")); +} + +#[test] +fn projection_handles_maximum_raw_sums_without_i64_overflow() { + let row = consumption(i64::MAX, r#"{"pricing_credits_per_usd":1}"#); + let result = project_consumption_statistic(std::iter::repeat_n(row, MAX_RECORDS), false); + assert_eq!(result.money.quota_usd.as_deref(), Some("92233720368547758070000")); +} + +#[test] +fn projection_does_not_publish_partial_known_totals() { + let result = project_consumption_statistic([ + consumption(500000, r#"{"pricing_credits_per_usd":500000}"#), + consumption(12, "{}"), + ], false); + assert!(!result.money.quota_money_available); + assert_eq!(result.money.quota_usd, None); + assert_eq!(result.unpriced_quota_records, 1); + assert!(!result.quota_projection_truncated); + assert_eq!(result.money.quota_money_basis, "aggregate_missing_credit_basis"); +} + +#[test] +fn projection_distinguishes_empty_zero_and_truncated() { + let empty = project_consumption_statistic([], false); + assert_eq!(empty.money.quota_usd.as_deref(), Some("0")); + assert_eq!(empty.money.quota_money_basis, "empty_result"); + let zero = project_log(consumption(0, r#"{"pricing_credits_per_usd":1}"#)); + assert_eq!(zero.quota_usd.as_deref(), Some("0")); + assert_eq!(zero.quota_money_basis, "captured_credits_per_usd"); + let truncated = project_consumption_statistic([], true); + assert!(truncated.quota_projection_truncated); + assert_eq!(truncated.money.quota_usd, None); +} + +#[test] +fn projection_bounds_rows_and_number_of_bases() { + let input = consumption(1, r#"{"pricing_credits_per_usd":1}"#); + assert!(project_consumption_statistic(std::iter::repeat_n(input, MAX_RECORDS + 1), false).quota_projection_truncated); + let metadata: Vec<_> = (1..=MAX_BASES + 1).map(|basis| format!(r#"{{"pricing_credits_per_usd":{basis}}}"#)).collect(); + assert!(!project_consumption_statistic(metadata[..MAX_BASES].iter().map(|other| consumption(1, other)), false).quota_projection_truncated); + assert!(project_consumption_statistic(metadata.iter().map(|other| consumption(1, other)), false).quota_projection_truncated); +} + +#[test] +fn projection_matches_versioned_golden_contract() { + let fixtures: Value = serde_json::from_str(include_str!("../src/history_read/fixtures/projection-v1.json")).unwrap(); + for case in fixtures["log_cases"].as_array().unwrap() { + let input = LogQuotaInput { + quota: case["quota"].as_i64(), + log_type: case["log_type"].as_i64().unwrap() as i32, + other: case["other"].as_str().unwrap(), + }; + assert_eq!(serde_json::to_value(project_log(input)).unwrap(), case["expected"], "{}", case["name"]); + } + for case in fixtures["statistic_cases"].as_array().unwrap() { + let rows = case["rows"].as_array().unwrap().iter().map(|row| LogQuotaInput { + quota: row["quota"].as_i64(), + log_type: row["log_type"].as_i64().unwrap() as i32, + other: row["other"].as_str().unwrap(), + }); + let result = project_consumption_statistic(rows, case["source_truncated"].as_bool().unwrap()); + assert_eq!(serde_json::to_value(result).unwrap(), case["expected"], "{}", case["name"]); + } +} + +#[test] +fn usage_preserves_separate_baseline_and_refund_units() { + let input = UsageArithmeticInput { + current_used: 250, historical_raw: 300, reversed_raw: 60, reversed_normalized: 20, + divisor: "3", rounding: UsageRounding::HalfAwayFromZero, + }; + let result = project_migrated_usage(&input).unwrap(); + assert_eq!(result.normalized_used_quota, 90); + assert_eq!(result.historical_raw_quota, 240); + assert_eq!(result.historical_normalized_quota, 80); + assert_eq!(result.post_migration_quota, 10); + // Dividing current_used by either a log's captured USD anchor or the + // migration divisor is not the normalized cumulative-usage calculation. + assert_ne!(result.normalized_used_quota, input.current_used / 3); +} + +#[test] +fn usage_rejects_unproven_reductions_and_bad_bases() { + let mut input = UsageArithmeticInput { + current_used: 299, historical_raw: 300, reversed_raw: 0, reversed_normalized: 0, + divisor: "3", rounding: UsageRounding::TowardZero, + }; + assert_eq!(project_migrated_usage(&input), Err(UsageError::UnprovenReduction)); + input.current_used = 300; + for divisor in ["", "1", "0", "-3", "3e0", "3.", "3.0000000000000000001", "NaN"] { + input.divisor = divisor; + assert_eq!(project_migrated_usage(&input), Err(UsageError::InvalidBasis)); + } + input.divisor = "3"; + input.reversed_raw = 301; + assert_eq!(project_migrated_usage(&input), Err(UsageError::InvalidBasis)); +} + +#[test] +fn usage_obeys_the_captured_rounding_policy() { + let mut input = UsageArithmeticInput { + current_used: 3, historical_raw: 3, reversed_raw: 0, reversed_normalized: 0, + divisor: "2", rounding: UsageRounding::TowardZero, + }; + assert_eq!(project_migrated_usage(&input).unwrap().normalized_used_quota, 1); + input.rounding = UsageRounding::HalfAwayFromZero; + assert_eq!(project_migrated_usage(&input).unwrap().normalized_used_quota, 2); +} + +async fn prepare(pool: &PgPool) { + sqlx::query("CREATE TABLE a04_history_fixture (id BIGINT PRIMARY KEY, user_id BIGINT NOT NULL, quota BIGINT NOT NULL)") + .execute(pool).await.unwrap(); + sqlx::query("INSERT INTO a04_history_fixture VALUES (1,7,100),(2,8,999)") + .execute(pool).await.unwrap(); +} + +#[sqlx::test(migrations = false)] +async fn pg_default_snapshot_exposes_the_count_page_counterexample(pool: PgPool) { + prepare(&pool).await; + let mut tx = pool.begin().await.unwrap(); + sqlx::query("SET TRANSACTION ISOLATION LEVEL READ COMMITTED").execute(&mut *tx).await.unwrap(); + let count: i64 = sqlx::query_scalar("SELECT count(*) FROM a04_history_fixture WHERE user_id=7") + .fetch_one(&mut *tx).await.unwrap(); + sqlx::query("INSERT INTO a04_history_fixture VALUES (3,7,200)").execute(&pool).await.unwrap(); + let rows: Vec<(i64,)> = sqlx::query_as("SELECT quota FROM a04_history_fixture WHERE user_id=7 ORDER BY id") + .fetch_all(&mut *tx).await.unwrap(); + assert_eq!(count, 1); + assert_eq!(rows.len(), 2); + tx.rollback().await.unwrap(); +} + +#[sqlx::test(migrations = false)] +async fn pg_read_snapshot_keeps_count_page_sum_consistent(pool: PgPool) { + prepare(&pool).await; + let mut snapshot = ReadSnapshot::begin(&pool).await.unwrap(); + let count: i64 = sqlx::query_scalar("SELECT count(*) FROM a04_history_fixture WHERE user_id=7") + .fetch_one(snapshot.connection()).await.unwrap(); + sqlx::query("INSERT INTO a04_history_fixture VALUES (3,7,200)").execute(&pool).await.unwrap(); + let rows: Vec<(i64,)> = sqlx::query_as("SELECT quota FROM a04_history_fixture WHERE user_id=7 ORDER BY id") + .fetch_all(snapshot.connection()).await.unwrap(); + let sum: String = sqlx::query_scalar("SELECT sum(quota)::text FROM a04_history_fixture WHERE user_id=7") + .fetch_one(snapshot.connection()).await.unwrap(); + assert_eq!(count, 1); + assert_eq!(rows, [(100,)]); + assert_eq!(sum, "100"); + snapshot.finish().await.unwrap(); + let current: i64 = sqlx::query_scalar("SELECT count(*) FROM a04_history_fixture WHERE user_id=7").fetch_one(&pool).await.unwrap(); + assert_eq!(current, 2); +} + +#[sqlx::test(migrations = false)] +async fn pg_read_snapshot_cannot_mutate_persistent_rows(pool: PgPool) { + prepare(&pool).await; + let mut snapshot = ReadSnapshot::begin(&pool).await.unwrap(); + let error = sqlx::query("UPDATE a04_history_fixture SET quota=0") + .execute(snapshot.connection()).await.unwrap_err(); + assert_eq!(error.as_database_error().unwrap().code().as_deref(), Some("25006")); + snapshot.finish().await.unwrap(); + let values: Vec<(i64,)> = sqlx::query_as("SELECT quota FROM a04_history_fixture ORDER BY id").fetch_all(&pool).await.unwrap(); + assert_eq!(values, [(100,), (999,)]); +} From c3fa657dfca216918f66219ab530b64b6a5e59be Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 12 Oct 2026 14:04:20 +0800 Subject: [PATCH 2/6] feat(core): wire A04 authorized history queries and complete bounded finance ZIP Use existing stored credential checks inside one read-only snapshot for scope, counts, pages, totals and all export documents. Refuse missing, writable or duplicate offline sources; preserve unknown historical money and raw quota. Add real PostgreSQL restricted-role and HTTP integration tests, plus a scoped read-only CI runner that retains resolved lockfile and exact tested input. The offline A03 source-view provider and catalog integration remain required; synthetic fixtures are not historical import or production readiness evidence. --- .github/workflows/a04-history.yml | 57 +++ services/core/Cargo.toml | 4 +- services/core/src/finance_export.rs | 6 +- services/core/src/finance_export/archive.rs | 192 ++++++++++ services/core/src/history_read.rs | 2 + services/core/src/history_read/http.rs | 121 ++++++ services/core/src/history_read/repository.rs | 347 ++++++++++++++++++ services/core/src/identity/mod.rs | 8 + services/core/src/lib.rs | 2 + services/core/src/main.rs | 3 + services/core/tests/finance_export.rs | 2 +- .../core/tests/fixtures/history_source_v1.sql | 41 +++ services/core/tests/history_read/check.sh | 31 ++ services/core/tests/history_repository.rs | 218 +++++++++++ 14 files changed, 1030 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/a04-history.yml create mode 100644 services/core/src/finance_export/archive.rs create mode 100644 services/core/src/history_read/http.rs create mode 100644 services/core/src/history_read/repository.rs create mode 100644 services/core/tests/fixtures/history_source_v1.sql create mode 100644 services/core/tests/history_read/check.sh create mode 100644 services/core/tests/history_repository.rs diff --git a/.github/workflows/a04-history.yml b/.github/workflows/a04-history.yml new file mode 100644 index 000000000..6e75ede32 --- /dev/null +++ b/.github/workflows/a04-history.yml @@ -0,0 +1,57 @@ +name: A04 history and finance acceptance +on: + pull_request: + branches: [wip/rust-core-go-extensions] + paths: + - .github/workflows/a04-history.yml + - services/core/** +permissions: + contents: read +concurrency: + group: a04-history-${{ github.ref }} + cancel-in-progress: true +jobs: + postgres: + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + shell: bash + services: + postgres: + image: postgres:17-alpine + env: + POSTGRES_PASSWORD: postgres + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U postgres" --health-interval 5s + --health-timeout 3s --health-retries 10 + env: + DATABASE_URL: postgres://postgres:postgres@127.0.0.1:5432/postgres + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + persist-credentials: false + - uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db + with: + version: '2026.10.8' + install: false + add_shims_to_path: false + env: false + - name: Isolated Rust and PostgreSQL acceptance + run: | + mkdir -p "$RUNNER_TEMP/a04-tools" + cd "$RUNNER_TEMP/a04-tools" + printf '[tools]\nrust = "1.99.0"\n' > mise.toml + mise trust mise.toml + mise install rust@1.99.0 + mise exec -- rustup component add --toolchain 1.99.0 rustfmt + mise exec -- bash "$GITHUB_WORKSPACE/services/core/tests/history_read/check.sh" + - name: Preserve exact input and all test results + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: a04-acceptance-${{ github.sha }} + path: ${{ runner.temp }}/a04-evidence + if-no-files-found: warn + retention-days: 7 diff --git a/services/core/Cargo.toml b/services/core/Cargo.toml index 646c55b37..38f557dcb 100644 --- a/services/core/Cargo.toml +++ b/services/core/Cargo.toml @@ -26,7 +26,9 @@ http-body = "=1.1.0" rand = "=0.8.5" reqwest = { version = "=0.12.28", default-features = false, features = ["json", "stream", "rustls-tls", "http2"] } serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" +serde_json = { version = "1.0", features = ["raw_value", "arbitrary_precision"] } +num-rational = "=0.4.2" +zip = { version = "=6.0.0", default-features = false } sha2 = "=0.10.9" sqlx = { version = "=0.8.6", default-features = false, features = ["runtime-tokio-rustls", "postgres", "json", "macros"] } tokio = { version = "1.47.1", features = ["macros", "net", "rt-multi-thread", "signal", "time", "sync", "io-util"] } diff --git a/services/core/src/finance_export.rs b/services/core/src/finance_export.rs index b681e140e..09ab1c67d 100644 --- a/services/core/src/finance_export.rs +++ b/services/core/src/finance_export.rs @@ -2,6 +2,8 @@ //! Allowlisted usage JSON for the finance ZIP. This is not an HTTP endpoint or //! a complete finance bundle. The caller must authorize and select rows under //! the same read snapshot; no raw storage object implements Serialize here. +pub mod archive; + use crate::history_read::{LogQuotaInput, QuotaMoneyProjection, project_log}; use serde::Serialize; use std::io::{self, Write}; @@ -25,7 +27,7 @@ pub struct UsageExportInput<'a> { pub completion_tokens: i64, pub use_time: i64, pub is_stream: bool, - pub channel_id: i64, + pub channel_id: Option, pub group: &'a str, pub other: &'a str, } @@ -48,7 +50,7 @@ struct UsageExportRow<'a> { completion_tokens: i64, use_time: i64, is_stream: bool, - channel_id: i64, + channel_id: Option, group: &'a str, } diff --git a/services/core/src/finance_export/archive.rs b/services/core/src/finance_export/archive.rs new file mode 100644 index 000000000..fcf813221 --- /dev/null +++ b/services/core/src/finance_export/archive.rs @@ -0,0 +1,192 @@ +//! Complete ZIP assembly, bounded across every document. Uses only the +//! authenticated snapshot and fixed document/field names. Never downloads +//! upstream data or fetches current FX to value historical records. +use super::{MAX_EXPORT_BYTES,MAX_EXPORT_ROWS,UsageExportInput,UsageJsonWriter}; +use crate::history_read::repository::{HistoryRead,ReadError,Result}; +use serde_json::{Value,json}; +use sqlx::Row; +use std::{collections::BTreeMap,io::{self,Cursor,Seek,SeekFrom,Write}}; +use zip::{ZipWriter,write::SimpleFileOptions,CompressionMethod}; + +/// Unknown source fields never enter an exported record. These are the main +/// finance DTO field names, not the underlying tables' entire JSON records. +pub struct Dataset { pub name:&'static str, pub fields:&'static str, pub timed:bool } +pub const DATASETS:&[Dataset]=&[ + Dataset {name:"users-balances",fields:"user_id username role status group quota used_quota affiliate_quota affiliate_history_quota request_count created_at last_api_activity_at trust_level_override normalized_used_quota usage_projection_available usage_projection_reason effective_group_ratio effective_topup_group_ratio",timed:false}, + Dataset {name:"channels-pricing",fields:"channel_id type status name weight created_time test_time response_time base_url balance balance_updated_time models model_mapping group used_quota priority auto_ban tag",timed:false}, + Dataset {name:"subscription-plans",fields:"plan_id title subtitle price_amount currency duration_unit duration_value custom_seconds enabled sort_order allow_balance_pay allow_wallet_overflow max_purchase_per_user upgrade_group downgrade_group total_amount quota_reset_period quota_reset_custom_seconds created_at updated_at",timed:false}, + Dataset {name:"topups",fields:"topup_id user_id amount credited_quota expected_amount_micros settled_amount_micros settlement_currency money payment_method payment_provider create_time complete_time status",timed:true}, + Dataset {name:"subscription-orders",fields:"order_id user_id plan_id plan_price plan_currency expected_amount_micros settlement_currency payment_method payment_provider provider_product_id provider_subscription_state current_period_start current_period_end refunded_amount_micros status create_time complete_time",timed:true}, + Dataset {name:"subscription-payment-events",fields:"payment_event_id subscription_order_id payment_provider settlement_currency settlement_amount_micros period_start period_end created_time",timed:true}, + Dataset {name:"bounty-ledger",fields:"ledger_id project_id challenge_id user_id counterparty_user_id kind quota note created_at",timed:true}, + Dataset {name:"checkins",fields:"checkin_id user_id checkin_date quota_awarded created_at",timed:true}, + Dataset {name:"redemptions",fields:"redemption_id created_by_user_id status name quota created_time redeemed_time used_user_id expired_time",timed:false}, + Dataset {name:"user-subscriptions",fields:"user_subscription_id user_id plan_id amount_total amount_used start_time end_time status source last_reset_time next_reset_time upgrade_group prev_user_group downgrade_group allow_wallet_overflow created_at updated_at",timed:false}, +]; +const OPTIONS:&[&str]=&["ModelPrice","ModelRatio","CacheRatio","CreateCacheRatio","CompletionRatio","ImageRatio","AudioRatio","AudioCompletionRatio","GroupRatio","GroupGroupRatio","TopupGroupRatio","UserUsableGroups","QuotaPerUnit","Price","USDExchangeRate","MinTopUp","DataExportInterval","payment_setting.amount_options","payment_setting.amount_discount","tool_price_setting.prices","billing_setting.billing_mode","billing_setting.billing_expr","checkin_setting.enabled","checkin_setting.min_quota","checkin_setting.max_quota","checkin_setting.level_multipliers"]; +const DERIVED:&[(&str,&str)]=&[("model_prices","ModelPrice"),("model_ratios","ModelRatio"),("completion_ratios","CompletionRatio"),("cache_ratios","CacheRatio"),("create_cache_ratios","CreateCacheRatio"),("image_ratios","ImageRatio"),("audio_ratios","AudioRatio"),("audio_completion_ratios","AudioCompletionRatio"),("tool_prices","tool_price_setting.prices"),("billing_modes","billing_setting.billing_mode"),("billing_expressions","billing_setting.billing_expr")]; +const EFFECTIVE_FIELDS:&str="model_name quota_type model_ratio model_price completion_ratio cache_ratio create_cache_ratio image_ratio audio_ratio audio_completion_ratio enable_groups supported_endpoint_types"; + +struct BoundedCursor { inner:Cursor>, maximum:u64 } +impl Write for BoundedCursor { + fn write(&mut self,bytes:&[u8])->io::Result { + let end=self.inner.position().checked_add(bytes.len() as u64).ok_or_else(||io::Error::other("export limit"))?; + if end>self.maximum {return Err(io::Error::other("export limit"));} + self.inner.write(bytes) + } + fn flush(&mut self)->io::Result<()> {Ok(())} +} +impl Seek for BoundedCursor { + fn seek(&mut self,to:SeekFrom)->io::Result { + let position=match to {SeekFrom::Start(n)=>i128::from(n),SeekFrom::Current(n)=>i128::from(self.inner.position())+i128::from(n),SeekFrom::End(n)=>self.inner.get_ref().len() as i128+i128::from(n)}; + if !(0..=i128::from(self.maximum)).contains(&position) {return Err(io::Error::other("export limit"));} + self.inner.seek(SeekFrom::Start(position as u64)) + } +} +fn start(zip:&mut ZipWriter,name:&str)->Result<()> { + zip.start_file(format!("{name}.json"),SimpleFileOptions::default().compression_method(CompressionMethod::Stored).unix_permissions(0o600)) + .map_err(|_|ReadError::Limit) +} +fn write_json(zip:&mut ZipWriter,value:&Value)->Result<()> { + serde_json::to_writer(zip,value).map_err(|_|ReadError::Limit) +} + +/// Strip every non-allowlisted field before serializing; reject structured +/// values in scalar fields instead of exporting a nested provider payload. +pub fn project_record(dataset:&Dataset,record:&Value)->Result { + let object=record.as_object().ok_or(ReadError::Unavailable)?; + let mut output=serde_json::Map::new(); + for key in dataset.fields.split_whitespace() { + let Some(value)=object.get(key) else {continue;}; + match value { + Value::Null|Value::Bool(_)=>{}, + Value::Number(n) if n.to_string().len()<=64=>{}, + Value::String(text) if text.len()<=4096=>{}, + Value::Array(items) if dataset.name=="effective-model-pricing" + && matches!(key,"enable_groups"|"supported_endpoint_types") + && items.len()<=64 && items.iter().all(|item|item.as_str().is_some_and(|s|s.len()<=256))=>{}, + _=>return Err(ReadError::Unavailable), + } + output.insert(key.to_owned(),value.clone()); + } + if dataset.name=="channels-pricing" { + // URL parsing is local only. Userinfo, paths, query and fragment never + // enter the archive, including percent-encoded credential forms. + let value=output.remove("base_url").and_then(|v|v.as_str().map(str::to_owned)) + .and_then(|v|reqwest::Url::parse(&v).ok()) + .filter(|v|matches!(v.scheme(),"http"|"https") && v.host().is_some()) + .map(|v|Value::String(v.origin().ascii_serialization())); + if let Some(value)=value {output.insert("base_url".into(),value);} + } + if dataset.name=="users-balances" && output.get("usage_projection_available")!=Some(&Value::Bool(true)) { + output.remove("normalized_used_quota"); + output.insert("usage_projection_available".into(),Value::Bool(false)); + } + Ok(Value::Object(output)) +} + +pub async fn build(read:&mut HistoryRead<'_>,max_bytes:usize)->Result> { + if !read.administrator() {return Err(ReadError::Forbidden);} + if !(1024..=MAX_EXPORT_BYTES).contains(&max_bytes) {return Err(ReadError::Invalid);} + let (begin,end)=read.time_range(); + if begin<=0 || end31_536_000 {return Err(ReadError::Invalid);} + // All required datasets must be qualified. Missing datasets are not empty + // arrays, and cannot silently produce a convincing partial ZIP. + for name in DATASETS.iter().map(|d|d.name).chain(["financial-options","effective-model-pricing"]) { + read.require_source(name).await?; + } + let mut zip=ZipWriter::new(BoundedCursor {inner:Cursor::new(Vec::new()),maximum:max_bytes as u64}); + let mut counts=BTreeMap::new(); + let mut total=0usize; + for dataset in DATASETS { + let count=write_dataset(read,&mut zip,dataset,begin,end,&mut total).await?; + counts.insert(dataset.name,count); + } + let usage_count=usize::try_from(read.export_count().await?).map_err(|_|ReadError::Limit)?; + if total.checked_add(usage_count).is_none_or(|v| v>MAX_EXPORT_ROWS) {return Err(ReadError::Limit);} + start(&mut zip,"usage-billing-records")?; + let mut usage=UsageJsonWriter::new(&mut zip,MAX_EXPORT_ROWS,max_bytes).map_err(|_|ReadError::Limit)?; + let mut after=(-1,0); + let mut seen=0; + loop { + let rows=read.export_batch(after,64).await?; + if rows.is_empty() {break;} + for row in rows { + usage.push(&UsageExportInput {log_id:row.id,user_id:row.user_id,created_at:row.created_at, + log_type:row.log_type,username:&row.username,token_name:&row.token_name,model_name:&row.model_name, + quota:row.quota,prompt_tokens:row.prompt_tokens,completion_tokens:row.completion_tokens, + use_time:row.use_time,is_stream:row.is_stream,channel_id:row.channel_id,group:&row.group_name,other:&row.other}) + .map_err(|_|ReadError::Unavailable)?; + after=(row.created_at,row.id);seen+=1; + } + } + if seen!=usage_count {return Err(ReadError::Unavailable);} + usage.finish().map_err(|_|ReadError::Limit)?; + counts.insert("usage",seen); + total+=seen; + let rows=sqlx::query("SELECT name,CASE WHEN octet_length(value)<=65536 THEN value END AS value FROM core_history.finance_options_v1 WHERE name=ANY($1) ORDER BY name LIMIT 64") + .bind(OPTIONS).fetch_all(read.connection()).await?; + let mut options=serde_json::Map::new(); + for row in rows { + let name:String=row.try_get("name")?; + let value:String=row.try_get("value")?; + if value.len()>65_536 || options.insert(name,Value::String(value)).is_some() {return Err(ReadError::Unavailable);} + } + counts.insert("financial-options",options.len()); + start(&mut zip,"financial-options")?; + write_json(&mut zip,&Value::Object(options.clone()))?; + let mut derived=serde_json::Map::new(); + for (field,key) in DERIVED { + let value=options.get(*key).and_then(Value::as_str).map(serde_json::from_str::).transpose().map_err(|_|ReadError::Unavailable)?; + derived.insert((*field).into(),value.unwrap_or(Value::Null)); + } + start(&mut zip,"model-prices-and-ratios")?; + write_json(&mut zip,&Value::Object(derived))?; + // Effective prices use a separate source contract; they are not a fallback + // for missing historical money. Reuse the same field whitelist gate. + let effective=Dataset {name:"effective-model-pricing",fields:EFFECTIVE_FIELDS,timed:false}; + let effective_count=write_dataset(read,&mut zip,&effective,begin,end,&mut total).await?; + counts.insert("effective-model-pricing",effective_count); + start(&mut zip,"manifest")?; + let generated:i64=sqlx::query_scalar("SELECT extract(epoch FROM transaction_timestamp())::bigint").fetch_one(read.connection()).await?; + write_json(&mut zip,&json!({"schema_version":"lmm-finance-export/v1","generated_at":generated, + "start_timestamp":begin,"end_timestamp":end,"rows":counts,"truncated":{}, + "redactions":["credentials, provider payloads, IPs, raw requests, opaque log metadata, URL credentials and query strings"], + "notes":["Raw quota is unchanged. Unknown historical money is not zero.","All datasets use one authorized read-only snapshot. Current pricing is not historical evidence."]}))?; + let output=zip.finish().map_err(|_|ReadError::Limit)?.inner.into_inner(); + Ok(output) +} + +async fn write_dataset(read:&mut HistoryRead<'_>,zip:&mut ZipWriter,dataset:&Dataset, + begin:i64,end:i64,total:&mut usize)->Result { + let count=sqlx::query("SELECT count(*)::bigint AS n,count(DISTINCT id)::bigint AS ids FROM core_history.finance_records_v1 WHERE dataset=$1 AND (NOT $4 OR created_at BETWEEN $2 AND $3)") + .bind(dataset.name).bind(begin).bind(end).bind(dataset.timed).fetch_one(read.connection()).await?; + let n:i64=count.try_get("n")?; + if n!=count.try_get::("ids")? {return Err(ReadError::Unavailable);} + let expected=usize::try_from(n).map_err(|_|ReadError::Limit)?; + if total.checked_add(expected).is_none_or(|n|n>MAX_EXPORT_ROWS) {return Err(ReadError::Limit);} + start(zip,dataset.name)?; + zip.write_all(b"[").map_err(|_|ReadError::Limit)?; + let mut last=0i64; + let mut seen=0usize; + let fields:Vec<&str>=dataset.fields.split_whitespace().collect(); + while seenk) FROM unnest($5::text[]) k WHERE record ? k) END AS record FROM core_history.finance_records_v1 WHERE dataset=$1 AND id>$2 AND (NOT $6 OR created_at BETWEEN $3 AND $4) ORDER BY id LIMIT 64") + .bind(dataset.name).bind(last).bind(begin).bind(end).bind(&fields).bind(dataset.timed) + .fetch_all(read.connection()).await?; + if rows.is_empty() {return Err(ReadError::Unavailable);} + for row in rows { + let id:i64=row.try_get("id")?; + if id<=last || seen==expected {return Err(ReadError::Unavailable);} + last=id; + let record:Option=row.try_get("record")?; + let record=project_record(dataset,&record.ok_or(ReadError::Unavailable)?)?; + if seen>0 {zip.write_all(b",").map_err(|_|ReadError::Limit)?;} + write_json(zip,&record)?; + seen+=1; + } + } + zip.write_all(b"]\n").map_err(|_|ReadError::Limit)?; + *total+=seen; + Ok(seen) +} diff --git a/services/core/src/history_read.rs b/services/core/src/history_read.rs index f61978851..fb88e731a 100644 --- a/services/core/src/history_read.rs +++ b/services/core/src/history_read.rs @@ -11,6 +11,8 @@ use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use std::collections::BTreeMap; +pub mod repository; +pub mod http; pub mod snapshot; pub mod usage; diff --git a/services/core/src/history_read/http.rs b/services/core/src/history_read/http.rs new file mode 100644 index 000000000..45c3363e0 --- /dev/null +++ b/services/core/src/history_read/http.rs @@ -0,0 +1,121 @@ +//! Existing console-compatible routes. Authority comes from the stored bearer +//! credential, not query parameters, model output or a client-supplied role. +use super::repository::{Audience,Filter,HistoryRead,ReadError}; +use crate::finance_export::{MAX_EXPORT_BYTES,archive}; +use axum::{Json,Router,body::{Body,Bytes},extract::{OriginalUri,Query,Request,State}, + http::{HeaderMap,HeaderValue,StatusCode,header},middleware::{self,Next}, + response::{IntoResponse,Response},routing::get}; +use serde_json::{Value,json}; +use sqlx::PgPool; +use std::{io,sync::Arc,time::{Duration,SystemTime,UNIX_EPOCH}}; +use tokio::sync::{Semaphore,mpsc}; +use tokio_stream::wrappers::ReceiverStream; + +#[derive(Clone)] +pub struct HistoryApi { pool:PgPool, exports:Arc } +impl HistoryApi { + pub fn new(pool:PgPool)->Self {Self {pool,exports:Arc::new(Semaphore::new(1))}} +} +pub fn router(state:HistoryApi)->Router { + let mut router=Router::new(); + for path in ["/api/log/","/api/log","/api/log/self","/api/log/token","/api/log/stat", + "/api/log/self/stat","/api/data/","/api/data","/api/data/users","/api/data/self", + "/api/data/flow","/api/data/flow/self","/api/finance/export"] { + router=router.route(path,get(handle)); + } + router.layer(middleware::from_fn(headers)).with_state(state) +} +async fn headers(request:Request,next:Next)->Response { + let mut response=if request.uri().query().is_some_and(|q|q.len()>4096) { + ReadError::Limit.into_response() + } else { next.run(request).await }; + for (name,value) in [(header::CACHE_CONTROL,"no-store"),(header::REFERRER_POLICY,"no-referrer"), + (header::X_CONTENT_TYPE_OPTIONS,"nosniff"),(header::CONTENT_SECURITY_POLICY,"default-src 'none'; frame-ancestors 'none'")] { + response.headers_mut().insert(name,HeaderValue::from_static(value)); + } + response +} +impl IntoResponse for ReadError { + fn into_response(self)->Response { + let (status,code,message)=match self { + Self::Unauthorized=>(StatusCode::UNAUTHORIZED,"HISTORY_UNAUTHORIZED","Authentication is required"), + Self::Forbidden=>(StatusCode::FORBIDDEN,"HISTORY_FORBIDDEN","Insufficient permission"), + Self::Invalid=>(StatusCode::BAD_REQUEST,"HISTORY_INVALID","Invalid history filter or time range"), + Self::Limit=>(StatusCode::PAYLOAD_TOO_LARGE,"HISTORY_LIMIT","Result exceeds a resource limit; narrow the query"), + Self::Busy=>(StatusCode::TOO_MANY_REQUESTS,"HISTORY_BUSY","Another financial export is in progress"), + Self::Unavailable=>(StatusCode::SERVICE_UNAVAILABLE,"HISTORY_SOURCE_UNAVAILABLE","The complete verified history source is unavailable"), + }; + (status,Json(json!({"success":false,"code":code,"message":message}))).into_response() + } +} +fn bearer(headers:&HeaderMap)->Result<&str,ReadError> { + if headers.get_all(header::AUTHORIZATION).iter().count()!=1 {return Err(ReadError::Unauthorized);} + headers.get(header::AUTHORIZATION).and_then(|h|h.to_str().ok()) + .and_then(|h|h.strip_prefix("Bearer ")).filter(|s|!s.is_empty()).ok_or(ReadError::Unauthorized) +} +async fn handle(State(state):State,OriginalUri(uri):OriginalUri, + headers:HeaderMap,query:Result,axum::extract::rejection::QueryRejection>)->Response { + let result=async { + let secret=bearer(&headers)?; + let Query(mut filter)=query.map_err(|_|ReadError::Invalid)?; + let path=uri.path(); + let audience=match path { + "/api/log/self"|"/api/log/self/stat"|"/api/data/self"|"/api/data/flow/self"=>Audience::SelfUser, + "/api/log/token"=>Audience::Token, + _=>Audience::Administrator, + }; + if path=="/api/finance/export" { + // A finance bundle is a complete administrator snapshot, never a + // partially filtered wallet/log selection labeled as full finance. + if filter.user_id.is_some() || filter.team_id.is_some() || filter.log_type!=0 + || filter.channel.is_some() || [&filter.username,&filter.token_name,&filter.model_name,&filter.group,&filter.request_id].iter().any(|v|!v.is_empty()) { + return Err(ReadError::Invalid); + } + if filter.end_timestamp==0 { + filter.end_timestamp=SystemTime::now().duration_since(UNIX_EPOCH).map_err(|_|ReadError::Unavailable)?.as_secs() as i64; + } + if filter.start_timestamp==0 {filter.start_timestamp=filter.end_timestamp.saturating_sub(2_592_000).max(1);} + let permit=state.exports.clone().try_acquire_owned().map_err(|_|ReadError::Busy)?; + let mut read=HistoryRead::begin(&state.pool,secret,audience,filter).await?; + let data=archive::build(&mut read,MAX_EXPORT_BYTES).await?; + read.finish().await?; + let length=data.len(); + let (sender,receiver)=mpsc::channel::>(1); + tokio::spawn(async move { + // The pool connection is already released. One producer owns + // the memory budget. Disconnect or timeout drops the remainder. + let _permit=permit; + let mut data=Bytes::from(data); + let send=async { + while !data.is_empty() { + let chunk=data.split_to(data.len().min(16*1024)); + if sender.send(Ok(chunk)).await.is_err() {break;} + } + }; + if tokio::time::timeout(Duration::from_secs(30),send).await.is_err() { + let _=sender.try_send(Err(io::Error::new(io::ErrorKind::TimedOut,"export download expired"))); + } + }); + let mut response=Body::from_stream(ReceiverStream::new(receiver)).into_response(); + response.headers_mut().insert(header::CONTENT_TYPE,HeaderValue::from_static("application/zip")); + response.headers_mut().insert(header::CONTENT_DISPOSITION,HeaderValue::from_static("attachment; filename=\"lmm-finance.zip\"")); + response.headers_mut().insert(header::CONTENT_LENGTH,HeaderValue::from_str(&length.to_string()).map_err(|_|ReadError::Unavailable)?); + return Ok(response); + } + let mut read=HistoryRead::begin(&state.pool,secret,audience,filter).await?; + let value:Value=if path.ends_with("/stat") { + serde_json::to_value(read.statistics().await?).map_err(|_|ReadError::Unavailable)? + } else if path.starts_with("/api/data") { + serde_json::to_value(read.buckets(path.ends_with("/users"),path.contains("/flow")).await?).map_err(|_|ReadError::Unavailable)? + } else { + serde_json::to_value(read.logs().await?).map_err(|_|ReadError::Unavailable)? + }; + read.finish().await?; + Ok::<_,ReadError>(Json(json!({"success":true,"message":"","data":value})).into_response()) + }; + match tokio::time::timeout(Duration::from_secs(30),result).await { + Ok(Ok(response))=>response, + Ok(Err(error))=>error.into_response(), + Err(_)=>ReadError::Unavailable.into_response(), + } +} diff --git a/services/core/src/history_read/repository.rs b/services/core/src/history_read/repository.rs new file mode 100644 index 000000000..097eb0ce6 --- /dev/null +++ b/services/core/src/history_read/repository.rs @@ -0,0 +1,347 @@ +//! Consumer-side read contract. The offline source owner supplies the versioned +//! views; missing, writable, duplicate or incomplete sources fail closed. +//! No runtime DDL, import, money mutation or best-effort partial history. +use super::{LogQuotaInput, MAX_METADATA_BYTES, MAX_RECORDS, QuotaMoneyProjection, + QuotaStatisticProjection, captured_credits_per_usd, project_consumption_statistic, project_log, + snapshot::ReadSnapshot}; +use crate::identity::{self, CredentialKind, IdentityError}; +use serde::{Deserialize, Serialize}; +use sqlx::{PgConnection, PgPool, Postgres, QueryBuilder, Row}; + +pub const MAX_PAGE: i64 = 100; +pub const MAX_OFFSET: i64 = 1_000_000; +const SAFE_INTEGER: i128 = 9_007_199_254_740_991; +const BATCH: i64 = 64; +const COLUMNS: &str = "id,event_key,user_id,owner_account_id,credential_id,created_at,log_type,username,token_name,model_name,quota,prompt_tokens,completion_tokens,use_time,is_stream,channel_id,group_name,request_id,CASE WHEN octet_length(other)<=65536 THEN other ELSE '' END AS other"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ReadError { Unauthorized, Forbidden, Invalid, Unavailable, Limit, Busy } +impl From for ReadError { fn from(_: sqlx::Error) -> Self { Self::Unavailable } } +impl From for ReadError { + fn from(e: IdentityError) -> Self { + match e { IdentityError::Unauthorized => Self::Unauthorized, IdentityError::Forbidden => Self::Forbidden, + IdentityError::Invalid => Self::Invalid, _ => Self::Unavailable } + } +} +pub type Result = std::result::Result; + +#[derive(Clone, Copy)] +pub enum Audience { Administrator, SelfUser, Token } + +/// HTTP filters are not authority. User/key/account constraints are resolved +/// from the stored credential before these filters reach a query. +#[derive(Clone, Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Filter { + #[serde(rename="type")] + pub log_type: i32, + pub start_timestamp: i64, + pub end_timestamp: i64, + pub user_id: Option, + pub team_id: Option, + pub username: String, + pub token_name: String, + pub model_name: String, + pub channel: Option, + pub group: String, + pub request_id: String, + pub p: i64, + pub page_size: i64, +} +impl Filter { + pub fn validate(&self) -> Result<()> { + if !(0..=6).contains(&self.log_type) || self.start_timestamp<0 || self.end_timestamp<0 + || (self.end_timestamp!=0 && self.start_timestamp>self.end_timestamp) + || self.user_id.is_some_and(|id| id<=0) || self.team_id.is_some_and(|id| id<=0) + || self.channel.is_some_and(|id| id<0) || self.p<0 || self.page_size<0 + || [&self.username,&self.token_name,&self.model_name,&self.group,&self.request_id] + .iter().any(|s| s.len()>256 || s.contains('\0')) { + return Err(ReadError::Invalid); + } + self.offset()?; + Ok(()) + } + pub fn page(&self) -> i64 { self.p.max(1) } + pub fn size(&self) -> i64 { if self.page_size==0 {10} else {self.page_size.clamp(10,MAX_PAGE)} } + fn offset(&self) -> Result { + self.page().checked_sub(1).and_then(|v| v.checked_mul(self.size())) + .filter(|v| *v<=MAX_OFFSET).ok_or(ReadError::Limit) + } +} + +#[derive(Serialize, sqlx::FromRow)] +pub struct LogRow { + pub id: i64, + #[serde(skip)] pub event_key: String, + pub user_id: i64, + #[serde(skip)] pub owner_account_id: i64, + #[serde(skip)] pub credential_id: Option, + pub created_at: i64, + #[serde(rename="type")] pub log_type: i32, + pub username: String, + pub token_name: String, + pub model_name: String, + pub quota: Option, + pub prompt_tokens: i64, + pub completion_tokens: i64, + pub use_time: i64, + pub is_stream: bool, + #[serde(skip_serializing_if="Option::is_none")] pub channel_id: Option, + #[serde(rename="group")] pub group_name: String, + pub request_id: String, + #[serde(skip)] pub other: String, +} +impl LogRow { + pub fn projection_input(&self) -> LogQuotaInput<'_> { + LogQuotaInput { quota:self.quota,log_type:self.log_type,other:&self.other } + } + fn validate(&self) -> Result<()> { + if self.id<=0 || self.user_id<=0 || self.owner_account_id<=0 || self.credential_id.is_some_and(|id| id<=0) + || self.created_at<0 || self.prompt_tokens<0 || self.completion_tokens<0 || self.use_time<0 + || !(1..=6).contains(&self.log_type) || self.event_key.is_empty() || self.event_key.len()>256 + || self.other.len()>MAX_METADATA_BYTES + || [&self.username,&self.token_name,&self.model_name,&self.group_name,&self.request_id] + .iter().any(|v| v.len()>1024) + || [self.quota.unwrap_or(0),self.prompt_tokens,self.completion_tokens,self.use_time] + .into_iter().any(|v| i128::from(v).abs()>SAFE_INTEGER) { + return Err(ReadError::Unavailable); + } + Ok(()) + } +} +#[derive(Serialize)] +pub struct DisplayLog { + #[serde(flatten)] pub row: LogRow, + #[serde(flatten)] pub money: QuotaMoneyProjection, +} +#[derive(Serialize)] +pub struct Page { + pub page: i64, pub page_size:i64, pub total:i64, pub items:Vec, +} +#[derive(Serialize)] +pub struct Statistics { + pub quota: Option, + pub rpm: i64, + pub tpm: i64, + pub total: i64, + #[serde(flatten)] pub projection: QuotaStatisticProjection, +} +/// Its only connection is the read-only transaction. It cannot construct a +/// second snapshot accidentally when running counts, pages, or projections. +pub struct HistoryRead<'a> { + snapshot: ReadSnapshot<'a>, + filter: Filter, + user: Option, + credential: Option, + account: Option, + administrator: bool, + root: bool, +} +impl<'a> HistoryRead<'a> { + pub async fn begin(pool:&'a PgPool, secret:&str, audience:Audience, filter:Filter) -> Result { + filter.validate()?; + let mut snapshot=ReadSnapshot::begin(pool).await?; + let actor=identity::authorize_read_in_transaction(snapshot.connection(),secret).await?; + let administrator=matches!(audience,Audience::Administrator); + match audience { + Audience::Administrator if actor.credential_kind!=CredentialKind::Session || actor.platform_level<5 + => return Err(ReadError::Forbidden), + Audience::SelfUser if actor.credential_kind!=CredentialKind::Session => return Err(ReadError::Forbidden), + Audience::Token if actor.credential_kind!=CredentialKind::ApiKey => return Err(ReadError::Forbidden), + _=>{} + } + if !administrator && (filter.user_id.is_some_and(|id| id!=actor.user_id) + || !filter.username.is_empty() || filter.channel.is_some()) { + return Err(ReadError::Forbidden); + } + let mut account=matches!(audience,Audience::Token).then_some(actor.owner_account_id); + if let Some(team)=filter.team_id { + let team_account:Option=sqlx::query_scalar( + "SELECT t.account_id FROM core_identity.teams t JOIN core_identity.accounts a ON a.id=t.account_id WHERE t.id=$1 AND t.active AND a.active AND ($3 OR t.owner_user_id=$2 OR EXISTS(SELECT 1 FROM core_identity.memberships m WHERE m.team_id=t.id AND m.user_id=$2 AND m.active))") + .bind(team).bind(actor.user_id).bind(administrator).fetch_optional(snapshot.connection()).await?; + let team_account=team_account.ok_or(ReadError::Forbidden)?; + if account.is_some_and(|id| id!=team_account) { return Err(ReadError::Forbidden); } + account=Some(team_account); + } + let user=if administrator {filter.user_id} else {Some(actor.user_id)}; + let credential=matches!(audience,Audience::Token).then_some(actor.credential_id); + let root=administrator && actor.platform_level==6; + let mut this=Self { snapshot,filter,user,credential,account,administrator,root }; + this.require_source("logs").await?; + if credential.is_some() {this.require_source("token_logs").await?;} + Ok(this) + } + pub fn connection(&mut self) -> &mut PgConnection { self.snapshot.connection() } + pub fn administrator(&self) -> bool { self.administrator } + pub async fn finish(self) -> Result<()> { self.snapshot.finish().await.map_err(Into::into) } + + /// Receipts are supplied by the offline importer. The network role must not + /// be able to manufacture them, or mutate any of the read contract objects. + pub async fn require_source(&mut self, dataset:&str) -> Result<()> { + let exists:bool=sqlx::query_scalar("SELECT to_regclass('core_history.source_contract_v1') IS NOT NULL") + .fetch_one(self.connection()).await?; + if !exists { return Err(ReadError::Unavailable); } + let receipt:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM core_history.source_contract_v1 WHERE dataset=$1 AND version=1 AND complete AND manifest_sha256~'^[0-9a-f]{64}$') AND NOT has_table_privilege(current_user,'core_history.source_contract_v1','INSERT,UPDATE,DELETE,TRUNCATE')") + .bind(dataset).fetch_one(self.connection()).await?; + if !receipt { return Err(ReadError::Unavailable); } + let relation=match dataset { + "logs"|"token_logs"=>"core_history.log_records_v1", + "quota_data"=>"core_history.quota_data_v1", + "financial-options"=>"core_history.finance_options_v1", + _=>"core_history.finance_records_v1", + }; + let writable:bool=sqlx::query_scalar("SELECT has_schema_privilege(current_user,'core_history','CREATE') OR has_table_privilege(current_user,$1,'INSERT,UPDATE,DELETE,TRUNCATE')") + .bind(relation).fetch_one(self.connection()).await?; + if writable {return Err(ReadError::Unavailable);} + Ok(()) + } + fn selection(&self, columns:&str, consumption:bool) -> QueryBuilder<'static,Postgres> { + let mut q=QueryBuilder::new(format!("SELECT {columns} FROM core_history.log_records_v1 WHERE TRUE")); + if let Some(id)=self.user { q.push(" AND user_id=").push_bind(id); } + if let Some(id)=self.credential { q.push(" AND credential_id=").push_bind(id); } + if let Some(id)=self.account { q.push(" AND owner_account_id=").push_bind(id); } + if consumption { q.push(" AND log_type=2"); } + else if self.filter.log_type!=0 { q.push(" AND log_type=").push_bind(self.filter.log_type); } + if self.filter.start_timestamp!=0 { q.push(" AND created_at>=").push_bind(self.filter.start_timestamp); } + if self.filter.end_timestamp!=0 { q.push(" AND created_at<=").push_bind(self.filter.end_timestamp); } + for (column,value) in [("username",&self.filter.username),("token_name",&self.filter.token_name), + ("group_name",&self.filter.group),("request_id",&self.filter.request_id)] { + if !value.is_empty() { q.push(format!(" AND {column}=")).push_bind(value.clone()); } + } + if !self.filter.model_name.is_empty() { + // Literal substring, not a caller-controlled LIKE expression. + q.push(" AND strpos(model_name,").push_bind(self.filter.model_name.clone()).push(")>0"); + } + if let Some(id)=self.filter.channel { q.push(" AND channel_id=").push_bind(id); } + q + } + async fn count(&mut self, consumption:bool) -> Result { + let mut q=self.selection("count(*)::bigint AS total,count(DISTINCT id)::bigint AS ids,count(DISTINCT event_key)::bigint AS events",consumption); + let row=q.build().fetch_one(self.connection()).await?; + let count:i64=row.try_get("total")?; + if count!=row.try_get::("ids")? || count!=row.try_get::("events")? { + // Never arbitrarily pick a legacy or native duplicate as authoritative. + return Err(ReadError::Unavailable); + } + Ok(count) + } + pub async fn logs(&mut self) -> Result { + let total=self.count(false).await?; + let mut query=self.selection(COLUMNS,false); + query.push(" ORDER BY id DESC LIMIT ").push_bind(self.filter.size()) + .push(" OFFSET ").push_bind(self.filter.offset()?); + let rows:Vec=query.build_query_as().fetch_all(self.connection()).await?; + let mut items=Vec::with_capacity(rows.len()); + for mut row in rows { + row.validate()?; + let money=project_log(row.projection_input()); + if !self.administrator {row.channel_id=None;} + items.push(DisplayLog {row,money}); + } + Ok(Page {page:self.filter.page(),page_size:self.filter.size(),total,items}) + } + pub async fn statistics(&mut self) -> Result { + let total=self.count(true).await?; + let mut query=self.selection("sum(quota::numeric)::text AS quota,count(quota)::bigint AS stored_count,count(*) FILTER(WHERE created_at>=extract(epoch FROM transaction_timestamp())::bigint-60)::bigint AS rpm,coalesce(sum(prompt_tokens::numeric+completion_tokens::numeric) FILTER(WHERE created_at>=extract(epoch FROM transaction_timestamp())::bigint-60),0)::text AS tpm",true); + let sums=query.build().fetch_one(self.connection()).await?; + let raw:Option=sums.try_get("quota")?; + let quota=if total==0 {Some(0)} else if sums.try_get::("stored_count")?!=total {None} else {raw.as_deref().map(exact).transpose()?}; + let tpm=exact(&sums.try_get::("tpm")?)?; + let mut inputs=Vec::new(); + let mut cursor=0i64; + while inputs.len()").push_bind(cursor).push(" ORDER BY id LIMIT ") + .push_bind(BATCH.min((MAX_RECORDS-inputs.len()) as i64)); + let rows=query.build().fetch_all(self.connection()).await?; + if rows.is_empty() {return Err(ReadError::Unavailable);} + for row in rows { + cursor=row.try_get("id")?; + let quota:Option=row.try_get("quota")?; + let other:String=row.try_get("other")?; + let basis=captured_credits_per_usd(&other).map(|b| format!("{{\"pricing_credits_per_usd\":{b}}}")).unwrap_or_default(); + inputs.push((quota,basis)); + } + } + let projection=project_consumption_statistic(inputs.iter().map(|(quota,basis)| LogQuotaInput {quota:*quota,log_type:2,other:basis}),total>MAX_RECORDS as i64); + Ok(Statistics {quota,rpm:sums.try_get("rpm")?,tpm,total,projection}) + } + /// Used by export without retaining opaque metadata or an unbounded result. + pub async fn export_batch(&mut self, after:(i64,i64), limit:i64) -> Result> { + if !(1..=BATCH).contains(&limit) {return Err(ReadError::Invalid);} + let mut q=self.selection(COLUMNS,false); + q.push(" AND log_type IN (1,2,3,4,6) AND (created_at,id)>(").push_bind(after.0).push(",").push_bind(after.1) + .push(") ORDER BY created_at,id LIMIT ").push_bind(limit); + let rows:Vec=q.build_query_as().fetch_all(self.connection()).await?; + for row in &rows { row.validate()?; } + Ok(rows) + } + pub async fn export_count(&mut self) -> Result { + let mut q=self.selection("count(*)::bigint AS total,count(DISTINCT id)::bigint AS ids,count(DISTINCT event_key)::bigint AS events",false); + q.push(" AND log_type IN (1,2,3,4,6)"); + let row=q.build().fetch_one(self.connection()).await?; + let count:i64=row.try_get("total")?; + if count!=row.try_get::("ids")? || count!=row.try_get::("events")? {return Err(ReadError::Unavailable);} + Ok(count) + } + pub fn time_range(&self) -> (i64,i64) { (self.filter.start_timestamp,self.filter.end_timestamp) } + + pub async fn buckets(&mut self, by_user:bool, flow:bool) -> Result> { + if self.credential.is_some() || (by_user && !self.administrator) { + return Err(ReadError::Forbidden); + } + if self.filter.start_timestamp==0 || self.filter.end_timestamp==0 + || (!self.administrator && self.filter.end_timestamp-self.filter.start_timestamp>2_592_000) + || self.filter.log_type!=0 || !self.filter.token_name.is_empty() || !self.filter.request_id.is_empty() { + return Err(ReadError::Invalid); + } + self.require_source("quota_data").await?; + let group=if flow { + if self.root {"user_id,username,node_name,token_id,token_name,use_group,model_name,channel_id,channel_name"} + else if self.administrator {"user_id,username,use_group,model_name,channel_id,channel_name"} + else {"token_id,token_name,use_group,model_name"} + } else if by_user {"user_id,username,created_at"} + else {"created_at,model_name"}; + let mut q=QueryBuilder::::new(format!("SELECT {group},CASE WHEN count(quota)=count(*) THEN sum(quota::numeric)::text END AS quota,sum(token_used::numeric)::text AS token_used,sum(count::numeric)::text AS count FROM core_history.quota_data_v1 WHERE created_at>=")); + q.push_bind(self.filter.start_timestamp).push(" AND created_at<=").push_bind(self.filter.end_timestamp); + if flow {q.push(" AND use_group<>''");} + if let Some(user)=self.user {q.push(" AND user_id=").push_bind(user);} + if let Some(account)=self.account {q.push(" AND owner_account_id=").push_bind(account);} + for (column,value) in [("username",&self.filter.username),("use_group",&self.filter.group)] { + if !value.is_empty() {q.push(format!(" AND {column}=")).push_bind(value.clone());} + } + if !self.filter.model_name.is_empty() {q.push(" AND strpos(model_name,").push_bind(self.filter.model_name.clone()).push(")>0");} + if let Some(channel)=self.filter.channel {q.push(" AND channel_id=").push_bind(channel);} + let order=if flow {format!("sum(quota::numeric) DESC NULLS LAST,{group}")} else {group.to_owned()}; + q.push(format!(" GROUP BY {group} ORDER BY {order} LIMIT 10001")); + let rows=q.build().fetch_all(self.connection()).await?; + if rows.len()>MAX_RECORDS {return Err(ReadError::Limit);} + let mut output=Vec::with_capacity(rows.len()); + for row in rows { + let mut value=serde_json::Map::new(); + for name in group.split(',') { + if ["user_id","token_id","channel_id","created_at"].contains(&name) { + value.insert(name.to_owned(),serde_json::json!(row.try_get::(name)?)); + } else { + let text:String=row.try_get(name)?; + if text.len()>1024 {return Err(ReadError::Unavailable);} + value.insert(name.to_owned(),serde_json::json!(text)); + } + } + let quota:Option=row.try_get("quota")?; + value.insert("quota".into(),serde_json::json!(quota.as_deref().map(exact).transpose()?)); + for name in ["token_used","count"] { + value.insert(name.into(),serde_json::json!(exact(&row.try_get::(name)?)?)); + } + value.insert("quota_money_available".into(),serde_json::json!(false)); + value.insert("quota_money_basis".into(),serde_json::json!("bucket_without_credit_basis")); + output.push(serde_json::Value::Object(value)); + } + Ok(output) + } +} +fn exact(value:&str) -> Result { + let value:i128=value.parse().map_err(|_| ReadError::Unavailable)?; + if value.abs()>SAFE_INTEGER {return Err(ReadError::Unavailable);} + i64::try_from(value).map_err(|_| ReadError::Unavailable) +} diff --git a/services/core/src/identity/mod.rs b/services/core/src/identity/mod.rs index adf50cc86..8a8f2690f 100644 --- a/services/core/src/identity/mod.rs +++ b/services/core/src/identity/mod.rs @@ -277,6 +277,14 @@ async fn authenticate(c: &mut PgConnection, secret: &str, write: bool) -> Result .ok_or(IdentityError::Unauthorized)?; principal(c, row).await } +/// Read authority without row locks in the caller's read-only snapshot. +/// This does not authorize financial writes or reserve money. +pub(crate) async fn authorize_read_in_transaction( + connection: &mut PgConnection, + secret: &str, +) -> Result { + authenticate(connection, secret, false).await +} /// Authenticate and hold current credential, user and account authority locks /// in the caller's transaction. Financial writes must use this same transaction. pub(crate) async fn authorize_in_transaction( diff --git a/services/core/src/lib.rs b/services/core/src/lib.rs index a7846d1ed..167250a23 100644 --- a/services/core/src/lib.rs +++ b/services/core/src/lib.rs @@ -10,7 +10,9 @@ pub mod config; pub mod console_compat; pub mod core_schema; pub mod events; +pub mod finance_export; pub mod financial_profile; +pub mod history_read; pub mod funding; pub mod http; pub mod identity; diff --git a/services/core/src/main.rs b/services/core/src/main.rs index 5a97ca5c5..62362e691 100644 --- a/services/core/src/main.rs +++ b/services/core/src/main.rs @@ -178,6 +178,9 @@ async fn run_core() -> Result<(), Box> { }; let mut routes = lmm_core::identity_http::router(native_store); if console_enabled { + routes = routes.merge(lmm_core::history_read::http::router( + lmm_core::history_read::http::HistoryApi::new(pool.clone()), + )); routes = routes.merge(lmm_core::console_compat::router( lmm_core::console_compat::ConsoleCompat::new(pool.clone(), store.clone()), )); diff --git a/services/core/tests/finance_export.rs b/services/core/tests/finance_export.rs index 539794715..78779428a 100644 --- a/services/core/tests/finance_export.rs +++ b/services/core/tests/finance_export.rs @@ -10,7 +10,7 @@ fn row(id: i64, other: &str) -> UsageExportInput<'_> { log_id: id, user_id: 7, created_at: 1700000000, log_type: 2, username: "synthetic-user", token_name: "synthetic-token", model_name: "local-model", quota: Some(500000), prompt_tokens: 5, completion_tokens: 2, use_time: 1, - is_stream: false, channel_id: 9, group: "default", other, + is_stream: false, channel_id: Some(9), group: "default", other, } } diff --git a/services/core/tests/fixtures/history_source_v1.sql b/services/core/tests/fixtures/history_source_v1.sql new file mode 100644 index 000000000..1f1bd5d54 --- /dev/null +++ b/services/core/tests/fixtures/history_source_v1.sql @@ -0,0 +1,41 @@ +-- SYNTHETIC ONLY. This is a test provider for the A04 consumer contract. +-- It is not an import plan, source qualification proof, or production schema. +CREATE SCHEMA a04_fixture; +CREATE SCHEMA core_history; +CREATE TABLE a04_fixture.contract ( + dataset text PRIMARY KEY, version integer NOT NULL DEFAULT 1, + complete boolean NOT NULL DEFAULT true, + manifest_sha256 text NOT NULL DEFAULT repeat('a',64) +); +CREATE VIEW core_history.source_contract_v1 AS SELECT * FROM a04_fixture.contract; +CREATE TABLE a04_fixture.logs ( + id bigint PRIMARY KEY, event_key text NOT NULL, user_id bigint NOT NULL, + owner_account_id bigint NOT NULL, credential_id bigint, + created_at bigint NOT NULL DEFAULT 1700000000, log_type integer NOT NULL DEFAULT 2, + username text NOT NULL DEFAULT 'synthetic-user', token_name text NOT NULL DEFAULT 'synthetic-key', + model_name text NOT NULL DEFAULT 'local-model', quota bigint, + prompt_tokens bigint NOT NULL DEFAULT 5, completion_tokens bigint NOT NULL DEFAULT 2, + use_time bigint NOT NULL DEFAULT 1, is_stream boolean NOT NULL DEFAULT false, + channel_id bigint, group_name text NOT NULL DEFAULT 'default', request_id text NOT NULL DEFAULT '', + other text NOT NULL DEFAULT '{"pricing_credits_per_usd":500000}' +); +CREATE VIEW core_history.log_records_v1 AS SELECT * FROM a04_fixture.logs; +CREATE TABLE a04_fixture.quota_data ( + id bigint PRIMARY KEY, user_id bigint NOT NULL, owner_account_id bigint NOT NULL, + username text NOT NULL, created_at bigint NOT NULL, model_name text NOT NULL, + token_id bigint NOT NULL, token_name text NOT NULL, use_group text NOT NULL, + channel_id bigint NOT NULL, channel_name text NOT NULL, node_name text NOT NULL, + quota bigint, token_used bigint NOT NULL, count bigint NOT NULL +); +CREATE VIEW core_history.quota_data_v1 AS SELECT * FROM a04_fixture.quota_data; +CREATE TABLE a04_fixture.finance_records ( + dataset text NOT NULL, id bigint NOT NULL, created_at bigint NOT NULL DEFAULT 1700000000, + record jsonb NOT NULL, PRIMARY KEY (dataset,id) +); +CREATE VIEW core_history.finance_records_v1 AS SELECT * FROM a04_fixture.finance_records; +CREATE TABLE a04_fixture.finance_options (name text PRIMARY KEY,value text NOT NULL); +CREATE VIEW core_history.finance_options_v1 AS SELECT * FROM a04_fixture.finance_options; +INSERT INTO a04_fixture.contract(dataset) VALUES + ('logs'),('token_logs'),('quota_data'),('users-balances'),('channels-pricing'),('subscription-plans'), + ('topups'),('subscription-orders'),('subscription-payment-events'),('bounty-ledger'), + ('checkins'),('redemptions'),('user-subscriptions'),('financial-options'),('effective-model-pricing'); diff --git a/services/core/tests/history_read/check.sh b/services/core/tests/history_read/check.sh new file mode 100644 index 000000000..4a2b0d49c --- /dev/null +++ b/services/core/tests/history_read/check.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Run only against a fresh CI database. Called through the private mise setup. +set -euo pipefail +cd "${GITHUB_WORKSPACE:?}/services/core" +: "${RUNNER_TEMP:?}" "${DATABASE_URL:?}" +case "$DATABASE_URL" in + postgres://postgres:postgres@127.0.0.1:*/postgres) ;; + *) printf '%s\n' 'A04 runner requires its dedicated synthetic CI database.' >&2; exit 2;; +esac +mkdir -p "$RUNNER_TEMP/a04-evidence" +evidence="$RUNNER_TEMP/a04-evidence" +export CARGO_TARGET_DIR="$RUNNER_TEMP/a04-target" +export CARGO_BUILD_JOBS=2 CARGO_INCREMENTAL=0 CARGO_PROFILE_TEST_DEBUG=0 CARGO_PROFILE_DEV_DEBUG=0 +rustc --version | tee "$evidence/rustc.txt" +cargo --version | tee "$evidence/cargo.txt" +grep -q '^rustc 1\.99\.0 ' "$evidence/rustc.txt" +grep -q '^cargo 1\.99\.0 ' "$evidence/cargo.txt" +printf '%s\n' "$(git rev-parse HEAD)" > "$evidence/source-sha.txt" +# First integration pass resolves the new exact dependencies. The resulting +# lockfile and formatted source are retained for review; no CI write token. +cargo metadata --format-version 1 >/dev/null +files=(src/history_read.rs src/history_read/*.rs src/finance_export.rs src/finance_export/*.rs tests/history_read.rs tests/finance_export.rs tests/history_repository.rs) +rustfmt --edition 2024 --config skip_children=true "${files[@]}" +git diff --binary -- Cargo.lock "${files[@]}" > "$evidence/generated.patch" +cp Cargo.lock "$evidence/Cargo.lock" +tar -czf "$evidence/checked-source.tar.gz" Cargo.toml Cargo.lock src tests schema build.rs +set +e +cargo test --locked --test history_read --test finance_export --test history_repository --test financial_profile --no-fail-fast 2>&1 | tee "$evidence/tests.log" +status=${PIPESTATUS[0]} +printf '%s\n' "$status" > "$evidence/test-exit-code.txt" +exit "$status" diff --git a/services/core/tests/history_repository.rs b/services/core/tests/history_repository.rs new file mode 100644 index 000000000..71ee7856d --- /dev/null +++ b/services/core/tests/history_repository.rs @@ -0,0 +1,218 @@ +//! Real PostgreSQL and stored credentials. The only synthetic boundary is the +//! offline A03 source-view provider, never an always-allow authorization mock. +use lmm_core::{accounts::{Account,AccountKind},core_schema,funding::FundingPolicy, + identity::{IdentityStore,IssuedCredential},history_read::{http::{self,HistoryApi}, + repository::{Audience,Filter,HistoryRead,ReadError}},finance_export::archive}; +use axum::{body::{Body,to_bytes},http::{Request,StatusCode}}; +use serde_json::{Value,json}; +use sqlx::PgPool; +use std::io::{Cursor,Read}; +use tower::ServiceExt; +#[path="common/runtime.rs"] +mod runtime; + +struct Fixture { + owner:PgPool, runtime:runtime::RuntimeRole, session:IssuedCredential, + other:IssuedCredential, admin:IssuedCredential, root:IssuedCredential, + key:IssuedCredential, second_key:IssuedCredential, account:i64, +} +impl Fixture { + async fn new(pool:PgPool)->Self { + core_schema::init(&pool).await.unwrap(); + let store=IdentityStore::from_pool(pool.clone()); + let session=store.bootstrap_user(7,1).await.unwrap(); + let other=store.bootstrap_user(8,1).await.unwrap(); + let admin=store.bootstrap_user(9,5).await.unwrap(); + let root=store.bootstrap_user(10,6).await.unwrap(); + let account=store.authorize(&session.secret).await.unwrap().owner_account_id; + let key=store.issue_key(&session.secret,Account {id:7,kind:AccountKind::Personal},FundingPolicy::new_personal_key(),3600).await.unwrap(); + let second_key=store.issue_key(&session.secret,Account {id:7,kind:AccountKind::Personal},FundingPolicy::new_personal_key(),3600).await.unwrap(); + sqlx::raw_sql(include_str!("fixtures/history_source_v1.sql")).execute(&pool).await.unwrap(); + let runtime=runtime::RuntimeRole::create(&pool).await; + sqlx::raw_sql(&format!("GRANT USAGE ON SCHEMA core_history TO {0}; GRANT SELECT ON ALL TABLES IN SCHEMA core_history TO {0}",runtime.name)) + .execute(&pool).await.unwrap(); + let this=Self {owner:pool,runtime,session,other,admin,root,key,second_key,account}; + this.insert(1,7,this.key.id,Some(500000),r#"{"pricing_credits_per_usd":500000,"api_key":"SYNTHETIC_SECRET"}"#).await; + this.insert(2,7,this.second_key.id,Some(1000000),r#"{"pricing_credits_per_usd":1000000}"#).await; + this.insert(3,8,this.other.id,Some(99000000),"{}").await; + this + } + async fn insert(&self,id:i64,user:i64,key:i64,quota:Option,other:&str) { + let account:i64=sqlx::query_scalar("SELECT personal_account_id FROM core_identity.users WHERE id=$1").bind(user).fetch_one(&self.owner).await.unwrap(); + sqlx::query("INSERT INTO a04_fixture.logs(id,event_key,user_id,owner_account_id,credential_id,quota,other,channel_id) VALUES($1,$2,$3,$4,$5,$6,$7,99)") + .bind(id).bind(format!("event-{id}")).bind(user).bind(account).bind(key).bind(quota).bind(other).execute(&self.owner).await.unwrap(); + } + async fn request(&self,path:&str,secret:&str)->(StatusCode,Value) { + let response=http::router(HistoryApi::new(self.runtime.pool.clone())).oneshot(Request::builder().uri(path).header("authorization",format!("Bearer {secret}")).body(Body::empty()).unwrap()).await.unwrap(); + let status=response.status(); + assert_eq!(response.headers()["cache-control"],"no-store"); + assert_eq!(response.headers()["x-content-type-options"],"nosniff"); + let bytes=to_bytes(response.into_body(),8*1024*1024).await.unwrap(); + (status,serde_json::from_slice(&bytes).unwrap()) + } + async fn cleanup(self) {self.runtime.cleanup().await;} +} +fn range()->Filter {Filter {start_timestamp:1_699_999_900,end_timestamp:1_700_000_100,..Filter::default()}} + +#[sqlx::test(migrations=false)] +async fn real_sessions_and_keys_never_broaden_history_scope(pool:PgPool) { + let f=Fixture::new(pool).await; + for path in ["/api/log/self?user_id=8","/api/log/","/api/log/stat","/api/data/users?start_timestamp=1&end_timestamp=2","/api/finance/export"] { + assert_eq!(f.request(path,&f.session.secret).await.0,StatusCode::FORBIDDEN,"{path}"); + } + for path in ["/api/log/self","/api/log/stat","/api/finance/export"] { + assert_eq!(f.request(path,&f.key.secret).await.0,StatusCode::FORBIDDEN,"{path}"); + } + let (status,body)=f.request("/api/log/token",&f.key.secret).await; + assert_eq!(status,StatusCode::OK);assert_eq!(body["data"]["total"],1); + assert_eq!(body["data"]["items"][0]["id"],1); + assert!(body["data"]["items"][0].get("channel_id").is_none()); + let (_,body)=f.request("/api/log/self",&f.session.secret).await; + assert_eq!(body["data"]["total"],2); + let text=body.to_string(); + assert!(!text.contains("SYNTHETIC_SECRET") && !text.contains("credential_id") && !text.contains("event_key")); + let (_,body)=f.request("/api/log/?user_id=8",&f.admin.secret).await; + assert_eq!(body["data"]["total"],1);assert_eq!(body["data"]["items"][0]["user_id"],8); + f.cleanup().await; +} + +#[sqlx::test(migrations=false)] +async fn count_page_money_and_authority_share_a_single_read_snapshot(pool:PgPool) { + let f=Fixture::new(pool).await; + let before:i64=sqlx::query_scalar("SELECT count(*) FROM core_identity.audit").fetch_one(&f.owner).await.unwrap(); + let mut read=HistoryRead::begin(&f.runtime.pool,&f.session.secret,Audience::SelfUser,Filter::default()).await.unwrap(); + assert_eq!(read.logs().await.unwrap().total,2); + f.insert(4,7,f.key.id,Some(500000),r#"{"pricing_credits_per_usd":500000}"#).await; + sqlx::query("UPDATE core_identity.credentials SET revoked_at=clock_timestamp() WHERE id=$1").bind(f.session.id).execute(&f.owner).await.unwrap(); + let page=read.logs().await.unwrap();let stat=read.statistics().await.unwrap(); + assert_eq!(page.total,2);assert_eq!(page.items.len(),2);assert_eq!(stat.total,2); + assert_eq!(stat.quota,Some(1500000));assert_eq!(stat.projection.money.quota_usd.as_deref(),Some("2")); + read.finish().await.unwrap(); + assert_eq!(f.request("/api/log/self",&f.session.secret).await.0,StatusCode::UNAUTHORIZED); + let after:i64=sqlx::query_scalar("SELECT count(*) FROM core_identity.audit").fetch_one(&f.owner).await.unwrap(); + assert_eq!(before,after); + f.cleanup().await; +} + +#[sqlx::test(migrations=false)] +async fn missing_incomplete_writable_or_duplicate_source_never_looks_empty(pool:PgPool) { + let f=Fixture::new(pool).await; + sqlx::query("UPDATE a04_fixture.contract SET complete=false WHERE dataset='logs'").execute(&f.owner).await.unwrap(); + assert_eq!(f.request("/api/log/self",&f.session.secret).await.0,StatusCode::SERVICE_UNAVAILABLE); + sqlx::query("UPDATE a04_fixture.contract SET complete=true WHERE dataset='logs'").execute(&f.owner).await.unwrap(); + sqlx::query("UPDATE a04_fixture.logs SET event_key='event-1' WHERE id=2").execute(&f.owner).await.unwrap(); + assert_eq!(f.request("/api/log/self",&f.session.secret).await.0,StatusCode::SERVICE_UNAVAILABLE); + sqlx::query("UPDATE a04_fixture.logs SET event_key='event-2' WHERE id=2").execute(&f.owner).await.unwrap(); + sqlx::raw_sql(&format!("GRANT UPDATE ON core_history.log_records_v1 TO {}",f.runtime.name)).execute(&f.owner).await.unwrap(); + assert_eq!(f.request("/api/log/self",&f.session.secret).await.0,StatusCode::SERVICE_UNAVAILABLE); + sqlx::raw_sql("DROP VIEW core_history.source_contract_v1").execute(&f.owner).await.unwrap(); + assert_eq!(f.request("/api/log/self",&f.session.secret).await.0,StatusCode::SERVICE_UNAVAILABLE); + f.cleanup().await; +} + +#[sqlx::test(migrations=false)] +async fn metadata_nulls_and_refunds_are_not_silently_repriced(pool:PgPool) { + let f=Fixture::new(pool).await; + f.insert(4,7,f.key.id,None,"{}").await; + f.insert(5,7,f.key.id,Some(-7),"{}").await; + sqlx::query("UPDATE a04_fixture.logs SET log_type=6 WHERE id=5").execute(&f.owner).await.unwrap(); + let (_,body)=f.request("/api/log/self/stat",&f.session.secret).await; + assert_eq!(body["data"]["total"],3); + assert_eq!(body["data"]["quota_money_available"],false); + assert!(body["data"].get("quota_usd").is_none()); + assert!(body["data"]["quota"].is_null()); + assert_eq!(body["data"]["unpriced_quota_records"],1); + let (_,page)=f.request("/api/log/self?type=6",&f.session.secret).await; + assert_eq!(page["data"]["items"][0]["quota"],-7); + assert_eq!(page["data"]["items"][0]["quota_money_basis"],"not_consumption"); + f.cleanup().await; +} + +#[sqlx::test(migrations=false)] +async fn team_filter_requires_membership_and_does_not_broaden_key_owner(pool:PgPool) { + let f=Fixture::new(pool).await; + let store=IdentityStore::from_pool(f.owner.clone()); + let team=store.create_team(&f.other.secret).await.unwrap(); + let path=format!("/api/log/self?team_id={team}"); + assert_eq!(f.request(&path,&f.session.secret).await.0,StatusCode::FORBIDDEN); + assert_eq!(f.request(&format!("/api/log/token?team_id={team}"),&f.key.secret).await.0,StatusCode::FORBIDDEN); + // The owner may select the team, but cannot read unrelated personal rows. + let (_,body)=f.request(&path,&f.other.secret).await; + assert_eq!(body["data"]["total"],0); + f.cleanup().await; +} + +#[sqlx::test(migrations=false)] +async fn bucket_and_flow_preserve_raw_quota_and_redact_admin_dimensions(pool:PgPool) { + let f=Fixture::new(pool).await; + sqlx::query("INSERT INTO a04_fixture.quota_data VALUES(1,7,$1,'synthetic-user',1700000000,'model',1,'key','default',99,'channel','node',500000,7,1)") + .bind(f.account).execute(&f.owner).await.unwrap(); + for path in ["/api/data/self","/api/data/flow/self"] { + let (_,body)=f.request(&format!("{path}?start_timestamp=1699999999&end_timestamp=1700000001"),&f.session.secret).await; + let row=&body["data"][0];assert_eq!(row["quota"],500000);assert_eq!(row["quota_money_available"],false); + assert!(row.get("channel_id").is_none() && row.get("node_name").is_none()); + } + let (_,body)=f.request("/api/data/flow?start_timestamp=1699999999&end_timestamp=1700000001",&f.admin.secret).await; + assert_eq!(body["data"][0]["channel_id"],99);assert!(body["data"][0].get("node_name").is_none()); + let (_,body)=f.request("/api/data/flow?start_timestamp=1699999999&end_timestamp=1700000001",&f.root.secret).await; + assert_eq!(body["data"][0]["node_name"],"node"); + f.cleanup().await; +} + +#[sqlx::test(migrations=false)] +async fn query_parameters_are_values_not_sql_or_authority(pool:PgPool) { + let f=Fixture::new(pool).await; + let (status,body)=f.request("/api/log/self?model_name=%27%20OR%201%3D1--",&f.session.secret).await; + assert_eq!(status,StatusCode::OK);assert_eq!(body["data"]["total"],0); + for path in ["/api/log/self?start_timestamp=-1","/api/log/self?start_timestamp=2&end_timestamp=1","/api/log/self?type=99","/api/log/self?role=admin"] { + assert_eq!(f.request(path,&f.session.secret).await.0,StatusCode::BAD_REQUEST,"{path}"); + } + assert_eq!(f.request("/api/log/self?p=9223372036854775807",&f.session.secret).await.0,StatusCode::PAYLOAD_TOO_LARGE); + f.cleanup().await; +} + +#[sqlx::test(migrations=false)] +async fn complete_zip_is_readable_redacted_and_uses_a_single_snapshot(pool:PgPool) { + let f=Fixture::new(pool).await; + sqlx::query("INSERT INTO a04_fixture.finance_records(dataset,id,record) VALUES ('topups',1,$1),('channels-pricing',1,$2)") + .bind(json!({"topup_id":1,"user_id":7,"settled_amount_micros":1000000,"api_key":"SYNTHETIC_SECRET"})) + .bind(json!({"channel_id":99,"base_url":"https://user:secret@example.test/path?token=secret#secret","channel_key":"SYNTHETIC_SECRET"})) + .execute(&f.owner).await.unwrap(); + let mut read=HistoryRead::begin(&f.runtime.pool,&f.admin.secret,Audience::Administrator,range()).await.unwrap(); + sqlx::query("UPDATE a04_fixture.finance_records SET record=record||'{\"settled_amount_micros\":2000000}'::jsonb WHERE dataset='topups'").execute(&f.owner).await.unwrap(); + let bytes=archive::build(&mut read,1024*1024).await.unwrap(); + read.finish().await.unwrap(); + let mut zip=zip::ZipArchive::new(Cursor::new(bytes)).unwrap(); + assert_eq!(zip.len(),15); + for index in 0..zip.len() { + let mut file=zip.by_index(index).unwrap();let mut text=String::new();file.read_to_string(&mut text).unwrap(); + assert!(!text.contains("SYNTHETIC_SECRET")); + let value:Value=serde_json::from_str(&text).unwrap(); + if file.name()=="topups.json" {assert_eq!(value[0]["settled_amount_micros"],1000000);} + if file.name()=="channels-pricing.json" {assert_eq!(value[0]["base_url"],"https://example.test");} + } + f.cleanup().await; +} + +#[sqlx::test(migrations=false)] +async fn archive_refuses_missing_documents_limits_and_runtime_writes(pool:PgPool) { + let f=Fixture::new(pool).await; + let mut read=HistoryRead::begin(&f.runtime.pool,&f.admin.secret,Audience::Administrator,range()).await.unwrap(); + assert!(matches!(archive::build(&mut read,1024).await,Err(ReadError::Limit))); + read.finish().await.unwrap(); + sqlx::query("DELETE FROM a04_fixture.contract WHERE dataset='topups'").execute(&f.owner).await.unwrap(); + let mut read=HistoryRead::begin(&f.runtime.pool,&f.admin.secret,Audience::Administrator,range()).await.unwrap(); + assert_eq!(archive::build(&mut read,1024*1024).await.unwrap_err(),ReadError::Unavailable); + let error=sqlx::query("UPDATE core_identity.users SET active=false WHERE id=7").execute(read.connection()).await.unwrap_err(); + assert_eq!(error.as_database_error().unwrap().code().as_deref(),Some("25006")); + read.finish().await.unwrap(); + assert!(IdentityStore::from_pool(f.owner.clone()).authorize(&f.session.secret).await.is_ok()); + f.cleanup().await; +} + +#[test] +fn finance_field_gate_never_serializes_nested_opaque_payloads() { + let topup=archive::DATASETS.iter().find(|d| d.name=="topups").unwrap(); + assert_eq!(archive::project_record(topup,&json!({"topup_id":1,"provider_payload":{"secret":"x"}})).unwrap(),json!({"topup_id":1})); + assert_eq!(archive::project_record(topup,&json!({"money":{"secret":"x"}})).unwrap_err(),ReadError::Unavailable); +} From 2fe46263bd9813c0a61eb93cfe771dd4bd5268d6 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 12 Oct 2026 18:49:21 +0800 Subject: [PATCH 3/6] fix(core): preserve finance export limit and IO error categories Keep byte limits through JSON and ZIP layers instead of converting them into missing-source errors. Retain real IO/source failures and poison incomplete usage exports. Follow up on PR #875 failing PG limit case. --- services/core/src/finance_export.rs | 74 ++- services/core/src/finance_export/archive.rs | 480 ++++++++++++++------ 2 files changed, 410 insertions(+), 144 deletions(-) diff --git a/services/core/src/finance_export.rs b/services/core/src/finance_export.rs index 09ab1c67d..85b48e669 100644 --- a/services/core/src/finance_export.rs +++ b/services/core/src/finance_export.rs @@ -57,7 +57,11 @@ struct UsageExportRow<'a> { impl<'a> From<&UsageExportInput<'a>> for UsageExportRow<'a> { fn from(row: &UsageExportInput<'a>) -> Self { Self { - money: project_log(LogQuotaInput { quota: row.quota, log_type: row.log_type, other: row.other }), + money: project_log(LogQuotaInput { + quota: row.quota, + log_type: row.log_type, + other: row.other, + }), log_id: row.log_id, user_id: row.user_id, created_at: row.created_at, @@ -88,6 +92,17 @@ pub enum ExportError { Aborted, } +impl From for ExportError { + fn from(error: io::Error) -> Self { + match error.kind() { + io::ErrorKind::FileTooLarge + | io::ErrorKind::StorageFull + | io::ErrorKind::QuotaExceeded => Self::ByteLimit, + _ => Self::Io, + } + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct ExportReceipt { pub rows: usize, @@ -112,11 +127,22 @@ pub struct UsageJsonWriter { impl UsageJsonWriter { pub fn new(mut writer: W, max_rows: usize, max_bytes: usize) -> Result { - if max_rows == 0 || max_rows > MAX_EXPORT_ROWS || !(2..=MAX_EXPORT_BYTES).contains(&max_bytes) { + if max_rows == 0 + || max_rows > MAX_EXPORT_ROWS + || !(2..=MAX_EXPORT_BYTES).contains(&max_bytes) + { return Err(ExportError::InvalidLimits); } - writer.write_all(b"[").map_err(|_| ExportError::Io)?; - Ok(Self { writer, max_rows, max_bytes, rows: 0, bytes: 1, last_order: None, failed: false }) + writer.write_all(b"[").map_err(ExportError::from)?; + Ok(Self { + writer, + max_rows, + max_bytes, + rows: 0, + bytes: 1, + last_order: None, + failed: false, + }) } pub fn push(&mut self, row: &UsageExportInput<'_>) -> Result<(), ExportError> { @@ -134,9 +160,13 @@ impl UsageJsonWriter { if self.rows == self.max_rows { return Err(ExportError::RowLimit); } - if row.log_id <= 0 || row.user_id <= 0 || row.created_at < 0 + if row.log_id <= 0 + || row.user_id <= 0 + || row.created_at < 0 || ![1, 2, 3, 4, 6].contains(&row.log_type) - || [row.username, row.token_name, row.model_name, row.group].into_iter().any(|value| value.len() > MAX_TEXT_BYTES) + || [row.username, row.token_name, row.model_name, row.group] + .into_iter() + .any(|value| value.len() > MAX_TEXT_BYTES) { return Err(ExportError::InvalidRow); } @@ -146,9 +176,14 @@ impl UsageJsonWriter { } let mut encoded = BoundedRow { bytes: Vec::new() }; serde_json::to_writer(&mut encoded, &UsageExportRow::from(row)) - .map_err(|_| ExportError::Encoding)?; + .map_err(|error| match error.io_error_kind() { + Some(kind) => ExportError::from(io::Error::from(kind)), + None => ExportError::Encoding, + })?; let separator = usize::from(self.rows > 0); - let required = self.bytes.checked_add(separator) + let required = self + .bytes + .checked_add(separator) .and_then(|value| value.checked_add(encoded.bytes.len())) .and_then(|value| value.checked_add(1)) // Reserve the closing bracket. .ok_or(ExportError::ByteLimit)?; @@ -156,9 +191,11 @@ impl UsageJsonWriter { return Err(ExportError::ByteLimit); } if separator != 0 { - self.writer.write_all(b",").map_err(|_| ExportError::Io)?; + self.writer.write_all(b",").map_err(ExportError::from)?; } - self.writer.write_all(&encoded.bytes).map_err(|_| ExportError::Io)?; + self.writer + .write_all(&encoded.bytes) + .map_err(ExportError::from)?; self.bytes = required - 1; self.rows += 1; self.last_order = Some(order); @@ -171,9 +208,15 @@ impl UsageJsonWriter { if self.failed { return Err(ExportError::Aborted); } - self.writer.write_all(b"]").map_err(|_| ExportError::Io)?; - self.writer.flush().map_err(|_| ExportError::Io)?; - Ok((self.writer, ExportReceipt { rows: self.rows, bytes: self.bytes + 1 })) + self.writer.write_all(b"]").map_err(ExportError::from)?; + self.writer.flush().map_err(ExportError::from)?; + Ok(( + self.writer, + ExportReceipt { + rows: self.rows, + bytes: self.bytes + 1, + }, + )) } } @@ -184,7 +227,10 @@ struct BoundedRow { impl Write for BoundedRow { fn write(&mut self, data: &[u8]) -> io::Result { if data.len() > MAX_EXPORT_ROW_BYTES.saturating_sub(self.bytes.len()) { - return Err(io::Error::other("finance export row limit")); + return Err(io::Error::new( + io::ErrorKind::FileTooLarge, + "finance export row limit", + )); } self.bytes.extend_from_slice(data); Ok(data.len()) diff --git a/services/core/src/finance_export/archive.rs b/services/core/src/finance_export/archive.rs index fcf813221..b1baf2f9f 100644 --- a/services/core/src/finance_export/archive.rs +++ b/services/core/src/finance_export/archive.rs @@ -1,192 +1,412 @@ //! Complete ZIP assembly, bounded across every document. Uses only the //! authenticated snapshot and fixed document/field names. Never downloads //! upstream data or fetches current FX to value historical records. -use super::{MAX_EXPORT_BYTES,MAX_EXPORT_ROWS,UsageExportInput,UsageJsonWriter}; -use crate::history_read::repository::{HistoryRead,ReadError,Result}; -use serde_json::{Value,json}; +use super::{ExportError, MAX_EXPORT_BYTES, MAX_EXPORT_ROWS, UsageExportInput, UsageJsonWriter}; +use crate::history_read::repository::{HistoryRead, ReadError, Result}; +use serde_json::{Value, json}; use sqlx::Row; -use std::{collections::BTreeMap,io::{self,Cursor,Seek,SeekFrom,Write}}; -use zip::{ZipWriter,write::SimpleFileOptions,CompressionMethod}; +use std::{ + collections::BTreeMap, + io::{self, Cursor, Seek, SeekFrom, Write}, +}; +use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; /// Unknown source fields never enter an exported record. These are the main /// finance DTO field names, not the underlying tables' entire JSON records. -pub struct Dataset { pub name:&'static str, pub fields:&'static str, pub timed:bool } -pub const DATASETS:&[Dataset]=&[ - Dataset {name:"users-balances",fields:"user_id username role status group quota used_quota affiliate_quota affiliate_history_quota request_count created_at last_api_activity_at trust_level_override normalized_used_quota usage_projection_available usage_projection_reason effective_group_ratio effective_topup_group_ratio",timed:false}, - Dataset {name:"channels-pricing",fields:"channel_id type status name weight created_time test_time response_time base_url balance balance_updated_time models model_mapping group used_quota priority auto_ban tag",timed:false}, - Dataset {name:"subscription-plans",fields:"plan_id title subtitle price_amount currency duration_unit duration_value custom_seconds enabled sort_order allow_balance_pay allow_wallet_overflow max_purchase_per_user upgrade_group downgrade_group total_amount quota_reset_period quota_reset_custom_seconds created_at updated_at",timed:false}, - Dataset {name:"topups",fields:"topup_id user_id amount credited_quota expected_amount_micros settled_amount_micros settlement_currency money payment_method payment_provider create_time complete_time status",timed:true}, - Dataset {name:"subscription-orders",fields:"order_id user_id plan_id plan_price plan_currency expected_amount_micros settlement_currency payment_method payment_provider provider_product_id provider_subscription_state current_period_start current_period_end refunded_amount_micros status create_time complete_time",timed:true}, - Dataset {name:"subscription-payment-events",fields:"payment_event_id subscription_order_id payment_provider settlement_currency settlement_amount_micros period_start period_end created_time",timed:true}, - Dataset {name:"bounty-ledger",fields:"ledger_id project_id challenge_id user_id counterparty_user_id kind quota note created_at",timed:true}, - Dataset {name:"checkins",fields:"checkin_id user_id checkin_date quota_awarded created_at",timed:true}, - Dataset {name:"redemptions",fields:"redemption_id created_by_user_id status name quota created_time redeemed_time used_user_id expired_time",timed:false}, - Dataset {name:"user-subscriptions",fields:"user_subscription_id user_id plan_id amount_total amount_used start_time end_time status source last_reset_time next_reset_time upgrade_group prev_user_group downgrade_group allow_wallet_overflow created_at updated_at",timed:false}, +pub struct Dataset { + pub name: &'static str, + pub fields: &'static str, + pub timed: bool, +} +pub const DATASETS: &[Dataset] = &[ + Dataset { + name: "users-balances", + fields: "user_id username role status group quota used_quota affiliate_quota affiliate_history_quota request_count created_at last_api_activity_at trust_level_override normalized_used_quota usage_projection_available usage_projection_reason effective_group_ratio effective_topup_group_ratio", + timed: false, + }, + Dataset { + name: "channels-pricing", + fields: "channel_id type status name weight created_time test_time response_time base_url balance balance_updated_time models model_mapping group used_quota priority auto_ban tag", + timed: false, + }, + Dataset { + name: "subscription-plans", + fields: "plan_id title subtitle price_amount currency duration_unit duration_value custom_seconds enabled sort_order allow_balance_pay allow_wallet_overflow max_purchase_per_user upgrade_group downgrade_group total_amount quota_reset_period quota_reset_custom_seconds created_at updated_at", + timed: false, + }, + Dataset { + name: "topups", + fields: "topup_id user_id amount credited_quota expected_amount_micros settled_amount_micros settlement_currency money payment_method payment_provider create_time complete_time status", + timed: true, + }, + Dataset { + name: "subscription-orders", + fields: "order_id user_id plan_id plan_price plan_currency expected_amount_micros settlement_currency payment_method payment_provider provider_product_id provider_subscription_state current_period_start current_period_end refunded_amount_micros status create_time complete_time", + timed: true, + }, + Dataset { + name: "subscription-payment-events", + fields: "payment_event_id subscription_order_id payment_provider settlement_currency settlement_amount_micros period_start period_end created_time", + timed: true, + }, + Dataset { + name: "bounty-ledger", + fields: "ledger_id project_id challenge_id user_id counterparty_user_id kind quota note created_at", + timed: true, + }, + Dataset { + name: "checkins", + fields: "checkin_id user_id checkin_date quota_awarded created_at", + timed: true, + }, + Dataset { + name: "redemptions", + fields: "redemption_id created_by_user_id status name quota created_time redeemed_time used_user_id expired_time", + timed: false, + }, + Dataset { + name: "user-subscriptions", + fields: "user_subscription_id user_id plan_id amount_total amount_used start_time end_time status source last_reset_time next_reset_time upgrade_group prev_user_group downgrade_group allow_wallet_overflow created_at updated_at", + timed: false, + }, +]; +const OPTIONS: &[&str] = &[ + "ModelPrice", + "ModelRatio", + "CacheRatio", + "CreateCacheRatio", + "CompletionRatio", + "ImageRatio", + "AudioRatio", + "AudioCompletionRatio", + "GroupRatio", + "GroupGroupRatio", + "TopupGroupRatio", + "UserUsableGroups", + "QuotaPerUnit", + "Price", + "USDExchangeRate", + "MinTopUp", + "DataExportInterval", + "payment_setting.amount_options", + "payment_setting.amount_discount", + "tool_price_setting.prices", + "billing_setting.billing_mode", + "billing_setting.billing_expr", + "checkin_setting.enabled", + "checkin_setting.min_quota", + "checkin_setting.max_quota", + "checkin_setting.level_multipliers", ]; -const OPTIONS:&[&str]=&["ModelPrice","ModelRatio","CacheRatio","CreateCacheRatio","CompletionRatio","ImageRatio","AudioRatio","AudioCompletionRatio","GroupRatio","GroupGroupRatio","TopupGroupRatio","UserUsableGroups","QuotaPerUnit","Price","USDExchangeRate","MinTopUp","DataExportInterval","payment_setting.amount_options","payment_setting.amount_discount","tool_price_setting.prices","billing_setting.billing_mode","billing_setting.billing_expr","checkin_setting.enabled","checkin_setting.min_quota","checkin_setting.max_quota","checkin_setting.level_multipliers"]; -const DERIVED:&[(&str,&str)]=&[("model_prices","ModelPrice"),("model_ratios","ModelRatio"),("completion_ratios","CompletionRatio"),("cache_ratios","CacheRatio"),("create_cache_ratios","CreateCacheRatio"),("image_ratios","ImageRatio"),("audio_ratios","AudioRatio"),("audio_completion_ratios","AudioCompletionRatio"),("tool_prices","tool_price_setting.prices"),("billing_modes","billing_setting.billing_mode"),("billing_expressions","billing_setting.billing_expr")]; -const EFFECTIVE_FIELDS:&str="model_name quota_type model_ratio model_price completion_ratio cache_ratio create_cache_ratio image_ratio audio_ratio audio_completion_ratio enable_groups supported_endpoint_types"; +const DERIVED: &[(&str, &str)] = &[ + ("model_prices", "ModelPrice"), + ("model_ratios", "ModelRatio"), + ("completion_ratios", "CompletionRatio"), + ("cache_ratios", "CacheRatio"), + ("create_cache_ratios", "CreateCacheRatio"), + ("image_ratios", "ImageRatio"), + ("audio_ratios", "AudioRatio"), + ("audio_completion_ratios", "AudioCompletionRatio"), + ("tool_prices", "tool_price_setting.prices"), + ("billing_modes", "billing_setting.billing_mode"), + ("billing_expressions", "billing_setting.billing_expr"), +]; +const EFFECTIVE_FIELDS: &str = "model_name quota_type model_ratio model_price completion_ratio cache_ratio create_cache_ratio image_ratio audio_ratio audio_completion_ratio enable_groups supported_endpoint_types"; -struct BoundedCursor { inner:Cursor>, maximum:u64 } +struct BoundedCursor { + inner: Cursor>, + maximum: u64, +} impl Write for BoundedCursor { - fn write(&mut self,bytes:&[u8])->io::Result { - let end=self.inner.position().checked_add(bytes.len() as u64).ok_or_else(||io::Error::other("export limit"))?; - if end>self.maximum {return Err(io::Error::other("export limit"));} + fn write(&mut self, bytes: &[u8]) -> io::Result { + let end = self + .inner + .position() + .checked_add(bytes.len() as u64) + .ok_or_else(|| io::Error::new(io::ErrorKind::FileTooLarge, "export limit"))?; + if end > self.maximum { + return Err(io::Error::new(io::ErrorKind::FileTooLarge, "export limit")); + } self.inner.write(bytes) } - fn flush(&mut self)->io::Result<()> {Ok(())} + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } } impl Seek for BoundedCursor { - fn seek(&mut self,to:SeekFrom)->io::Result { - let position=match to {SeekFrom::Start(n)=>i128::from(n),SeekFrom::Current(n)=>i128::from(self.inner.position())+i128::from(n),SeekFrom::End(n)=>self.inner.get_ref().len() as i128+i128::from(n)}; - if !(0..=i128::from(self.maximum)).contains(&position) {return Err(io::Error::other("export limit"));} + fn seek(&mut self, to: SeekFrom) -> io::Result { + let position = match to { + SeekFrom::Start(n) => i128::from(n), + SeekFrom::Current(n) => i128::from(self.inner.position()) + i128::from(n), + SeekFrom::End(n) => self.inner.get_ref().len() as i128 + i128::from(n), + }; + if position < 0 { + return Err(io::Error::from(io::ErrorKind::InvalidInput)); + } + if position > i128::from(self.maximum) { + return Err(io::Error::new(io::ErrorKind::FileTooLarge, "export limit")); + } self.inner.seek(SeekFrom::Start(position as u64)) } } -fn start(zip:&mut ZipWriter,name:&str)->Result<()> { - zip.start_file(format!("{name}.json"),SimpleFileOptions::default().compression_method(CompressionMethod::Stored).unix_permissions(0o600)) - .map_err(|_|ReadError::Limit) +fn export_error(error: ExportError) -> ReadError { + match error { + ExportError::RowLimit | ExportError::ByteLimit => ReadError::Limit, + ExportError::InvalidLimits => ReadError::Invalid, + _ => ReadError::Unavailable, + } +} +fn io_error(error: io::Error) -> ReadError { + export_error(ExportError::from(error)) } -fn write_json(zip:&mut ZipWriter,value:&Value)->Result<()> { - serde_json::to_writer(zip,value).map_err(|_|ReadError::Limit) +fn zip_error(error: zip::result::ZipError) -> ReadError { + match error { + zip::result::ZipError::Io(error) => io_error(error), + _ => ReadError::Unavailable, + } +} +fn start(zip: &mut ZipWriter, name: &str) -> Result<()> { + zip.start_file( + format!("{name}.json"), + SimpleFileOptions::default() + .compression_method(CompressionMethod::Stored) + .unix_permissions(0o600), + ) + .map_err(zip_error) +} +fn write_json(zip: &mut ZipWriter, value: &Value) -> Result<()> { + serde_json::to_writer(zip, value).map_err(|error| match error.io_error_kind() { + Some(kind) => io_error(io::Error::from(kind)), + None => ReadError::Unavailable, + }) } /// Strip every non-allowlisted field before serializing; reject structured -/// values in scalar fields instead of exporting a nested provider payload. -pub fn project_record(dataset:&Dataset,record:&Value)->Result { - let object=record.as_object().ok_or(ReadError::Unavailable)?; - let mut output=serde_json::Map::new(); +/// values in scalar fields instead of exporting a nested provider payloads. +pub fn project_record(dataset: &Dataset, record: &Value) -> Result { + let object = record.as_object().ok_or(ReadError::Unavailable)?; + let mut output = serde_json::Map::new(); for key in dataset.fields.split_whitespace() { - let Some(value)=object.get(key) else {continue;}; + let Some(value) = object.get(key) else { + continue; + }; match value { - Value::Null|Value::Bool(_)=>{}, - Value::Number(n) if n.to_string().len()<=64=>{}, - Value::String(text) if text.len()<=4096=>{}, - Value::Array(items) if dataset.name=="effective-model-pricing" - && matches!(key,"enable_groups"|"supported_endpoint_types") - && items.len()<=64 && items.iter().all(|item|item.as_str().is_some_and(|s|s.len()<=256))=>{}, - _=>return Err(ReadError::Unavailable), + Value::Null | Value::Bool(_) => {} + Value::Number(n) if n.to_string().len() <= 64 => {} + Value::String(text) if text.len() <= 4096 => {} + Value::Array(items) + if dataset.name == "effective-model-pricing" + && matches!(key, "enable_groups" | "supported_endpoint_types") + && items.len() <= 64 + && items + .iter() + .all(|item| item.as_str().is_some_and(|s| s.len() <= 256)) => {} + _ => return Err(ReadError::Unavailable), } - output.insert(key.to_owned(),value.clone()); + output.insert(key.to_owned(), value.clone()); } - if dataset.name=="channels-pricing" { + if dataset.name == "channels-pricing" { // URL parsing is local only. Userinfo, paths, query and fragment never // enter the archive, including percent-encoded credential forms. - let value=output.remove("base_url").and_then(|v|v.as_str().map(str::to_owned)) - .and_then(|v|reqwest::Url::parse(&v).ok()) - .filter(|v|matches!(v.scheme(),"http"|"https") && v.host().is_some()) - .map(|v|Value::String(v.origin().ascii_serialization())); - if let Some(value)=value {output.insert("base_url".into(),value);} + let value = output + .remove("base_url") + .and_then(|v| v.as_str().map(str::to_owned)) + .and_then(|v| reqwest::Url::parse(&v).ok()) + .filter(|v| matches!(v.scheme(), "http" | "https") && v.host().is_some()) + .map(|v| Value::String(v.origin().ascii_serialization())); + if let Some(value) = value { + output.insert("base_url".into(), value); + } } - if dataset.name=="users-balances" && output.get("usage_projection_available")!=Some(&Value::Bool(true)) { + if dataset.name == "users-balances" + && output.get("usage_projection_available") != Some(&Value::Bool(true)) + { output.remove("normalized_used_quota"); - output.insert("usage_projection_available".into(),Value::Bool(false)); + output.insert("usage_projection_available".into(), Value::Bool(false)); } Ok(Value::Object(output)) } -pub async fn build(read:&mut HistoryRead<'_>,max_bytes:usize)->Result> { - if !read.administrator() {return Err(ReadError::Forbidden);} - if !(1024..=MAX_EXPORT_BYTES).contains(&max_bytes) {return Err(ReadError::Invalid);} - let (begin,end)=read.time_range(); - if begin<=0 || end31_536_000 {return Err(ReadError::Invalid);} +pub async fn build(read: &mut HistoryRead<'_>, max_bytes: usize) -> Result> { + if !read.administrator() { + return Err(ReadError::Forbidden); + } + if !(1024..=MAX_EXPORT_BYTES).contains(&max_bytes) { + return Err(ReadError::Invalid); + } + let (begin, end) = read.time_range(); + if begin <= 0 || end < begin || end - begin > 31_536_000 { + return Err(ReadError::Invalid); + } // All required datasets must be qualified. Missing datasets are not empty // arrays, and cannot silently produce a convincing partial ZIP. - for name in DATASETS.iter().map(|d|d.name).chain(["financial-options","effective-model-pricing"]) { + for name in DATASETS + .iter() + .map(|d| d.name) + .chain(["financial-options", "effective-model-pricing"]) + { read.require_source(name).await?; } - let mut zip=ZipWriter::new(BoundedCursor {inner:Cursor::new(Vec::new()),maximum:max_bytes as u64}); - let mut counts=BTreeMap::new(); - let mut total=0usize; + let mut zip = ZipWriter::new(BoundedCursor { + inner: Cursor::new(Vec::new()), + maximum: max_bytes as u64, + }); + let mut counts = BTreeMap::new(); + let mut total = 0usize; for dataset in DATASETS { - let count=write_dataset(read,&mut zip,dataset,begin,end,&mut total).await?; - counts.insert(dataset.name,count); - } - let usage_count=usize::try_from(read.export_count().await?).map_err(|_|ReadError::Limit)?; - if total.checked_add(usage_count).is_none_or(|v| v>MAX_EXPORT_ROWS) {return Err(ReadError::Limit);} - start(&mut zip,"usage-billing-records")?; - let mut usage=UsageJsonWriter::new(&mut zip,MAX_EXPORT_ROWS,max_bytes).map_err(|_|ReadError::Limit)?; - let mut after=(-1,0); - let mut seen=0; + let count = write_dataset(read, &mut zip, dataset, begin, end, &mut total).await?; + counts.insert(dataset.name, count); + } + let usage_count = usize::try_from(read.export_count().await?).map_err(|_| ReadError::Limit)?; + if total + .checked_add(usage_count) + .is_none_or(|v| v > MAX_EXPORT_ROWS) + { + return Err(ReadError::Limit); + } + start(&mut zip, "usage-billing-records")?; + let mut usage = + UsageJsonWriter::new(&mut zip, MAX_EXPORT_ROWS, max_bytes).map_err(export_error)?; + let mut after = (-1, 0); + let mut seen = 0; loop { - let rows=read.export_batch(after,64).await?; - if rows.is_empty() {break;} + let rows = read.export_batch(after, 64).await?; + if rows.is_empty() { + break; + } for row in rows { - usage.push(&UsageExportInput {log_id:row.id,user_id:row.user_id,created_at:row.created_at, - log_type:row.log_type,username:&row.username,token_name:&row.token_name,model_name:&row.model_name, - quota:row.quota,prompt_tokens:row.prompt_tokens,completion_tokens:row.completion_tokens, - use_time:row.use_time,is_stream:row.is_stream,channel_id:row.channel_id,group:&row.group_name,other:&row.other}) - .map_err(|_|ReadError::Unavailable)?; - after=(row.created_at,row.id);seen+=1; + usage + .push(&UsageExportInput { + log_id: row.id, + user_id: row.user_id, + created_at: row.created_at, + log_type: row.log_type, + username: &row.username, + token_name: &row.token_name, + model_name: &row.model_name, + quota: row.quota, + prompt_tokens: row.prompt_tokens, + completion_tokens: row.completion_tokens, + use_time: row.use_time, + is_stream: row.is_stream, + channel_id: row.channel_id, + group: &row.group_name, + other: &row.other, + }) + .map_err(export_error)?; + after = (row.created_at, row.id); + seen += 1; } } - if seen!=usage_count {return Err(ReadError::Unavailable);} - usage.finish().map_err(|_|ReadError::Limit)?; - counts.insert("usage",seen); - total+=seen; + if seen != usage_count { + return Err(ReadError::Unavailable); + } + usage.finish().map_err(export_error)?; + counts.insert("usage", seen); + total += seen; let rows=sqlx::query("SELECT name,CASE WHEN octet_length(value)<=65536 THEN value END AS value FROM core_history.finance_options_v1 WHERE name=ANY($1) ORDER BY name LIMIT 64") .bind(OPTIONS).fetch_all(read.connection()).await?; - let mut options=serde_json::Map::new(); + let mut options = serde_json::Map::new(); for row in rows { - let name:String=row.try_get("name")?; - let value:String=row.try_get("value")?; - if value.len()>65_536 || options.insert(name,Value::String(value)).is_some() {return Err(ReadError::Unavailable);} - } - counts.insert("financial-options",options.len()); - start(&mut zip,"financial-options")?; - write_json(&mut zip,&Value::Object(options.clone()))?; - let mut derived=serde_json::Map::new(); - for (field,key) in DERIVED { - let value=options.get(*key).and_then(Value::as_str).map(serde_json::from_str::).transpose().map_err(|_|ReadError::Unavailable)?; - derived.insert((*field).into(),value.unwrap_or(Value::Null)); - } - start(&mut zip,"model-prices-and-ratios")?; - write_json(&mut zip,&Value::Object(derived))?; + let name: String = row.try_get("name")?; + let value: String = row.try_get("value")?; + if value.len() > 65_536 || options.insert(name, Value::String(value)).is_some() { + return Err(ReadError::Unavailable); + } + } + counts.insert("financial-options", options.len()); + start(&mut zip, "financial-options")?; + write_json(&mut zip, &Value::Object(options.clone()))?; + let mut derived = serde_json::Map::new(); + for (field, key) in DERIVED { + let value = options + .get(*key) + .and_then(Value::as_str) + .map(serde_json::from_str::) + .transpose() + .map_err(|_| ReadError::Unavailable)?; + derived.insert((*field).into(), value.unwrap_or(Value::Null)); + } + start(&mut zip, "model-prices-and-ratios")?; + write_json(&mut zip, &Value::Object(derived))?; // Effective prices use a separate source contract; they are not a fallback // for missing historical money. Reuse the same field whitelist gate. - let effective=Dataset {name:"effective-model-pricing",fields:EFFECTIVE_FIELDS,timed:false}; - let effective_count=write_dataset(read,&mut zip,&effective,begin,end,&mut total).await?; - counts.insert("effective-model-pricing",effective_count); - start(&mut zip,"manifest")?; - let generated:i64=sqlx::query_scalar("SELECT extract(epoch FROM transaction_timestamp())::bigint").fetch_one(read.connection()).await?; - write_json(&mut zip,&json!({"schema_version":"lmm-finance-export/v1","generated_at":generated, + let effective = Dataset { + name: "effective-model-pricing", + fields: EFFECTIVE_FIELDS, + timed: false, + }; + let effective_count = write_dataset(read, &mut zip, &effective, begin, end, &mut total).await?; + counts.insert("effective-model-pricing", effective_count); + start(&mut zip, "manifest")?; + let generated: i64 = + sqlx::query_scalar("SELECT extract(epoch FROM transaction_timestamp())::bigint") + .fetch_one(read.connection()) + .await?; + write_json( + &mut zip, + &json!({"schema_version":"lmm-finance-export/v1","generated_at":generated, "start_timestamp":begin,"end_timestamp":end,"rows":counts,"truncated":{}, "redactions":["credentials, provider payloads, IPs, raw requests, opaque log metadata, URL credentials and query strings"], - "notes":["Raw quota is unchanged. Unknown historical money is not zero.","All datasets use one authorized read-only snapshot. Current pricing is not historical evidence."]}))?; - let output=zip.finish().map_err(|_|ReadError::Limit)?.inner.into_inner(); + "notes":["Raw quota is unchanged. Unknown historical money is not zero.","All datasets use one authorized read-only snapshot. Current pricing is not historical evidence."]}), + )?; + let output = zip + .finish() + .map_err(zip_error)? + .inner + .into_inner(); Ok(output) } -async fn write_dataset(read:&mut HistoryRead<'_>,zip:&mut ZipWriter,dataset:&Dataset, - begin:i64,end:i64,total:&mut usize)->Result { +async fn write_dataset( + read: &mut HistoryRead<'_>, + zip: &mut ZipWriter, + dataset: &Dataset, + begin: i64, + end: i64, + total: &mut usize, +) -> Result { let count=sqlx::query("SELECT count(*)::bigint AS n,count(DISTINCT id)::bigint AS ids FROM core_history.finance_records_v1 WHERE dataset=$1 AND (NOT $4 OR created_at BETWEEN $2 AND $3)") .bind(dataset.name).bind(begin).bind(end).bind(dataset.timed).fetch_one(read.connection()).await?; - let n:i64=count.try_get("n")?; - if n!=count.try_get::("ids")? {return Err(ReadError::Unavailable);} - let expected=usize::try_from(n).map_err(|_|ReadError::Limit)?; - if total.checked_add(expected).is_none_or(|n|n>MAX_EXPORT_ROWS) {return Err(ReadError::Limit);} - start(zip,dataset.name)?; - zip.write_all(b"[").map_err(|_|ReadError::Limit)?; - let mut last=0i64; - let mut seen=0usize; - let fields:Vec<&str>=dataset.fields.split_whitespace().collect(); - while seen("ids")? { + return Err(ReadError::Unavailable); + } + let expected = usize::try_from(n).map_err(|_| ReadError::Limit)?; + if total + .checked_add(expected) + .is_none_or(|n| n > MAX_EXPORT_ROWS) + { + return Err(ReadError::Limit); + } + start(zip, dataset.name)?; + zip.write_all(b"[").map_err(io_error)?; + let mut last = 0i64; + let mut seen = 0usize; + let fields: Vec<&str> = dataset.fields.split_whitespace().collect(); + while seen < expected { let rows=sqlx::query("SELECT id,CASE WHEN octet_length(record::text)<=32768 THEN (SELECT jsonb_object_agg(k,record->k) FROM unnest($5::text[]) k WHERE record ? k) END AS record FROM core_history.finance_records_v1 WHERE dataset=$1 AND id>$2 AND (NOT $6 OR created_at BETWEEN $3 AND $4) ORDER BY id LIMIT 64") .bind(dataset.name).bind(last).bind(begin).bind(end).bind(&fields).bind(dataset.timed) .fetch_all(read.connection()).await?; - if rows.is_empty() {return Err(ReadError::Unavailable);} + if rows.is_empty() { + return Err(ReadError::Unavailable); + } for row in rows { - let id:i64=row.try_get("id")?; - if id<=last || seen==expected {return Err(ReadError::Unavailable);} - last=id; - let record:Option=row.try_get("record")?; - let record=project_record(dataset,&record.ok_or(ReadError::Unavailable)?)?; - if seen>0 {zip.write_all(b",").map_err(|_|ReadError::Limit)?;} - write_json(zip,&record)?; - seen+=1; + let id: i64 = row.try_get("id")?; + if id <= last || seen == expected { + return Err(ReadError::Unavailable); + } + last = id; + let record: Option = row.try_get("record")?; + let record = project_record(dataset, &record.ok_or(ReadError::Unavailable)?)?; + if seen > 0 { + zip.write_all(b",").map_err(io_error)?; + } + write_json(zip, &record)?; + seen += 1; } } - zip.write_all(b"]\n").map_err(|_|ReadError::Limit)?; - *total+=seen; + zip.write_all(b"]\n").map_err(io_error)?; + *total += seen; Ok(seen) } From 0d2aaa2f766c77f15890f2a2b6c17aaeca84a4fa Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 12 Oct 2026 19:05:47 +0800 Subject: [PATCH 4/6] fix(core): reject writable history columns and conflicting source receipts Check column grants on both source views and qualification receipts. Add real PostgreSQL regressions for column grants, duplicate receipts, ZIP footer bounds and malformed usage, plus typed sink IO tests. Freeze the S11 lock integration patch in the isolated acceptance runner instead of resolving a different dependency graph on each CI run. --- services/core/src/history_read/repository.rs | 632 +++++++++++----- services/core/tests/finance_export.rs | 143 +++- .../core/tests/history_read/cargo-lock.patch | 124 ++++ services/core/tests/history_read/check.sh | 20 +- services/core/tests/history_repository.rs | 684 ++++++++++++++---- 5 files changed, 1264 insertions(+), 339 deletions(-) create mode 100644 services/core/tests/history_read/cargo-lock.patch diff --git a/services/core/src/history_read/repository.rs b/services/core/src/history_read/repository.rs index 097eb0ce6..5e6454aca 100644 --- a/services/core/src/history_read/repository.rs +++ b/services/core/src/history_read/repository.rs @@ -1,9 +1,10 @@ //! Consumer-side read contract. The offline source owner supplies the versioned //! views; missing, writable, duplicate or incomplete sources fail closed. //! No runtime DDL, import, money mutation or best-effort partial history. -use super::{LogQuotaInput, MAX_METADATA_BYTES, MAX_RECORDS, QuotaMoneyProjection, - QuotaStatisticProjection, captured_credits_per_usd, project_consumption_statistic, project_log, - snapshot::ReadSnapshot}; +use super::{ + LogQuotaInput, MAX_METADATA_BYTES, MAX_RECORDS, QuotaMoneyProjection, QuotaStatisticProjection, + captured_credits_per_usd, project_consumption_statistic, project_log, snapshot::ReadSnapshot, +}; use crate::identity::{self, CredentialKind, IdentityError}; use serde::{Deserialize, Serialize}; use sqlx::{PgConnection, PgPool, Postgres, QueryBuilder, Row}; @@ -15,25 +16,44 @@ const BATCH: i64 = 64; const COLUMNS: &str = "id,event_key,user_id,owner_account_id,credential_id,created_at,log_type,username,token_name,model_name,quota,prompt_tokens,completion_tokens,use_time,is_stream,channel_id,group_name,request_id,CASE WHEN octet_length(other)<=65536 THEN other ELSE '' END AS other"; #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ReadError { Unauthorized, Forbidden, Invalid, Unavailable, Limit, Busy } -impl From for ReadError { fn from(_: sqlx::Error) -> Self { Self::Unavailable } } +pub enum ReadError { + Unauthorized, + Forbidden, + Invalid, + Unavailable, + Limit, + Busy, +} +impl From for ReadError { + fn from(_: sqlx::Error) -> Self { + Self::Unavailable + } +} impl From for ReadError { fn from(e: IdentityError) -> Self { - match e { IdentityError::Unauthorized => Self::Unauthorized, IdentityError::Forbidden => Self::Forbidden, - IdentityError::Invalid => Self::Invalid, _ => Self::Unavailable } + match e { + IdentityError::Unauthorized => Self::Unauthorized, + IdentityError::Forbidden => Self::Forbidden, + IdentityError::Invalid => Self::Invalid, + _ => Self::Unavailable, + } } } pub type Result = std::result::Result; #[derive(Clone, Copy)] -pub enum Audience { Administrator, SelfUser, Token } +pub enum Audience { + Administrator, + SelfUser, + Token, +} /// HTTP filters are not authority. User/key/account constraints are resolved /// from the stored credential before these filters reach a query. #[derive(Clone, Default, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct Filter { - #[serde(rename="type")] + #[serde(rename = "type")] pub log_type: i32, pub start_timestamp: i64, pub end_timestamp: i64, @@ -50,34 +70,62 @@ pub struct Filter { } impl Filter { pub fn validate(&self) -> Result<()> { - if !(0..=6).contains(&self.log_type) || self.start_timestamp<0 || self.end_timestamp<0 - || (self.end_timestamp!=0 && self.start_timestamp>self.end_timestamp) - || self.user_id.is_some_and(|id| id<=0) || self.team_id.is_some_and(|id| id<=0) - || self.channel.is_some_and(|id| id<0) || self.p<0 || self.page_size<0 - || [&self.username,&self.token_name,&self.model_name,&self.group,&self.request_id] - .iter().any(|s| s.len()>256 || s.contains('\0')) { + if !(0..=6).contains(&self.log_type) + || self.start_timestamp < 0 + || self.end_timestamp < 0 + || (self.end_timestamp != 0 && self.start_timestamp > self.end_timestamp) + || self.user_id.is_some_and(|id| id <= 0) + || self.team_id.is_some_and(|id| id <= 0) + || self.channel.is_some_and(|id| id < 0) + || self.p < 0 + || self.page_size < 0 + || [ + &self.username, + &self.token_name, + &self.model_name, + &self.group, + &self.request_id, + ] + .iter() + .any(|s| s.len() > 256 || s.contains('\0')) + { return Err(ReadError::Invalid); } self.offset()?; Ok(()) } - pub fn page(&self) -> i64 { self.p.max(1) } - pub fn size(&self) -> i64 { if self.page_size==0 {10} else {self.page_size.clamp(10,MAX_PAGE)} } + pub fn page(&self) -> i64 { + self.p.max(1) + } + pub fn size(&self) -> i64 { + if self.page_size == 0 { + 10 + } else { + self.page_size.clamp(10, MAX_PAGE) + } + } fn offset(&self) -> Result { - self.page().checked_sub(1).and_then(|v| v.checked_mul(self.size())) - .filter(|v| *v<=MAX_OFFSET).ok_or(ReadError::Limit) + self.page() + .checked_sub(1) + .and_then(|v| v.checked_mul(self.size())) + .filter(|v| *v <= MAX_OFFSET) + .ok_or(ReadError::Limit) } } #[derive(Serialize, sqlx::FromRow)] pub struct LogRow { pub id: i64, - #[serde(skip)] pub event_key: String, + #[serde(skip)] + pub event_key: String, pub user_id: i64, - #[serde(skip)] pub owner_account_id: i64, - #[serde(skip)] pub credential_id: Option, + #[serde(skip)] + pub owner_account_id: i64, + #[serde(skip)] + pub credential_id: Option, pub created_at: i64, - #[serde(rename="type")] pub log_type: i32, + #[serde(rename = "type")] + pub log_type: i32, pub username: String, pub token_name: String, pub model_name: String, @@ -86,24 +134,53 @@ pub struct LogRow { pub completion_tokens: i64, pub use_time: i64, pub is_stream: bool, - #[serde(skip_serializing_if="Option::is_none")] pub channel_id: Option, - #[serde(rename="group")] pub group_name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub channel_id: Option, + #[serde(rename = "group")] + pub group_name: String, pub request_id: String, - #[serde(skip)] pub other: String, + #[serde(skip)] + pub other: String, } impl LogRow { pub fn projection_input(&self) -> LogQuotaInput<'_> { - LogQuotaInput { quota:self.quota,log_type:self.log_type,other:&self.other } + LogQuotaInput { + quota: self.quota, + log_type: self.log_type, + other: &self.other, + } } fn validate(&self) -> Result<()> { - if self.id<=0 || self.user_id<=0 || self.owner_account_id<=0 || self.credential_id.is_some_and(|id| id<=0) - || self.created_at<0 || self.prompt_tokens<0 || self.completion_tokens<0 || self.use_time<0 - || !(1..=6).contains(&self.log_type) || self.event_key.is_empty() || self.event_key.len()>256 - || self.other.len()>MAX_METADATA_BYTES - || [&self.username,&self.token_name,&self.model_name,&self.group_name,&self.request_id] - .iter().any(|v| v.len()>1024) - || [self.quota.unwrap_or(0),self.prompt_tokens,self.completion_tokens,self.use_time] - .into_iter().any(|v| i128::from(v).abs()>SAFE_INTEGER) { + if self.id <= 0 + || self.user_id <= 0 + || self.owner_account_id <= 0 + || self.credential_id.is_some_and(|id| id <= 0) + || self.created_at < 0 + || self.prompt_tokens < 0 + || self.completion_tokens < 0 + || self.use_time < 0 + || !(1..=6).contains(&self.log_type) + || self.event_key.is_empty() + || self.event_key.len() > 256 + || self.other.len() > MAX_METADATA_BYTES + || [ + &self.username, + &self.token_name, + &self.model_name, + &self.group_name, + &self.request_id, + ] + .iter() + .any(|v| v.len() > 1024) + || [ + self.quota.unwrap_or(0), + self.prompt_tokens, + self.completion_tokens, + self.use_time, + ] + .into_iter() + .any(|v| i128::from(v).abs() > SAFE_INTEGER) + { return Err(ReadError::Unavailable); } Ok(()) @@ -111,12 +188,17 @@ impl LogRow { } #[derive(Serialize)] pub struct DisplayLog { - #[serde(flatten)] pub row: LogRow, - #[serde(flatten)] pub money: QuotaMoneyProjection, + #[serde(flatten)] + pub row: LogRow, + #[serde(flatten)] + pub money: QuotaMoneyProjection, } #[derive(Serialize)] pub struct Page { - pub page: i64, pub page_size:i64, pub total:i64, pub items:Vec, + pub page: i64, + pub page_size: i64, + pub total: i64, + pub items: Vec, } #[derive(Serialize)] pub struct Statistics { @@ -124,7 +206,8 @@ pub struct Statistics { pub rpm: i64, pub tpm: i64, pub total: i64, - #[serde(flatten)] pub projection: QuotaStatisticProjection, + #[serde(flatten)] + pub projection: QuotaStatisticProjection, } /// Its only connection is the read-only transaction. It cannot construct a /// second snapshot accidentally when running counts, pages, or projections. @@ -138,210 +221,403 @@ pub struct HistoryRead<'a> { root: bool, } impl<'a> HistoryRead<'a> { - pub async fn begin(pool:&'a PgPool, secret:&str, audience:Audience, filter:Filter) -> Result { + pub async fn begin( + pool: &'a PgPool, + secret: &str, + audience: Audience, + filter: Filter, + ) -> Result { filter.validate()?; - let mut snapshot=ReadSnapshot::begin(pool).await?; - let actor=identity::authorize_read_in_transaction(snapshot.connection(),secret).await?; - let administrator=matches!(audience,Audience::Administrator); + let mut snapshot = ReadSnapshot::begin(pool).await?; + let actor = identity::authorize_read_in_transaction(snapshot.connection(), secret).await?; + let administrator = matches!(audience, Audience::Administrator); match audience { - Audience::Administrator if actor.credential_kind!=CredentialKind::Session || actor.platform_level<5 - => return Err(ReadError::Forbidden), - Audience::SelfUser if actor.credential_kind!=CredentialKind::Session => return Err(ReadError::Forbidden), - Audience::Token if actor.credential_kind!=CredentialKind::ApiKey => return Err(ReadError::Forbidden), - _=>{} - } - if !administrator && (filter.user_id.is_some_and(|id| id!=actor.user_id) - || !filter.username.is_empty() || filter.channel.is_some()) { + Audience::Administrator + if actor.credential_kind != CredentialKind::Session || actor.platform_level < 5 => + { + return Err(ReadError::Forbidden); + } + Audience::SelfUser if actor.credential_kind != CredentialKind::Session => { + return Err(ReadError::Forbidden); + } + Audience::Token if actor.credential_kind != CredentialKind::ApiKey => { + return Err(ReadError::Forbidden); + } + _ => {} + } + if !administrator + && (filter.user_id.is_some_and(|id| id != actor.user_id) + || !filter.username.is_empty() + || filter.channel.is_some()) + { return Err(ReadError::Forbidden); } - let mut account=matches!(audience,Audience::Token).then_some(actor.owner_account_id); - if let Some(team)=filter.team_id { + let mut account = matches!(audience, Audience::Token).then_some(actor.owner_account_id); + if let Some(team) = filter.team_id { let team_account:Option=sqlx::query_scalar( "SELECT t.account_id FROM core_identity.teams t JOIN core_identity.accounts a ON a.id=t.account_id WHERE t.id=$1 AND t.active AND a.active AND ($3 OR t.owner_user_id=$2 OR EXISTS(SELECT 1 FROM core_identity.memberships m WHERE m.team_id=t.id AND m.user_id=$2 AND m.active))") .bind(team).bind(actor.user_id).bind(administrator).fetch_optional(snapshot.connection()).await?; - let team_account=team_account.ok_or(ReadError::Forbidden)?; - if account.is_some_and(|id| id!=team_account) { return Err(ReadError::Forbidden); } - account=Some(team_account); - } - let user=if administrator {filter.user_id} else {Some(actor.user_id)}; - let credential=matches!(audience,Audience::Token).then_some(actor.credential_id); - let root=administrator && actor.platform_level==6; - let mut this=Self { snapshot,filter,user,credential,account,administrator,root }; + let team_account = team_account.ok_or(ReadError::Forbidden)?; + if account.is_some_and(|id| id != team_account) { + return Err(ReadError::Forbidden); + } + account = Some(team_account); + } + let user = if administrator { + filter.user_id + } else { + Some(actor.user_id) + }; + let credential = matches!(audience, Audience::Token).then_some(actor.credential_id); + let root = administrator && actor.platform_level == 6; + let mut this = Self { + snapshot, + filter, + user, + credential, + account, + administrator, + root, + }; this.require_source("logs").await?; - if credential.is_some() {this.require_source("token_logs").await?;} + if credential.is_some() { + this.require_source("token_logs").await?; + } Ok(this) } - pub fn connection(&mut self) -> &mut PgConnection { self.snapshot.connection() } - pub fn administrator(&self) -> bool { self.administrator } - pub async fn finish(self) -> Result<()> { self.snapshot.finish().await.map_err(Into::into) } + pub fn connection(&mut self) -> &mut PgConnection { + self.snapshot.connection() + } + pub fn administrator(&self) -> bool { + self.administrator + } + pub async fn finish(self) -> Result<()> { + self.snapshot.finish().await.map_err(Into::into) + } /// Receipts are supplied by the offline importer. The network role must not /// be able to manufacture them, or mutate any of the read contract objects. - pub async fn require_source(&mut self, dataset:&str) -> Result<()> { - let exists:bool=sqlx::query_scalar("SELECT to_regclass('core_history.source_contract_v1') IS NOT NULL") - .fetch_one(self.connection()).await?; - if !exists { return Err(ReadError::Unavailable); } - let receipt:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM core_history.source_contract_v1 WHERE dataset=$1 AND version=1 AND complete AND manifest_sha256~'^[0-9a-f]{64}$') AND NOT has_table_privilege(current_user,'core_history.source_contract_v1','INSERT,UPDATE,DELETE,TRUNCATE')") - .bind(dataset).fetch_one(self.connection()).await?; - if !receipt { return Err(ReadError::Unavailable); } - let relation=match dataset { - "logs"|"token_logs"=>"core_history.log_records_v1", - "quota_data"=>"core_history.quota_data_v1", - "financial-options"=>"core_history.finance_options_v1", - _=>"core_history.finance_records_v1", + pub async fn require_source(&mut self, dataset: &str) -> Result<()> { + let exists: bool = + sqlx::query_scalar("SELECT to_regclass('core_history.source_contract_v1') IS NOT NULL") + .fetch_one(self.connection()) + .await?; + if !exists { + return Err(ReadError::Unavailable); + } + self.require_read_only("core_history.source_contract_v1").await?; + let receipt: bool = sqlx::query_scalar( + "SELECT count(*)=1 AND coalesce(bool_and(version=1 AND complete AND manifest_sha256~'^[0-9a-f]{64}$'),false) FROM core_history.source_contract_v1 WHERE dataset=$1", + ) + .bind(dataset) + .fetch_one(self.connection()) + .await?; + if !receipt { + return Err(ReadError::Unavailable); + } + let relation = match dataset { + "logs" | "token_logs" => "core_history.log_records_v1", + "quota_data" => "core_history.quota_data_v1", + "financial-options" => "core_history.finance_options_v1", + _ => "core_history.finance_records_v1", }; - let writable:bool=sqlx::query_scalar("SELECT has_schema_privilege(current_user,'core_history','CREATE') OR has_table_privilege(current_user,$1,'INSERT,UPDATE,DELETE,TRUNCATE')") - .bind(relation).fetch_one(self.connection()).await?; - if writable {return Err(ReadError::Unavailable);} + self.require_read_only(relation).await + } + + async fn require_read_only(&mut self, relation: &str) -> Result<()> { + // A table-level check alone misses INSERT/UPDATE grants on individual + // columns. Neither source rows nor their receipt may be writable. + let writable: bool = sqlx::query_scalar( + "SELECT has_schema_privilege(current_user,'core_history','CREATE') OR has_table_privilege(current_user,$1,'INSERT,UPDATE,DELETE,TRUNCATE') OR has_any_column_privilege(current_user,$1,'INSERT,UPDATE')", + ) + .bind(relation) + .fetch_one(self.connection()) + .await?; + if writable { + return Err(ReadError::Unavailable); + } Ok(()) } - fn selection(&self, columns:&str, consumption:bool) -> QueryBuilder<'static,Postgres> { - let mut q=QueryBuilder::new(format!("SELECT {columns} FROM core_history.log_records_v1 WHERE TRUE")); - if let Some(id)=self.user { q.push(" AND user_id=").push_bind(id); } - if let Some(id)=self.credential { q.push(" AND credential_id=").push_bind(id); } - if let Some(id)=self.account { q.push(" AND owner_account_id=").push_bind(id); } - if consumption { q.push(" AND log_type=2"); } - else if self.filter.log_type!=0 { q.push(" AND log_type=").push_bind(self.filter.log_type); } - if self.filter.start_timestamp!=0 { q.push(" AND created_at>=").push_bind(self.filter.start_timestamp); } - if self.filter.end_timestamp!=0 { q.push(" AND created_at<=").push_bind(self.filter.end_timestamp); } - for (column,value) in [("username",&self.filter.username),("token_name",&self.filter.token_name), - ("group_name",&self.filter.group),("request_id",&self.filter.request_id)] { - if !value.is_empty() { q.push(format!(" AND {column}=")).push_bind(value.clone()); } + fn selection(&self, columns: &str, consumption: bool) -> QueryBuilder<'static, Postgres> { + let mut q = QueryBuilder::new(format!( + "SELECT {columns} FROM core_history.log_records_v1 WHERE TRUE" + )); + if let Some(id) = self.user { + q.push(" AND user_id=").push_bind(id); + } + if let Some(id) = self.credential { + q.push(" AND credential_id=").push_bind(id); + } + if let Some(id) = self.account { + q.push(" AND owner_account_id=").push_bind(id); + } + if consumption { + q.push(" AND log_type=2"); + } else if self.filter.log_type != 0 { + q.push(" AND log_type=").push_bind(self.filter.log_type); + } + if self.filter.start_timestamp != 0 { + q.push(" AND created_at>=") + .push_bind(self.filter.start_timestamp); + } + if self.filter.end_timestamp != 0 { + q.push(" AND created_at<=") + .push_bind(self.filter.end_timestamp); + } + for (column, value) in [ + ("username", &self.filter.username), + ("token_name", &self.filter.token_name), + ("group_name", &self.filter.group), + ("request_id", &self.filter.request_id), + ] { + if !value.is_empty() { + q.push(format!(" AND {column}=")).push_bind(value.clone()); + } } if !self.filter.model_name.is_empty() { // Literal substring, not a caller-controlled LIKE expression. - q.push(" AND strpos(model_name,").push_bind(self.filter.model_name.clone()).push(")>0"); + q.push(" AND strpos(model_name,") + .push_bind(self.filter.model_name.clone()) + .push(")>0"); + } + if let Some(id) = self.filter.channel { + q.push(" AND channel_id=").push_bind(id); } - if let Some(id)=self.filter.channel { q.push(" AND channel_id=").push_bind(id); } q } - async fn count(&mut self, consumption:bool) -> Result { + async fn count(&mut self, consumption: bool) -> Result { let mut q=self.selection("count(*)::bigint AS total,count(DISTINCT id)::bigint AS ids,count(DISTINCT event_key)::bigint AS events",consumption); - let row=q.build().fetch_one(self.connection()).await?; - let count:i64=row.try_get("total")?; - if count!=row.try_get::("ids")? || count!=row.try_get::("events")? { + let row = q.build().fetch_one(self.connection()).await?; + let count: i64 = row.try_get("total")?; + if count != row.try_get::("ids")? || count != row.try_get::("events")? { // Never arbitrarily pick a legacy or native duplicate as authoritative. return Err(ReadError::Unavailable); } Ok(count) } pub async fn logs(&mut self) -> Result { - let total=self.count(false).await?; - let mut query=self.selection(COLUMNS,false); - query.push(" ORDER BY id DESC LIMIT ").push_bind(self.filter.size()) - .push(" OFFSET ").push_bind(self.filter.offset()?); - let rows:Vec=query.build_query_as().fetch_all(self.connection()).await?; - let mut items=Vec::with_capacity(rows.len()); + let total = self.count(false).await?; + let mut query = self.selection(COLUMNS, false); + query + .push(" ORDER BY id DESC LIMIT ") + .push_bind(self.filter.size()) + .push(" OFFSET ") + .push_bind(self.filter.offset()?); + let rows: Vec = query.build_query_as().fetch_all(self.connection()).await?; + let mut items = Vec::with_capacity(rows.len()); for mut row in rows { row.validate()?; - let money=project_log(row.projection_input()); - if !self.administrator {row.channel_id=None;} - items.push(DisplayLog {row,money}); + let money = project_log(row.projection_input()); + if !self.administrator { + row.channel_id = None; + } + items.push(DisplayLog { row, money }); } - Ok(Page {page:self.filter.page(),page_size:self.filter.size(),total,items}) + Ok(Page { + page: self.filter.page(), + page_size: self.filter.size(), + total, + items, + }) } pub async fn statistics(&mut self) -> Result { - let total=self.count(true).await?; + let total = self.count(true).await?; let mut query=self.selection("sum(quota::numeric)::text AS quota,count(quota)::bigint AS stored_count,count(*) FILTER(WHERE created_at>=extract(epoch FROM transaction_timestamp())::bigint-60)::bigint AS rpm,coalesce(sum(prompt_tokens::numeric+completion_tokens::numeric) FILTER(WHERE created_at>=extract(epoch FROM transaction_timestamp())::bigint-60),0)::text AS tpm",true); - let sums=query.build().fetch_one(self.connection()).await?; - let raw:Option=sums.try_get("quota")?; - let quota=if total==0 {Some(0)} else if sums.try_get::("stored_count")?!=total {None} else {raw.as_deref().map(exact).transpose()?}; - let tpm=exact(&sums.try_get::("tpm")?)?; - let mut inputs=Vec::new(); - let mut cursor=0i64; - while inputs.len()").push_bind(cursor).push(" ORDER BY id LIMIT ") - .push_bind(BATCH.min((MAX_RECORDS-inputs.len()) as i64)); - let rows=query.build().fetch_all(self.connection()).await?; - if rows.is_empty() {return Err(ReadError::Unavailable);} + let sums = query.build().fetch_one(self.connection()).await?; + let raw: Option = sums.try_get("quota")?; + let quota = if total == 0 { + Some(0) + } else if sums.try_get::("stored_count")? != total { + None + } else { + raw.as_deref().map(exact).transpose()? + }; + let tpm = exact(&sums.try_get::("tpm")?)?; + let mut inputs = Vec::new(); + let mut cursor = 0i64; + while inputs.len() < MAX_RECORDS && (inputs.len() as i64) < total { + let mut query = self.selection( + "id,quota,CASE WHEN octet_length(other)<=65536 THEN other ELSE '' END AS other", + true, + ); + query + .push(" AND id>") + .push_bind(cursor) + .push(" ORDER BY id LIMIT ") + .push_bind(BATCH.min((MAX_RECORDS - inputs.len()) as i64)); + let rows = query.build().fetch_all(self.connection()).await?; + if rows.is_empty() { + return Err(ReadError::Unavailable); + } for row in rows { - cursor=row.try_get("id")?; - let quota:Option=row.try_get("quota")?; - let other:String=row.try_get("other")?; - let basis=captured_credits_per_usd(&other).map(|b| format!("{{\"pricing_credits_per_usd\":{b}}}")).unwrap_or_default(); - inputs.push((quota,basis)); + cursor = row.try_get("id")?; + let quota: Option = row.try_get("quota")?; + let other: String = row.try_get("other")?; + let basis = captured_credits_per_usd(&other) + .map(|b| format!("{{\"pricing_credits_per_usd\":{b}}}")) + .unwrap_or_default(); + inputs.push((quota, basis)); } } - let projection=project_consumption_statistic(inputs.iter().map(|(quota,basis)| LogQuotaInput {quota:*quota,log_type:2,other:basis}),total>MAX_RECORDS as i64); - Ok(Statistics {quota,rpm:sums.try_get("rpm")?,tpm,total,projection}) + let projection = project_consumption_statistic( + inputs.iter().map(|(quota, basis)| LogQuotaInput { + quota: *quota, + log_type: 2, + other: basis, + }), + total > MAX_RECORDS as i64, + ); + Ok(Statistics { + quota, + rpm: sums.try_get("rpm")?, + tpm, + total, + projection, + }) } /// Used by export without retaining opaque metadata or an unbounded result. - pub async fn export_batch(&mut self, after:(i64,i64), limit:i64) -> Result> { - if !(1..=BATCH).contains(&limit) {return Err(ReadError::Invalid);} - let mut q=self.selection(COLUMNS,false); - q.push(" AND log_type IN (1,2,3,4,6) AND (created_at,id)>(").push_bind(after.0).push(",").push_bind(after.1) - .push(") ORDER BY created_at,id LIMIT ").push_bind(limit); - let rows:Vec=q.build_query_as().fetch_all(self.connection()).await?; - for row in &rows { row.validate()?; } + pub async fn export_batch(&mut self, after: (i64, i64), limit: i64) -> Result> { + if !(1..=BATCH).contains(&limit) { + return Err(ReadError::Invalid); + } + let mut q = self.selection(COLUMNS, false); + q.push(" AND log_type IN (1,2,3,4,6) AND (created_at,id)>(") + .push_bind(after.0) + .push(",") + .push_bind(after.1) + .push(") ORDER BY created_at,id LIMIT ") + .push_bind(limit); + let rows: Vec = q.build_query_as().fetch_all(self.connection()).await?; + for row in &rows { + row.validate()?; + } Ok(rows) } pub async fn export_count(&mut self) -> Result { let mut q=self.selection("count(*)::bigint AS total,count(DISTINCT id)::bigint AS ids,count(DISTINCT event_key)::bigint AS events",false); q.push(" AND log_type IN (1,2,3,4,6)"); - let row=q.build().fetch_one(self.connection()).await?; - let count:i64=row.try_get("total")?; - if count!=row.try_get::("ids")? || count!=row.try_get::("events")? {return Err(ReadError::Unavailable);} + let row = q.build().fetch_one(self.connection()).await?; + let count: i64 = row.try_get("total")?; + if count != row.try_get::("ids")? || count != row.try_get::("events")? { + return Err(ReadError::Unavailable); + } Ok(count) } - pub fn time_range(&self) -> (i64,i64) { (self.filter.start_timestamp,self.filter.end_timestamp) } + pub fn time_range(&self) -> (i64, i64) { + (self.filter.start_timestamp, self.filter.end_timestamp) + } - pub async fn buckets(&mut self, by_user:bool, flow:bool) -> Result> { + pub async fn buckets(&mut self, by_user: bool, flow: bool) -> Result> { if self.credential.is_some() || (by_user && !self.administrator) { return Err(ReadError::Forbidden); } - if self.filter.start_timestamp==0 || self.filter.end_timestamp==0 - || (!self.administrator && self.filter.end_timestamp-self.filter.start_timestamp>2_592_000) - || self.filter.log_type!=0 || !self.filter.token_name.is_empty() || !self.filter.request_id.is_empty() { + if self.filter.start_timestamp == 0 + || self.filter.end_timestamp == 0 + || (!self.administrator + && self.filter.end_timestamp - self.filter.start_timestamp > 2_592_000) + || self.filter.log_type != 0 + || !self.filter.token_name.is_empty() + || !self.filter.request_id.is_empty() + { return Err(ReadError::Invalid); } self.require_source("quota_data").await?; - let group=if flow { - if self.root {"user_id,username,node_name,token_id,token_name,use_group,model_name,channel_id,channel_name"} - else if self.administrator {"user_id,username,use_group,model_name,channel_id,channel_name"} - else {"token_id,token_name,use_group,model_name"} - } else if by_user {"user_id,username,created_at"} - else {"created_at,model_name"}; - let mut q=QueryBuilder::::new(format!("SELECT {group},CASE WHEN count(quota)=count(*) THEN sum(quota::numeric)::text END AS quota,sum(token_used::numeric)::text AS token_used,sum(count::numeric)::text AS count FROM core_history.quota_data_v1 WHERE created_at>=")); - q.push_bind(self.filter.start_timestamp).push(" AND created_at<=").push_bind(self.filter.end_timestamp); - if flow {q.push(" AND use_group<>''");} - if let Some(user)=self.user {q.push(" AND user_id=").push_bind(user);} - if let Some(account)=self.account {q.push(" AND owner_account_id=").push_bind(account);} - for (column,value) in [("username",&self.filter.username),("use_group",&self.filter.group)] { - if !value.is_empty() {q.push(format!(" AND {column}=")).push_bind(value.clone());} - } - if !self.filter.model_name.is_empty() {q.push(" AND strpos(model_name,").push_bind(self.filter.model_name.clone()).push(")>0");} - if let Some(channel)=self.filter.channel {q.push(" AND channel_id=").push_bind(channel);} - let order=if flow {format!("sum(quota::numeric) DESC NULLS LAST,{group}")} else {group.to_owned()}; + let group = if flow { + if self.root { + "user_id,username,node_name,token_id,token_name,use_group,model_name,channel_id,channel_name" + } else if self.administrator { + "user_id,username,use_group,model_name,channel_id,channel_name" + } else { + "token_id,token_name,use_group,model_name" + } + } else if by_user { + "user_id,username,created_at" + } else { + "created_at,model_name" + }; + let mut q = QueryBuilder::::new(format!( + "SELECT {group},CASE WHEN count(quota)=count(*) THEN sum(quota::numeric)::text END AS quota,sum(token_used::numeric)::text AS token_used,sum(count::numeric)::text AS count FROM core_history.quota_data_v1 WHERE created_at>=" + )); + q.push_bind(self.filter.start_timestamp) + .push(" AND created_at<=") + .push_bind(self.filter.end_timestamp); + if flow { + q.push(" AND use_group<>''"); + } + if let Some(user) = self.user { + q.push(" AND user_id=").push_bind(user); + } + if let Some(account) = self.account { + q.push(" AND owner_account_id=").push_bind(account); + } + for (column, value) in [ + ("username", &self.filter.username), + ("use_group", &self.filter.group), + ] { + if !value.is_empty() { + q.push(format!(" AND {column}=")).push_bind(value.clone()); + } + } + if !self.filter.model_name.is_empty() { + q.push(" AND strpos(model_name,") + .push_bind(self.filter.model_name.clone()) + .push(")>0"); + } + if let Some(channel) = self.filter.channel { + q.push(" AND channel_id=").push_bind(channel); + } + let order = if flow { + format!("sum(quota::numeric) DESC NULLS LAST,{group}") + } else { + group.to_owned() + }; q.push(format!(" GROUP BY {group} ORDER BY {order} LIMIT 10001")); - let rows=q.build().fetch_all(self.connection()).await?; - if rows.len()>MAX_RECORDS {return Err(ReadError::Limit);} - let mut output=Vec::with_capacity(rows.len()); + let rows = q.build().fetch_all(self.connection()).await?; + if rows.len() > MAX_RECORDS { + return Err(ReadError::Limit); + } + let mut output = Vec::with_capacity(rows.len()); for row in rows { - let mut value=serde_json::Map::new(); + let mut value = serde_json::Map::new(); for name in group.split(',') { - if ["user_id","token_id","channel_id","created_at"].contains(&name) { - value.insert(name.to_owned(),serde_json::json!(row.try_get::(name)?)); + if ["user_id", "token_id", "channel_id", "created_at"].contains(&name) { + value.insert( + name.to_owned(), + serde_json::json!(row.try_get::(name)?), + ); } else { - let text:String=row.try_get(name)?; - if text.len()>1024 {return Err(ReadError::Unavailable);} - value.insert(name.to_owned(),serde_json::json!(text)); + let text: String = row.try_get(name)?; + if text.len() > 1024 { + return Err(ReadError::Unavailable); + } + value.insert(name.to_owned(), serde_json::json!(text)); } } - let quota:Option=row.try_get("quota")?; - value.insert("quota".into(),serde_json::json!(quota.as_deref().map(exact).transpose()?)); - for name in ["token_used","count"] { - value.insert(name.into(),serde_json::json!(exact(&row.try_get::(name)?)?)); + let quota: Option = row.try_get("quota")?; + value.insert( + "quota".into(), + serde_json::json!(quota.as_deref().map(exact).transpose()?), + ); + for name in ["token_used", "count"] { + value.insert( + name.into(), + serde_json::json!(exact(&row.try_get::(name)?)?), + ); } - value.insert("quota_money_available".into(),serde_json::json!(false)); - value.insert("quota_money_basis".into(),serde_json::json!("bucket_without_credit_basis")); + value.insert("quota_money_available".into(), serde_json::json!(false)); + value.insert( + "quota_money_basis".into(), + serde_json::json!("bucket_without_credit_basis"), + ); output.push(serde_json::Value::Object(value)); } Ok(output) } } -fn exact(value:&str) -> Result { - let value:i128=value.parse().map_err(|_| ReadError::Unavailable)?; - if value.abs()>SAFE_INTEGER {return Err(ReadError::Unavailable);} +fn exact(value: &str) -> Result { + let value: i128 = value.parse().map_err(|_| ReadError::Unavailable)?; + if value.abs() > SAFE_INTEGER { + return Err(ReadError::Unavailable); + } i64::try_from(value).map_err(|_| ReadError::Unavailable) } diff --git a/services/core/tests/finance_export.rs b/services/core/tests/finance_export.rs index 78779428a..54055010b 100644 --- a/services/core/tests/finance_export.rs +++ b/services/core/tests/finance_export.rs @@ -1,22 +1,36 @@ // SPDX-License-Identifier: AGPL-3.0-or-later use lmm_core::finance_export::{ - ExportError, UsageExportInput, UsageJsonWriter, MAX_EXPORT_BYTES, MAX_EXPORT_ROWS, + ExportError, MAX_EXPORT_BYTES, MAX_EXPORT_ROWS, UsageExportInput, UsageJsonWriter, }; use serde_json::Value; use std::io::{self, Write}; fn row(id: i64, other: &str) -> UsageExportInput<'_> { UsageExportInput { - log_id: id, user_id: 7, created_at: 1700000000, log_type: 2, - username: "synthetic-user", token_name: "synthetic-token", model_name: "local-model", - quota: Some(500000), prompt_tokens: 5, completion_tokens: 2, use_time: 1, - is_stream: false, channel_id: Some(9), group: "default", other, + log_id: id, + user_id: 7, + created_at: 1700000000, + log_type: 2, + username: "synthetic-user", + token_name: "synthetic-token", + model_name: "local-model", + quota: Some(500000), + prompt_tokens: 5, + completion_tokens: 2, + use_time: 1, + is_stream: false, + channel_id: Some(9), + group: "default", + other, } } #[test] fn export_has_an_explicit_field_allowlist() { - let input = row(1, r#"{"pricing_credits_per_usd":500000,"api_key":"SYNTHETIC_SECRET","admin_info":{"user_id":8},"provider_payload":"SYNTHETIC_PROVIDER_DATA"}"#); + let input = row( + 1, + r#"{"pricing_credits_per_usd":500000,"api_key":"SYNTHETIC_SECRET","admin_info":{"user_id":8},"provider_payload":"SYNTHETIC_PROVIDER_DATA"}"#, + ); let mut writer = UsageJsonWriter::new(Vec::new(), 10, 10000).unwrap(); writer.push(&input).unwrap(); let (bytes, receipt) = writer.finish().unwrap(); @@ -24,9 +38,32 @@ fn export_has_an_explicit_field_allowlist() { assert!(!text.contains("SYNTHETIC_SECRET")); assert!(!text.contains("SYNTHETIC_PROVIDER_DATA")); let decoded: Value = serde_json::from_str(&text).unwrap(); - let mut fields: Vec<_> = decoded[0].as_object().unwrap().keys().map(String::as_str).collect(); + let mut fields: Vec<_> = decoded[0] + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); fields.sort_unstable(); - let mut expected = vec!["log_id", "user_id", "created_at", "type", "username", "token_name", "model_name", "quota", "prompt_tokens", "completion_tokens", "use_time", "is_stream", "channel_id", "group", "quota_usd", "quota_money_available", "quota_money_basis"]; + let mut expected = vec![ + "log_id", + "user_id", + "created_at", + "type", + "username", + "token_name", + "model_name", + "quota", + "prompt_tokens", + "completion_tokens", + "use_time", + "is_stream", + "channel_id", + "group", + "quota_usd", + "quota_money_available", + "quota_money_basis", + ]; expected.sort_unstable(); assert_eq!(fields, expected); assert_eq!(decoded[0]["quota"], 500000); @@ -84,8 +121,16 @@ fn export_enforces_row_order_and_duplicate_rejection() { #[test] fn export_limits_cannot_be_raised_past_the_server_ceiling() { - for (rows, bytes) in [(0, 2), (1, 1), (MAX_EXPORT_ROWS + 1, 100), (1, MAX_EXPORT_BYTES + 1)] { - assert!(matches!(UsageJsonWriter::new(Vec::new(), rows, bytes), Err(ExportError::InvalidLimits))); + for (rows, bytes) in [ + (0, 2), + (1, 1), + (MAX_EXPORT_ROWS + 1, 100), + (1, MAX_EXPORT_BYTES + 1), + ] { + assert!(matches!( + UsageJsonWriter::new(Vec::new(), rows, bytes), + Err(ExportError::InvalidLimits) + )); } } @@ -129,17 +174,24 @@ fn export_empty_result_is_a_real_empty_json_array() { assert_eq!(receipt.bytes, 2); } -struct BrokenSink { remaining: usize } +struct BrokenSink { + remaining: usize, +} impl Write for BrokenSink { fn write(&mut self, bytes: &[u8]) -> io::Result { if self.remaining == 0 { - return Err(io::Error::new(io::ErrorKind::BrokenPipe, "synthetic disconnect")); + return Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "synthetic disconnect", + )); } let written = self.remaining.min(bytes.len()); self.remaining -= written; Ok(written) } - fn flush(&mut self) -> io::Result<()> { Ok(()) } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } } #[test] @@ -149,3 +201,68 @@ fn export_writer_failure_cannot_be_retried_as_success() { assert_eq!(writer.push(&row(2, "{}")), Err(ExportError::Aborted)); assert!(matches!(writer.finish(), Err(ExportError::Aborted))); } + +struct FailingSink { + writes: usize, + fail_on_write: usize, + kind: io::ErrorKind, + fail_flush: bool, +} +impl Write for FailingSink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.writes += 1; + if self.writes == self.fail_on_write { + return Err(io::Error::from(self.kind)); + } + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + if self.fail_flush { + Err(io::Error::from(self.kind)) + } else { + Ok(()) + } + } +} + +#[test] +fn export_distinguishes_sink_limits_from_other_io_failures() { + for (kind, expected) in [ + (io::ErrorKind::FileTooLarge, ExportError::ByteLimit), + (io::ErrorKind::StorageFull, ExportError::ByteLimit), + (io::ErrorKind::QuotaExceeded, ExportError::ByteLimit), + (io::ErrorKind::BrokenPipe, ExportError::Io), + (io::ErrorKind::PermissionDenied, ExportError::Io), + ] { + for fail_on_write in 1..=3 { + let sink = FailingSink { + writes: 0, + fail_on_write, + kind, + fail_flush: false, + }; + let result = UsageJsonWriter::new(sink, 10, 10000); + if fail_on_write == 1 { + assert!(matches!(result, Err(error) if error == expected)); + continue; + } + let mut writer = result.unwrap(); + let result = writer.push(&row(1, "{}")); + if fail_on_write == 2 { + assert_eq!(result, Err(expected)); + assert!(matches!(writer.finish(), Err(ExportError::Aborted))); + } else { + result.unwrap(); + assert!(matches!(writer.finish(), Err(error) if error == expected)); + } + } + let sink = FailingSink { + writes: 0, + fail_on_write: usize::MAX, + kind, + fail_flush: true, + }; + let writer = UsageJsonWriter::new(sink, 10, 10000).unwrap(); + assert!(matches!(writer.finish(), Err(error) if error == expected)); + } +} diff --git a/services/core/tests/history_read/cargo-lock.patch b/services/core/tests/history_read/cargo-lock.patch new file mode 100644 index 000000000..d81fd51e3 --- /dev/null +++ b/services/core/tests/history_read/cargo-lock.patch @@ -0,0 +1,124 @@ +diff --git a/services/core/Cargo.lock b/services/core/Cargo.lock +index e571b17..b54ce58 100644 +--- a/services/core/Cargo.lock ++++ b/services/core/Cargo.lock +@@ -58,6 +58,15 @@ version = "1.0.104" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + ++[[package]] ++name = "arbitrary" ++version = "1.5.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "3bc62ac97cc33321f50863d514c3bc38a453947a8f9e781137e47c7401020aed" ++dependencies = [ ++ "derive_arbitrary", ++] ++ + [[package]] + name = "argon2" + version = "0.5.3" +@@ -331,6 +340,15 @@ version = "2.5.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + ++[[package]] ++name = "crc32fast" ++version = "1.5.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" ++dependencies = [ ++ "cfg-if", ++] ++ + [[package]] + name = "crossbeam-queue" + version = "0.3.14" +@@ -376,6 +394,17 @@ dependencies = [ + "zeroize", + ] + ++[[package]] ++name = "derive_arbitrary" ++version = "1.5.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "1b034bd7d5f032402a2479444dcc6f74e36a03f31854d41680fb240ef682a1ac" ++dependencies = [ ++ "proc-macro2", ++ "quote", ++ "syn 3.0.6", ++] ++ + [[package]] + name = "digest" + version = "0.10.7" +@@ -1095,6 +1124,7 @@ dependencies = [ + "hmac", + "http-body", + "jsonwebtoken", ++ "num-rational", + "prost", + "protoc-bin-vendored", + "rand 0.8.5", +@@ -1111,6 +1141,7 @@ dependencies = [ + "tonic-prost", + "tonic-prost-build", + "tower", ++ "zip", + ] + + [[package]] +@@ -1179,6 +1210,16 @@ version = "0.10.1" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + ++[[package]] ++name = "num-bigint" ++version = "0.4.8" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" ++dependencies = [ ++ "num-integer", ++ "num-traits", ++] ++ + [[package]] + name = "num-bigint-dig" + version = "0.8.6" +@@ -1214,6 +1255,17 @@ dependencies = [ + "num-traits", + ] + ++[[package]] ++name = "num-rational" ++version = "0.4.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" ++dependencies = [ ++ "num-bigint", ++ "num-integer", ++ "num-traits", ++] ++ + [[package]] + name = "num-traits" + version = "0.2.19" +@@ -3131,6 +3183,18 @@ dependencies = [ + "syn 3.0.6", + ] + ++[[package]] ++name = "zip" ++version = "6.0.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" ++dependencies = [ ++ "arbitrary", ++ "crc32fast", ++ "indexmap", ++ "memchr", ++] ++ + [[package]] + name = "zmij" + version = "1.0.23" diff --git a/services/core/tests/history_read/check.sh b/services/core/tests/history_read/check.sh index 4a2b0d49c..5b27f40bc 100644 --- a/services/core/tests/history_read/check.sh +++ b/services/core/tests/history_read/check.sh @@ -16,14 +16,28 @@ cargo --version | tee "$evidence/cargo.txt" grep -q '^rustc 1\.99\.0 ' "$evidence/rustc.txt" grep -q '^cargo 1\.99\.0 ' "$evidence/cargo.txt" printf '%s\n' "$(git rev-parse HEAD)" > "$evidence/source-sha.txt" -# First integration pass resolves the new exact dependencies. The resulting -# lockfile and formatted source are retained for review; no CI write token. -cargo metadata --format-version 1 >/dev/null +# Shared lock integration is owned by S11. Test its frozen patch in this +# disposable checkout, not a freshly resolved dependency graph on every run. +# Both accepted hashes are exact Cargo-generated inputs retained by run +# 38210812027. Unexpected lock changes require a new reviewed patch. +case "$(git hash-object Cargo.lock)" in + e571b17a29f201e321f005eba20c2b4fa0bf3bf0) + patch="$PWD/tests/history_read/cargo-lock.patch" + git -C "$GITHUB_WORKSPACE" apply --check "$patch" + git -C "$GITHUB_WORKSPACE" apply "$patch" + ;; + b54ce5875919b916743171f6ce9b72a1d08449a7) ;; + *) printf '%s\n' 'A04 lock input changed; reconcile the integration patch.' >&2; exit 2;; +esac +test "$(git hash-object Cargo.lock)" = b54ce5875919b916743171f6ce9b72a1d08449a7 +cargo metadata --locked --format-version 1 >/dev/null +printf '%s\n' 'isolated: frozen S11 lock patch and scoped rustfmt applied' > "$evidence/tested-source-mode.txt" files=(src/history_read.rs src/history_read/*.rs src/finance_export.rs src/finance_export/*.rs tests/history_read.rs tests/finance_export.rs tests/history_repository.rs) rustfmt --edition 2024 --config skip_children=true "${files[@]}" git diff --binary -- Cargo.lock "${files[@]}" > "$evidence/generated.patch" cp Cargo.lock "$evidence/Cargo.lock" tar -czf "$evidence/checked-source.tar.gz" Cargo.toml Cargo.lock src tests schema build.rs +sha256sum "$evidence/checked-source.tar.gz" > "$evidence/checked-source.sha256" set +e cargo test --locked --test history_read --test finance_export --test history_repository --test financial_profile --no-fail-fast 2>&1 | tee "$evidence/tests.log" status=${PIPESTATUS[0]} diff --git a/services/core/tests/history_repository.rs b/services/core/tests/history_repository.rs index 71ee7856d..2922c462e 100644 --- a/services/core/tests/history_repository.rs +++ b/services/core/tests/history_repository.rs @@ -1,218 +1,612 @@ //! Real PostgreSQL and stored credentials. The only synthetic boundary is the //! offline A03 source-view provider, never an always-allow authorization mock. -use lmm_core::{accounts::{Account,AccountKind},core_schema,funding::FundingPolicy, - identity::{IdentityStore,IssuedCredential},history_read::{http::{self,HistoryApi}, - repository::{Audience,Filter,HistoryRead,ReadError}},finance_export::archive}; -use axum::{body::{Body,to_bytes},http::{Request,StatusCode}}; -use serde_json::{Value,json}; +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, +}; +use lmm_core::{ + accounts::{Account, AccountKind}, + core_schema, + finance_export::archive, + funding::FundingPolicy, + history_read::{ + http::{self, HistoryApi}, + repository::{Audience, Filter, HistoryRead, ReadError}, + }, + identity::{IdentityStore, IssuedCredential}, +}; +use serde_json::{Value, json}; use sqlx::PgPool; -use std::io::{Cursor,Read}; +use std::io::{Cursor, Read}; use tower::ServiceExt; -#[path="common/runtime.rs"] +#[allow(dead_code)] +#[path = "common/runtime.rs"] mod runtime; struct Fixture { - owner:PgPool, runtime:runtime::RuntimeRole, session:IssuedCredential, - other:IssuedCredential, admin:IssuedCredential, root:IssuedCredential, - key:IssuedCredential, second_key:IssuedCredential, account:i64, + owner: PgPool, + runtime: runtime::RuntimeRole, + session: IssuedCredential, + other: IssuedCredential, + admin: IssuedCredential, + root: IssuedCredential, + key: IssuedCredential, + second_key: IssuedCredential, + account: i64, } impl Fixture { - async fn new(pool:PgPool)->Self { + async fn new(pool: PgPool) -> Self { core_schema::init(&pool).await.unwrap(); - let store=IdentityStore::from_pool(pool.clone()); - let session=store.bootstrap_user(7,1).await.unwrap(); - let other=store.bootstrap_user(8,1).await.unwrap(); - let admin=store.bootstrap_user(9,5).await.unwrap(); - let root=store.bootstrap_user(10,6).await.unwrap(); - let account=store.authorize(&session.secret).await.unwrap().owner_account_id; - let key=store.issue_key(&session.secret,Account {id:7,kind:AccountKind::Personal},FundingPolicy::new_personal_key(),3600).await.unwrap(); - let second_key=store.issue_key(&session.secret,Account {id:7,kind:AccountKind::Personal},FundingPolicy::new_personal_key(),3600).await.unwrap(); - sqlx::raw_sql(include_str!("fixtures/history_source_v1.sql")).execute(&pool).await.unwrap(); - let runtime=runtime::RuntimeRole::create(&pool).await; + let store = IdentityStore::from_pool(pool.clone()); + let session = store.bootstrap_user(7, 1).await.unwrap(); + let other = store.bootstrap_user(8, 1).await.unwrap(); + let admin = store.bootstrap_user(9, 5).await.unwrap(); + let root = store.bootstrap_user(10, 6).await.unwrap(); + let account = store + .authorize(&session.secret) + .await + .unwrap() + .owner_account_id; + let key = store + .issue_key( + &session.secret, + Account { + id: 7, + kind: AccountKind::Personal, + }, + FundingPolicy::new_personal_key(), + 3600, + ) + .await + .unwrap(); + let second_key = store + .issue_key( + &session.secret, + Account { + id: 7, + kind: AccountKind::Personal, + }, + FundingPolicy::new_personal_key(), + 3600, + ) + .await + .unwrap(); + sqlx::raw_sql(include_str!("fixtures/history_source_v1.sql")) + .execute(&pool) + .await + .unwrap(); + let runtime = runtime::RuntimeRole::create(&pool).await; sqlx::raw_sql(&format!("GRANT USAGE ON SCHEMA core_history TO {0}; GRANT SELECT ON ALL TABLES IN SCHEMA core_history TO {0}",runtime.name)) .execute(&pool).await.unwrap(); - let this=Self {owner:pool,runtime,session,other,admin,root,key,second_key,account}; - this.insert(1,7,this.key.id,Some(500000),r#"{"pricing_credits_per_usd":500000,"api_key":"SYNTHETIC_SECRET"}"#).await; - this.insert(2,7,this.second_key.id,Some(1000000),r#"{"pricing_credits_per_usd":1000000}"#).await; - this.insert(3,8,this.other.id,Some(99000000),"{}").await; + let this = Self { + owner: pool, + runtime, + session, + other, + admin, + root, + key, + second_key, + account, + }; + this.insert( + 1, + 7, + this.key.id, + Some(500000), + r#"{"pricing_credits_per_usd":500000,"api_key":"SYNTHETIC_SECRET"}"#, + ) + .await; + this.insert( + 2, + 7, + this.second_key.id, + Some(1000000), + r#"{"pricing_credits_per_usd":1000000}"#, + ) + .await; + this.insert(3, 8, this.other.id, Some(99000000), "{}").await; this } - async fn insert(&self,id:i64,user:i64,key:i64,quota:Option,other:&str) { - let account:i64=sqlx::query_scalar("SELECT personal_account_id FROM core_identity.users WHERE id=$1").bind(user).fetch_one(&self.owner).await.unwrap(); + async fn insert(&self, id: i64, user: i64, key: i64, quota: Option, other: &str) { + let account: i64 = + sqlx::query_scalar("SELECT personal_account_id FROM core_identity.users WHERE id=$1") + .bind(user) + .fetch_one(&self.owner) + .await + .unwrap(); sqlx::query("INSERT INTO a04_fixture.logs(id,event_key,user_id,owner_account_id,credential_id,quota,other,channel_id) VALUES($1,$2,$3,$4,$5,$6,$7,99)") .bind(id).bind(format!("event-{id}")).bind(user).bind(account).bind(key).bind(quota).bind(other).execute(&self.owner).await.unwrap(); } - async fn request(&self,path:&str,secret:&str)->(StatusCode,Value) { - let response=http::router(HistoryApi::new(self.runtime.pool.clone())).oneshot(Request::builder().uri(path).header("authorization",format!("Bearer {secret}")).body(Body::empty()).unwrap()).await.unwrap(); - let status=response.status(); - assert_eq!(response.headers()["cache-control"],"no-store"); - assert_eq!(response.headers()["x-content-type-options"],"nosniff"); - let bytes=to_bytes(response.into_body(),8*1024*1024).await.unwrap(); - (status,serde_json::from_slice(&bytes).unwrap()) + async fn request(&self, path: &str, secret: &str) -> (StatusCode, Value) { + let response = http::router(HistoryApi::new(self.runtime.pool.clone())) + .oneshot( + Request::builder() + .uri(path) + .header("authorization", format!("Bearer {secret}")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + assert_eq!(response.headers()["cache-control"], "no-store"); + assert_eq!(response.headers()["x-content-type-options"], "nosniff"); + let bytes = to_bytes(response.into_body(), 8 * 1024 * 1024) + .await + .unwrap(); + (status, serde_json::from_slice(&bytes).unwrap()) + } + async fn cleanup(self) { + self.runtime.cleanup().await; + } +} +fn range() -> Filter { + Filter { + start_timestamp: 1_699_999_900, + end_timestamp: 1_700_000_100, + ..Filter::default() } - async fn cleanup(self) {self.runtime.cleanup().await;} } -fn range()->Filter {Filter {start_timestamp:1_699_999_900,end_timestamp:1_700_000_100,..Filter::default()}} -#[sqlx::test(migrations=false)] -async fn real_sessions_and_keys_never_broaden_history_scope(pool:PgPool) { - let f=Fixture::new(pool).await; - for path in ["/api/log/self?user_id=8","/api/log/","/api/log/stat","/api/data/users?start_timestamp=1&end_timestamp=2","/api/finance/export"] { - assert_eq!(f.request(path,&f.session.secret).await.0,StatusCode::FORBIDDEN,"{path}"); +#[sqlx::test(migrations = false)] +async fn real_sessions_and_keys_never_broaden_history_scope(pool: PgPool) { + let f = Fixture::new(pool).await; + for path in [ + "/api/log/self?user_id=8", + "/api/log/", + "/api/log/stat", + "/api/data/users?start_timestamp=1&end_timestamp=2", + "/api/finance/export", + ] { + assert_eq!( + f.request(path, &f.session.secret).await.0, + StatusCode::FORBIDDEN, + "{path}" + ); } - for path in ["/api/log/self","/api/log/stat","/api/finance/export"] { - assert_eq!(f.request(path,&f.key.secret).await.0,StatusCode::FORBIDDEN,"{path}"); + for path in ["/api/log/self", "/api/log/stat", "/api/finance/export"] { + assert_eq!( + f.request(path, &f.key.secret).await.0, + StatusCode::FORBIDDEN, + "{path}" + ); } - let (status,body)=f.request("/api/log/token",&f.key.secret).await; - assert_eq!(status,StatusCode::OK);assert_eq!(body["data"]["total"],1); - assert_eq!(body["data"]["items"][0]["id"],1); + let (status, body) = f.request("/api/log/token", &f.key.secret).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["data"]["total"], 1); + assert_eq!(body["data"]["items"][0]["id"], 1); assert!(body["data"]["items"][0].get("channel_id").is_none()); - let (_,body)=f.request("/api/log/self",&f.session.secret).await; - assert_eq!(body["data"]["total"],2); - let text=body.to_string(); - assert!(!text.contains("SYNTHETIC_SECRET") && !text.contains("credential_id") && !text.contains("event_key")); - let (_,body)=f.request("/api/log/?user_id=8",&f.admin.secret).await; - assert_eq!(body["data"]["total"],1);assert_eq!(body["data"]["items"][0]["user_id"],8); + let (_, body) = f.request("/api/log/self", &f.session.secret).await; + assert_eq!(body["data"]["total"], 2); + let text = body.to_string(); + assert!( + !text.contains("SYNTHETIC_SECRET") + && !text.contains("credential_id") + && !text.contains("event_key") + ); + let (_, body) = f.request("/api/log/?user_id=8", &f.admin.secret).await; + assert_eq!(body["data"]["total"], 1); + assert_eq!(body["data"]["items"][0]["user_id"], 8); f.cleanup().await; } -#[sqlx::test(migrations=false)] -async fn count_page_money_and_authority_share_a_single_read_snapshot(pool:PgPool) { - let f=Fixture::new(pool).await; - let before:i64=sqlx::query_scalar("SELECT count(*) FROM core_identity.audit").fetch_one(&f.owner).await.unwrap(); - let mut read=HistoryRead::begin(&f.runtime.pool,&f.session.secret,Audience::SelfUser,Filter::default()).await.unwrap(); - assert_eq!(read.logs().await.unwrap().total,2); - f.insert(4,7,f.key.id,Some(500000),r#"{"pricing_credits_per_usd":500000}"#).await; - sqlx::query("UPDATE core_identity.credentials SET revoked_at=clock_timestamp() WHERE id=$1").bind(f.session.id).execute(&f.owner).await.unwrap(); - let page=read.logs().await.unwrap();let stat=read.statistics().await.unwrap(); - assert_eq!(page.total,2);assert_eq!(page.items.len(),2);assert_eq!(stat.total,2); - assert_eq!(stat.quota,Some(1500000));assert_eq!(stat.projection.money.quota_usd.as_deref(),Some("2")); +#[sqlx::test(migrations = false)] +async fn count_page_money_and_authority_share_a_single_read_snapshot(pool: PgPool) { + let f = Fixture::new(pool).await; + let before: i64 = sqlx::query_scalar("SELECT count(*) FROM core_identity.audit") + .fetch_one(&f.owner) + .await + .unwrap(); + let mut read = HistoryRead::begin( + &f.runtime.pool, + &f.session.secret, + Audience::SelfUser, + Filter::default(), + ) + .await + .unwrap(); + assert_eq!(read.logs().await.unwrap().total, 2); + f.insert( + 4, + 7, + f.key.id, + Some(500000), + r#"{"pricing_credits_per_usd":500000}"#, + ) + .await; + sqlx::query("UPDATE core_identity.credentials SET revoked_at=clock_timestamp() WHERE id=$1") + .bind(f.session.id) + .execute(&f.owner) + .await + .unwrap(); + let page = read.logs().await.unwrap(); + let stat = read.statistics().await.unwrap(); + assert_eq!(page.total, 2); + assert_eq!(page.items.len(), 2); + assert_eq!(stat.total, 2); + assert_eq!(stat.quota, Some(1500000)); + assert_eq!(stat.projection.money.quota_usd.as_deref(), Some("2")); read.finish().await.unwrap(); - assert_eq!(f.request("/api/log/self",&f.session.secret).await.0,StatusCode::UNAUTHORIZED); - let after:i64=sqlx::query_scalar("SELECT count(*) FROM core_identity.audit").fetch_one(&f.owner).await.unwrap(); - assert_eq!(before,after); + assert_eq!( + f.request("/api/log/self", &f.session.secret).await.0, + StatusCode::UNAUTHORIZED + ); + let after: i64 = sqlx::query_scalar("SELECT count(*) FROM core_identity.audit") + .fetch_one(&f.owner) + .await + .unwrap(); + assert_eq!(before, after); f.cleanup().await; } -#[sqlx::test(migrations=false)] -async fn missing_incomplete_writable_or_duplicate_source_never_looks_empty(pool:PgPool) { - let f=Fixture::new(pool).await; - sqlx::query("UPDATE a04_fixture.contract SET complete=false WHERE dataset='logs'").execute(&f.owner).await.unwrap(); - assert_eq!(f.request("/api/log/self",&f.session.secret).await.0,StatusCode::SERVICE_UNAVAILABLE); - sqlx::query("UPDATE a04_fixture.contract SET complete=true WHERE dataset='logs'").execute(&f.owner).await.unwrap(); - sqlx::query("UPDATE a04_fixture.logs SET event_key='event-1' WHERE id=2").execute(&f.owner).await.unwrap(); - assert_eq!(f.request("/api/log/self",&f.session.secret).await.0,StatusCode::SERVICE_UNAVAILABLE); - sqlx::query("UPDATE a04_fixture.logs SET event_key='event-2' WHERE id=2").execute(&f.owner).await.unwrap(); - sqlx::raw_sql(&format!("GRANT UPDATE ON core_history.log_records_v1 TO {}",f.runtime.name)).execute(&f.owner).await.unwrap(); - assert_eq!(f.request("/api/log/self",&f.session.secret).await.0,StatusCode::SERVICE_UNAVAILABLE); - sqlx::raw_sql("DROP VIEW core_history.source_contract_v1").execute(&f.owner).await.unwrap(); - assert_eq!(f.request("/api/log/self",&f.session.secret).await.0,StatusCode::SERVICE_UNAVAILABLE); +#[sqlx::test(migrations = false)] +async fn missing_incomplete_writable_or_duplicate_source_never_looks_empty(pool: PgPool) { + let f = Fixture::new(pool).await; + sqlx::query("UPDATE a04_fixture.contract SET complete=false WHERE dataset='logs'") + .execute(&f.owner) + .await + .unwrap(); + assert_eq!( + f.request("/api/log/self", &f.session.secret).await.0, + StatusCode::SERVICE_UNAVAILABLE + ); + sqlx::query("UPDATE a04_fixture.contract SET complete=true WHERE dataset='logs'") + .execute(&f.owner) + .await + .unwrap(); + sqlx::query("UPDATE a04_fixture.logs SET event_key='event-1' WHERE id=2") + .execute(&f.owner) + .await + .unwrap(); + assert_eq!( + f.request("/api/log/self", &f.session.secret).await.0, + StatusCode::SERVICE_UNAVAILABLE + ); + sqlx::query("UPDATE a04_fixture.logs SET event_key='event-2' WHERE id=2") + .execute(&f.owner) + .await + .unwrap(); + sqlx::raw_sql(&format!( + "GRANT UPDATE ON core_history.log_records_v1 TO {}", + f.runtime.name + )) + .execute(&f.owner) + .await + .unwrap(); + assert_eq!( + f.request("/api/log/self", &f.session.secret).await.0, + StatusCode::SERVICE_UNAVAILABLE + ); + sqlx::raw_sql("DROP VIEW core_history.source_contract_v1") + .execute(&f.owner) + .await + .unwrap(); + assert_eq!( + f.request("/api/log/self", &f.session.secret).await.0, + StatusCode::SERVICE_UNAVAILABLE + ); f.cleanup().await; } -#[sqlx::test(migrations=false)] -async fn metadata_nulls_and_refunds_are_not_silently_repriced(pool:PgPool) { - let f=Fixture::new(pool).await; - f.insert(4,7,f.key.id,None,"{}").await; - f.insert(5,7,f.key.id,Some(-7),"{}").await; - sqlx::query("UPDATE a04_fixture.logs SET log_type=6 WHERE id=5").execute(&f.owner).await.unwrap(); - let (_,body)=f.request("/api/log/self/stat",&f.session.secret).await; - assert_eq!(body["data"]["total"],3); - assert_eq!(body["data"]["quota_money_available"],false); +#[sqlx::test(migrations = false)] +async fn metadata_nulls_and_refunds_are_not_silently_repriced(pool: PgPool) { + let f = Fixture::new(pool).await; + f.insert(4, 7, f.key.id, None, "{}").await; + f.insert(5, 7, f.key.id, Some(-7), "{}").await; + sqlx::query("UPDATE a04_fixture.logs SET log_type=6 WHERE id=5") + .execute(&f.owner) + .await + .unwrap(); + let (_, body) = f.request("/api/log/self/stat", &f.session.secret).await; + assert_eq!(body["data"]["total"], 3); + assert_eq!(body["data"]["quota_money_available"], false); assert!(body["data"].get("quota_usd").is_none()); assert!(body["data"]["quota"].is_null()); - assert_eq!(body["data"]["unpriced_quota_records"],1); - let (_,page)=f.request("/api/log/self?type=6",&f.session.secret).await; - assert_eq!(page["data"]["items"][0]["quota"],-7); - assert_eq!(page["data"]["items"][0]["quota_money_basis"],"not_consumption"); + assert_eq!(body["data"]["unpriced_quota_records"], 1); + let (_, page) = f.request("/api/log/self?type=6", &f.session.secret).await; + assert_eq!(page["data"]["items"][0]["quota"], -7); + assert_eq!( + page["data"]["items"][0]["quota_money_basis"], + "not_consumption" + ); f.cleanup().await; } -#[sqlx::test(migrations=false)] -async fn team_filter_requires_membership_and_does_not_broaden_key_owner(pool:PgPool) { - let f=Fixture::new(pool).await; - let store=IdentityStore::from_pool(f.owner.clone()); - let team=store.create_team(&f.other.secret).await.unwrap(); - let path=format!("/api/log/self?team_id={team}"); - assert_eq!(f.request(&path,&f.session.secret).await.0,StatusCode::FORBIDDEN); - assert_eq!(f.request(&format!("/api/log/token?team_id={team}"),&f.key.secret).await.0,StatusCode::FORBIDDEN); +#[sqlx::test(migrations = false)] +async fn team_filter_requires_membership_and_does_not_broaden_key_owner(pool: PgPool) { + let f = Fixture::new(pool).await; + let store = IdentityStore::from_pool(f.owner.clone()); + let team = store.create_team(&f.other.secret).await.unwrap(); + let path = format!("/api/log/self?team_id={team}"); + assert_eq!( + f.request(&path, &f.session.secret).await.0, + StatusCode::FORBIDDEN + ); + assert_eq!( + f.request(&format!("/api/log/token?team_id={team}"), &f.key.secret) + .await + .0, + StatusCode::FORBIDDEN + ); // The owner may select the team, but cannot read unrelated personal rows. - let (_,body)=f.request(&path,&f.other.secret).await; - assert_eq!(body["data"]["total"],0); + let (_, body) = f.request(&path, &f.other.secret).await; + assert_eq!(body["data"]["total"], 0); f.cleanup().await; } -#[sqlx::test(migrations=false)] -async fn bucket_and_flow_preserve_raw_quota_and_redact_admin_dimensions(pool:PgPool) { - let f=Fixture::new(pool).await; +#[sqlx::test(migrations = false)] +async fn bucket_and_flow_preserve_raw_quota_and_redact_admin_dimensions(pool: PgPool) { + let f = Fixture::new(pool).await; sqlx::query("INSERT INTO a04_fixture.quota_data VALUES(1,7,$1,'synthetic-user',1700000000,'model',1,'key','default',99,'channel','node',500000,7,1)") .bind(f.account).execute(&f.owner).await.unwrap(); - for path in ["/api/data/self","/api/data/flow/self"] { - let (_,body)=f.request(&format!("{path}?start_timestamp=1699999999&end_timestamp=1700000001"),&f.session.secret).await; - let row=&body["data"][0];assert_eq!(row["quota"],500000);assert_eq!(row["quota_money_available"],false); + for path in ["/api/data/self", "/api/data/flow/self"] { + let (_, body) = f + .request( + &format!("{path}?start_timestamp=1699999999&end_timestamp=1700000001"), + &f.session.secret, + ) + .await; + let row = &body["data"][0]; + assert_eq!(row["quota"], 500000); + assert_eq!(row["quota_money_available"], false); assert!(row.get("channel_id").is_none() && row.get("node_name").is_none()); } - let (_,body)=f.request("/api/data/flow?start_timestamp=1699999999&end_timestamp=1700000001",&f.admin.secret).await; - assert_eq!(body["data"][0]["channel_id"],99);assert!(body["data"][0].get("node_name").is_none()); - let (_,body)=f.request("/api/data/flow?start_timestamp=1699999999&end_timestamp=1700000001",&f.root.secret).await; - assert_eq!(body["data"][0]["node_name"],"node"); + let (_, body) = f + .request( + "/api/data/flow?start_timestamp=1699999999&end_timestamp=1700000001", + &f.admin.secret, + ) + .await; + assert_eq!(body["data"][0]["channel_id"], 99); + assert!(body["data"][0].get("node_name").is_none()); + let (_, body) = f + .request( + "/api/data/flow?start_timestamp=1699999999&end_timestamp=1700000001", + &f.root.secret, + ) + .await; + assert_eq!(body["data"][0]["node_name"], "node"); f.cleanup().await; } -#[sqlx::test(migrations=false)] -async fn query_parameters_are_values_not_sql_or_authority(pool:PgPool) { - let f=Fixture::new(pool).await; - let (status,body)=f.request("/api/log/self?model_name=%27%20OR%201%3D1--",&f.session.secret).await; - assert_eq!(status,StatusCode::OK);assert_eq!(body["data"]["total"],0); - for path in ["/api/log/self?start_timestamp=-1","/api/log/self?start_timestamp=2&end_timestamp=1","/api/log/self?type=99","/api/log/self?role=admin"] { - assert_eq!(f.request(path,&f.session.secret).await.0,StatusCode::BAD_REQUEST,"{path}"); +#[sqlx::test(migrations = false)] +async fn query_parameters_are_values_not_sql_or_authority(pool: PgPool) { + let f = Fixture::new(pool).await; + let (status, body) = f + .request( + "/api/log/self?model_name=%27%20OR%201%3D1--", + &f.session.secret, + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["data"]["total"], 0); + for path in [ + "/api/log/self?start_timestamp=-1", + "/api/log/self?start_timestamp=2&end_timestamp=1", + "/api/log/self?type=99", + "/api/log/self?role=admin", + ] { + assert_eq!( + f.request(path, &f.session.secret).await.0, + StatusCode::BAD_REQUEST, + "{path}" + ); } - assert_eq!(f.request("/api/log/self?p=9223372036854775807",&f.session.secret).await.0,StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!( + f.request("/api/log/self?p=9223372036854775807", &f.session.secret) + .await + .0, + StatusCode::PAYLOAD_TOO_LARGE + ); f.cleanup().await; } -#[sqlx::test(migrations=false)] -async fn complete_zip_is_readable_redacted_and_uses_a_single_snapshot(pool:PgPool) { - let f=Fixture::new(pool).await; +#[sqlx::test(migrations = false)] +async fn complete_zip_is_readable_redacted_and_uses_a_single_snapshot(pool: PgPool) { + let f = Fixture::new(pool).await; sqlx::query("INSERT INTO a04_fixture.finance_records(dataset,id,record) VALUES ('topups',1,$1),('channels-pricing',1,$2)") .bind(json!({"topup_id":1,"user_id":7,"settled_amount_micros":1000000,"api_key":"SYNTHETIC_SECRET"})) .bind(json!({"channel_id":99,"base_url":"https://user:secret@example.test/path?token=secret#secret","channel_key":"SYNTHETIC_SECRET"})) .execute(&f.owner).await.unwrap(); - let mut read=HistoryRead::begin(&f.runtime.pool,&f.admin.secret,Audience::Administrator,range()).await.unwrap(); + let mut read = HistoryRead::begin( + &f.runtime.pool, + &f.admin.secret, + Audience::Administrator, + range(), + ) + .await + .unwrap(); sqlx::query("UPDATE a04_fixture.finance_records SET record=record||'{\"settled_amount_micros\":2000000}'::jsonb WHERE dataset='topups'").execute(&f.owner).await.unwrap(); - let bytes=archive::build(&mut read,1024*1024).await.unwrap(); + let bytes = archive::build(&mut read, 1024 * 1024).await.unwrap(); read.finish().await.unwrap(); - let mut zip=zip::ZipArchive::new(Cursor::new(bytes)).unwrap(); - assert_eq!(zip.len(),15); + let mut zip = zip::ZipArchive::new(Cursor::new(bytes)).unwrap(); + assert_eq!(zip.len(), 15); for index in 0..zip.len() { - let mut file=zip.by_index(index).unwrap();let mut text=String::new();file.read_to_string(&mut text).unwrap(); + let mut file = zip.by_index(index).unwrap(); + let mut text = String::new(); + file.read_to_string(&mut text).unwrap(); assert!(!text.contains("SYNTHETIC_SECRET")); - let value:Value=serde_json::from_str(&text).unwrap(); - if file.name()=="topups.json" {assert_eq!(value[0]["settled_amount_micros"],1000000);} - if file.name()=="channels-pricing.json" {assert_eq!(value[0]["base_url"],"https://example.test");} + let value: Value = serde_json::from_str(&text).unwrap(); + if file.name() == "topups.json" { + assert_eq!(value[0]["settled_amount_micros"], 1000000); + } + if file.name() == "channels-pricing.json" { + assert_eq!(value[0]["base_url"], "https://example.test"); + } } f.cleanup().await; } -#[sqlx::test(migrations=false)] -async fn archive_refuses_missing_documents_limits_and_runtime_writes(pool:PgPool) { - let f=Fixture::new(pool).await; - let mut read=HistoryRead::begin(&f.runtime.pool,&f.admin.secret,Audience::Administrator,range()).await.unwrap(); - assert!(matches!(archive::build(&mut read,1024).await,Err(ReadError::Limit))); +#[sqlx::test(migrations = false)] +async fn archive_refuses_missing_documents_limits_and_runtime_writes(pool: PgPool) { + let f = Fixture::new(pool).await; + let mut read = HistoryRead::begin( + &f.runtime.pool, + &f.admin.secret, + Audience::Administrator, + range(), + ) + .await + .unwrap(); + assert!(matches!( + archive::build(&mut read, 1024).await, + Err(ReadError::Limit) + )); read.finish().await.unwrap(); - sqlx::query("DELETE FROM a04_fixture.contract WHERE dataset='topups'").execute(&f.owner).await.unwrap(); - let mut read=HistoryRead::begin(&f.runtime.pool,&f.admin.secret,Audience::Administrator,range()).await.unwrap(); - assert_eq!(archive::build(&mut read,1024*1024).await.unwrap_err(),ReadError::Unavailable); - let error=sqlx::query("UPDATE core_identity.users SET active=false WHERE id=7").execute(read.connection()).await.unwrap_err(); - assert_eq!(error.as_database_error().unwrap().code().as_deref(),Some("25006")); + sqlx::query("DELETE FROM a04_fixture.contract WHERE dataset='topups'") + .execute(&f.owner) + .await + .unwrap(); + let mut read = HistoryRead::begin( + &f.runtime.pool, + &f.admin.secret, + Audience::Administrator, + range(), + ) + .await + .unwrap(); + assert_eq!( + archive::build(&mut read, 1024 * 1024).await.unwrap_err(), + ReadError::Unavailable + ); + let error = sqlx::query("UPDATE core_identity.users SET active=false WHERE id=7") + .execute(read.connection()) + .await + .unwrap_err(); + assert_eq!( + error.as_database_error().unwrap().code().as_deref(), + Some("25006") + ); read.finish().await.unwrap(); - assert!(IdentityStore::from_pool(f.owner.clone()).authorize(&f.session.secret).await.is_ok()); + assert!( + IdentityStore::from_pool(f.owner.clone()) + .authorize(&f.session.secret) + .await + .is_ok() + ); f.cleanup().await; } #[test] fn finance_field_gate_never_serializes_nested_opaque_payloads() { - let topup=archive::DATASETS.iter().find(|d| d.name=="topups").unwrap(); - assert_eq!(archive::project_record(topup,&json!({"topup_id":1,"provider_payload":{"secret":"x"}})).unwrap(),json!({"topup_id":1})); - assert_eq!(archive::project_record(topup,&json!({"money":{"secret":"x"}})).unwrap_err(),ReadError::Unavailable); + let topup = archive::DATASETS + .iter() + .find(|d| d.name == "topups") + .unwrap(); + assert_eq!( + archive::project_record( + topup, + &json!({"topup_id":1,"provider_payload":{"secret":"x"}}) + ) + .unwrap(), + json!({"topup_id":1}) + ); + assert_eq!( + archive::project_record(topup, &json!({"money":{"secret":"x"}})).unwrap_err(), + ReadError::Unavailable + ); +} + +#[sqlx::test(migrations = false)] +async fn archive_limit_in_footer_is_not_a_source_failure(pool: PgPool) { + let f = Fixture::new(pool).await; + let mut read = HistoryRead::begin( + &f.runtime.pool, + &f.admin.secret, + Audience::Administrator, + range(), + ) + .await + .unwrap(); + let bytes = archive::build(&mut read, 1024 * 1024).await.unwrap(); + // One byte less than the complete archive must fail even if every JSON + // document fits; ZIP directory and footer bytes share the same limit. + assert_eq!( + archive::build(&mut read, bytes.len() - 1).await.unwrap_err(), + ReadError::Limit + ); + assert_eq!(archive::build(&mut read, bytes.len()).await.unwrap(), bytes); + read.finish().await.unwrap(); + f.cleanup().await; +} + +#[sqlx::test(migrations = false)] +async fn archive_invalid_usage_remains_a_source_failure(pool: PgPool) { + let f = Fixture::new(pool).await; + sqlx::query("UPDATE a04_fixture.logs SET prompt_tokens=-1 WHERE id=1") + .execute(&f.owner) + .await + .unwrap(); + let mut read = HistoryRead::begin( + &f.runtime.pool, + &f.admin.secret, + Audience::Administrator, + range(), + ) + .await + .unwrap(); + assert_eq!( + archive::build(&mut read, 1024 * 1024).await.unwrap_err(), + ReadError::Unavailable + ); + read.finish().await.unwrap(); + f.cleanup().await; +} + +#[sqlx::test(migrations = false)] +async fn column_grants_cannot_bypass_source_immutability(pool: PgPool) { + let f = Fixture::new(pool).await; + for (relation, column) in [ + ("core_history.log_records_v1", "quota"), + ("core_history.source_contract_v1", "complete"), + ] { + sqlx::raw_sql(&format!( + "GRANT UPDATE({column}) ON {relation} TO {}", + f.runtime.name + )) + .execute(&f.owner) + .await + .unwrap(); + let table_write: bool = sqlx::query_scalar( + "SELECT has_table_privilege(current_user,$1,'UPDATE')", + ) + .bind(relation) + .fetch_one(&f.runtime.pool) + .await + .unwrap(); + assert!(!table_write, "column grant must exercise the old blind spot"); + assert_eq!( + f.request("/api/log/self", &f.session.secret).await.0, + StatusCode::SERVICE_UNAVAILABLE + ); + sqlx::raw_sql(&format!( + "REVOKE UPDATE({column}) ON {relation} FROM {}", + f.runtime.name + )) + .execute(&f.owner) + .await + .unwrap(); + assert_eq!( + f.request("/api/log/self", &f.session.secret).await.0, + StatusCode::OK + ); + } + f.cleanup().await; +} + +#[sqlx::test(migrations = false)] +async fn conflicting_source_receipts_do_not_qualify_history(pool: PgPool) { + let f = Fixture::new(pool).await; + sqlx::raw_sql( + "CREATE OR REPLACE VIEW core_history.source_contract_v1 AS SELECT * FROM a04_fixture.contract UNION ALL SELECT dataset,version,false,repeat('b',64) FROM a04_fixture.contract WHERE dataset='logs'", + ) + .execute(&f.owner) + .await + .unwrap(); + assert_eq!( + f.request("/api/log/self", &f.session.secret).await.0, + StatusCode::SERVICE_UNAVAILABLE + ); + f.cleanup().await; } From 3fce505c3494d604b1746530f33ac714c00defed Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 12 Oct 2026 19:11:19 +0800 Subject: [PATCH 5/6] feat(core): extend A04 native reads, CSV export and bounded projections Preserve the concurrent A04 PostgreSQL/HTTP/ZIP implementation. Add native-only settled charge reads with strict account/key scope and synthetic PG regressions. Add allowlisted CSV output with spreadsheet formula protection and metadata/row validation. Bound historical aggregate metadata processing without reporting partial money totals. A03 legacy source completeness and full integration remain blocked. --- services/core/src/finance_export.rs | 39 +- services/core/src/finance_export/csv.rs | 129 +++++++ services/core/src/history_read.rs | 128 ++++--- services/core/src/history_read/native.rs | 395 +++++++++++++++++++++ services/core/tests/finance_export.rs | 3 + services/core/tests/finance_export/csv.rs | 121 +++++++ services/core/tests/history_read.rs | 374 +++++++++++++++---- services/core/tests/history_read/native.rs | 225 ++++++++++++ 8 files changed, 1289 insertions(+), 125 deletions(-) create mode 100644 services/core/src/finance_export/csv.rs create mode 100644 services/core/src/history_read/native.rs create mode 100644 services/core/tests/finance_export/csv.rs create mode 100644 services/core/tests/history_read/native.rs diff --git a/services/core/src/finance_export.rs b/services/core/src/finance_export.rs index 85b48e669..fe620aaa5 100644 --- a/services/core/src/finance_export.rs +++ b/services/core/src/finance_export.rs @@ -3,11 +3,13 @@ //! a complete finance bundle. The caller must authorize and select rows under //! the same read snapshot; no raw storage object implements Serialize here. pub mod archive; +pub mod csv; use crate::history_read::{LogQuotaInput, QuotaMoneyProjection, project_log}; use serde::Serialize; use std::io::{self, Write}; +pub const MAX_EXPORT_METADATA_BYTES: usize = 8 * 1024 * 1024; pub const MAX_EXPORT_ROWS: usize = 200_000; pub const MAX_EXPORT_BYTES: usize = 64 * 1024 * 1024; pub const MAX_EXPORT_ROW_BYTES: usize = 32 * 1024; @@ -121,6 +123,7 @@ pub struct UsageJsonWriter { max_bytes: usize, rows: usize, bytes: usize, + metadata_bytes: usize, last_order: Option<(i64, i64)>, failed: bool, } @@ -140,6 +143,7 @@ impl UsageJsonWriter { max_bytes, rows: 0, bytes: 1, + metadata_bytes: 0, last_order: None, failed: false, }) @@ -160,16 +164,12 @@ impl UsageJsonWriter { if self.rows == self.max_rows { return Err(ExportError::RowLimit); } - if row.log_id <= 0 - || row.user_id <= 0 - || row.created_at < 0 - || ![1, 2, 3, 4, 6].contains(&row.log_type) - || [row.username, row.token_name, row.model_name, row.group] - .into_iter() - .any(|value| value.len() > MAX_TEXT_BYTES) - { - return Err(ExportError::InvalidRow); - } + validate_row(row)?; + let metadata_bytes = self + .metadata_bytes + .checked_add(row.other.len()) + .filter(|value| *value <= MAX_EXPORT_METADATA_BYTES) + .ok_or(ExportError::ByteLimit)?; let order = (row.created_at, row.log_id); if self.last_order.is_some_and(|last| order <= last) { return Err(ExportError::OutOfOrder); @@ -198,6 +198,7 @@ impl UsageJsonWriter { .map_err(ExportError::from)?; self.bytes = required - 1; self.rows += 1; + self.metadata_bytes = metadata_bytes; self.last_order = Some(order); Ok(()) } @@ -220,6 +221,24 @@ impl UsageJsonWriter { } } +fn validate_row(row: &UsageExportInput<'_>) -> Result<(), ExportError> { + if row.log_id <= 0 + || row.user_id <= 0 + || !(0..=253_402_300_799).contains(&row.created_at) + || ![1, 2, 3, 4, 6].contains(&row.log_type) + || [row.prompt_tokens, row.completion_tokens, row.use_time] + .into_iter() + .any(|value| value < 0) + || row.channel_id.is_some_and(|value| value < 0) + || [row.username, row.token_name, row.model_name, row.group] + .into_iter() + .any(|value| value.len() > MAX_TEXT_BYTES || value.contains('\0')) + { + return Err(ExportError::InvalidRow); + } + Ok(()) +} + struct BoundedRow { bytes: Vec, } diff --git a/services/core/src/finance_export/csv.rs b/services/core/src/finance_export/csv.rs new file mode 100644 index 000000000..60a13c474 --- /dev/null +++ b/services/core/src/finance_export/csv.rs @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! UTF-8 CSV from the same allowlist as JSON. Raw source rows are never serialized. +use super::{ + BoundedRow, ExportError, ExportReceipt, MAX_EXPORT_BYTES, MAX_EXPORT_METADATA_BYTES, + MAX_EXPORT_ROWS, UsageExportInput, UsageExportRow, validate_row, +}; +use std::io::Write; + +const COLUMNS: [&str; 17] = [ + "log_id", "user_id", "created_at", "type", "username", "token_name", "model_name", + "quota", "prompt_tokens", "completion_tokens", "use_time", "is_stream", "channel_id", + "group", "quota_usd", "quota_money_available", "quota_money_basis", +]; + +/// The caller discards its spool on ANY error. CSV has no end marker, so the +/// receipt, not whether an early row can be parsed, determines completion. +pub struct UsageCsvWriter { + writer: W, + max_rows: usize, + max_bytes: usize, + rows: usize, + bytes: usize, + metadata_bytes: usize, + last_order: Option<(i64, i64)>, + failed: bool, +} + +impl UsageCsvWriter { + pub fn new(mut writer: W, max_rows: usize, max_bytes: usize, utf8_bom: bool) -> Result { + if max_rows == 0 || max_rows > MAX_EXPORT_ROWS || max_bytes > MAX_EXPORT_BYTES { + return Err(ExportError::InvalidLimits); + } + let mut header = BoundedRow { bytes: Vec::new() }; + if utf8_bom { + header.write_all(b"\xef\xbb\xbf").map_err(|_| ExportError::Encoding)?; + } + let columns: Vec<_> = COLUMNS.iter().map(|name| (*name, false)).collect(); + encode_record(&mut header, &columns)?; + if header.bytes.len() > max_bytes { + return Err(ExportError::InvalidLimits); + } + writer.write_all(&header.bytes).map_err(|_| ExportError::Io)?; + Ok(Self { + writer, max_rows, max_bytes, rows: 0, bytes: header.bytes.len(), + metadata_bytes: 0, last_order: None, failed: false, + }) + } + + pub fn push(&mut self, row: &UsageExportInput<'_>) -> Result<(), ExportError> { + if self.failed { + return Err(ExportError::Aborted); + } + let result = self.push_checked(row); + if result.is_err() { + self.failed = true; + } + result + } + + fn push_checked(&mut self, row: &UsageExportInput<'_>) -> Result<(), ExportError> { + if self.rows == self.max_rows { + return Err(ExportError::RowLimit); + } + validate_row(row)?; + let order = (row.created_at, row.log_id); + if self.last_order.is_some_and(|last| order <= last) { + return Err(ExportError::OutOfOrder); + } + let metadata_bytes = self.metadata_bytes.checked_add(row.other.len()) + .filter(|bytes| *bytes <= MAX_EXPORT_METADATA_BYTES).ok_or(ExportError::ByteLimit)?; + // The bounded serializer runs BEFORE Value conversion. This guarantees + // the map allocation cannot contain opaque or unlimited source metadata. + let mut json = BoundedRow { bytes: Vec::new() }; + serde_json::to_writer(&mut json, &UsageExportRow::from(row)).map_err(|_| ExportError::Encoding)?; + let value: serde_json::Value = serde_json::from_slice(&json.bytes).map_err(|_| ExportError::Encoding)?; + let mut strings = Vec::with_capacity(COLUMNS.len()); + for key in COLUMNS { + strings.push(match &value[key] { + serde_json::Value::Null => (String::new(), false), + serde_json::Value::String(text) => (text.clone(), true), + serde_json::Value::Number(number) => (number.to_string(), false), + serde_json::Value::Bool(flag) => (flag.to_string(), false), + _ => return Err(ExportError::Encoding), + }); + } + let cells: Vec<_> = strings.iter().map(|(text, protect)| (text.as_str(), *protect)).collect(); + let mut encoded = BoundedRow { bytes: Vec::new() }; + encode_record(&mut encoded, &cells)?; + let bytes = self.bytes.checked_add(encoded.bytes.len()) + .filter(|bytes| *bytes <= self.max_bytes).ok_or(ExportError::ByteLimit)?; + self.writer.write_all(&encoded.bytes).map_err(|_| ExportError::Io)?; + self.bytes = bytes; + self.rows += 1; + self.metadata_bytes = metadata_bytes; + self.last_order = Some(order); + Ok(()) + } + + pub fn finish(mut self) -> Result<(W, ExportReceipt), ExportError> { + if self.failed { + return Err(ExportError::Aborted); + } + self.writer.flush().map_err(|_| ExportError::Io)?; + Ok((self.writer, ExportReceipt { rows: self.rows, bytes: self.bytes })) + } +} + +fn encode_record(writer: &mut BoundedRow, cells: &[(&str, bool)]) -> Result<(), ExportError> { + for (index, (cell, protect)) in cells.iter().enumerate() { + if index != 0 { + writer.write_all(b",").map_err(|_| ExportError::Encoding)?; + } + writer.write_all(b"\"").map_err(|_| ExportError::Encoding)?; + let visible = cell.trim_start_matches(|c: char| c.is_whitespace() || c.is_control() || c == '\u{feff}'); + // Quoting alone does not stop spreadsheet formulas. Prefix before the + // ORIGINAL text; never remove or normalize raw stored text for JSON. + if *protect && (visible.starts_with(['=', '+', '-', '@']) || cell.starts_with(['\t', '\r', '\n'])) { + writer.write_all(b"'").map_err(|_| ExportError::Encoding)?; + } + for part in cell.split_inclusive('"') { + writer.write_all(part.as_bytes()).map_err(|_| ExportError::Encoding)?; + if part.ends_with('"') { + writer.write_all(b"\"").map_err(|_| ExportError::Encoding)?; + } + } + writer.write_all(b"\"").map_err(|_| ExportError::Encoding)?; + } + writer.write_all(b"\r\n").map_err(|_| ExportError::Encoding) +} diff --git a/services/core/src/history_read.rs b/services/core/src/history_read.rs index fb88e731a..af0f50b18 100644 --- a/services/core/src/history_read.rs +++ b/services/core/src/history_read.rs @@ -11,6 +11,7 @@ use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use std::collections::BTreeMap; +pub mod native; pub mod repository; pub mod http; pub mod snapshot; @@ -19,6 +20,8 @@ pub mod usage; pub const MAX_RECORDS: usize = 10_000; pub const MAX_BASES: usize = 64; pub const MAX_METADATA_BYTES: usize = 64 * 1024; +/// Total inspected metadata per aggregation, independent of the row ceiling. +pub const MAX_PROJECTION_METADATA_BYTES: usize = 8 * 1024 * 1024; pub const MAX_CREDIT_BASIS: i64 = 9_007_199_254_740_991; pub const LOG_TYPE_CONSUME: i32 = 2; @@ -147,65 +150,100 @@ pub fn project_log(input: LogQuotaInput<'_>) -> QuotaMoneyProjection { ) } -/// Project a complete, authorized consumption selection. The repository must -/// use the same filters and snapshot for the raw sum, count, page and this scan. -/// Do not pass only a page and describe its sum as the all-history statistic. -/// Non-consumption rows in this input are invalid, not silently excluded. -pub fn project_consumption_statistic<'a>( - rows: impl IntoIterator>, - source_truncated: bool, -) -> QuotaStatisticProjection { - let mut sums = BTreeMap::::new(); - let mut count = 0; - let mut unpriced = 0; - let limited = |unknown| QuotaStatisticProjection { - money: QuotaMoneyProjection::unknown("projection_limit"), - unpriced_quota_records: unknown, - quota_projection_truncated: true, - }; - for input in rows { - count += 1; - if count > MAX_RECORDS { - return limited(unpriced); +/// Incremental, bounded projection. The caller can discard each raw metadata +/// value after push, instead of retaining all selected provider payloads. +#[derive(Default)] +pub struct QuotaStatisticAccumulator { + sums: BTreeMap, + count: usize, + metadata_bytes: usize, + unpriced: usize, + truncated: bool, +} + +impl QuotaStatisticAccumulator { + /// False means stop the source scan: a resource ceiling was reached. Once + /// limited, the accumulator cannot produce a partial known monetary total. + pub fn push(&mut self, input: LogQuotaInput<'_>) -> bool { + if self.truncated { + return false; + } + if self.count == MAX_RECORDS { + self.truncated = true; + return false; } + self.count += 1; + let Some(bytes) = self.metadata_bytes.checked_add(input.other.len()) + .filter(|bytes| *bytes <= MAX_PROJECTION_METADATA_BYTES) + else { + self.truncated = true; + return false; + }; + self.metadata_bytes = bytes; let quota = input.quota.filter(|value| *value >= 0); let basis = captured_credits_per_usd(input.other); let (Some(quota), Some(basis)) = (quota, basis) else { - unpriced += 1; - continue; + self.unpriced += 1; + return true; }; if input.log_type != LOG_TYPE_CONSUME { - unpriced += 1; - continue; + self.unpriced += 1; + return true; } - if !sums.contains_key(&basis) && sums.len() == MAX_BASES { - return limited(unpriced); + if !self.sums.contains_key(&basis) && self.sums.len() == MAX_BASES { + self.truncated = true; + return false; } - let sum = sums.entry(basis).or_default(); + let sum = self.sums.entry(basis).or_default(); let Some(next) = sum.checked_add(i128::from(quota)) else { - return limited(unpriced); + self.truncated = true; + return false; }; *sum = next; + true } - if source_truncated { - return limited(unpriced); + + pub fn finish(self, source_truncated: bool) -> QuotaStatisticProjection { + if self.truncated || source_truncated { + return QuotaStatisticProjection { + money: QuotaMoneyProjection::unknown("projection_limit"), + unpriced_quota_records: self.unpriced, + quota_projection_truncated: true, + }; + } + let money = if self.unpriced > 0 { + QuotaMoneyProjection::unknown("aggregate_missing_credit_basis") + } else { + let total = self.sums.into_iter().fold( + BigRational::from_integer(0.into()), + |total, (basis, quota)| total + BigRational::new(quota.into(), basis.into()), + ); + QuotaMoneyProjection::known( + &total, + if self.count == 0 { "empty_result" } else { "captured_credits_per_usd" }, + ) + }; + QuotaStatisticProjection { + money, + unpriced_quota_records: self.unpriced, + quota_projection_truncated: false, + } } - let money = if unpriced > 0 { - QuotaMoneyProjection::unknown("aggregate_missing_credit_basis") - } else { - let total = sums.into_iter().fold(BigRational::from_integer(0.into()), |total, (basis, quota)| { - total + BigRational::new(quota.into(), basis.into()) - }); - QuotaMoneyProjection::known( - &total, - if count == 0 { "empty_result" } else { "captured_credits_per_usd" }, - ) - }; - QuotaStatisticProjection { - money, - unpriced_quota_records: unpriced, - quota_projection_truncated: false, +} + +/// Project a complete, authorized consumption selection, not only one page. +/// A repository must use the same filters and snapshot for all of its reads. +pub fn project_consumption_statistic<'a>( + rows: impl IntoIterator>, + source_truncated: bool, +) -> QuotaStatisticProjection { + let mut accumulator = QuotaStatisticAccumulator::default(); + for row in rows { + if !accumulator.push(row) { + break; + } } + accumulator.finish(source_truncated) } pub fn project_bucket_without_components() -> QuotaMoneyProjection { diff --git a/services/core/src/history_read/native.rs b/services/core/src/history_read/native.rs new file mode 100644 index 000000000..543960680 --- /dev/null +++ b/services/core/src/history_read/native.rs @@ -0,0 +1,395 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! Reads committed native model charges, not legacy logs or opening counters. +//! +//! Do not mount this as a legacy `/api/log` or all-history statistics endpoint. +//! The explicit `includes_legacy = false` is part of every response. Historical +//! sources still require the A03 ownership, unit and duplicate-mapping contract. +use super::snapshot::ReadSnapshot; +use crate::financial_profile::{self, FinancialProfile}; +use crate::identity::{self, CredentialKind, IdentityError}; +use serde::Serialize; +use sqlx::{PgPool, Postgres, QueryBuilder, Row, postgres::PgRow}; + +pub const MAX_PAGE_SIZE: u32 = 200; +pub const MAX_PAGE_OFFSET: u64 = 1_000_000; +pub const MAX_FLOW_BUCKETS: i64 = 2_048; +const CREDITS_PER_USD: i128 = 500_000; +const SOURCE: &str = "core_charging.requests/v1"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ReadError { + Unauthorized, + Forbidden, + Invalid, + Unavailable, + Storage, +} +impl From for ReadError { + fn from(_: sqlx::Error) -> Self { + Self::Storage + } +} +impl From for ReadError { + fn from(error: IdentityError) -> Self { + match error { + IdentityError::Unauthorized => Self::Unauthorized, + IdentityError::Forbidden => Self::Forbidden, + IdentityError::Invalid => Self::Invalid, + _ => Self::Storage, + } + } +} + +/// The server selects a mode for its route, never from a JSON permission field. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ReadMode { + SelfSession, + AdministratorSession, + ApiKey, +} + +/// All filters only narrow server-derived authority. No payer/team expansion. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NativeFilter { + pub start: i64, + pub end: i64, + pub user_id: Option, + pub key_id: Option, + pub model: Option, + pub group: Option, +} +impl NativeFilter { + fn validate(&self) -> Result<(), ReadError> { + if self.start < -62_135_596_800 + || self.end > 253_402_300_800 + || self.start >= self.end + || [self.user_id, self.key_id].into_iter().flatten().any(|id| id <= 0) + || [&self.model, &self.group] + .into_iter() + .flatten() + .any(|text| text.is_empty() || text.len() > 128 || text.chars().any(char::is_control)) + { + return Err(ReadError::Invalid); + } + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct NativePageRequest { + pub page: u32, + pub size: u32, +} +impl NativePageRequest { + fn offset(self) -> Result { + if self.page == 0 || self.size == 0 || self.size > MAX_PAGE_SIZE { + return Err(ReadError::Invalid); + } + let offset = u64::from(self.page - 1) * u64::from(self.size); + if offset > MAX_PAGE_OFFSET { + return Err(ReadError::Invalid); + } + i64::try_from(offset).map_err(|_| ReadError::Invalid) + } +} + +/// Native amounts are decimal strings so a browser cannot round BIGINT values. +/// Gross, refund and net amounts remain separate; a refund is not another call. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct NativeUsageTotals { + pub source: &'static str, + pub includes_legacy: bool, + pub request_count: String, + pub settled_credits: String, + pub refunded_credits: String, + pub net_credits: String, + pub settled_usd: String, + pub refunded_usd: String, + pub net_usd: String, + pub input_tokens: String, + pub cached_tokens: String, + pub output_tokens: String, +} + +/// No raw price JSON, fingerprint, worker digest, provider metadata or payer ID. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct NativeUsageRow { + pub id: String, + pub authorized_at: i64, + pub model: String, + pub group: String, + pub price_version: String, + pub settled_credits: String, + pub refunded_credits: String, + pub net_credits: String, + pub net_usd: String, + pub input_tokens: String, + pub cached_tokens: String, + pub output_tokens: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct NativeUsagePage { + pub totals: NativeUsageTotals, + pub page: u32, + pub page_size: u32, + pub items: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct NativeUsageBucket { + pub start: i64, + pub totals: NativeUsageTotals, +} + +// This object cannot be constructed from caller-supplied actor/account values. +// Its connection and authorized filters stay private for its entire lifetime. +pub struct NativeUsageRead<'a> { + snapshot: ReadSnapshot<'a>, + filter: NativeFilter, + owner_account_id: Option, + actor_user_id: i64, + mode: ReadMode, +} + +const TOTAL_COLUMNS: &str = "count(*)::text AS request_count, \ + COALESCE(sum(r.settled::numeric),0)::text AS settled, \ + COALESCE(sum(r.refunded::numeric),0)::text AS refunded, \ + COALESCE(sum(r.input_tokens::numeric),0)::text AS input_tokens, \ + COALESCE(sum(r.cached_tokens::numeric),0)::text AS cached_tokens, \ + COALESCE(sum(r.output_tokens::numeric),0)::text AS output_tokens"; + +impl<'a> NativeUsageRead<'a> { + pub async fn begin( + pool: &'a PgPool, + secret: &str, + mode: ReadMode, + mut filter: NativeFilter, + ) -> Result { + filter.validate()?; + let mut snapshot = ReadSnapshot::begin(pool).await?; + // S11's narrow integration helper reuses the existing identity decoder, + // including revocation/expiry/grants, without forbidden FOR SHARE locks. + let actor = identity::authorize_read_in_transaction(snapshot.connection(), secret).await?; + let owner_account_id = match mode { + ReadMode::SelfSession => { + if actor.credential_kind != CredentialKind::Session + || filter.user_id.is_some_and(|id| id != actor.user_id) + { + return Err(ReadError::Forbidden); + } + filter.user_id = Some(actor.user_id); + None + } + ReadMode::AdministratorSession => { + if actor.credential_kind != CredentialKind::Session { + return Err(ReadError::Forbidden); + } + // A paid service level is NOT a platform administrator role. + let allowed: bool = sqlx::query_scalar( + "SELECT platform_role IN ('admin','superadmin') FROM core_identity.users WHERE id=$1 AND active", + ) + .bind(actor.user_id) + .fetch_one(snapshot.connection()) + .await?; + if !allowed { + return Err(ReadError::Forbidden); + } + None + } + ReadMode::ApiKey => { + if actor.credential_kind != CredentialKind::ApiKey + || filter.user_id.is_some_and(|id| id != actor.user_id) + || filter.key_id.is_some_and(|id| id != actor.credential_id) + { + return Err(ReadError::Forbidden); + } + filter.user_id = Some(actor.user_id); + filter.key_id = Some(actor.credential_id); + Some(actor.owner_account_id) + } + }; + Ok(Self { + snapshot, + filter, + owner_account_id, + actor_user_id: actor.user_id, + mode, + }) + } + + pub async fn totals(&mut self) -> Result { + let mut query = QueryBuilder::::new("SELECT "); + query.push(TOTAL_COLUMNS); + selection(&mut query, &self.filter, self.owner_account_id); + let row = query.build().fetch_one(self.snapshot.connection()).await?; + totals_from_row(&row) + } + + /// Count and page share the authority, filters AND database snapshot. + pub async fn page(&mut self, page: NativePageRequest) -> Result { + let offset = page.offset()?; + let totals = self.totals().await?; + let mut query = QueryBuilder::::new( + "SELECT r.id,r.authorized_at,r.settled,r.refunded,r.input_tokens,r.cached_tokens,r.output_tokens, \ + CASE WHEN octet_length(r.price->>'model')<=128 THEN r.price->>'model' END AS model, \ + CASE WHEN octet_length(r.price->>'group')<=128 THEN r.price->>'group' END AS group_name, \ + CASE WHEN octet_length(r.price->>'version')<=128 THEN r.price->>'version' END AS price_version", + ); + selection(&mut query, &self.filter, self.owner_account_id); + query.push(" ORDER BY r.authorized_at DESC,r.id COLLATE \"C\" DESC LIMIT "); + query.push_bind(i64::from(page.size)); + query.push(" OFFSET "); + query.push_bind(offset); + let rows = query.build().fetch_all(self.snapshot.connection()).await?; + let items = rows.iter().map(usage_from_row).collect::, _>>()?; + Ok(NativeUsagePage { totals, page: page.page, page_size: page.size, items }) + } + + /// UTC fixed-duration buckets, not calendar/time-zone subscription periods. + pub async fn flow(&mut self, seconds: i64) -> Result, ReadError> { + if ![900, 3600, 86400].contains(&seconds) + || (self.filter.end - 1).div_euclid(seconds) + - self.filter.start.div_euclid(seconds) + 1 > MAX_FLOW_BUCKETS + { + return Err(ReadError::Invalid); + } + let mut query = QueryBuilder::::new("SELECT (floor(r.authorized_at::numeric/"); + query.push_bind(seconds); + query.push(")*"); + query.push_bind(seconds); + query.push(")::bigint AS bucket_start,"); + query.push(TOTAL_COLUMNS); + selection(&mut query, &self.filter, self.owner_account_id); + query.push(" GROUP BY 1 ORDER BY 1"); + let rows = query.build().fetch_all(self.snapshot.connection()).await?; + rows.iter().map(|row| Ok(NativeUsageBucket { + start: row.try_get("bucket_start")?, + totals: totals_from_row(row)?, + })).collect() + } + + /// A key authorizes key history only, never the user's complete wallet. + pub async fn self_profile(&mut self) -> Result { + if self.mode != ReadMode::SelfSession { + return Err(ReadError::Forbidden); + } + financial_profile::read(self.snapshot.connection(), self.actor_user_id) + .await + .map_err(|error| match error { + financial_profile::ProfileError::Invalid => ReadError::Invalid, + financial_profile::ProfileError::Unavailable => ReadError::Unavailable, + financial_profile::ProfileError::Storage => ReadError::Storage, + }) + } + + pub async fn finish(self) -> Result<(), ReadError> { + self.snapshot.finish().await.map_err(Into::into) + } +} + +fn selection<'q>(query: &mut QueryBuilder<'q, Postgres>, filter: &'q NativeFilter, owner: Option) { + query.push(" FROM core_charging.requests r WHERE r.state='settled' AND r.authorized_at>="); + query.push_bind(filter.start); + query.push(" AND r.authorized_at<"); + query.push_bind(filter.end); + for (name, value) in [("r.actor_user_id", filter.user_id), ("r.key_id", filter.key_id), ("r.owner_account_id", owner)] { + if let Some(value) = value { + query.push(" AND ").push(name).push("=").push_bind(value); + } + } + for (name, value) in [("model", &filter.model), ("group", &filter.group)] { + if let Some(value) = value { + query.push(" AND r.price->>'").push(name).push("'=").push_bind(value); + } + } +} + +fn nonnegative(value: &str) -> Result { + value.parse::().ok().filter(|amount| *amount >= 0).ok_or(ReadError::Unavailable) +} + +fn credit_usd(value: i128) -> String { + // All values are checked nonnegative first. 1/500000 USD is exactly 2e-6. + let whole = value / CREDITS_PER_USD; + let fraction = (value % CREDITS_PER_USD) * 2; + if fraction == 0 { + whole.to_string() + } else { + format!("{whole}.{fraction:06}").trim_end_matches('0').to_owned() + } +} + +fn totals_from_row(row: &PgRow) -> Result { + let amount = |name| -> Result { nonnegative(&row.try_get::(name)?) }; + let settled = amount("settled")?; + let refunded = amount("refunded")?; + let net = settled.checked_sub(refunded).filter(|value| *value >= 0).ok_or(ReadError::Storage)?; + Ok(NativeUsageTotals { + source: SOURCE, + includes_legacy: false, + request_count: amount("request_count")?.to_string(), + settled_credits: settled.to_string(), + refunded_credits: refunded.to_string(), + net_credits: net.to_string(), + settled_usd: credit_usd(settled), + refunded_usd: credit_usd(refunded), + net_usd: credit_usd(net), + input_tokens: amount("input_tokens")?.to_string(), + cached_tokens: amount("cached_tokens")?.to_string(), + output_tokens: amount("output_tokens")?.to_string(), + }) +} + +fn usage_from_row(row: &PgRow) -> Result { + let amount = |name| -> Result { + let value: i64 = row.try_get(name)?; + (value >= 0).then_some(i128::from(value)).ok_or(ReadError::Storage) + }; + let text = |name| -> Result { + let value: Option = row.try_get(name)?; + value.filter(|v| !v.is_empty() && v.len() <= 128 && !v.chars().any(char::is_control)) + .ok_or(ReadError::Storage) + }; + let settled = amount("settled")?; + let refunded = amount("refunded")?; + let net = settled.checked_sub(refunded).filter(|value| *value >= 0).ok_or(ReadError::Storage)?; + Ok(NativeUsageRow { + id: text("id")?, + authorized_at: row.try_get("authorized_at")?, + model: text("model")?, + group: text("group_name")?, + price_version: text("price_version")?, + settled_credits: settled.to_string(), + refunded_credits: refunded.to_string(), + net_credits: net.to_string(), + net_usd: credit_usd(net), + input_tokens: amount("input_tokens")?.to_string(), + cached_tokens: amount("cached_tokens")?.to_string(), + output_tokens: amount("output_tokens")?.to_string(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn native_money_is_exact_and_never_uses_a_legacy_divisor() { + for (credits, expected) in [(0, "0"), (1, "0.000002"), (3, "0.000006"), (500_001, "1.000002")] { + assert_eq!(credit_usd(credits), expected); + } + assert_eq!(nonnegative("-1"), Err(ReadError::Unavailable)); + assert_eq!(nonnegative("1.0"), Err(ReadError::Unavailable)); + assert_eq!(nonnegative("170141183460469231731687303715884105728"), Err(ReadError::Unavailable)); + } + + #[test] + fn pagination_rejects_zero_and_offset_overflow() { + for page in [NativePageRequest { page: 0, size: 1 }, NativePageRequest { page: 1, size: 0 }, + NativePageRequest { page: 1, size: MAX_PAGE_SIZE + 1 }, NativePageRequest { page: u32::MAX, size: 200 }] { + assert_eq!(page.offset(), Err(ReadError::Invalid)); + } + assert_eq!(NativePageRequest { page: 2, size: 200 }.offset(), Ok(200)); + } +} diff --git a/services/core/tests/finance_export.rs b/services/core/tests/finance_export.rs index 54055010b..a4282b2d0 100644 --- a/services/core/tests/finance_export.rs +++ b/services/core/tests/finance_export.rs @@ -266,3 +266,6 @@ fn export_distinguishes_sink_limits_from_other_io_failures() { assert!(matches!(writer.finish(), Err(error) if error == expected)); } } + +#[path = "finance_export/csv.rs"] +mod csv; diff --git a/services/core/tests/finance_export/csv.rs b/services/core/tests/finance_export/csv.rs new file mode 100644 index 000000000..5df563308 --- /dev/null +++ b/services/core/tests/finance_export/csv.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +use super::{BrokenSink, row}; +use lmm_core::finance_export::{ + ExportError, MAX_EXPORT_METADATA_BYTES, UsageJsonWriter, csv::UsageCsvWriter, +}; + +#[test] +fn csv_quotes_unicode_commas_quotes_and_embedded_newlines() { + let mut input = row(1, r#"{"pricing_credits_per_usd":500000,"secret":"PRIVATE"}"#); + input.username = "用户,\"test\"\nline"; + let mut writer = UsageCsvWriter::new(Vec::new(), 10, 10_000, true).unwrap(); + writer.push(&input).unwrap(); + let (bytes, receipt) = writer.finish().unwrap(); + assert!(bytes.starts_with(b"\xef\xbb\xbf")); + assert_eq!(receipt.rows, 1); + assert_eq!(receipt.bytes, bytes.len()); + let text = String::from_utf8(bytes).unwrap(); + assert!(text.contains("\"用户,\"\"test\"\"\nline\"")); + assert!(text.ends_with("\r\n")); + assert!(!text.contains("PRIVATE")); + assert!(!text.contains("secret")); +} + +#[test] +fn csv_formula_guard_cannot_be_bypassed_by_leading_space_or_control() { + for text in ["=1+1", "+1+1", "-1+1", "@SUM(A1)", " =1", "\t=1", "\r@SUM(A1)", "\n+1", "\u{feff}=1"] { + let mut input = row(1, "{}"); + input.username = text; + let mut writer = UsageCsvWriter::new(Vec::new(), 10, 10_000, false).unwrap(); + writer.push(&input).unwrap(); + let (bytes, _) = writer.finish().unwrap(); + let decoded = String::from_utf8(bytes).unwrap(); + assert!(decoded.contains(&format!("\"'{text}\"")), "{text:?}"); + assert_eq!(input.username, text); + let mut json = UsageJsonWriter::new(Vec::new(), 10, 10_000).unwrap(); + json.push(&input).unwrap(); + let value: serde_json::Value = serde_json::from_slice(&json.finish().unwrap().0).unwrap(); + assert_eq!(value[0]["username"], text); + } +} + +#[test] +fn csv_does_not_change_a_typed_negative_raw_quota_into_a_formula_or_zero() { + let mut input = row(1, "{}"); + input.log_type = 3; + input.quota = Some(-17); + let mut writer = UsageCsvWriter::new(Vec::new(), 10, 10_000, false).unwrap(); + writer.push(&input).unwrap(); + let text = String::from_utf8(writer.finish().unwrap().0).unwrap(); + assert!(text.contains(",\"-17\",")); + assert!(!text.contains("\"'-17\"")); + assert!(!text.contains("captured_credits_per_usd")); +} + +#[test] +fn csv_unknown_money_is_blank_with_an_explicit_unavailable_flag() { + let mut writer = UsageCsvWriter::new(Vec::new(), 10, 10_000, false).unwrap(); + writer.push(&row(1, "{}")).unwrap(); + let text = String::from_utf8(writer.finish().unwrap().0).unwrap(); + assert!(text.ends_with("\"\",\"false\",\"missing_credit_basis\"\r\n")); +} + +#[test] +fn csv_limits_and_duplicate_order_abort_the_whole_export() { + let mut writer = UsageCsvWriter::new(Vec::new(), 1, 10_000, false).unwrap(); + writer.push(&row(1, "{}")).unwrap(); + assert_eq!(writer.push(&row(2, "{}")), Err(ExportError::RowLimit)); + assert!(matches!(writer.finish(), Err(ExportError::Aborted))); + let mut writer = UsageCsvWriter::new(Vec::new(), 10, 10_000, false).unwrap(); + writer.push(&row(1, "{}")).unwrap(); + assert_eq!(writer.push(&row(1, "{}")), Err(ExportError::OutOfOrder)); + assert!(matches!(writer.finish(), Err(ExportError::Aborted))); + assert!(matches!(UsageCsvWriter::new(Vec::new(), 10, 1, false), Err(ExportError::InvalidLimits))); +} + +#[test] +fn csv_final_byte_and_header_bom_are_included_in_the_limit() { + let mut complete = UsageCsvWriter::new(Vec::new(), 1, 10_000, true).unwrap(); + complete.push(&row(1, "{}")).unwrap(); + let expected = complete.finish().unwrap().0; + let mut exact = UsageCsvWriter::new(Vec::new(), 1, expected.len(), true).unwrap(); + exact.push(&row(1, "{}")).unwrap(); + assert_eq!(exact.finish().unwrap().0, expected); + let mut small = UsageCsvWriter::new(Vec::new(), 1, expected.len() - 1, true).unwrap(); + assert_eq!(small.push(&row(1, "{}")), Err(ExportError::ByteLimit)); + assert!(matches!(small.finish(), Err(ExportError::Aborted))); +} + +#[test] +fn csv_io_failure_never_produces_a_completion_receipt() { + let header = UsageCsvWriter::new(Vec::new(), 10, 10_000, false).unwrap().finish().unwrap().0; + let mut writer = UsageCsvWriter::new(BrokenSink { remaining: header.len() + 5 }, 10, 10_000, false).unwrap(); + assert_eq!(writer.push(&row(1, "{}")), Err(ExportError::Io)); + assert_eq!(writer.push(&row(2, "{}")), Err(ExportError::Aborted)); + assert!(matches!(writer.finish(), Err(ExportError::Aborted))); +} + +#[test] +fn both_formats_bound_total_metadata_work_and_reject_invalid_scalars() { + let padding = "x".repeat(MAX_EXPORT_METADATA_BYTES + 1); + let input = row(1, &padding); + let mut csv = UsageCsvWriter::new(Vec::new(), 10, 10_000, false).unwrap(); + assert_eq!(csv.push(&input), Err(ExportError::ByteLimit)); + let mut json = UsageJsonWriter::new(Vec::new(), 10, 10_000).unwrap(); + assert_eq!(json.push(&input), Err(ExportError::ByteLimit)); + for which in 0..6 { + let mut invalid = row(1, "{}"); + match which { + 0 => invalid.prompt_tokens = -1, + 1 => invalid.completion_tokens = -1, + 2 => invalid.use_time = -1, + 3 => invalid.channel_id = Some(-1), + 4 => invalid.created_at = 253_402_300_800, + _ => invalid.username = "nul\0name", + } + let mut csv = UsageCsvWriter::new(Vec::new(), 10, 10_000, false).unwrap(); + assert_eq!(csv.push(&invalid), Err(ExportError::InvalidRow)); + let mut json = UsageJsonWriter::new(Vec::new(), 10, 10_000).unwrap(); + assert_eq!(json.push(&invalid), Err(ExportError::InvalidRow)); + } +} diff --git a/services/core/tests/history_read.rs b/services/core/tests/history_read.rs index 366c56498..aa2a0002a 100644 --- a/services/core/tests/history_read.rs +++ b/services/core/tests/history_read.rs @@ -3,8 +3,8 @@ //! databases; run them only under A09's exclusive synthetic resource lease. use lmm_core::history_read::{ LogQuotaInput, MAX_BASES, MAX_CREDIT_BASIS, MAX_METADATA_BYTES, MAX_RECORDS, - captured_credits_per_usd, project_bucket_without_components, - project_consumption_statistic, project_log, + captured_credits_per_usd, project_bucket_without_components, project_consumption_statistic, + project_log, snapshot::ReadSnapshot, usage::{UsageArithmeticInput, UsageError, UsageRounding, project_migrated_usage}, }; @@ -12,54 +12,122 @@ use serde_json::Value; use sqlx::PgPool; fn consumption(quota: i64, other: &str) -> LogQuotaInput<'_> { - LogQuotaInput { quota: Some(quota), log_type: 2, other } + LogQuotaInput { + quota: Some(quota), + log_type: 2, + other, + } } #[test] fn projection_accepts_exact_integer_json_forms() { - for raw in ["500000", "5e5", "500000.0", "500000000e-3", "0.0000000000005e18"] { + for raw in [ + "500000", + "5e5", + "500000.0", + "500000000e-3", + "0.0000000000005e18", + ] { let other = format!(r#"{{"pricing_credits_per_usd":{raw}}}"#); assert_eq!(captured_credits_per_usd(&other), Some(500000), "{raw}"); } - assert_eq!(captured_credits_per_usd(&format!(r#"{{"pricing_credits_per_usd":{MAX_CREDIT_BASIS}}}"#)), Some(MAX_CREDIT_BASIS)); + assert_eq!( + captured_credits_per_usd(&format!( + r#"{{"pricing_credits_per_usd":{MAX_CREDIT_BASIS}}}"# + )), + Some(MAX_CREDIT_BASIS) + ); } #[test] fn projection_rejects_unbounded_ambiguous_or_inexact_anchors() { - for raw in ["0", "-1", "true", "null", "[]", "{}", r#""500000""#, "1.5", "9007199254740991.1", "9007199254740992", "1e19", "1e-19", "1e999999999999999999999", "0.000000000000000001"] { - assert_eq!(captured_credits_per_usd(&format!(r#"{{"pricing_credits_per_usd":{raw}}}"#)), None, "{raw}"); + for raw in [ + "0", + "-1", + "true", + "null", + "[]", + "{}", + r#""500000""#, + "1.5", + "9007199254740991.1", + "9007199254740992", + "1e19", + "1e-19", + "1e999999999999999999999", + "0.000000000000000001", + ] { + assert_eq!( + captured_credits_per_usd(&format!(r#"{{"pricing_credits_per_usd":{raw}}}"#)), + None, + "{raw}" + ); } - for other in ["{}", "[]", "[500000]", "null", "not JSON", r#"{"pricing_credits_per_usd":1,"pricing_credits_per_usd":2}"#, r#"{"pricing_credits_per_usd":1} {}"#] { + for other in [ + "{}", + "[]", + "[500000]", + "null", + "not JSON", + r#"{"pricing_credits_per_usd":1,"pricing_credits_per_usd":2}"#, + r#"{"pricing_credits_per_usd":1} {}"#, + ] { assert_eq!(captured_credits_per_usd(other), None, "{other}"); } - let large = format!(r#"{{"pricing_credits_per_usd":1,"padding":"{}"}}"#, "x".repeat(MAX_METADATA_BYTES)); + let large = format!( + r#"{{"pricing_credits_per_usd":1,"padding":"{}"}}"#, + "x".repeat(MAX_METADATA_BYTES) + ); assert_eq!(captured_credits_per_usd(&large), None); } #[test] fn projection_preserves_raw_quota_and_unknown_semantics() { let input = consumption(1, r#"{"pricing_credits_per_usd":3}"#); - assert_eq!(project_log(input).quota_usd.as_deref(), Some("0.333333333333333333")); + assert_eq!( + project_log(input).quota_usd.as_deref(), + Some("0.333333333333333333") + ); assert_eq!(input.quota, Some(1)); assert_eq!(project_log(consumption(0, "{}")).quota_usd, None); - assert_eq!(project_log(consumption(-1, r#"{"pricing_credits_per_usd":3}"#)).quota_money_basis, "missing_credit_basis"); - assert_eq!(project_log(LogQuotaInput { quota: None, ..input }).quota_money_basis, "invalid_stored_quota"); + assert_eq!( + project_log(consumption(-1, r#"{"pricing_credits_per_usd":3}"#)).quota_money_basis, + "missing_credit_basis" + ); + assert_eq!( + project_log(LogQuotaInput { + quota: None, + ..input + }) + .quota_money_basis, + "invalid_stored_quota" + ); for log_type in [0, 1, 3, 4, 5, 6, 7] { - assert_eq!(project_log(LogQuotaInput { log_type, ..input }).quota_money_basis, "not_consumption"); + assert_eq!( + project_log(LogQuotaInput { log_type, ..input }).quota_money_basis, + "not_consumption" + ); } - assert_eq!(project_bucket_without_components().quota_money_basis, "bucket_without_credit_basis"); + assert_eq!( + project_bucket_without_components().quota_money_basis, + "bucket_without_credit_basis" + ); } #[test] fn projection_rounds_only_once_after_adding_different_bases() { - let result = project_consumption_statistic([ - consumption(1, r#"{"pricing_credits_per_usd":3}"#), - consumption(1, r#"{"pricing_credits_per_usd":6}"#), - ], false); + let result = project_consumption_statistic( + [ + consumption(1, r#"{"pricing_credits_per_usd":3}"#), + consumption(1, r#"{"pricing_credits_per_usd":6}"#), + ], + false, + ); assert_eq!(result.money.quota_usd.as_deref(), Some("0.5")); - let result = project_consumption_statistic([ - consumption(1, r#"{"pricing_credits_per_usd":3}"#); 3 - ], false); + let result = project_consumption_statistic( + [consumption(1, r#"{"pricing_credits_per_usd":3}"#); 3], + false, + ); assert_eq!(result.money.quota_usd.as_deref(), Some("1")); } @@ -69,28 +137,49 @@ fn projection_rounds_an_exact_mixed_basis_half_tie_away_from_zero() { consumption(1, r#"{"pricing_credits_per_usd":524288}"#), consumption(1, r#"{"pricing_credits_per_usd":3814697265625}"#), ]; - assert_eq!(project_consumption_statistic(rows, false).money.quota_usd.as_deref(), Some("0.000001907348894957")); - assert_eq!(project_consumption_statistic(rows.into_iter().rev(), false).money.quota_usd.as_deref(), Some("0.000001907348894957")); + assert_eq!( + project_consumption_statistic(rows, false) + .money + .quota_usd + .as_deref(), + Some("0.000001907348894957") + ); + assert_eq!( + project_consumption_statistic(rows.into_iter().rev(), false) + .money + .quota_usd + .as_deref(), + Some("0.000001907348894957") + ); } #[test] fn projection_handles_maximum_raw_sums_without_i64_overflow() { let row = consumption(i64::MAX, r#"{"pricing_credits_per_usd":1}"#); let result = project_consumption_statistic(std::iter::repeat_n(row, MAX_RECORDS), false); - assert_eq!(result.money.quota_usd.as_deref(), Some("92233720368547758070000")); + assert_eq!( + result.money.quota_usd.as_deref(), + Some("92233720368547758070000") + ); } #[test] fn projection_does_not_publish_partial_known_totals() { - let result = project_consumption_statistic([ - consumption(500000, r#"{"pricing_credits_per_usd":500000}"#), - consumption(12, "{}"), - ], false); + let result = project_consumption_statistic( + [ + consumption(500000, r#"{"pricing_credits_per_usd":500000}"#), + consumption(12, "{}"), + ], + false, + ); assert!(!result.money.quota_money_available); assert_eq!(result.money.quota_usd, None); assert_eq!(result.unpriced_quota_records, 1); assert!(!result.quota_projection_truncated); - assert_eq!(result.money.quota_money_basis, "aggregate_missing_credit_basis"); + assert_eq!( + result.money.quota_money_basis, + "aggregate_missing_credit_basis" + ); } #[test] @@ -109,39 +198,77 @@ fn projection_distinguishes_empty_zero_and_truncated() { #[test] fn projection_bounds_rows_and_number_of_bases() { let input = consumption(1, r#"{"pricing_credits_per_usd":1}"#); - assert!(project_consumption_statistic(std::iter::repeat_n(input, MAX_RECORDS + 1), false).quota_projection_truncated); - let metadata: Vec<_> = (1..=MAX_BASES + 1).map(|basis| format!(r#"{{"pricing_credits_per_usd":{basis}}}"#)).collect(); - assert!(!project_consumption_statistic(metadata[..MAX_BASES].iter().map(|other| consumption(1, other)), false).quota_projection_truncated); - assert!(project_consumption_statistic(metadata.iter().map(|other| consumption(1, other)), false).quota_projection_truncated); + assert!( + project_consumption_statistic(std::iter::repeat_n(input, MAX_RECORDS + 1), false) + .quota_projection_truncated + ); + let metadata: Vec<_> = (1..=MAX_BASES + 1) + .map(|basis| format!(r#"{{"pricing_credits_per_usd":{basis}}}"#)) + .collect(); + assert!( + !project_consumption_statistic( + metadata[..MAX_BASES] + .iter() + .map(|other| consumption(1, other)), + false + ) + .quota_projection_truncated + ); + assert!( + project_consumption_statistic(metadata.iter().map(|other| consumption(1, other)), false) + .quota_projection_truncated + ); } #[test] fn projection_matches_versioned_golden_contract() { - let fixtures: Value = serde_json::from_str(include_str!("../src/history_read/fixtures/projection-v1.json")).unwrap(); + let fixtures: Value = serde_json::from_str(include_str!( + "../src/history_read/fixtures/projection-v1.json" + )) + .unwrap(); for case in fixtures["log_cases"].as_array().unwrap() { let input = LogQuotaInput { quota: case["quota"].as_i64(), log_type: case["log_type"].as_i64().unwrap() as i32, other: case["other"].as_str().unwrap(), }; - assert_eq!(serde_json::to_value(project_log(input)).unwrap(), case["expected"], "{}", case["name"]); + assert_eq!( + serde_json::to_value(project_log(input)).unwrap(), + case["expected"], + "{}", + case["name"] + ); } for case in fixtures["statistic_cases"].as_array().unwrap() { - let rows = case["rows"].as_array().unwrap().iter().map(|row| LogQuotaInput { - quota: row["quota"].as_i64(), - log_type: row["log_type"].as_i64().unwrap() as i32, - other: row["other"].as_str().unwrap(), - }); - let result = project_consumption_statistic(rows, case["source_truncated"].as_bool().unwrap()); - assert_eq!(serde_json::to_value(result).unwrap(), case["expected"], "{}", case["name"]); + let rows = case["rows"] + .as_array() + .unwrap() + .iter() + .map(|row| LogQuotaInput { + quota: row["quota"].as_i64(), + log_type: row["log_type"].as_i64().unwrap() as i32, + other: row["other"].as_str().unwrap(), + }); + let result = + project_consumption_statistic(rows, case["source_truncated"].as_bool().unwrap()); + assert_eq!( + serde_json::to_value(result).unwrap(), + case["expected"], + "{}", + case["name"] + ); } } #[test] fn usage_preserves_separate_baseline_and_refund_units() { let input = UsageArithmeticInput { - current_used: 250, historical_raw: 300, reversed_raw: 60, reversed_normalized: 20, - divisor: "3", rounding: UsageRounding::HalfAwayFromZero, + current_used: 250, + historical_raw: 300, + reversed_raw: 60, + reversed_normalized: 20, + divisor: "3", + rounding: UsageRounding::HalfAwayFromZero, }; let result = project_migrated_usage(&input).unwrap(); assert_eq!(result.normalized_used_quota, 90); @@ -156,48 +283,97 @@ fn usage_preserves_separate_baseline_and_refund_units() { #[test] fn usage_rejects_unproven_reductions_and_bad_bases() { let mut input = UsageArithmeticInput { - current_used: 299, historical_raw: 300, reversed_raw: 0, reversed_normalized: 0, - divisor: "3", rounding: UsageRounding::TowardZero, + current_used: 299, + historical_raw: 300, + reversed_raw: 0, + reversed_normalized: 0, + divisor: "3", + rounding: UsageRounding::TowardZero, }; - assert_eq!(project_migrated_usage(&input), Err(UsageError::UnprovenReduction)); + assert_eq!( + project_migrated_usage(&input), + Err(UsageError::UnprovenReduction) + ); input.current_used = 300; - for divisor in ["", "1", "0", "-3", "3e0", "3.", "3.0000000000000000001", "NaN"] { + for divisor in [ + "", + "1", + "0", + "-3", + "3e0", + "3.", + "3.0000000000000000001", + "NaN", + ] { input.divisor = divisor; - assert_eq!(project_migrated_usage(&input), Err(UsageError::InvalidBasis)); + assert_eq!( + project_migrated_usage(&input), + Err(UsageError::InvalidBasis) + ); } input.divisor = "3"; input.reversed_raw = 301; - assert_eq!(project_migrated_usage(&input), Err(UsageError::InvalidBasis)); + assert_eq!( + project_migrated_usage(&input), + Err(UsageError::InvalidBasis) + ); } #[test] fn usage_obeys_the_captured_rounding_policy() { let mut input = UsageArithmeticInput { - current_used: 3, historical_raw: 3, reversed_raw: 0, reversed_normalized: 0, - divisor: "2", rounding: UsageRounding::TowardZero, + current_used: 3, + historical_raw: 3, + reversed_raw: 0, + reversed_normalized: 0, + divisor: "2", + rounding: UsageRounding::TowardZero, }; - assert_eq!(project_migrated_usage(&input).unwrap().normalized_used_quota, 1); + assert_eq!( + project_migrated_usage(&input) + .unwrap() + .normalized_used_quota, + 1 + ); input.rounding = UsageRounding::HalfAwayFromZero; - assert_eq!(project_migrated_usage(&input).unwrap().normalized_used_quota, 2); + assert_eq!( + project_migrated_usage(&input) + .unwrap() + .normalized_used_quota, + 2 + ); } async fn prepare(pool: &PgPool) { sqlx::query("CREATE TABLE a04_history_fixture (id BIGINT PRIMARY KEY, user_id BIGINT NOT NULL, quota BIGINT NOT NULL)") .execute(pool).await.unwrap(); sqlx::query("INSERT INTO a04_history_fixture VALUES (1,7,100),(2,8,999)") - .execute(pool).await.unwrap(); + .execute(pool) + .await + .unwrap(); } #[sqlx::test(migrations = false)] async fn pg_default_snapshot_exposes_the_count_page_counterexample(pool: PgPool) { prepare(&pool).await; let mut tx = pool.begin().await.unwrap(); - sqlx::query("SET TRANSACTION ISOLATION LEVEL READ COMMITTED").execute(&mut *tx).await.unwrap(); + sqlx::query("SET TRANSACTION ISOLATION LEVEL READ COMMITTED") + .execute(&mut *tx) + .await + .unwrap(); let count: i64 = sqlx::query_scalar("SELECT count(*) FROM a04_history_fixture WHERE user_id=7") - .fetch_one(&mut *tx).await.unwrap(); - sqlx::query("INSERT INTO a04_history_fixture VALUES (3,7,200)").execute(&pool).await.unwrap(); - let rows: Vec<(i64,)> = sqlx::query_as("SELECT quota FROM a04_history_fixture WHERE user_id=7 ORDER BY id") - .fetch_all(&mut *tx).await.unwrap(); + .fetch_one(&mut *tx) + .await + .unwrap(); + sqlx::query("INSERT INTO a04_history_fixture VALUES (3,7,200)") + .execute(&pool) + .await + .unwrap(); + let rows: Vec<(i64,)> = + sqlx::query_as("SELECT quota FROM a04_history_fixture WHERE user_id=7 ORDER BY id") + .fetch_all(&mut *tx) + .await + .unwrap(); assert_eq!(count, 1); assert_eq!(rows.len(), 2); tx.rollback().await.unwrap(); @@ -208,17 +384,32 @@ async fn pg_read_snapshot_keeps_count_page_sum_consistent(pool: PgPool) { prepare(&pool).await; let mut snapshot = ReadSnapshot::begin(&pool).await.unwrap(); let count: i64 = sqlx::query_scalar("SELECT count(*) FROM a04_history_fixture WHERE user_id=7") - .fetch_one(snapshot.connection()).await.unwrap(); - sqlx::query("INSERT INTO a04_history_fixture VALUES (3,7,200)").execute(&pool).await.unwrap(); - let rows: Vec<(i64,)> = sqlx::query_as("SELECT quota FROM a04_history_fixture WHERE user_id=7 ORDER BY id") - .fetch_all(snapshot.connection()).await.unwrap(); - let sum: String = sqlx::query_scalar("SELECT sum(quota)::text FROM a04_history_fixture WHERE user_id=7") - .fetch_one(snapshot.connection()).await.unwrap(); + .fetch_one(snapshot.connection()) + .await + .unwrap(); + sqlx::query("INSERT INTO a04_history_fixture VALUES (3,7,200)") + .execute(&pool) + .await + .unwrap(); + let rows: Vec<(i64,)> = + sqlx::query_as("SELECT quota FROM a04_history_fixture WHERE user_id=7 ORDER BY id") + .fetch_all(snapshot.connection()) + .await + .unwrap(); + let sum: String = + sqlx::query_scalar("SELECT sum(quota)::text FROM a04_history_fixture WHERE user_id=7") + .fetch_one(snapshot.connection()) + .await + .unwrap(); assert_eq!(count, 1); assert_eq!(rows, [(100,)]); assert_eq!(sum, "100"); snapshot.finish().await.unwrap(); - let current: i64 = sqlx::query_scalar("SELECT count(*) FROM a04_history_fixture WHERE user_id=7").fetch_one(&pool).await.unwrap(); + let current: i64 = + sqlx::query_scalar("SELECT count(*) FROM a04_history_fixture WHERE user_id=7") + .fetch_one(&pool) + .await + .unwrap(); assert_eq!(current, 2); } @@ -227,9 +418,52 @@ async fn pg_read_snapshot_cannot_mutate_persistent_rows(pool: PgPool) { prepare(&pool).await; let mut snapshot = ReadSnapshot::begin(&pool).await.unwrap(); let error = sqlx::query("UPDATE a04_history_fixture SET quota=0") - .execute(snapshot.connection()).await.unwrap_err(); - assert_eq!(error.as_database_error().unwrap().code().as_deref(), Some("25006")); + .execute(snapshot.connection()) + .await + .unwrap_err(); + assert_eq!( + error.as_database_error().unwrap().code().as_deref(), + Some("25006") + ); snapshot.finish().await.unwrap(); - let values: Vec<(i64,)> = sqlx::query_as("SELECT quota FROM a04_history_fixture ORDER BY id").fetch_all(&pool).await.unwrap(); + let values: Vec<(i64,)> = sqlx::query_as("SELECT quota FROM a04_history_fixture ORDER BY id") + .fetch_all(&pool) + .await + .unwrap(); assert_eq!(values, [(100,), (999,)]); } + +#[path = "history_read/native.rs"] +mod native; + +#[test] +fn metadata_work_is_bounded_even_below_the_record_ceiling() { + use lmm_core::history_read::{MAX_PROJECTION_METADATA_BYTES, QuotaStatisticAccumulator}; + let prefix = r#"{"pricing_credits_per_usd":1,"padding":""#; + let metadata = format!("{}{}{}", prefix, "x".repeat(MAX_METADATA_BYTES - prefix.len() - 2), "\"}"); + assert_eq!(metadata.len(), MAX_METADATA_BYTES); + let mut accumulator = QuotaStatisticAccumulator::default(); + for _ in 0..MAX_PROJECTION_METADATA_BYTES / MAX_METADATA_BYTES { + assert!(accumulator.push(consumption(1, &metadata))); + } + assert!(!accumulator.push(consumption(1, &metadata))); + assert!(!accumulator.push(consumption(1, r#"{"pricing_credits_per_usd":1}"#))); + let result = accumulator.finish(false); + assert!(result.quota_projection_truncated); + assert!(!result.money.quota_money_available); + assert_eq!(result.money.quota_usd, None); +} + +#[test] +fn streamed_projection_matches_the_same_complete_selection() { + use lmm_core::history_read::QuotaStatisticAccumulator; + let rows = [ + consumption(1, r#"{"pricing_credits_per_usd":3}"#), + consumption(1, r#"{"pricing_credits_per_usd":6}"#), + ]; + let mut accumulator = QuotaStatisticAccumulator::default(); + for row in rows { + assert!(accumulator.push(row)); + } + assert_eq!(accumulator.finish(false), project_consumption_statistic(rows, false)); +} diff --git a/services/core/tests/history_read/native.rs b/services/core/tests/history_read/native.rs new file mode 100644 index 000000000..28ee04615 --- /dev/null +++ b/services/core/tests/history_read/native.rs @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +//! Actual identity, charging and ledger fixtures, all in SQLx disposable PG. +//! These are test definitions, not evidence until the commands have executed. +use lmm_core::{ + accounts::{Account, AccountKind}, + billing::{ + Credits, + charging::{Charging, Outcome, Price, Request, Usage}, + ledger::{Action, Ledger, Request as LedgerRequest}, + transactional_ledger::TransactionalLedger, + }, + core_schema, + funding::FundingPolicy, + history_read::native::{NativeFilter, NativePageRequest, NativeUsageRead, ReadError, ReadMode}, + identity::{IdentityStore, IssuedCredential}, +}; +use sqlx::PgPool; +use std::sync::Arc; + +struct Fixture { + identity: IdentityStore, + session: IssuedCredential, + other: IssuedCredential, + key: IssuedCredential, + second_key: IssuedCredential, + charging: Charging, + filter: NativeFilter, +} + +async fn fixture(pool: &PgPool) -> Fixture { + core_schema::init(pool).await.unwrap(); + let identity = IdentityStore::from_pool(pool.clone()); + let session = identity.bootstrap_user(7, 1).await.unwrap(); + let other = identity.bootstrap_user(8, 1).await.unwrap(); + let owner = Account { kind: AccountKind::Personal, id: 7 }; + let key = identity.issue_key(&session.secret, owner, FundingPolicy::inherit_account_settings(), 3600).await.unwrap(); + let second_key = identity.issue_key(&session.secret, owner, FundingPolicy::inherit_account_settings(), 3600).await.unwrap(); + let payer = identity.authorize(&key.secret).await.unwrap().owner_account_id; + let ledger = Ledger::from_pool(pool.clone()); + ledger.open_wallet(payer).await.unwrap(); + ledger.execute(&LedgerRequest { + scope: "a04-native-tests".into(), + operation_key: "synthetic-credit".into(), + actor_user_id: Some(7), + reason: "Synthetic fixture only".into(), + action: Action::Credit { + account_id: payer, + amount_units: Credits::try_from(100_000).unwrap(), + payment_reference: "synthetic-credit".into(), + }, + }).await.unwrap(); + let now: i64 = sqlx::query_scalar("SELECT floor(extract(epoch from clock_timestamp()))::bigint") + .fetch_one(pool).await.unwrap(); + Fixture { + identity, + session, + other, + key, + second_key, + charging: Charging::new(pool.clone(), Arc::new(TransactionalLedger)), + filter: NativeFilter { start: now - 60, end: now + 600, user_id: None, key_id: None, model: None, group: None }, + } +} + +async fn settle(fixture: &Fixture, secret: &str, id: &str) { + let request = Request { + id: id.into(), + fingerprint: [3; 32], + reserve: 100, + lease_seconds: 300, + maximum_seconds: 600, + price: Price { + version: "a04-synthetic-v1".into(), + model: "local-model".into(), + group: "default".into(), + input_per_million: 1_000_000, + cached_per_million: 500_000, + output_per_million: 2_000_000, + }, + }; + fixture.charging.reserve(secret, &request, &[5; 32]).await.unwrap(); + fixture.charging.start(id, &[5; 32]).await.unwrap(); + fixture.charging.settle(id, &[5; 32], Usage { input: 10, cached: 2, output: 20 }, Outcome::Completed).await.unwrap(); +} + +#[sqlx::test(migrations = false)] +async fn native_queries_share_authority_count_page_flow_and_refund_units(pool: PgPool) { + let f = fixture(&pool).await; + settle(&f, &f.key.secret, "native-charge").await; + f.charging.refund("native-charge", "refund-once", 9, "Synthetic correction").await.unwrap(); + let mut read = NativeUsageRead::begin(&pool, &f.session.secret, ReadMode::SelfSession, f.filter.clone()).await.unwrap(); + let page = read.page(NativePageRequest { page: 1, size: 20 }).await.unwrap(); + assert_eq!(page.totals.request_count, "1"); + assert_eq!(page.totals.settled_credits, "49"); + assert_eq!(page.totals.refunded_credits, "9"); + assert_eq!(page.totals.net_credits, "40"); + assert_eq!(page.totals.net_usd, "0.00008"); + assert!(!page.totals.includes_legacy); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].net_credits, "40"); + assert_eq!(read.self_profile().await.unwrap().normalized_used_quota, Some(40)); + let flow = read.flow(3600).await.unwrap(); + assert_eq!(flow.len(), 1); + assert_eq!(flow[0].totals, page.totals); + read.finish().await.unwrap(); + let mut other = NativeUsageRead::begin(&pool, &f.other.secret, ReadMode::SelfSession, f.filter.clone()).await.unwrap(); + assert_eq!(other.totals().await.unwrap().request_count, "0"); + other.finish().await.unwrap(); +} + +#[sqlx::test(migrations = false)] +async fn session_and_key_cannot_change_their_authority_with_filters(pool: PgPool) { + let f = fixture(&pool).await; + let mut filter = f.filter.clone(); + filter.user_id = Some(8); + assert!(matches!(NativeUsageRead::begin(&pool, &f.session.secret, ReadMode::SelfSession, filter.clone()).await, Err(ReadError::Forbidden))); + assert!(matches!(NativeUsageRead::begin(&pool, &f.key.secret, ReadMode::ApiKey, filter).await, Err(ReadError::Forbidden))); + let mut filter = f.filter.clone(); + filter.key_id = Some(f.second_key.id); + assert!(matches!(NativeUsageRead::begin(&pool, &f.key.secret, ReadMode::ApiKey, filter).await, Err(ReadError::Forbidden))); + assert!(matches!(NativeUsageRead::begin(&pool, &f.key.secret, ReadMode::SelfSession, f.filter.clone()).await, Err(ReadError::Forbidden))); + assert!(matches!(NativeUsageRead::begin(&pool, &f.session.secret, ReadMode::ApiKey, f.filter.clone()).await, Err(ReadError::Forbidden))); + let mut read = NativeUsageRead::begin(&pool, &f.key.secret, ReadMode::ApiKey, f.filter.clone()).await.unwrap(); + assert!(matches!(read.self_profile().await, Err(ReadError::Forbidden))); + read.finish().await.unwrap(); +} + +#[sqlx::test(migrations = false)] +async fn platform_role_not_service_level_controls_administrator_reads(pool: PgPool) { + let f = fixture(&pool).await; + sqlx::query("UPDATE core_identity.accounts SET service_level=4 WHERE id=(SELECT personal_account_id FROM core_identity.users WHERE id=7)") + .execute(&pool).await.unwrap(); + assert!(matches!(NativeUsageRead::begin(&pool, &f.session.secret, ReadMode::AdministratorSession, f.filter.clone()).await, Err(ReadError::Forbidden))); + let admin = f.identity.bootstrap_user(9, 5).await.unwrap(); + let mut read = NativeUsageRead::begin(&pool, &admin.secret, ReadMode::AdministratorSession, f.filter.clone()).await.unwrap(); + assert_eq!(read.totals().await.unwrap().request_count, "0"); + read.finish().await.unwrap(); +} + +#[sqlx::test(migrations = false)] +async fn key_reads_never_include_another_key_or_team_key(pool: PgPool) { + let f = fixture(&pool).await; + settle(&f, &f.key.secret, "first-key-charge").await; + settle(&f, &f.second_key.secret, "second-key-charge").await; + let team = f.identity.create_team(&f.session.secret).await.unwrap(); + let team_key = f.identity.issue_key(&f.session.secret, Account { kind: AccountKind::Team, id: team }, FundingPolicy::inherit_account_settings(), 3600).await.unwrap(); + // The team key owns a different account and has no charge, even though its + // human owner has two personal-key charges in exactly the same interval. + let mut team_read = NativeUsageRead::begin(&pool, &team_key.secret, ReadMode::ApiKey, f.filter.clone()).await.unwrap(); + assert_eq!(team_read.totals().await.unwrap().request_count, "0"); + team_read.finish().await.unwrap(); + let mut read = NativeUsageRead::begin(&pool, &f.key.secret, ReadMode::ApiKey, f.filter.clone()).await.unwrap(); + let page = read.page(NativePageRequest { page: 1, size: 20 }).await.unwrap(); + assert_eq!(page.totals.request_count, "1"); + assert_eq!(page.items[0].id, "first-key-charge"); + read.finish().await.unwrap(); +} + +#[sqlx::test(migrations = false)] +async fn concurrent_settlement_and_refund_cannot_change_an_existing_snapshot(pool: PgPool) { + let f = fixture(&pool).await; + settle(&f, &f.key.secret, "before-snapshot").await; + let mut read = NativeUsageRead::begin(&pool, &f.session.secret, ReadMode::SelfSession, f.filter.clone()).await.unwrap(); + let before = read.totals().await.unwrap(); + settle(&f, &f.key.secret, "after-snapshot").await; + f.charging.refund("before-snapshot", "concurrent-refund", 9, "Synthetic correction").await.unwrap(); + let page = read.page(NativePageRequest { page: 1, size: 20 }).await.unwrap(); + assert_eq!(page.totals, before); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].refunded_credits, "0"); + read.finish().await.unwrap(); + let mut next = NativeUsageRead::begin(&pool, &f.session.secret, ReadMode::SelfSession, f.filter.clone()).await.unwrap(); + let totals = next.totals().await.unwrap(); + assert_eq!(totals.request_count, "2"); + assert_eq!(totals.net_credits, "89"); + next.finish().await.unwrap(); +} + +#[sqlx::test(migrations = false)] +async fn filters_are_bound_and_have_identical_statistics_and_page_scope(pool: PgPool) { + let f = fixture(&pool).await; + settle(&f, &f.key.secret, "filter-charge").await; + let mut filter = f.filter.clone(); + filter.model = Some("local-model' OR true --".into()); + let mut read = NativeUsageRead::begin(&pool, &f.session.secret, ReadMode::SelfSession, filter).await.unwrap(); + let page = read.page(NativePageRequest { page: 1, size: 20 }).await.unwrap(); + assert_eq!(page.totals.request_count, "0"); + assert!(page.items.is_empty()); + assert!(read.flow(900).await.unwrap().is_empty()); + read.finish().await.unwrap(); +} + +#[sqlx::test(migrations = false)] +async fn revoked_or_expired_key_cannot_start_a_new_snapshot(pool: PgPool) { + let f = fixture(&pool).await; + sqlx::query("UPDATE core_identity.credentials SET revoked_at=clock_timestamp() WHERE id=$1") + .bind(f.key.id).execute(&pool).await.unwrap(); + assert!(matches!(NativeUsageRead::begin(&pool, &f.key.secret, ReadMode::ApiKey, f.filter.clone()).await, Err(ReadError::Unauthorized))); + sqlx::query("UPDATE core_identity.credentials SET expires_at=clock_timestamp()-interval '1 second' WHERE id=$1") + .bind(f.second_key.id).execute(&pool).await.unwrap(); + assert!(matches!(NativeUsageRead::begin(&pool, &f.second_key.secret, ReadMode::ApiKey, f.filter.clone()).await, Err(ReadError::Unauthorized))); +} + +#[sqlx::test(migrations = false)] +async fn native_response_has_no_raw_secrets_and_reads_do_not_write_state(pool: PgPool) { + let f = fixture(&pool).await; + settle(&f, &f.key.secret, "allowlisted-charge").await; + let sql = "SELECT (SELECT count(*) FROM core_identity.audit),(SELECT count(*) FROM core_billing.ledger_journals), \ + (SELECT sum(balance_units::numeric)::text FROM core_billing.balance_state), \ + (SELECT sum(revision)::text FROM core_charging.requests)"; + let before: (i64, i64, String, String) = sqlx::query_as(sql).fetch_one(&pool).await.unwrap(); + let mut read = NativeUsageRead::begin(&pool, &f.session.secret, ReadMode::SelfSession, f.filter.clone()).await.unwrap(); + let page = read.page(NativePageRequest { page: 1, size: 20 }).await.unwrap(); + let json = serde_json::to_value(page).unwrap(); + let fields = json["items"][0].as_object().unwrap(); + let mut names: Vec<_> = fields.keys().map(String::as_str).collect(); + names.sort_unstable(); + let mut expected = vec!["id", "authorized_at", "model", "group", "price_version", "settled_credits", "refunded_credits", "net_credits", "net_usd", "input_tokens", "cached_tokens", "output_tokens"]; + expected.sort_unstable(); + assert_eq!(names, expected); + assert!(!json.to_string().contains(&f.key.secret)); + read.finish().await.unwrap(); + let after: (i64, i64, String, String) = sqlx::query_as(sql).fetch_one(&pool).await.unwrap(); + assert_eq!(before, after); +} From 0d44fb95c1bb7d11f9e8bc74e963b936f7a0542e Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 12 Oct 2026 19:15:24 +0800 Subject: [PATCH 6/6] fix(core): preserve JSON control roundtrips while rejecting CSV NUL Allow safe JSON escaping of control characters as before. Reject NUL only for CSV, where spreadsheet readers may truncate input. Verify the separate format policies with regression cases from the A04 CI failure. --- services/core/src/finance_export.rs | 2 +- services/core/src/finance_export/csv.rs | 7 +++++++ services/core/tests/finance_export/csv.rs | 10 +++++++++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/services/core/src/finance_export.rs b/services/core/src/finance_export.rs index fe620aaa5..6e0952cba 100644 --- a/services/core/src/finance_export.rs +++ b/services/core/src/finance_export.rs @@ -232,7 +232,7 @@ fn validate_row(row: &UsageExportInput<'_>) -> Result<(), ExportError> { || row.channel_id.is_some_and(|value| value < 0) || [row.username, row.token_name, row.model_name, row.group] .into_iter() - .any(|value| value.len() > MAX_TEXT_BYTES || value.contains('\0')) + .any(|value| value.len() > MAX_TEXT_BYTES) { return Err(ExportError::InvalidRow); } diff --git a/services/core/src/finance_export/csv.rs b/services/core/src/finance_export/csv.rs index 60a13c474..ebe0cc5fd 100644 --- a/services/core/src/finance_export/csv.rs +++ b/services/core/src/finance_export/csv.rs @@ -62,6 +62,13 @@ impl UsageCsvWriter { return Err(ExportError::RowLimit); } validate_row(row)?; + // JSON may safely escape NUL; CSV clients can truncate at it. + if [row.username, row.token_name, row.model_name, row.group] + .into_iter() + .any(|value| value.contains('\0')) + { + return Err(ExportError::InvalidRow); + } let order = (row.created_at, row.log_id); if self.last_order.is_some_and(|last| order <= last) { return Err(ExportError::OutOfOrder); diff --git a/services/core/tests/finance_export/csv.rs b/services/core/tests/finance_export/csv.rs index 5df563308..9f746a876 100644 --- a/services/core/tests/finance_export/csv.rs +++ b/services/core/tests/finance_export/csv.rs @@ -116,6 +116,14 @@ fn both_formats_bound_total_metadata_work_and_reject_invalid_scalars() { let mut csv = UsageCsvWriter::new(Vec::new(), 10, 10_000, false).unwrap(); assert_eq!(csv.push(&invalid), Err(ExportError::InvalidRow)); let mut json = UsageJsonWriter::new(Vec::new(), 10, 10_000).unwrap(); - assert_eq!(json.push(&invalid), Err(ExportError::InvalidRow)); + if which == 5 { + // NUL is escaped by JSON, but must never reach CSV clients. + json.push(&invalid).unwrap(); + let (bytes, _) = json.finish().unwrap(); + let decoded: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(decoded[0]["username"], "nul\0name"); + } else { + assert_eq!(json.push(&invalid), Err(ExportError::InvalidRow)); + } } }