diff --git a/.gitignore b/.gitignore index 0fad045..49f22a5 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,9 @@ Cargo.lock.bk # Python virtual environment .venv/ +# Git worktrees +.worktrees/ + # IDE/editor/devtool files /.vscode/ /.idea/ diff --git a/deploy/kind/kind-config.yaml b/deploy/kind/kind-config.yaml index e1affb2..52e3f7e 100644 --- a/deploy/kind/kind-config.yaml +++ b/deploy/kind/kind-config.yaml @@ -15,6 +15,9 @@ nodes: - containerPort: 30000 hostPort: 13000 protocol: TCP + - containerPort: 30001 + hostPort: 13001 + protocol: TCP - role: worker extraMounts: - hostPath: /tmp/hyperbytedb-data/worker-0 diff --git a/docs/developer-guide/system-architecture.md b/docs/developer-guide/system-architecture.md index 3935857..ecd541b 100644 --- a/docs/developer-guide/system-architecture.md +++ b/docs/developer-guide/system-architecture.md @@ -449,7 +449,16 @@ All queries use `OutputFormat::JSONEachRow` — one JSON object per result row. The query language module is `src/timeseriesql/` (Influx-compatible TimeseriesQL). -The parser is a **hand-rolled recursive descent parser** (no parser generator). It lives in `src/timeseriesql/parser.rs`. +Parsing uses **two statement grammars** on top of a shared masking scanner (`scan.rs`): + +| Path | Module | Mechanism | +|------|--------|-----------| +| `SELECT` | `parser.rs` | Clause scanner finds `FROM`/`WHERE`/… keywords; expression parser for fields and predicates | +| DDL/SHOW | `ddl_parser.rs` + `lexer.rs` | Token stream + `TokenCursor` for keyword-sequence statements | + +Both paths share `scan.rs` for quote/regex/paren masking and `lexer::split_statements` for multi-statement input. DDL `WHERE` clauses and CQ/MV inner queries delegate to `parser::parse_expr`. + +The SELECT expression parser is **hand-rolled recursive descent** (no parser generator). ### Parse flow @@ -528,6 +537,10 @@ Key AST nodes (in `src/timeseriesql/ast.rs`): - `FillOption` — Null, None, Previous, Linear, Value(f64) - `Measurement` — optional database, optional RP, name or regex +**SLIMIT/SOFFSET:** Parsed into the AST but not translated to ClickHouse SQL. The query service applies series-level pagination after merging result series (`query_service.rs`). + +**Time bounds for fill:** `extract_time_bounds` (in `to_clickhouse/time_bounds.rs`) derives min/max epoch nanoseconds from WHERE clauses for WITH FILL grid anchoring. AND predicates intersect bounds; OR predicates use the envelope of disjuncts only when every branch defines that side (min and/or max); if any OR branch lacks a time predicate, or any branch is missing a lower or upper cap, the corresponding bound is omitted (conservative). + --- ## 13. ClickHouse SQL Translator diff --git a/hyperbytedb/src/timeseriesql/lexer.rs b/hyperbytedb/src/timeseriesql/lexer.rs index 8fc92a6..7da3da4 100644 --- a/hyperbytedb/src/timeseriesql/lexer.rs +++ b/hyperbytedb/src/timeseriesql/lexer.rs @@ -5,6 +5,7 @@ use crate::error::HyperbytedbError; use crate::timeseriesql::ast::{Duration, DurationUnit}; +use crate::timeseriesql::scan::{ScannedChar, is_regex_start_at, scan_chars}; /// Lexer token with source span. #[derive(Debug, Clone, PartialEq)] @@ -71,93 +72,30 @@ pub fn tokenize(input: &str) -> Result, HyperbytedbError> { Ok(tokens) } -/// Split multi-statement input on `;` outside quotes and BEGIN…END blocks. +/// Split multi-statement input on `;` outside quotes, regex literals, and +/// BEGIN…END blocks. pub fn split_statements(input: &str) -> Result, HyperbytedbError> { + let scan = scan_chars(input)?; let mut statements = Vec::new(); let mut start = 0usize; - let mut i = 0usize; - let bytes = input.as_bytes(); - let mut in_single = false; - let mut in_double = false; - let mut in_regex = false; let mut begin_depth = 0i32; - // Last significant (non-whitespace) char outside string/regex literals, - // used to decide whether a `/` is in operand position (regex start). - let mut prev_sig: Option = None; - - while i < bytes.len() { - let c = bytes[i] as char; - if in_regex { - if c == '\\' { - i += 2; - continue; - } - if c == '/' { - in_regex = false; - prev_sig = Some('/'); - } - i += 1; - continue; - } - if in_single { - if c == '\'' { - if i + 1 < bytes.len() && bytes[i + 1] == b'\'' { - i += 2; - continue; - } - in_single = false; - prev_sig = Some('\''); + for (si, sc) in scan.iter().enumerate() { + if !sc.masked && sc.depth == 0 { + if matches_unmasked_keyword_at(input, &scan, si, "BEGIN") { + begin_depth += 1; + } else if begin_depth > 0 && matches_unmasked_keyword_at(input, &scan, si, "END") { + begin_depth -= 1; } - i += 1; - continue; } - if in_double { - if c == '"' { - if i + 1 < bytes.len() && bytes[i + 1] == b'"' { - i += 2; - continue; - } - in_double = false; - prev_sig = Some('"'); - } - i += 1; - continue; - } - - let at_word_boundary = i == 0 || !is_ident_continue(bytes[i - 1] as char); - match c { - '\'' => in_single = true, - '"' => in_double = true, - // `/` in operand position (after `=~`, `!~`, `(`, `,` or `=`) - // starts a regex literal; a `;` inside it must not split. - '/' if matches!(prev_sig, Some('~') | Some('(') | Some(',') | Some('=')) => { - in_regex = true; - } - ';' if begin_depth == 0 => { - let slice = input[start..i].trim(); - if !slice.is_empty() { - statements.push(slice.to_string()); - } - start = i + 1; - } - _ if is_ident_start(c) && at_word_boundary && matches_keyword_at(input, i, "BEGIN") => { - begin_depth += 1 + if sc.ch == ';' && begin_depth == 0 && !sc.masked && sc.depth == 0 { + let slice = input[start..sc.idx].trim(); + if !slice.is_empty() { + statements.push(slice.to_string()); } - _ if is_ident_start(c) - && at_word_boundary - && begin_depth > 0 - && matches_keyword_at(input, i, "END") => - { - begin_depth -= 1; - } - _ => {} - } - if !c.is_whitespace() { - prev_sig = Some(c); + start = sc.idx + sc.ch.len_utf8(); } - i += 1; } let tail = input[start..].trim(); @@ -167,15 +105,34 @@ pub fn split_statements(input: &str) -> Result, HyperbytedbError> { Ok(statements) } -fn matches_keyword_at(input: &str, start: usize, kw: &str) -> bool { - // Byte-wise compare: slicing `rest[..kw.len()]` panics when a multibyte - // char straddles the boundary (e.g. an identifier containing `fi`). - let rest = input.as_bytes().get(start..); - let Some(rest) = rest else { return false }; - if rest.len() < kw.len() || !rest[..kw.len()].eq_ignore_ascii_case(kw.as_bytes()) { +/// Match an ASCII keyword at scan index `si` when it is not inside a masked span. +fn matches_unmasked_keyword_at(input: &str, scan: &[ScannedChar], si: usize, kw: &str) -> bool { + let sc = &scan[si]; + if sc.masked || sc.depth != 0 { return false; } - !matches!(rest.get(kw.len()), Some(b) if is_ident_continue(*b as char)) + if si > 0 { + let prev = scan[si - 1].ch; + if prev.is_alphanumeric() || prev == '_' { + return false; + } + } + + let start = sc.idx; + let bytes = input.as_bytes(); + if start + kw.len() > bytes.len() + || !bytes[start..start + kw.len()].eq_ignore_ascii_case(kw.as_bytes()) + { + return false; + } + + if start + kw.len() < bytes.len() { + let next = bytes[start + kw.len()] as char; + if next.is_alphanumeric() || next == '_' { + return false; + } + } + true } /// Sum compound duration text (e.g. `1h30m`, `0`, `INF`) to nanoseconds. @@ -401,9 +358,6 @@ struct Lexer<'a> { input: &'a str, chars: Vec<(usize, char)>, pos: usize, - /// Kind of the previously emitted token, used to decide whether a `/` - /// begins a regex literal (operand position after `=~`/`!~`) or division. - last_kind: Option, } impl<'a> Lexer<'a> { @@ -412,7 +366,6 @@ impl<'a> Lexer<'a> { input, chars: input.char_indices().collect(), pos: 0, - last_kind: None, } } @@ -439,9 +392,7 @@ impl<'a> Lexer<'a> { } fn next_token(&mut self) -> Result { - let tok = self.scan_token()?; - self.last_kind = Some(tok.kind.clone()); - Ok(tok) + self.scan_token() } fn scan_token(&mut self) -> Result { @@ -592,15 +543,7 @@ impl<'a> Lexer<'a> { '\'' => self.read_string_lit(start), '"' => self.read_ident_quoted(start), '/' => { - // A `/` only starts a regex literal in operand position, i.e. - // immediately after a regex-match operator. Anywhere else it is - // division; treating every `/` as a regex made a lone `/` (e.g. - // arithmetic in a WHERE clause) abort tokenization of the whole - // statement. - if matches!( - self.last_kind, - Some(TokenKind::MatchRegex) | Some(TokenKind::NotMatchRegex) - ) { + if is_regex_start_at(self.input, start) { self.read_regex(start) } else { self.bump_char(); @@ -992,6 +935,28 @@ mod tests { ); } + #[test] + fn regex_measurement_after_from_keyword() { + let toks = tokenize("SHOW SERIES FROM /^cpu/").unwrap(); + assert!( + toks.iter() + .any(|t| matches!(&t.kind, TokenKind::Regex(r) if r == "^cpu")), + "FROM /^cpu/ must tokenize as regex measurement: {toks:?}" + ); + } + + #[test] + fn split_and_tokenize_agree_on_regex_measurement() { + let input = "SHOW SERIES FROM /^cpu/; SHOW DATABASES"; + let stmts = split_statements(input).unwrap(); + assert_eq!(stmts.len(), 2); + let toks = tokenize(&stmts[0]).unwrap(); + assert!( + toks.iter() + .any(|t| matches!(&t.kind, TokenKind::Regex(r) if r == "^cpu")) + ); + } + #[test] fn regex_swallows_keyword_like_content() { // The inner LIMIT belongs to the regex; only the trailing one is a @@ -1030,4 +995,48 @@ mod tests { assert_eq!(stmts.len(), 2); assert!(stmts[0].ends_with("END")); } + + #[test] + fn split_statements_ignores_semicolon_inside_parens() { + let stmts = split_statements("SELECT * FROM t WHERE x IN (1; 2); SHOW DATABASES").unwrap(); + assert_eq!(stmts.len(), 2); + assert_eq!(stmts[0], "SELECT * FROM t WHERE x IN (1; 2)"); + assert_eq!(stmts[1], "SHOW DATABASES"); + } + + #[test] + fn split_statements_fails_on_unclosed_paren_in_batch() { + let err = split_statements("SELECT ( FROM cpu; SHOW DATABASES").unwrap_err(); + assert!( + err.to_string().contains("unclosed"), + "expected unclosed paren error, got: {err}" + ); + } + + #[test] + fn split_statements_ignores_begin_end_inside_string_literals() { + let stmts = split_statements( + "CREATE CONTINUOUS QUERY cq ON db BEGIN SELECT * FROM m WHERE msg = 'BEGIN; END'; END; SHOW DATABASES", + ) + .unwrap(); + assert_eq!( + stmts.len(), + 2, + "BEGIN/END substrings inside quoted literals must not affect block depth: {stmts:?}" + ); + assert!(stmts[0].contains("'BEGIN; END'")); + assert_eq!(stmts[1], "SHOW DATABASES"); + } + + #[test] + fn split_statements_does_not_treat_quoted_begin_as_block_open() { + let stmts = split_statements("SELECT 'BEGIN'; SHOW DATABASES").unwrap(); + assert_eq!( + stmts.len(), + 2, + "quoted BEGIN must not suppress splitting: {stmts:?}" + ); + assert_eq!(stmts[0], "SELECT 'BEGIN'"); + assert_eq!(stmts[1], "SHOW DATABASES"); + } } diff --git a/hyperbytedb/src/timeseriesql/mod.rs b/hyperbytedb/src/timeseriesql/mod.rs index 46c3ace..c9e3c78 100644 --- a/hyperbytedb/src/timeseriesql/mod.rs +++ b/hyperbytedb/src/timeseriesql/mod.rs @@ -3,6 +3,7 @@ pub mod ddl_parser; pub mod digest; pub mod lexer; pub mod parser; +pub(crate) mod scan; pub mod to_clickhouse; use crate::error::HyperbytedbError; diff --git a/hyperbytedb/src/timeseriesql/parser.rs b/hyperbytedb/src/timeseriesql/parser.rs index f2ce43e..01aab6d 100644 --- a/hyperbytedb/src/timeseriesql/parser.rs +++ b/hyperbytedb/src/timeseriesql/parser.rs @@ -2,6 +2,10 @@ use crate::error::HyperbytedbError; use crate::timeseriesql::ast::*; use crate::timeseriesql::ddl_parser; use crate::timeseriesql::lexer; +use crate::timeseriesql::scan::{ + find_keyword_position, find_top_level_operator, match_keyword_at, rfind_top_level_ci, + scan_chars, split_top_level_commas, +}; pub fn parse_query(input: &str) -> Result, HyperbytedbError> { let input = input.trim(); @@ -173,266 +177,6 @@ fn parse_select(input: &str) -> Result { Ok(Statement::Select(stmt)) } -/// Per-character scan info produced by [`scan_chars`]. -#[derive(Debug, Clone, Copy)] -struct ScannedChar { - /// Byte offset of the character in the original input (valid for slicing). - idx: usize, - ch: char, - /// Paren depth: 0 for top-level characters (the outermost parens - /// themselves included), > 0 strictly inside parentheses. - depth: i32, - /// True when the character is part of a single-quoted string, a - /// double-quoted identifier, or a regex literal (delimiters included). - masked: bool, -} - -/// Masking scanner shared by all SELECT-parsing string primitives. -/// -/// Walks the ORIGINAL string char by char (never an uppercased copy, whose -/// byte offsets can diverge for chars like `ı`/`fi`) and tracks: -/// - single-quoted string literals, honoring both `\'` and `''` escapes, -/// - double-quoted identifiers (`""` escape) as an independent state — a -/// quote char inside the other quote kind does not toggle, -/// - regex literals `/.../` (with `\/` escape), distinguished from division -/// by [`slash_is_regex_start`], -/// - parenthesis depth. -/// -/// The output has exactly one entry per input char, in order. -fn scan_chars(input: &str) -> Result, HyperbytedbError> { - let chars: Vec<(usize, char)> = input.char_indices().collect(); - let mut out = Vec::with_capacity(chars.len()); - let mut depth: u32 = 0; - let mut i = 0usize; - while i < chars.len() { - let (idx, ch) = chars[i]; - match ch { - '\'' | '"' => { - let quote = ch; - out.push(ScannedChar { - idx, - ch, - depth: depth as i32, - masked: true, - }); - i += 1; - while i < chars.len() { - let (jdx, c) = chars[i]; - out.push(ScannedChar { - idx: jdx, - ch: c, - depth: depth as i32, - masked: true, - }); - i += 1; - if quote == '\'' && c == '\\' && i < chars.len() { - // Backslash escape (`\'`, `\\`) inside a string literal. - let (kdx, k) = chars[i]; - out.push(ScannedChar { - idx: kdx, - ch: k, - depth: depth as i32, - masked: true, - }); - i += 1; - } else if c == quote { - if i < chars.len() && chars[i].1 == quote { - // Doubled-quote escape: '' or "". - let (kdx, k) = chars[i]; - out.push(ScannedChar { - idx: kdx, - ch: k, - depth: depth as i32, - masked: true, - }); - i += 1; - } else { - break; - } - } - } - } - '/' if slash_is_regex_start(&chars, i) => { - out.push(ScannedChar { - idx, - ch, - depth: depth as i32, - masked: true, - }); - i += 1; - while i < chars.len() { - let (jdx, c) = chars[i]; - out.push(ScannedChar { - idx: jdx, - ch: c, - depth: depth as i32, - masked: true, - }); - i += 1; - if c == '\\' && i < chars.len() { - let (kdx, k) = chars[i]; - out.push(ScannedChar { - idx: kdx, - ch: k, - depth: depth as i32, - masked: true, - }); - i += 1; - } else if c == '/' { - break; - } - } - } - '(' => { - out.push(ScannedChar { - idx, - ch, - depth: depth as i32, - masked: false, - }); - depth += 1; - i += 1; - } - ')' => { - if depth == 0 { - return Err(HyperbytedbError::QueryParse(format!( - "unbalanced ')' in expression: {input}" - ))); - } - depth -= 1; - out.push(ScannedChar { - idx, - ch, - depth: depth as i32, - masked: false, - }); - i += 1; - } - _ => { - out.push(ScannedChar { - idx, - ch, - depth: depth as i32, - masked: false, - }); - i += 1; - } - } - } - if depth != 0 { - return Err(HyperbytedbError::QueryParse(format!( - "unclosed '(' in expression: {input}" - ))); - } - Ok(out) -} - -/// Whether a `/` at `chars[pos]` begins a regex literal rather than division. -/// Division follows an operand (identifier, number, `)` or a quoted value); -/// a regex follows start-of-input, an operator/comma/open paren, or a clause -/// keyword that puts the slash in operand position (`FROM /re/`, -/// `GROUP BY /re/`). -fn slash_is_regex_start(chars: &[(usize, char)], pos: usize) -> bool { - let mut j = pos; - while j > 0 && chars[j - 1].1.is_whitespace() { - j -= 1; - } - if j == 0 { - return true; - } - let prev = chars[j - 1].1; - if matches!(prev, ')' | '"' | '\'') { - return false; - } - if prev.is_alphanumeric() || prev == '_' { - let end = j; - let mut start = j; - while start > 0 && (chars[start - 1].1.is_alphanumeric() || chars[start - 1].1 == '_') { - start -= 1; - } - let word: String = chars[start..end].iter().map(|&(_, c)| c).collect(); - return ["FROM", "WHERE", "BY", "AND", "OR"] - .iter() - .any(|kw| word.eq_ignore_ascii_case(kw)); - } - true -} - -fn is_keyword_boundary_before(c: char) -> bool { - c.is_whitespace() || matches!(c, ')' | '\'' | '"') -} - -fn is_keyword_boundary_after(c: char) -> bool { - c.is_whitespace() || matches!(c, '(' | '\'' | '"' | '/') -} - -/// Match an ASCII `keyword` ("LIMIT", "GROUP BY", …) at scan index `i`, -/// case-insensitively on the original string. Two-word keywords accept any -/// whitespace run between the words. Only unmasked, top-level (paren depth 0) -/// text matches, and the keyword must be delimited by whitespace, a paren, or -/// a quote on either side (so `(a=1)AND(b=2)` works). Returns the matched -/// byte range `(start, end)`. -fn match_keyword_at( - input: &str, - scan: &[ScannedChar], - i: usize, - keyword: &str, -) -> Option<(usize, usize)> { - let sc = scan[i]; - if sc.masked || sc.depth != 0 { - return None; - } - if i > 0 && !is_keyword_boundary_before(scan[i - 1].ch) { - return None; - } - - let bytes = input.as_bytes(); - let mut words = keyword.split_ascii_whitespace(); - let first = words.next()?; - let start = sc.idx; - if start + first.len() > bytes.len() - || !bytes[start..start + first.len()].eq_ignore_ascii_case(first.as_bytes()) - { - return None; - } - // The matched region is ASCII, so scan indices advance one per byte. - let mut j = i + first.len(); - for word in words { - let ws_start = j; - while j < scan.len() && scan[j].ch.is_whitespace() { - j += 1; - } - if j == ws_start || j >= scan.len() { - return None; - } - let word_start = scan[j].idx; - if word_start + word.len() > bytes.len() - || !bytes[word_start..word_start + word.len()].eq_ignore_ascii_case(word.as_bytes()) - { - return None; - } - j += word.len(); - } - if j < scan.len() && !is_keyword_boundary_after(scan[j].ch) { - return None; - } - let end = if j < scan.len() { - scan[j].idx - } else { - input.len() - }; - Some((start, end)) -} - -/// Byte range of the first top-level occurrence of `keyword` in `input`. -fn find_keyword_position( - input: &str, - scan: &[ScannedChar], - keyword: &str, -) -> Option<(usize, usize)> { - (0..scan.len()).find_map(|i| match_keyword_at(input, scan, i, keyword)) -} - /// Split a SELECT body into clause segments using case-insensitive keyword /// matching on the original string. Keywords inside strings, quoted /// identifiers, regex literals, or parentheses (subqueries) are ignored. @@ -517,20 +261,6 @@ fn parse_field_list(input: &str) -> Result, HyperbytedbError> { Ok(fields) } -fn split_top_level_commas(input: &str) -> Result, HyperbytedbError> { - let scan = scan_chars(input)?; - let mut parts = Vec::new(); - let mut last = 0; - for sc in &scan { - if sc.ch == ',' && !sc.masked && sc.depth == 0 { - parts.push(&input[last..sc.idx]); - last = sc.idx + 1; - } - } - parts.push(&input[last..]); - Ok(parts) -} - fn parse_field_expr(input: &str) -> Result { let input = input.trim(); @@ -692,38 +422,6 @@ fn try_parse_arithmetic_expr(input: &str) -> Result, HyperbytedbErr Ok(None) } -/// Find the first top-level (unmasked, paren depth 0) occurrence of a -/// symbolic operator, refusing matches that are part of a longer operator -/// (`=` inside `>=`/`!=`/`=~`, `<` inside `<=`/`<>`, `>` inside `>=`/`<>`). -fn find_top_level_operator(input: &str, scan: &[ScannedChar], op: &str) -> Option { - let bytes = input.as_bytes(); - let op_bytes = op.as_bytes(); - for sc in scan { - if sc.masked || sc.depth != 0 { - continue; - } - let i = sc.idx; - if i + op_bytes.len() > bytes.len() || &bytes[i..i + op_bytes.len()] != op_bytes { - continue; - } - let prev = i.checked_sub(1).map(|p| bytes[p]); - let next = bytes.get(i + op_bytes.len()).copied(); - let standalone = match op { - "=" => { - !matches!(prev, Some(b'!' | b'<' | b'>' | b'=')) - && !matches!(next, Some(b'~' | b'=')) - } - "<" => !matches!(next, Some(b'=' | b'>')), - ">" => !matches!(prev, Some(b'<')) && !matches!(next, Some(b'=')), - _ => true, - }; - if standalone { - return Some(i); - } - } - None -} - fn parse_atom(input: &str) -> Result { let input = input.trim(); @@ -985,32 +683,6 @@ fn unquote(s: &str) -> String { } } -/// Byte offset of the last unmasked, top-level, ASCII-case-insensitive -/// occurrence of `needle` that does not continue an identifier. -fn rfind_top_level_ci(input: &str, scan: &[ScannedChar], needle: &str) -> Option { - let bytes = input.as_bytes(); - let needle_bytes = needle.as_bytes(); - for (k, sc) in scan.iter().enumerate().rev() { - if sc.masked || sc.depth != 0 { - continue; - } - let i = sc.idx; - if i + needle_bytes.len() > bytes.len() - || !bytes[i..i + needle_bytes.len()].eq_ignore_ascii_case(needle_bytes) - { - continue; - } - if k > 0 { - let prev = scan[k - 1].ch; - if prev.is_alphanumeric() || prev == '_' || prev == '"' { - continue; - } - } - return Some(i); - } - None -} - fn parse_group_by_clause(input: &str) -> Result<(GroupBy, Option), HyperbytedbError> { let mut fill = None; let mut dims_str = input.to_string(); @@ -1850,9 +1522,6 @@ mod tests { other => panic!("expected BinaryExpr, got {:?}", other), } - // Goes through parse_select directly: lexer::split_statements (out of - // scope for the parser fix) still has a byte-boundary panic on this - // input (`rest[..kw.len()]` at lexer.rs:138). let s = match parse_select(r#"SELECT "fix" FROM cpu"#).unwrap() { Statement::Select(s) => s, other => panic!("expected SELECT, got {:?}", other), @@ -1864,6 +1533,36 @@ mod tests { assert_eq!(s.from[0].name_str(), Some("cpu")); } + #[test] + fn test_parse_query_multibyte_quoted_identifier() { + let stmts = parse_query(r#"SELECT "fix" FROM cpu"#).unwrap(); + match &stmts[0] { + Statement::Select(s) => { + match &s.fields[0].expr { + Expr::Identifier(name) => assert_eq!(name, "fix"), + other => panic!("expected identifier, got {:?}", other), + } + assert_eq!(s.from[0].name_str(), Some("cpu")); + } + other => panic!("expected SELECT, got {:?}", other), + } + } + + #[test] + fn test_parse_slimit_soffset() { + let s = select_stmt("SELECT * FROM cpu SLIMIT 10 SOFFSET 5"); + assert_eq!(s.slimit, Some(10)); + assert_eq!(s.soffset, Some(5)); + } + + #[test] + fn test_parse_limit_and_slimit_together() { + let s = select_stmt("SELECT * FROM cpu LIMIT 100 SLIMIT 10"); + assert_eq!(s.limit, Some(100)); + assert_eq!(s.slimit, Some(10)); + assert!(s.soffset.is_none()); + } + #[test] fn test_fill_after_non_ascii_string() { let s = select_stmt(r#"SELECT last("v") FROM m WHERE city = 'ığdır' fill(null)"#); diff --git a/hyperbytedb/src/timeseriesql/scan.rs b/hyperbytedb/src/timeseriesql/scan.rs new file mode 100644 index 0000000..62dd485 --- /dev/null +++ b/hyperbytedb/src/timeseriesql/scan.rs @@ -0,0 +1,384 @@ +//! Quote-, regex-, and paren-aware masking scanner shared by SELECT parsing, +//! statement splitting, and DDL tokenization. + +use crate::error::HyperbytedbError; + +/// Per-character scan info produced by [`scan_chars`]. +#[derive(Debug, Clone, Copy)] +pub struct ScannedChar { + /// Byte offset of the character in the original input (valid for slicing). + pub idx: usize, + pub ch: char, + /// Paren depth: 0 for top-level characters (the outermost parens + /// themselves included), > 0 strictly inside parentheses. + pub depth: i32, + /// True when the character is part of a single-quoted string, a + /// double-quoted identifier, or a regex literal (delimiters included). + pub masked: bool, +} + +/// Masking scanner shared by all InfluxQL string primitives. +/// +/// Walks the ORIGINAL string char by char (never an uppercased copy, whose +/// byte offsets can diverge for chars like `ı`/`fi`) and tracks: +/// - single-quoted string literals, honoring both `\'` and `''` escapes, +/// - double-quoted identifiers (`""` escape) as an independent state — a +/// quote char inside the other quote kind does not toggle, +/// - regex literals `/.../` (with `\/` escape), distinguished from division +/// by [`slash_is_regex_start`], +/// - parenthesis depth. +/// +/// The output has exactly one entry per input char, in order. +pub fn scan_chars(input: &str) -> Result, HyperbytedbError> { + let chars: Vec<(usize, char)> = input.char_indices().collect(); + let mut out = Vec::with_capacity(chars.len()); + let mut depth: u32 = 0; + let mut i = 0usize; + while i < chars.len() { + let (idx, ch) = chars[i]; + match ch { + '\'' | '"' => { + let quote = ch; + out.push(ScannedChar { + idx, + ch, + depth: depth as i32, + masked: true, + }); + i += 1; + while i < chars.len() { + let (jdx, c) = chars[i]; + out.push(ScannedChar { + idx: jdx, + ch: c, + depth: depth as i32, + masked: true, + }); + i += 1; + if quote == '\'' && c == '\\' && i < chars.len() { + let (kdx, k) = chars[i]; + out.push(ScannedChar { + idx: kdx, + ch: k, + depth: depth as i32, + masked: true, + }); + i += 1; + } else if c == quote { + if i < chars.len() && chars[i].1 == quote { + let (kdx, k) = chars[i]; + out.push(ScannedChar { + idx: kdx, + ch: k, + depth: depth as i32, + masked: true, + }); + i += 1; + } else { + break; + } + } + } + } + '/' if slash_is_regex_start(&chars, i) => { + out.push(ScannedChar { + idx, + ch, + depth: depth as i32, + masked: true, + }); + i += 1; + while i < chars.len() { + let (jdx, c) = chars[i]; + out.push(ScannedChar { + idx: jdx, + ch: c, + depth: depth as i32, + masked: true, + }); + i += 1; + if c == '\\' && i < chars.len() { + let (kdx, k) = chars[i]; + out.push(ScannedChar { + idx: kdx, + ch: k, + depth: depth as i32, + masked: true, + }); + i += 1; + } else if c == '/' { + break; + } + } + } + '(' => { + out.push(ScannedChar { + idx, + ch, + depth: depth as i32, + masked: false, + }); + depth += 1; + i += 1; + } + ')' => { + if depth == 0 { + return Err(HyperbytedbError::QueryParse(format!( + "unbalanced ')' in expression: {input}" + ))); + } + depth -= 1; + out.push(ScannedChar { + idx, + ch, + depth: depth as i32, + masked: false, + }); + i += 1; + } + _ => { + out.push(ScannedChar { + idx, + ch, + depth: depth as i32, + masked: false, + }); + i += 1; + } + } + } + if depth != 0 { + return Err(HyperbytedbError::QueryParse(format!( + "unclosed '(' in expression: {input}" + ))); + } + Ok(out) +} + +/// Whether a `/` at `chars[pos]` begins a regex literal rather than division. +pub fn slash_is_regex_start(chars: &[(usize, char)], pos: usize) -> bool { + let mut j = pos; + while j > 0 && chars[j - 1].1.is_whitespace() { + j -= 1; + } + if j == 0 { + return true; + } + let prev = chars[j - 1].1; + if matches!(prev, ')' | '"' | '\'') { + return false; + } + if prev.is_alphanumeric() || prev == '_' { + let end = j; + let mut start = j; + while start > 0 && (chars[start - 1].1.is_alphanumeric() || chars[start - 1].1 == '_') { + start -= 1; + } + let word: String = chars[start..end].iter().map(|&(_, c)| c).collect(); + return ["FROM", "WHERE", "BY", "AND", "OR"] + .iter() + .any(|kw| word.eq_ignore_ascii_case(kw)); + } + true +} + +/// Whether a `/` at `byte_pos` in `input` begins a regex literal. +pub fn is_regex_start_at(input: &str, byte_pos: usize) -> bool { + let chars: Vec<(usize, char)> = input.char_indices().collect(); + let Some(pos) = chars.iter().position(|(idx, _)| *idx == byte_pos) else { + return false; + }; + slash_is_regex_start(&chars, pos) +} + +pub(crate) fn is_keyword_boundary_before(c: char) -> bool { + c.is_whitespace() || matches!(c, ')' | '\'' | '"') +} + +pub(crate) fn is_keyword_boundary_after(c: char) -> bool { + c.is_whitespace() || matches!(c, '(' | '\'' | '"' | '/') +} + +/// Match an ASCII `keyword` at scan index `i`, case-insensitively on the original string. +pub fn match_keyword_at( + input: &str, + scan: &[ScannedChar], + i: usize, + keyword: &str, +) -> Option<(usize, usize)> { + let sc = scan[i]; + if sc.masked || sc.depth != 0 { + return None; + } + if i > 0 && !is_keyword_boundary_before(scan[i - 1].ch) { + return None; + } + + let bytes = input.as_bytes(); + let mut words = keyword.split_ascii_whitespace(); + let first = words.next()?; + let start = sc.idx; + if start + first.len() > bytes.len() + || !bytes[start..start + first.len()].eq_ignore_ascii_case(first.as_bytes()) + { + return None; + } + let mut j = i + first.len(); + for word in words { + let ws_start = j; + while j < scan.len() && scan[j].ch.is_whitespace() { + j += 1; + } + if j == ws_start || j >= scan.len() { + return None; + } + let word_start = scan[j].idx; + if word_start + word.len() > bytes.len() + || !bytes[word_start..word_start + word.len()].eq_ignore_ascii_case(word.as_bytes()) + { + return None; + } + j += word.len(); + } + if j < scan.len() && !is_keyword_boundary_after(scan[j].ch) { + return None; + } + let end = if j < scan.len() { + scan[j].idx + } else { + input.len() + }; + Some((start, end)) +} + +/// Byte range of the first top-level occurrence of `keyword` in `input`. +pub fn find_keyword_position( + input: &str, + scan: &[ScannedChar], + keyword: &str, +) -> Option<(usize, usize)> { + (0..scan.len()).find_map(|i| match_keyword_at(input, scan, i, keyword)) +} + +/// Find the first top-level symbolic operator occurrence. +pub fn find_top_level_operator(input: &str, scan: &[ScannedChar], op: &str) -> Option { + let bytes = input.as_bytes(); + let op_bytes = op.as_bytes(); + for sc in scan { + if sc.masked || sc.depth != 0 { + continue; + } + let i = sc.idx; + if i + op_bytes.len() > bytes.len() || &bytes[i..i + op_bytes.len()] != op_bytes { + continue; + } + let prev = i.checked_sub(1).map(|p| bytes[p]); + let next = bytes.get(i + op_bytes.len()).copied(); + let standalone = match op { + "=" => { + !matches!(prev, Some(b'!' | b'<' | b'>' | b'=')) + && !matches!(next, Some(b'~' | b'=')) + } + "<" => !matches!(next, Some(b'=' | b'>')), + ">" => !matches!(prev, Some(b'<')) && !matches!(next, Some(b'=')), + _ => true, + }; + if standalone { + return Some(i); + } + } + None +} + +/// Byte offset of the last unmasked, top-level case-insensitive `needle`. +pub fn rfind_top_level_ci(input: &str, scan: &[ScannedChar], needle: &str) -> Option { + let bytes = input.as_bytes(); + let needle_bytes = needle.as_bytes(); + for (k, sc) in scan.iter().enumerate().rev() { + if sc.masked || sc.depth != 0 { + continue; + } + let i = sc.idx; + if i + needle_bytes.len() > bytes.len() + || !bytes[i..i + needle_bytes.len()].eq_ignore_ascii_case(needle_bytes) + { + continue; + } + if k > 0 { + let prev = scan[k - 1].ch; + if prev.is_alphanumeric() || prev == '_' || prev == '"' { + continue; + } + } + return Some(i); + } + None +} + +/// Split on top-level commas outside quotes, regex literals, and parentheses. +pub fn split_top_level_commas(input: &str) -> Result, HyperbytedbError> { + let scan = scan_chars(input)?; + let mut parts = Vec::new(); + let mut last = 0; + for sc in &scan { + if sc.ch == ',' && !sc.masked && sc.depth == 0 { + parts.push(&input[last..sc.idx]); + last = sc.idx + 1; + } + } + parts.push(&input[last..]); + Ok(parts) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn division_is_not_regex() { + let input = "10/2"; + let scan = scan_chars(input).unwrap(); + assert!(scan.iter().any(|sc| sc.ch == '/' && !sc.masked)); + } + + #[test] + fn regex_after_match_operator_is_masked() { + let input = "=~ /foo/"; + let scan = scan_chars(input).unwrap(); + assert!(scan.iter().all(|sc| sc.ch == '/' + || sc.ch == 'f' + || sc.ch == 'o' + || sc.ch == '~' + || sc.ch == '=' + || sc.masked + || sc.ch.is_whitespace())); + let slashes: Vec<_> = scan.iter().filter(|sc| sc.ch == '/').collect(); + assert_eq!(slashes.len(), 2); + assert!(slashes.iter().all(|sc| sc.masked)); + } + + #[test] + fn regex_after_from_keyword_is_masked() { + let input = "FROM /^cpu/"; + let scan = scan_chars(input).unwrap(); + let slash = scan.iter().find(|sc| sc.ch == '/').unwrap(); + assert!(slash.masked); + } + + #[test] + fn is_regex_start_at_matches_scan_chars() { + let input = "SELECT * FROM /^cpu/"; + let slash_pos = input.find('/').unwrap(); + assert!(is_regex_start_at(input, slash_pos)); + assert!(!is_regex_start_at("10/2", 2)); + assert!(is_regex_start_at("=~ /foo/", "=~ /foo/".find('/').unwrap())); + } + + #[test] + fn semicolon_inside_regex_is_masked() { + let input = r#"host =~ /a;b/"#; + let scan = scan_chars(input).unwrap(); + let semi = scan.iter().find(|sc| sc.ch == ';').unwrap(); + assert!(semi.masked); + } +} diff --git a/hyperbytedb/src/timeseriesql/to_clickhouse.rs b/hyperbytedb/src/timeseriesql/to_clickhouse.rs deleted file mode 100644 index 2f87442..0000000 --- a/hyperbytedb/src/timeseriesql/to_clickhouse.rs +++ /dev/null @@ -1,3540 +0,0 @@ -use crate::domain::chdb_naming::QuotedTableName; -use crate::domain::column_mapping::ColumnMapping; -use crate::domain::rollup::{RollupCombine, aggregate_source_field_name, mean_rollup_column_names}; -use crate::error::HyperbytedbError; -use crate::timeseriesql::ast::*; -use std::fmt::Write; - -/// Extract (min_time_nanos, max_time_nanos) from a WHERE clause, if present. -/// Returns `(Option, Option)`. -pub fn extract_time_bounds(condition: Option<&Expr>) -> (Option, Option) { - let mut min_time: Option = None; - let mut max_time: Option = None; - - if let Some(expr) = condition { - collect_time_bounds(expr, &mut min_time, &mut max_time); - } - (min_time, max_time) -} - -fn collect_time_bounds(expr: &Expr, min_time: &mut Option, max_time: &mut Option) { - if let Expr::BinaryExpr(be) = expr { - if matches!(be.op, BinaryOp::And) { - collect_time_bounds(&be.left, min_time, max_time); - collect_time_bounds(&be.right, min_time, max_time); - return; - } - - if !is_time_epoch_comparison(be) { - return; - } - - let (time_is_left, epoch_expr) = if is_time_identifier(&be.left) { - (true, &be.right) - } else { - (false, &be.left) - }; - - let nanos = match epoch_expr { - Expr::DurationLiteral(d) => d.to_nanos(), - Expr::IntegerLiteral(n) => *n, - _ => return, - }; - - // Normalize the operator so it's always `time value` - let effective_op = if time_is_left { - &be.op - } else { - &match be.op { - BinaryOp::Gt => BinaryOp::Lt, - BinaryOp::Gte => BinaryOp::Lte, - BinaryOp::Lt => BinaryOp::Gt, - BinaryOp::Lte => BinaryOp::Gte, - ref other => other.clone(), - } - }; - - // ANDed bounds intersect: keep the tightest lower bound (max) and the - // tightest upper bound (min). - match effective_op { - BinaryOp::Gte | BinaryOp::Gt | BinaryOp::Eq => { - *min_time = Some(min_time.map_or(nanos, |cur| cur.max(nanos))); - } - _ => {} - } - match effective_op { - BinaryOp::Lte | BinaryOp::Lt | BinaryOp::Eq => { - *max_time = Some(max_time.map_or(nanos, |cur| cur.min(nanos))); - } - _ => {} - } - } -} - -/// `SELECT ... INTO` requires `GROUP BY time()` so results are bucketed -/// before writing to the destination measurement. -pub fn validate_select_into(stmt: &SelectStatement) -> Result<(), HyperbytedbError> { - if stmt.into.is_none() { - return Ok(()); - } - let Some(gb) = stmt.group_by.as_ref() else { - return Err(HyperbytedbError::QueryParse( - "SELECT INTO requires GROUP BY time()".to_string(), - )); - }; - if gb.time_dimension().is_none() { - return Err(HyperbytedbError::QueryParse( - "SELECT INTO requires GROUP BY time()".to_string(), - )); - } - Ok(()) -} - -/// The per-measurement series (tag dimension) table to join for tag resolution. -/// In the `series_id` layout the fact table no longer stores tag columns; when a -/// query references a tag we re-attach the tag columns from this table. -#[derive(Debug, Clone, Copy)] -pub struct SeriesJoin<'a> { - /// Backtick-quoted `___series` table name. - pub table: &'a QuotedTableName, - /// Force the inline tag-rejoin view even when the query body references no - /// tag. Set when tombstone predicates (spliced into WHERE post-translation) - /// reference tag columns that must be present in the FROM source. - pub force: bool, - /// Physical column names that actually exist in the series table. - /// When empty, all tags from the ColumnMapping are projected (backward-compat). - pub tag_columns: &'a [String], -} - -/// Translate against a native MergeTree table (or other pre-formatted FROM -/// source). This is the sole public translate entry for production queries. -/// -/// When `series` is provided and the query references any tag, the fact table is -/// wrapped in an inline view that re-attaches the dimension table's tag columns -/// (see [`build_from_source`]); otherwise it is queried directly. -pub fn translate_native_table( - stmt: &SelectStatement, - table_source: &str, - mapping: Option<&ColumnMapping>, - series: Option>, - time_bounds: Option<(Option, Option)>, -) -> Result { - translate_inner(stmt, table_source, mapping, series, time_bounds) -} - -fn translate_inner( - stmt: &SelectStatement, - from_source: &str, - mapping: Option<&ColumnMapping>, - series: Option>, - time_bounds: Option<(Option, Option)>, -) -> Result { - let mut out = String::new(); - - // InfluxQL treats a GROUP BY time() query without an explicit fill() as - // fill(null): every bucket in the queried range is emitted, with NULL - // aggregates for empty buckets. Writes (`SELECT ... INTO` / CQ runs) keep - // the absent-fill case as "no fill" so synthetic NULL rows are never - // inserted into the destination. - let effective_fill = match (&stmt.fill, &stmt.into) { - (Some(f), _) => f.clone(), - (None, Some(_)) => FillOption::None, - (None, None) => FillOption::Null, - }; - - // Only `fill()` coerces NULL aggregates to a numeric default in SQL. - // `fill(null)` must leave NULL so JSON shows null, not 0. - let use_ifnull_fill = matches!(effective_fill, FillOption::Value(_)); - let needs_with_fill = !matches!(effective_fill, FillOption::None); - let fill_value = match &effective_fill { - FillOption::Value(v) => *v, - _ => 0.0, - }; - - // tz() flows into every bucketing expression (SELECT / GROUP BY / ORDER BY - // and the WITH FILL grid anchors) so buckets align on local-time boundaries, - // including 23/25-hour DST days. - let tz = stmt.timezone.as_deref(); - - // Collect field alias names for the INTERPOLATE clause. These must match the - // output column names emitted by `translate_field` exactly — otherwise - // `fill(previous)`/`fill(linear)` reference a non-existent identifier (e.g. - // `INTERPOLATE (MEAN)` while the column is `mean_value`) and chDB errors out. - let field_aliases: Vec = stmt - .fields - .iter() - .filter_map(select_output_field_name) - .collect(); - - // SELECT - prepend the time bucket column when GROUP BY time() is present - write!(out, "SELECT ")?; - let mut select_parts: Vec = Vec::new(); - - let has_group_by_time = stmt - .group_by - .as_ref() - .and_then(|gb| gb.time_dimension()) - .is_some(); - - if let Some(ref gb) = stmt.group_by { - if let Some(Dimension::Time { interval, offset }) = gb.time_dimension() { - let time_expr = time_bucket_expr(interval, offset.as_ref(), tz); - // Use __time alias to avoid collision with the raw `time` column, - // then rename back to `time` in the result parser. - select_parts.push(format!("{} AS __time", time_expr)); - } - - // Include GROUP BY tag columns in SELECT so they appear in the result - // and can be used to split rows into separate InfluxDB series. - for tag in gb.tag_dimensions() { - select_parts.push(select_tag_column_sql(tag, mapping)?); - } - } - - let has_aggregate = stmt.fields.iter().any(|f| expr_contains_call(&f.expr)); - let has_star = stmt - .fields - .iter() - .any(|f| matches!(f.expr, Expr::Star | Expr::Wildcard)); - // True aggregates collapse rows; bare window transforms (difference("v"), - // moving_average("v", n), ...) stay per-point and must keep the raw `time` - // column and per-point ordering like raw selects. - let has_true_aggregate = stmt.fields.iter().any(|f| expr_contains_aggregate(&f.expr)); - let has_raw_transform = stmt - .fields - .iter() - .any(|f| expr_contains_raw_transform(&f.expr)); - - // Raw (non-aggregate) selects return one row per point and must carry the - // point's `time` column, like InfluxDB. `SELECT *` already projects `time`, - // and GROUP BY time() / aggregate queries get their time column elsewhere. - let is_raw_select = !has_group_by_time && !has_star && !has_aggregate; - let projects_point_time = - is_raw_select || (has_raw_transform && !has_group_by_time && !has_star); - if projects_point_time { - select_parts.insert(0, quote_phys_identifier("time")); - } - - let field_strs: Vec = stmt - .fields - .iter() - .map(|f| { - translate_field( - f, - use_ifnull_fill, - fill_value, - stmt.group_by.as_ref(), - mapping, - ) - }) - .collect::, HyperbytedbError>>()?; - select_parts.extend(field_strs); - write!(out, "{}", select_parts.join(", "))?; - - // FROM — wrapped in the tag-rejoin inline view when needed. - let from = build_from_source(from_source, series, mapping, stmt); - write!(out, "\nFROM {}", from)?; - - // WHERE - if let Some(ref cond) = stmt.condition { - write!(out, "\nWHERE ")?; - translate_expr(cond, &mut out, true, mapping)?; - } - - // GROUP BY - if let Some(ref gb) = stmt.group_by { - let mut gb_parts = Vec::new(); - - if let Some(Dimension::Time { interval, offset }) = gb.time_dimension() { - gb_parts.push(time_bucket_expr(interval, offset.as_ref(), tz)); - } - - // Tag dimensions only group the SQL when a true aggregate is present. - // Raw selects / bare window transforms keep one row per point: their - // tag columns stay projected (for per-series splitting in the result - // parser and PARTITION BY in window clauses) but grouping by them - // would be NOT_AN_AGGREGATE in ClickHouse. - if has_true_aggregate { - for tag in gb.tag_dimensions() { - // Must match the SELECT expression: physical column name (handles the - // `__tag__` collision prefix). Previously emitted the logical name, - // which is wrong for collision-renamed tags. - gb_parts.push(group_by_tag_sql(tag, mapping)?); - } - } - - if !gb_parts.is_empty() { - write!(out, "\nGROUP BY {}", gb_parts.join(", "))?; - } - } - - // Compute time column expression for ORDER BY - let time_col = stmt.group_by.as_ref().and_then(|gb| { - if let Some(Dimension::Time { interval, offset }) = gb.time_dimension() { - Some(time_bucket_expr(interval, offset.as_ref(), tz)) - } else { - None - } - }); - - // InfluxDB orders every result by time ascending by default; an explicit - // ORDER BY only changes the direction. Order whenever there is a time column - // to sort on: GROUP BY time() buckets, raw per-point selects (incl. `*`), or - // bare window transforms (which are per-point and project raw `time`). - // Aggregates without GROUP BY time() collapse to one row and need no ordering. - let has_orderable_time = - time_col.is_some() || (!has_aggregate && !has_group_by_time) || projects_point_time; - let time_desc = stmt.order_by.as_ref().is_some_and(|o| o.time_desc); - let do_fill = needs_with_fill && time_col.is_some(); - // ClickHouse WITH FILL on a DESC-ordered column never matches the ascending - // FROM/TO anchors we emit, so no fill rows are generated. Fill ascending in - // this (inner) SELECT and re-order descending in a wrapper below. - let wrap_desc_fill = time_desc && do_fill; - - if has_orderable_time { - write!(out, "\nORDER BY ")?; - - // When filling a tag-grouped query, the tag columns must precede the - // time-fill column in ORDER BY so ClickHouse fills each tag group - // independently. Without this, WITH FILL fills globally: gap buckets are - // emitted with empty tag values (a phantom all-NULL series) and the real - // per-tag series is never filled — which surfaces as "no data" in Grafana. - if do_fill && let Some(ref gb) = stmt.group_by { - for tag in gb.tag_dimensions() { - write!(out, "{} ASC, ", group_by_tag_sql(tag, mapping)?)?; - } - } - - if let Some(ref tc) = time_col { - write!(out, "{}", tc)?; - } else { - write!(out, "time")?; - } - if time_desc && !wrap_desc_fill { - write!(out, " DESC")?; - } else { - write!(out, " ASC")?; - } - - if do_fill - && let Some(ref gb) = stmt.group_by - && let Some(Dimension::Time { interval, offset }) = gb.time_dimension() - { - let step = interval.to_clickhouse_interval(); - write!(out, " WITH FILL")?; - if let Some((min_nanos, max_nanos)) = time_bounds - && let (Some(min), Some(max)) = (min_nanos, max_nanos) - { - // The grid anchors must use the same bucket shape (offset + - // timezone) as the bucket expression, or the generated grid - // interleaves phantom buckets. `WITH FILL ... TO` is exclusive, - // so extend one step past the bucket containing the upper WHERE - // bound to emit the final bucket. - let from_anchor = - time_bucket_expr_on(&nanos_to_ch_timestamp(min), interval, offset.as_ref(), tz); - let to_anchor = - time_bucket_expr_on(&nanos_to_ch_timestamp(max), interval, offset.as_ref(), tz); - write!(out, " FROM {from_anchor} TO {to_anchor} + {step}")?; - } - write!(out, " STEP {}", step)?; - - match effective_fill { - // fill(previous): use INTERPOLATE to carry forward last known value - FillOption::Previous if !field_aliases.is_empty() => { - let interp_cols: Vec = field_aliases - .iter() - .map(|a| quote_identifier(a)) - .collect::, HyperbytedbError>>()?; - write!(out, " INTERPOLATE ({})", interp_cols.join(", "))?; - } - // fill(linear): use INTERPOLATE with linear expressions - FillOption::Linear if !field_aliases.is_empty() => { - let interp_cols: Vec = field_aliases - .iter() - .map(|a| -> Result { - let q = quote_identifier(a)?; - Ok(format!("{q} AS {q}")) - }) - .collect::, HyperbytedbError>>()?; - write!(out, " INTERPOLATE ({})", interp_cols.join(", "))?; - } - // fill(): WITH FILL-generated rows get column defaults - // (NULL) that the ifNull() around the aggregate can't reach; a - // constant INTERPOLATE expression sets generated rows — - // including leading gaps — to the fill value. - FillOption::Value(v) if !field_aliases.is_empty() => { - let interp_cols: Vec = field_aliases - .iter() - .map(|a| Ok(format!("{} AS {}", quote_identifier(a)?, format_float(v)))) - .collect::, HyperbytedbError>>()?; - write!(out, " INTERPOLATE ({})", interp_cols.join(", "))?; - } - _ => {} - } - } - } - - // GROUP BY tag dimensions carry InfluxQL per-series LIMIT semantics and - // outer ordering. These are the logical (output) column names. - let tag_dims: Vec<&str> = stmt - .group_by - .as_ref() - .map(|gb| gb.tag_dimensions()) - .unwrap_or_default(); - - if wrap_desc_fill { - // Re-order the ascending filled grid descending, tags first (matching - // the tag-first fill ordering above). Outer clauses stay on the `)` - // line so tombstone WHERE-splicing targets only the inner query. - let mut order_parts: Vec = tag_dims - .iter() - .map(|t| Ok(format!("{} ASC", quote_identifier(t)?))) - .collect::, HyperbytedbError>>()?; - order_parts.push("__time DESC".to_string()); - out = format!( - "SELECT * FROM (\n{out}\n) ORDER BY {}", - order_parts.join(", ") - ); - } else if has_raw_transform && !has_group_by_time { - // InfluxQL omits rows where a per-point window transform has no value - // yet (difference/derivative/elapsed first point, moving_average until - // the window is full). Those surface as NULL transform outputs here; - // filter them in a wrapper. Rows where every named transform output is - // NULL are dropped — in InfluxDB a point with a null input field would - // not exist in that field's series at all. - let transform_aliases: Vec = stmt - .fields - .iter() - .filter(|f| expr_contains_raw_transform(&f.expr)) - .filter_map(select_output_field_name) - .collect(); - if !transform_aliases.is_empty() { - let cond = transform_aliases - .iter() - .map(|a| Ok(format!("{} IS NOT NULL", quote_identifier(a)?))) - .collect::, HyperbytedbError>>()? - .join(" OR "); - let dir = if time_desc { "DESC" } else { "ASC" }; - out = format!( - "SELECT * FROM (\n{out}\n) WHERE {cond} ORDER BY {} {dir}", - quote_phys_identifier("time") - ); - } - } - - // LIMIT / OFFSET — InfluxQL LIMIT/OFFSET paginate points *per series*; with - // tag dimensions in GROUP BY that maps to ClickHouse `LIMIT [m,] n BY tags`. - // Without tag grouping the whole result is one series, so plain LIMIT works. - if !tag_dims.is_empty() && stmt.limit.is_some() { - let by_cols = tag_dims - .iter() - .map(|t| quote_identifier(t)) - .collect::, HyperbytedbError>>()? - .join(", "); - let limit = stmt.limit.unwrap_or(0); - match stmt.offset { - Some(offset) => write!(out, "\nLIMIT {offset}, {limit} BY ({by_cols})")?, - None => write!(out, "\nLIMIT {limit} BY ({by_cols})")?, - } - } else { - if let Some(limit) = stmt.limit { - write!(out, "\nLIMIT {}", limit)?; - } - if let Some(offset) = stmt.offset { - write!(out, "\nOFFSET {}", offset)?; - } - } - - Ok(out) -} - -/// Whether a call is a per-point window transform (translated to a ClickHouse -/// window function) rather than a true aggregate. -fn is_window_transform_call(name: &str) -> bool { - matches!( - name.to_ascii_uppercase().as_str(), - "DERIVATIVE" - | "NON_NEGATIVE_DERIVATIVE" - | "DIFFERENCE" - | "NON_NEGATIVE_DIFFERENCE" - | "MOVING_AVERAGE" - | "CUMULATIVE_SUM" - | "ELAPSED" - ) -} - -/// Whether an expression contains a row-collapsing aggregate. Window transforms -/// only count when they wrap a nested aggregate (e.g. `difference(mean(v))`). -fn expr_contains_aggregate(expr: &Expr) -> bool { - match expr { - Expr::Call(fc) if is_window_transform_call(&fc.name) => { - fc.args.first().is_some_and(|a| matches!(a, Expr::Call(_))) - } - Expr::Call(_) => true, - Expr::BinaryExpr(be) => { - expr_contains_aggregate(&be.left) || expr_contains_aggregate(&be.right) - } - Expr::UnaryExpr(_, e) => expr_contains_aggregate(e), - _ => false, - } -} - -/// Whether an expression contains a window transform applied directly to a raw -/// field (no nested aggregate) — a per-point transform. -fn expr_contains_raw_transform(expr: &Expr) -> bool { - match expr { - Expr::Call(fc) if is_window_transform_call(&fc.name) => { - !fc.args.first().is_some_and(|a| matches!(a, Expr::Call(_))) - } - Expr::Call(_) => false, - Expr::BinaryExpr(be) => { - expr_contains_raw_transform(&be.left) || expr_contains_raw_transform(&be.right) - } - Expr::UnaryExpr(_, e) => expr_contains_raw_transform(e), - _ => false, - } -} - -/// Rename the internal `__time` bucket alias to `time`, for INSERT ... SELECT -/// destinations and subquery FROM sources. Only standalone `__time` tokens are -/// rewritten (bare, `"__time"`, or `` `__time` ``); identifiers that merely -/// contain the substring (e.g. `"cpu__time"`) are preserved. -#[must_use] -pub fn rename_time_bucket_alias(sql: &str) -> String { - let bytes = sql.as_bytes(); - let is_ident = |c: u8| c.is_ascii_alphanumeric() || c == b'_'; - let mut out = String::with_capacity(sql.len()); - let mut last = 0usize; - for (pos, _) in sql.match_indices("__time") { - if pos < last { - continue; - } - let prev = if pos == 0 { None } else { Some(bytes[pos - 1]) }; - let next = bytes.get(pos + "__time".len()).copied(); - // The exact quoted identifier, or a bare token not embedded in a longer - // (possibly quoted) identifier. - let exact_quoted = matches!( - (prev, next), - (Some(b'"'), Some(b'"')) | (Some(b'`'), Some(b'`')) - ); - let bare = prev.is_none_or(|c| !is_ident(c) && c != b'"' && c != b'`') - && next.is_none_or(|c| !is_ident(c) && c != b'"' && c != b'`'); - if exact_quoted || bare { - out.push_str(&sql[last..pos]); - out.push_str("time"); - last = pos + "__time".len(); - } - } - out.push_str(&sql[last..]); - out -} - -/// Wrap a translated SELECT as `INSERT INTO SELECT ...`, renaming `__time` to `time` -/// for the destination measurement schema. -pub fn translate_select_into( - stmt: &SelectStatement, - dest_table: &QuotedTableName, - source: &str, - mapping: Option<&ColumnMapping>, -) -> Result { - validate_select_into(stmt)?; - let select_sql = translate_inner(stmt, source, mapping, None, None)?; - let select_sql = rename_time_bucket_alias(&select_sql); - Ok(format!("INSERT INTO {dest_table}\n{select_sql}")) -} - -fn translate_materialized_view_field( - field: &Field, - group_by: Option<&GroupBy>, - mapping: &ColumnMapping, -) -> Result { - if let Expr::Call(func) = &field.expr - && func.name.eq_ignore_ascii_case("mean") - { - let source = aggregate_source_field_name(func)?; - let col = mapping.physical_select_identifier(&source); - let col_q = quote_phys_identifier(&col); - let (sum_col, count_col) = mean_rollup_column_names(&source); - return Ok(format!( - "sum({col_q}) AS {}, count({col_q}) AS {}", - quote_phys_identifier(&sum_col), - quote_phys_identifier(&count_col) - )); - } - translate_field(field, false, 0.0, group_by, Some(mapping)) -} - -/// Ensure coalesced MV source rows expose every field referenced in the SELECT. -fn mapping_with_mv_aggregate_fields(mapping: &ColumnMapping, fields: &[Field]) -> ColumnMapping { - let mut expanded = mapping.clone(); - for field in fields { - if let Expr::Call(func) = &field.expr - && let Ok(source) = aggregate_source_field_name(func) - { - expanded - .field_names - .insert(mapping.physical_select_identifier(&source)); - } - } - expanded -} - -/// ClickHouse `SELECT` body for a fact-table materialized view. Joins the source -/// series dimension, groups by the MV's `GROUP BY time(...)` bucket and tag -/// dimensions (dropping tags omitted from the GROUP BY, e.g. `server_id`), and -/// assigns a destination `series_id` via [`crate::domain::series::series_id_ch_sql`]. -pub fn translate_materialized_view_select( - stmt: &SelectStatement, - source_fact: &QuotedTableName, - source_series: &QuotedTableName, - dest_measurement: &str, - mapping: &ColumnMapping, -) -> Result { - validate_select_into(stmt)?; - let gb = stmt - .group_by - .as_ref() - .ok_or_else(|| HyperbytedbError::QueryParse("MV requires GROUP BY".to_string()))?; - let Some(Dimension::Time { interval, offset }) = gb.time_dimension() else { - return Err(HyperbytedbError::QueryParse( - "MV requires GROUP BY time(...)".to_string(), - )); - }; - let time_bucket = time_bucket_expr_on( - "t.time", - interval, - offset.as_ref(), - stmt.timezone.as_deref(), - ); - - let mut grouped_tags: Vec = gb - .tag_dimensions() - .iter() - .map(|s| (*s).to_string()) - .collect(); - grouped_tags.sort(); - - let series_id_expr = crate::domain::series::series_id_ch_sql_for_tags( - dest_measurement, - &grouped_tags, - |tag| quote_phys_identifier(&mapping.physical_tag_column_name(tag)), - "s", - ); - - // Field columns must appear in sorted-by-name order to match the - // destination fact table's DDL column order (build_create_table_sql - // sorts fields by physical name). ClickHouse INSERT matches by position - // when no explicit column list is given in the TO clause. - // mean() expands to two columns (sum_col, count_col) — flatten them - // individually so the interleaved sort is correct. - let mut field_expr_by_name: std::collections::BTreeMap = - std::collections::BTreeMap::new(); - for field in &stmt.fields { - if let Expr::Call(func) = &field.expr - && func.name.eq_ignore_ascii_case("mean") - { - let source = aggregate_source_field_name(func)?; - let col = mapping.physical_select_identifier(&source); - let col_q = quote_phys_identifier(&col); - let (sum_col, count_col) = mean_rollup_column_names(&source); - let sum_expr = format!("sum({col_q}) AS {}", quote_phys_identifier(&sum_col)); - let count_expr = format!("count({col_q}) AS {}", quote_phys_identifier(&count_col)); - field_expr_by_name.insert(sum_col.clone(), sum_expr); - field_expr_by_name.insert(count_col.clone(), count_expr); - } else { - let expr = translate_materialized_view_field(field, stmt.group_by.as_ref(), mapping)?; - let name = select_output_field_name(field).ok_or_else(|| { - HyperbytedbError::QueryParse( - "materialized view field requires a name or alias".to_string(), - ) - })?; - field_expr_by_name.insert(name, expr); - } - } - let sorted_field_strs: Vec = field_expr_by_name.into_values().collect(); - - let mut select_parts = vec![ - format!("{time_bucket} AS time"), - "any(t.`_mv_src_origin_node_id`) AS origin_node_id".to_string(), - "max(t.`_mv_src_ingest_seq`) AS ingest_seq".to_string(), - format!("min({series_id_expr}) AS series_id"), - ]; - select_parts.extend(sorted_field_strs); - - let mut group_parts = vec![time_bucket.clone()]; - for tag in &grouped_tags { - group_parts.push(format!( - "s.{}", - quote_phys_identifier(&mapping.physical_tag_column_name(tag)) - )); - } - - let mut out = String::new(); - write!(out, "SELECT {}", select_parts.join(", "))?; - let source_mapping = mapping_with_mv_aggregate_fields(mapping, &stmt.fields); - let coalesced_source = build_coalesced_fact_view_with_row_meta(source_fact, &source_mapping); - // ANY LEFT JOIN for consistency with the query path: fact rows whose series - // row hasn't landed yet must not be silently dropped from the rollup. - write!( - out, - "\nFROM {coalesced_source} AS t ANY LEFT JOIN {source_series} AS s ON t.`series_id` = s.`series_id`" - )?; - - if let Some(ref cond) = stmt.condition { - write!(out, "\nWHERE ")?; - translate_expr(cond, &mut out, true, Some(mapping))?; - } - - write!(out, "\nGROUP BY {}", group_parts.join(", "))?; - Ok(out) -} - -/// ClickHouse `SELECT` for the destination series-dimension MV: one row per -/// rolled-up tag combination (tags not listed in the MV GROUP BY are dropped). -/// -/// `tag_name_mapping` controls how logical tag keys map to physical column -/// names (tag-field collision prefix). The source mapping uses the *source* -/// measurement's field names for collision detection, but the *destination* -/// series table may have a different set of field columns (MV aliases rename -/// fields), so callers should pass a dedicated mapping (or set of field names) -/// that reflects the destination schema for correct physical column naming. -pub fn translate_materialized_view_series_select( - stmt: &SelectStatement, - source_series: &QuotedTableName, - dest_measurement: &str, - mapping: &ColumnMapping, - dest_field_names: Option<&std::collections::HashSet>, -) -> Result { - let gb = stmt - .group_by - .as_ref() - .ok_or_else(|| HyperbytedbError::QueryParse("MV requires GROUP BY".to_string()))?; - let mut grouped_tags: Vec = gb - .tag_dimensions() - .iter() - .map(|s| (*s).to_string()) - .collect(); - grouped_tags.sort(); - - if grouped_tags.is_empty() { - return Ok(format!( - "SELECT min({}) AS series_id FROM {source_series} AS s GROUP BY tuple()", - crate::domain::series::series_id_ch_sql(dest_measurement, &[] as &[String]) - )); - } - - // Resolve physical tag column names: use destination field names when - // provided (the destination series table's column naming depends on the - // destination's field set, not the source's). - let tag_phys_name = |tag: &str| -> String { - match dest_field_names { - Some(dfn) => { - let fields: std::collections::HashSet<&str> = - dfn.iter().map(|s| s.as_str()).collect(); - crate::domain::chdb_naming::tag_column_name(tag, &fields) - } - None => mapping.physical_tag_column_name(tag), - } - }; - - let series_id_expr = crate::domain::series::series_id_ch_sql_for_tags( - dest_measurement, - &grouped_tags, - |tag| quote_phys_identifier(&tag_phys_name(tag)), - "s", - ); - - let tag_cols: Vec = grouped_tags - .iter() - .map(|tag| format!("s.{}", quote_phys_identifier(&tag_phys_name(tag)))) - .collect(); - - let mut select_parts = vec![format!("min({series_id_expr}) AS series_id")]; - select_parts.extend(tag_cols.iter().cloned()); - - let mut out = String::new(); - write!(out, "SELECT {}", select_parts.join(", "))?; - write!(out, "\nFROM {source_series} AS s")?; - write!(out, "\nGROUP BY {}", tag_cols.join(", "))?; - Ok(out) -} - -/// `INSERT INTO SELECT ...` for one-time MV backfill of historical data. -pub fn translate_materialized_view_backfill( - stmt: &SelectStatement, - dest_table: &QuotedTableName, - source_fact: &QuotedTableName, - source_series: &QuotedTableName, - dest_measurement: &str, - mapping: &ColumnMapping, -) -> Result { - let select_sql = translate_materialized_view_select( - stmt, - source_fact, - source_series, - dest_measurement, - mapping, - )?; - let insert_cols = materialized_view_dest_insert_columns(stmt)?; - Ok(format!( - "INSERT INTO {dest_table} ({insert_cols})\nSELECT {insert_cols}\nFROM (\n{select_sql}\n)" - )) -} - -/// Destination fact columns in physical DDL order (matches [`build_create_table_sql`]). -fn materialized_view_dest_insert_columns( - stmt: &SelectStatement, -) -> Result { - let mut cols = vec![ - quote_phys_identifier("time"), - quote_phys_identifier("origin_node_id"), - quote_phys_identifier("ingest_seq"), - quote_phys_identifier("series_id"), - ]; - let mut field_names = materialized_view_dest_field_names(stmt)?; - field_names.sort(); - cols.extend(field_names.into_iter().map(|n| quote_phys_identifier(&n))); - Ok(cols.join(", ")) -} - -/// Output column names for MV destination fields (expands `mean()` to sum/count pairs). -fn materialized_view_dest_field_names( - stmt: &SelectStatement, -) -> Result, HyperbytedbError> { - let mut names = Vec::new(); - for field in &stmt.fields { - if let Expr::Call(func) = &field.expr - && func.name.eq_ignore_ascii_case("mean") - { - let source = aggregate_source_field_name(func)?; - let (sum_col, count_col) = mean_rollup_column_names(&source); - names.push(sum_col); - names.push(count_col); - continue; - } - names.push(select_output_field_name(field).ok_or_else(|| { - HyperbytedbError::QueryParse( - "materialized view field requires a name or alias".to_string(), - ) - })?); - } - Ok(names) -} - -/// Full `CREATE MATERIALIZED VIEW ... TO ... AS SELECT ...` DDL for the fact MV. -pub fn build_create_fact_materialized_view( - mv_name: &QuotedTableName, - dest_table: &QuotedTableName, - select_sql: &str, -) -> String { - format!("CREATE MATERIALIZED VIEW {mv_name} TO {dest_table} AS\n{select_sql}") -} - -/// Full `CREATE MATERIALIZED VIEW ... TO ... AS SELECT ...` for the series MV. -pub fn build_create_series_materialized_view( - mv_name: &QuotedTableName, - dest_series: &QuotedTableName, - select_sql: &str, -) -> String { - format!("CREATE MATERIALIZED VIEW {mv_name} TO {dest_series} AS\n{select_sql}") -} - -/// Like [`translate_select_into`], targeting a native MergeTree table source. -/// `series` lets a tag-grouped continuous query resolve tags from the source -/// measurement's dimension table. -pub fn translate_select_into_native( - stmt: &SelectStatement, - dest_table: &QuotedTableName, - source_table: &QuotedTableName, - mapping: Option<&ColumnMapping>, - series: Option>, -) -> Result { - validate_select_into(stmt)?; - let select_sql = translate_inner(stmt, source_table.as_str(), mapping, series, None)?; - let select_sql = rename_time_bucket_alias(&select_sql); - Ok(format!("INSERT INTO {dest_table}\n{select_sql}")) -} - -/// Like translate, but uses a custom source expression instead of file() - used for subqueries. -pub fn translate_with_source( - stmt: &SelectStatement, - source: &str, -) -> Result { - translate_inner(stmt, source, None, None, None) -} - -/// Whether `expr` references a tag (so the query needs the series join). Treats a -/// name present in `tag_keys` as a tag even if it also collides with a field name -/// — over-inclusive is safe (the field still resolves via `t.*`). -fn expr_references_tag(expr: &Expr, m: &ColumnMapping) -> bool { - match expr { - Expr::Identifier(name) => m.tag_keys.contains(name), - Expr::FieldRef { name, typ } => { - matches!(typ, Some(FieldType::Tag)) || m.tag_keys.contains(name) - } - Expr::BinaryExpr(be) => { - expr_references_tag(&be.left, m) || expr_references_tag(&be.right, m) - } - Expr::UnaryExpr(_, e) => expr_references_tag(e, m), - Expr::Call(fc) => fc.args.iter().any(|a| expr_references_tag(a, m)), - _ => false, - } -} - -/// Whether the query references any tag (in SELECT, WHERE, or GROUP BY) — or uses -/// `SELECT *`, which in InfluxDB includes tags. Determines whether the series -/// dimension table must be joined. -fn query_references_tag(stmt: &SelectStatement, m: &ColumnMapping) -> bool { - if stmt - .group_by - .as_ref() - .is_some_and(|gb| gb.references_tags()) - { - return true; - } - if stmt - .fields - .iter() - .any(|f| matches!(f.expr, Expr::Star | Expr::Wildcard) || expr_references_tag(&f.expr, m)) - { - return true; - } - stmt.condition - .as_ref() - .is_some_and(|c| expr_references_tag(c, m)) -} - -/// Build a query-time view that collapses duplicate `(series_id, time)` rows -/// to the single row with the highest `ingest_seq`. -/// -/// Partial Telegraf lines are merged at ingest (`coalesce_points_and_origins`); -/// at query time we must not merge fields independently across rows — per-field -/// `argMaxIf` can stitch a correct `available` from one row with a corrupt -/// `used_percent` from another (e.g. async replication writing a second row -/// for the same instant), which produces nonsense Grafana percentages. -pub fn build_coalesced_fact_view(fact_table: &QuotedTableName, mapping: &ColumnMapping) -> String { - build_coalesced_fact_view_impl(fact_table, mapping, false) -} - -/// Like [`build_coalesced_fact_view`], but preserves `ingest_seq` / `origin_node_id` for -/// downstream aggregates (materialized view source dedup). -pub fn build_coalesced_fact_view_with_row_meta( - fact_table: &QuotedTableName, - mapping: &ColumnMapping, -) -> String { - build_coalesced_fact_view_impl(fact_table, mapping, true) -} - -fn build_coalesced_fact_view_impl( - fact_table: &QuotedTableName, - mapping: &ColumnMapping, - include_row_metadata: bool, -) -> String { - let mut field_cols: Vec<&String> = mapping.field_names.iter().collect(); - field_cols.sort(); - let field_aggs: Vec = field_cols - .iter() - .map(|f| { - let q = quote_phys_identifier(f); - let agg = match mapping.field_rollups.get(*f) { - Some(RollupCombine::Sum) => format!("sum({q})"), - Some(RollupCombine::Min) => format!("min({q})"), - Some(RollupCombine::Max) => format!("max({q})"), - Some(RollupCombine::First) => format!("argMin({q}, `time`)"), - Some(RollupCombine::Last) | None => format!("argMax({q}, `ingest_seq`)"), - }; - format!("{agg} AS {q}") - }) - .collect(); - let select_fields = if field_aggs.is_empty() { - String::new() - } else { - format!(", {}", field_aggs.join(", ")) - }; - let row_meta = if include_row_metadata { - ", max(`ingest_seq`) AS `_mv_src_ingest_seq`, any(`origin_node_id`) AS `_mv_src_origin_node_id`" - } else { - "" - }; - format!( - "(SELECT `series_id`, `time`{row_meta}{select_fields} FROM {fact_table} GROUP BY `series_id`, `time`)" - ) -} - -/// Build the FROM source. When `mapping` is present the fact table is wrapped in -/// a coalesced view so partial-field rows merge before aggregation. When `series` -/// is set and the query references a tag, the coalesced fact table is wrapped in -/// an inline view that re-attaches the tag columns from the dimension table. -/// `ANY LEFT JOIN` takes at most one matching dimension row (so pre-merge duplicate -/// `ReplacingMergeTree` series rows can't fan out fact rows) and preserves fact -/// rows whose series row is briefly missing. Tag columns are exposed under their -/// physical names, so the rest of the translator — which already references tags -/// by physical name — is unchanged. -fn build_from_source( - fact_table: &str, - series: Option>, - mapping: Option<&ColumnMapping>, - stmt: &SelectStatement, -) -> String { - let fact = match mapping { - Some(m) => { - build_coalesced_fact_view(&QuotedTableName::new_quoted(fact_table.to_string()), m) - } - None => fact_table.to_string(), - }; - let (Some(sj), Some(m)) = (series, mapping) else { - return fact; - }; - if !sj.force && !query_references_tag(stmt, m) { - return fact; - } - let mut tag_cols: Vec = m - .tag_keys - .iter() - .map(|t| m.physical_tag_column_name(t)) - .collect(); - if tag_cols.is_empty() { - return fact; - } - // Only project tag columns that actually exist in the series table. - // MV destinations may have a subset of source tags (GROUP BY columns only). - if !sj.tag_columns.is_empty() { - tag_cols.retain(|c| sj.tag_columns.contains(c)); - } - if tag_cols.is_empty() { - return fact; - } - tag_cols.sort(); - let projected = tag_cols - .iter() - .map(|c| format!("s.{}", quote_phys_identifier(c))) - .collect::>() - .join(", "); - format!( - "(SELECT t.*, {projected} FROM {fact} AS t ANY LEFT JOIN {series} AS s ON t.`series_id` = s.`series_id`)", - series = sj.table, - ) -} - -/// GROUP BY expression for a tag: the physical column name, matching the SELECT -/// side. Without a mapping, falls back to the logical name (unchanged behaviour). -fn group_by_tag_sql( - tag: &str, - mapping: Option<&ColumnMapping>, -) -> Result { - match mapping { - Some(m) => Ok(quote_phys_identifier(&m.physical_tag_column_name(tag))), - None => quote_identifier(tag), - } -} - -fn time_bucket_expr(interval: &Duration, offset: Option<&Duration>, tz: Option<&str>) -> String { - time_bucket_expr_on("time", interval, offset, tz) -} - -/// Bucketing expression over an arbitrary time expression. `tz` (from `tz()`) -/// makes `toStartOfInterval` bucket on local-time boundaries in that zone, -/// which is what keeps `GROUP BY time(1d)` correct across 23/25-hour DST days. -fn time_bucket_expr_on( - time_col: &str, - interval: &Duration, - offset: Option<&Duration>, - tz: Option<&str>, -) -> String { - let interval_str = interval.to_clickhouse_interval(); - let tz_arg = tz - .map(|t| format!(", {}", quote_string(t))) - .unwrap_or_default(); - if let Some(off) = offset { - let off_str = off.to_clickhouse_interval(); - format!( - "toStartOfInterval({time_col} - {}, {}{tz_arg}) + {}", - off_str, interval_str, off_str - ) - } else { - format!("toStartOfInterval({time_col}, {}{tz_arg})", interval_str) - } -} - -fn select_tag_column_sql( - tag: &str, - mapping: Option<&ColumnMapping>, -) -> Result { - let Some(m) = mapping else { - return quote_identifier(tag); - }; - let phys = m.physical_tag_column_name(tag); - if phys == tag { - quote_identifier(tag) - } else { - Ok(format!( - "{} AS {}", - quote_phys_identifier(&phys), - quote_identifier(tag)? - )) - } -} - -fn translate_field( - field: &Field, - use_fill: bool, - fill_value: f64, - group_by: Option<&GroupBy>, - mapping: Option<&ColumnMapping>, -) -> Result { - let sql = translate_field_expr(&field.expr, use_fill, fill_value, group_by, mapping)?; - let alias = field - .alias - .clone() - .or_else(|| default_field_alias(&field.expr)); - Ok(match alias { - Some(a) => format!("{} AS {}", sql, quote_identifier(&a)?), - None => sql, - }) -} - -/// Output column name for a SELECT field (explicit alias or Influx-style default). -#[must_use] -pub fn select_output_field_name(field: &Field) -> Option { - field - .alias - .clone() - .or_else(|| default_field_alias(&field.expr)) -} - -/// Generate a default column alias matching InfluxDB conventions. -/// Single-arg aggregates include the field name for uniqueness: -/// `mean("usage_idle")` → `"mean_usage_idle"`, `count("x")` → `"count_x"`. -/// No-arg calls use just the function name: `count()` → `"count"`. -/// Non-call expressions get no alias. -fn default_field_alias(expr: &Expr) -> Option { - match expr { - Expr::Call(func) => { - let base = func.name.to_lowercase(); - if let Some(Expr::Identifier(field_name)) = func.args.first() { - Some(format!("{}_{}", base, field_name)) - } else { - Some(base) - } - } - _ => None, - } -} - -/// Whether an expression tree contains a function call (aggregate, selector, or -/// transform). Used to distinguish raw per-point selects from aggregate queries. -fn expr_contains_call(expr: &Expr) -> bool { - match expr { - Expr::Call(_) => true, - Expr::BinaryExpr(be) => expr_contains_call(&be.left) || expr_contains_call(&be.right), - Expr::UnaryExpr(_, e) => expr_contains_call(e), - _ => false, - } -} - -fn translate_field_expr( - expr: &Expr, - use_fill: bool, - fill_value: f64, - group_by: Option<&GroupBy>, - mapping: Option<&ColumnMapping>, -) -> Result { - match expr { - Expr::Star => Ok("*".to_string()), - Expr::Identifier(name) => { - let col = mapping - .map(|m| m.physical_select_identifier(name)) - .unwrap_or_else(|| name.clone()); - Ok(quote_phys_identifier(&col)) - } - Expr::FieldRef { name, .. } => { - let col = mapping - .map(|m| m.physical_select_identifier(name)) - .unwrap_or_else(|| name.clone()); - Ok(quote_phys_identifier(&col)) - } - Expr::Call(func) => translate_aggregate_call(func, use_fill, fill_value, group_by, mapping), - Expr::BinaryExpr(be) => translate_binary_expr(be, use_fill, fill_value, group_by, mapping), - Expr::UnaryExpr(op, e) => { - let inner = translate_field_expr(e, use_fill, fill_value, group_by, mapping)?; - Ok(match op { - UnaryOp::Neg => format!("(-{})", inner), - UnaryOp::Not => format!("(NOT {})", inner), - }) - } - Expr::StringLiteral(s) => Ok(quote_string(s)), - Expr::IntegerLiteral(n) => Ok(n.to_string()), - Expr::FloatLiteral(f) => Ok(f.to_string()), - Expr::BooleanLiteral(b) => Ok(if *b { "true" } else { "false" }.to_string()), - Expr::DurationLiteral(d) => Ok(d.to_clickhouse_interval()), - Expr::TimeLiteral(s) => Ok(quote_string(s)), - Expr::Regex(r) => Ok(format!( - "'{}'", - r.replace('\\', "\\\\").replace('\'', "\\'") - )), - Expr::Wildcard => Ok("*".to_string()), - Expr::Now => Ok("now64()".to_string()), - } -} - -fn translate_binary_expr( - be: &BinaryExpr, - use_fill: bool, - fill_value: f64, - group_by: Option<&GroupBy>, - mapping: Option<&ColumnMapping>, -) -> Result { - let left = translate_field_expr(&be.left, use_fill, fill_value, group_by, mapping)?; - let right = translate_field_expr(&be.right, use_fill, fill_value, group_by, mapping)?; - Ok(format!( - "({} {} {})", - left, - binary_op_to_clickhouse(&be.op), - right - )) -} - -fn translate_aggregate_call( - func: &FunctionCall, - use_fill: bool, - fill_value: f64, - group_by: Option<&GroupBy>, - mapping: Option<&ColumnMapping>, -) -> Result { - let name_upper = func.name.to_uppercase(); - let wrap_fill = |s: String| -> String { - if use_fill && group_by.is_some() { - format!("ifNull({}, {})", s, format_float(fill_value)) - } else { - s - } - }; - - let result = match name_upper.as_str() { - "MEAN" => { - let arg = get_single_arg(func, "MEAN")?; - if let Some(m) = mapping - && let Expr::Identifier(name) | Expr::FieldRef { name, .. } = arg - && let Some(mean_def) = m.mean_fields.get(name) - { - let sum_q = quote_phys_identifier(&mean_def.sum_col); - let count_q = quote_phys_identifier(&mean_def.count_col); - return Ok(wrap_fill(format!( - "(sum({sum_q}) / nullIf(sum({count_q}), 0))" - ))); - } - let f = translate_aggregate_arg(arg, mapping)?; - wrap_fill(format!("avg({})", f)) - } - "MEDIAN" => { - let arg = get_single_arg(func, "MEDIAN")?; - let f = translate_aggregate_arg(arg, mapping)?; - // InfluxQL median averages the two middle values on even counts; - // quantileExactInclusive(0.5) matches that exactly (ClickHouse - // `median` is sampling-based and approximate). - wrap_fill(format!("quantileExactInclusive(0.5)({})", f)) - } - "COUNT" => { - let arg = get_single_arg(func, "COUNT")?; - // count(distinct("v")) → exact distinct count. - if let Expr::Call(inner) = arg - && inner.name.eq_ignore_ascii_case("distinct") - { - let inner_arg = get_single_arg(inner, "DISTINCT")?; - let f = translate_aggregate_arg(inner_arg, mapping)?; - wrap_fill(format!("uniqExact({})", f)) - } else { - let f = translate_aggregate_arg(arg, mapping)?; - wrap_fill(format!("count({})", f)) - } - } - "SUM" => { - let arg = get_single_arg(func, "SUM")?; - let f = translate_aggregate_arg(arg, mapping)?; - wrap_fill(format!("sum({})", f)) - } - "MIN" => { - let arg = get_single_arg(func, "MIN")?; - let f = translate_aggregate_arg(arg, mapping)?; - wrap_fill(format!("min({})", f)) - } - "MAX" => { - let arg = get_single_arg(func, "MAX")?; - let f = translate_aggregate_arg(arg, mapping)?; - wrap_fill(format!("max({})", f)) - } - "FIRST" => { - let arg = get_single_arg(func, "FIRST")?; - let f = translate_aggregate_arg(arg, mapping)?; - wrap_fill(format!("argMin({}, time)", f)) - } - "LAST" => { - let arg = get_single_arg(func, "LAST")?; - let f = translate_aggregate_arg(arg, mapping)?; - wrap_fill(format!("argMax({}, time)", f)) - } - "PERCENTILE" => { - let (field_arg, pct_arg) = get_two_args(func, "PERCENTILE")?; - let f = translate_aggregate_arg(field_arg, mapping)?; - let pct = match &pct_arg { - Expr::IntegerLiteral(n) => (*n as f64) / 100.0, - Expr::FloatLiteral(f) => *f / 100.0, - _ => { - return Err(HyperbytedbError::QueryParse(format!( - "PERCENTILE second argument must be numeric, got {:?}", - pct_arg - ))); - } - }; - // InfluxQL percentile is nearest-rank and returns an actual sample - // (for [10,20,30,40] p50 = 20); quantileExactLow matches that. - wrap_fill(format!("quantileExactLow({})({})", format_float(pct), f)) - } - "SPREAD" => { - let arg = get_single_arg(func, "SPREAD")?; - let f = translate_aggregate_arg(arg, mapping)?; - wrap_fill(format!("(max({}) - min({}))", f, f)) - } - "STDDEV" => { - let arg = get_single_arg(func, "STDDEV")?; - let f = translate_aggregate_arg(arg, mapping)?; - // InfluxQL stddev is the *sample* standard deviation. - wrap_fill(format!("stddevSamp({})", f)) - } - "MODE" => { - let arg = get_single_arg(func, "MODE")?; - let f = translate_aggregate_arg(arg, mapping)?; - // topKWeighted returns an Array; unwrap to a scalar. Still - // approximate and tie-breaking is unspecified, unlike InfluxQL's - // lowest-value tie-break. - wrap_fill(format!("arrayElement(topKWeighted(1)({}, 1), 1)", f)) - } - "DISTINCT" => { - let arg = get_single_arg(func, "DISTINCT")?; - let f = translate_aggregate_arg(arg, mapping)?; - // arrayJoin(groupUniqArray(...)) yields one row per distinct value - // and — unlike `SELECT DISTINCT` — stays valid inside GROUP BY time(). - format!("arrayJoin(groupUniqArray({}))", f) - } - "DERIVATIVE" | "NON_NEGATIVE_DERIVATIVE" => { - let field_arg = get_single_arg(func, &name_upper)?; - let f = translate_field_or_nested(field_arg, group_by, mapping)?; - let window = build_window_clause(group_by, mapping)?; - let unit_nanos: i64 = if func.args.len() >= 2 { - match &func.args[1] { - Expr::DurationLiteral(d) => d.to_nanos(), - _ => 1_000_000_000, - } - } else { - 1_000_000_000 - }; - let unit_seconds = format_float(unit_nanos as f64 / 1_000_000_000.0); - let delta_value = format!("({f} - lagInFrame({f}, 1) {window})"); - // Use toFloat64() to get Unix timestamps as seconds (Float64) - // for correct arithmetic regardless of DateTime/DateTime64 type. - let time_ref = window_time_ref(group_by); - let delta_time = - format!("(toFloat64({time_ref}) - toFloat64(lagInFrame({time_ref}, 1) {window}))"); - let deriv = format!("{delta_value} / ({delta_time} / {unit_seconds})"); - if name_upper == "NON_NEGATIVE_DERIVATIVE" { - format!("if(({deriv}) >= 0, ({deriv}), NULL)") - } else { - deriv - } - } - "DIFFERENCE" | "NON_NEGATIVE_DIFFERENCE" => { - let arg = get_single_arg(func, &name_upper)?; - let f = translate_field_or_nested(arg, group_by, mapping)?; - let window = build_window_clause(group_by, mapping)?; - let diff = format!("({f} - lagInFrame({f}, 1) {window})"); - if name_upper == "NON_NEGATIVE_DIFFERENCE" { - format!("if({diff} >= 0, {diff}, NULL)") - } else { - diff - } - } - "MOVING_AVERAGE" => { - let (field_arg, n_arg) = get_two_args(func, "MOVING_AVERAGE")?; - let f = translate_field_or_nested(field_arg, group_by, mapping)?; - let time_ref = window_time_ref(group_by); - let n = match &n_arg { - Expr::IntegerLiteral(n) => *n, - _ => { - return Err(HyperbytedbError::QueryParse( - "MOVING_AVERAGE second argument must be integer".to_string(), - )); - } - }; - let partition_tags: Vec<&str> = - group_by.map(|gb| gb.tag_dimensions()).unwrap_or_default(); - let partition_clause = if partition_tags.is_empty() { - String::new() - } else { - let p = partition_tags - .iter() - .map(|t| { - let phys = mapping - .map(|m| m.physical_tag_column_name(t)) - .unwrap_or_else(|| t.to_string()); - Ok(quote_phys_identifier(&phys)) - }) - .collect::, HyperbytedbError>>()? - .join(", "); - format!("PARTITION BY {p} ") - }; - // InfluxQL emits moving_average values only once the window holds N - // points; gate on the frame's non-null count so shorter leading - // frames yield NULL (filtered for per-point transforms). - let frame = format!( - "({partition_clause}ORDER BY {time_ref} ROWS BETWEEN {preceding} PRECEDING AND CURRENT ROW)", - preceding = n - 1 - ); - format!("if(count({f}) OVER {frame} >= {n}, avg({f}) OVER {frame}, NULL)") - } - "CUMULATIVE_SUM" => { - let arg = get_single_arg(func, "CUMULATIVE_SUM")?; - let f = translate_field_or_nested(arg, group_by, mapping)?; - let time_ref = window_time_ref(group_by); - let partition_tags: Vec<&str> = - group_by.map(|gb| gb.tag_dimensions()).unwrap_or_default(); - let partition_clause = if partition_tags.is_empty() { - String::new() - } else { - let p = partition_tags - .iter() - .map(|t| { - let phys = mapping - .map(|m| m.physical_tag_column_name(t)) - .unwrap_or_else(|| t.to_string()); - Ok(quote_phys_identifier(&phys)) - }) - .collect::, HyperbytedbError>>()? - .join(", "); - format!("PARTITION BY {p} ") - }; - format!( - "sum({f}) OVER ({partition_clause}ORDER BY {time_ref} ROWS UNBOUNDED PRECEDING)" - ) - } - "ELAPSED" => { - let _field_arg = get_single_arg(func, "ELAPSED")?; - let time_ref = window_time_ref(group_by); - let window = build_window_clause(group_by, mapping)?; - let unit_nanos: i64 = if func.args.len() >= 2 { - match &func.args[1] { - Expr::DurationLiteral(d) => d.to_nanos(), - _ => 1_000_000_000, - } - } else { - 1_000_000_000 - }; - let unit_seconds = format_float(unit_nanos as f64 / 1_000_000_000.0); - // toNullable: lagInFrame on the non-Nullable time column would - // default to epoch 0 out-of-frame, making the first row a huge - // elapsed value instead of NULL (InfluxQL omits the first point). - format!( - "((toFloat64({time_ref}) - toFloat64(lagInFrame(toNullable({time_ref}), 1) {window})) / {unit_seconds})" - ) - } - _ => { - return Err(HyperbytedbError::QueryParse(format!( - "unsupported aggregate function: {}", - func.name - ))); - } - }; - - Ok(result) -} - -fn translate_aggregate_arg( - expr: &Expr, - mapping: Option<&ColumnMapping>, -) -> Result { - match expr { - Expr::Identifier(name) | Expr::FieldRef { name, .. } => { - let col = mapping - .map(|m| m.physical_select_identifier(name)) - .unwrap_or_else(|| name.clone()); - Ok(quote_phys_identifier(&col)) - } - Expr::Star => Ok("*".to_string()), - _ => Err(HyperbytedbError::QueryParse(format!( - "aggregate argument must be identifier or *, got {:?}", - expr - ))), - } -} - -/// Translate the first argument of a transform function (derivative, difference, etc.). -/// Accepts either a plain identifier or a nested aggregate like mean("reads"). -fn translate_field_or_nested( - expr: &Expr, - group_by: Option<&GroupBy>, - mapping: Option<&ColumnMapping>, -) -> Result { - match expr { - Expr::Call(inner_func) => { - translate_aggregate_call(inner_func, false, 0.0, group_by, mapping) - } - _ => translate_aggregate_arg(expr, mapping), - } -} - -/// Return the time column reference for window function ORDER BY clauses. -/// Uses `__time` (the time bucket alias) when GROUP BY time() is present, -/// raw `time` otherwise. -fn window_time_ref(group_by: Option<&GroupBy>) -> &'static str { - if group_by.and_then(|gb| gb.time_dimension()).is_some() { - "__time" - } else { - "time" - } -} - -/// Build the OVER (...) window clause for transform functions. -/// Includes PARTITION BY for GROUP BY tag dimensions so that window -/// functions (lagInFrame, etc.) operate within each series independently. -fn build_window_clause( - group_by: Option<&GroupBy>, - mapping: Option<&ColumnMapping>, -) -> Result { - let time_ref = window_time_ref(group_by); - let partition_tags: Vec<&str> = group_by.map(|gb| gb.tag_dimensions()).unwrap_or_default(); - - if partition_tags.is_empty() { - Ok(format!("OVER (ORDER BY {time_ref})")) - } else { - let partition = partition_tags - .iter() - .map(|t| { - let phys = mapping - .map(|m| m.physical_tag_column_name(t)) - .unwrap_or_else(|| t.to_string()); - Ok(quote_phys_identifier(&phys)) - }) - .collect::, HyperbytedbError>>()? - .join(", "); - Ok(format!( - "OVER (PARTITION BY {partition} ORDER BY {time_ref})" - )) - } -} - -fn get_single_arg<'a>(func: &'a FunctionCall, name: &str) -> Result<&'a Expr, HyperbytedbError> { - func.args.first().ok_or_else(|| { - HyperbytedbError::QueryParse(format!("{} requires exactly one argument", name)) - }) -} - -fn get_two_args<'a>( - func: &'a FunctionCall, - name: &str, -) -> Result<(&'a Expr, &'a Expr), HyperbytedbError> { - if func.args.len() < 2 { - return Err(HyperbytedbError::QueryParse(format!( - "{} requires exactly two arguments", - name - ))); - } - Ok((&func.args[0], &func.args[1])) -} - -/// Translate a WHERE condition expression to ClickHouse SQL. -/// Used by the DELETE statement handler to serialize tombstone predicates. -/// Tag identifiers are resolved to their physical column names so the spliced -/// WHERE clause matches the tag columns exposed by the series-rejoin inline view. -pub fn translate_condition( - expr: &Expr, - mapping: &ColumnMapping, - out: &mut String, -) -> Result<(), HyperbytedbError> { - translate_expr(expr, out, true, Some(mapping)) -} - -fn tag_field_collision(m: &ColumnMapping, name: &str) -> bool { - m.tag_keys.contains(name) && m.field_names.contains(name) -} - -fn is_where_literal(e: &Expr) -> bool { - matches!( - e, - Expr::IntegerLiteral(_) - | Expr::FloatLiteral(_) - | Expr::StringLiteral(_) - | Expr::BooleanLiteral(_) - ) -} - -fn where_identifier_physical_name( - m: &ColumnMapping, - name: &str, - other: &Expr, -) -> Result { - if !tag_field_collision(m, name) { - return quote_identifier(name); - } - match other { - Expr::IntegerLiteral(_) | Expr::FloatLiteral(_) | Expr::BooleanLiteral(_) => { - quote_identifier(name) - } - Expr::StringLiteral(_) | Expr::Regex(_) => { - Ok(quote_phys_identifier(&m.physical_tag_column_name(name))) - } - _ => Ok(quote_phys_identifier(&m.physical_tag_column_name(name))), - } -} - -fn regex_match_column_name( - left: &Expr, - mapping: Option<&ColumnMapping>, -) -> Result { - match left { - Expr::FieldRef { - name, - typ: Some(FieldType::Tag), - } => { - let col = mapping - .map(|m| m.physical_tag_column_name(name)) - .unwrap_or_else(|| name.clone()); - Ok(quote_phys_identifier(&col)) - } - Expr::FieldRef { - name, - typ: Some(FieldType::Field), - } => quote_identifier(name), - Expr::FieldRef { name, typ: None } => { - let col = mapping - .map(|m| m.physical_tag_column_name(name)) - .unwrap_or_else(|| name.clone()); - Ok(quote_phys_identifier(&col)) - } - Expr::Identifier(n) => { - let col = if let Some(m) = mapping { - if tag_field_collision(m, n) { - m.physical_tag_column_name(n) - } else { - m.physical_select_identifier(n) - } - } else { - n.clone() - }; - Ok(quote_phys_identifier(&col)) - } - _ => Err(HyperbytedbError::QueryParse( - "regex operator =~ / !~ requires identifier and regex".to_string(), - )), - } -} - -fn try_translate_where_binary_expr( - be: &BinaryExpr, - out: &mut String, - m: &ColumnMapping, -) -> Result { - let (name, lit, id_on_left, explicit_tag) = match (&be.left, &be.right) { - (Expr::Identifier(n), rhs) if is_where_literal(rhs) => (n.as_str(), rhs, true, false), - (Expr::FieldRef { name, typ: None }, rhs) if is_where_literal(rhs) => { - (name.as_str(), rhs, true, false) - } - ( - Expr::FieldRef { - name, - typ: Some(FieldType::Tag), - }, - rhs, - ) if is_where_literal(rhs) => (name.as_str(), rhs, true, true), - (lhs, Expr::Identifier(n)) if is_where_literal(lhs) => (n.as_str(), lhs, false, false), - (lhs, Expr::FieldRef { name, typ: None }) if is_where_literal(lhs) => { - (name.as_str(), lhs, false, false) - } - ( - lhs, - Expr::FieldRef { - name, - typ: Some(FieldType::Tag), - }, - ) if is_where_literal(lhs) => (name.as_str(), lhs, false, true), - _ => return Ok(false), - }; - if matches!(be.op, BinaryOp::And | BinaryOp::Or) { - return Ok(false); - } - // Tags are strings; comparing one to a numeric literal never matches in - // InfluxQL (and would be a type error in ClickHouse). Emit constant-false - // so the query runs and returns an empty result. - let is_pure_tag = explicit_tag || (m.tag_keys.contains(name) && !m.field_names.contains(name)); - if is_pure_tag && matches!(lit, Expr::IntegerLiteral(_) | Expr::FloatLiteral(_)) { - write!(out, "1 = 0")?; - return Ok(true); - } - if !tag_field_collision(m, name) { - return Ok(false); - } - let col = where_identifier_physical_name(m, name, lit)?; - if id_on_left { - write!(out, "{}", col)?; - write!(out, " {} ", binary_op_to_clickhouse(&be.op))?; - translate_expr(lit, out, true, Some(m))?; - } else { - translate_expr(lit, out, true, Some(m))?; - write!(out, " {} ", binary_op_to_clickhouse(&be.op))?; - write!(out, "{}", col)?; - } - Ok(true) -} - -fn translate_expr( - expr: &Expr, - out: &mut String, - in_where: bool, - mapping: Option<&ColumnMapping>, -) -> Result<(), HyperbytedbError> { - match expr { - Expr::Identifier(name) => { - if in_where && name.to_lowercase() == "time" { - write!(out, "time")?; - } else if in_where { - if let Some(m) = mapping { - if tag_field_collision(m, name) { - write!( - out, - "{}", - quote_phys_identifier(&m.physical_tag_column_name(name)) - )?; - } else { - write!(out, "{}", quote_identifier(name)?)?; - } - } else { - write!(out, "{}", quote_identifier(name)?)?; - } - } else { - write!(out, "{}", quote_identifier(name)?)?; - } - } - Expr::FieldRef { name, typ } => { - let s = match typ { - Some(FieldType::Tag) => { - if let Some(m) = mapping { - quote_phys_identifier(&m.physical_tag_column_name(name)) - } else { - quote_identifier(name)? - } - } - Some(FieldType::Field) => quote_identifier(name)?, - None => { - if let Some(m) = mapping { - if tag_field_collision(m, name) { - quote_phys_identifier(&m.physical_tag_column_name(name)) - } else { - quote_identifier(name)? - } - } else { - quote_identifier(name)? - } - } - }; - write!(out, "{}", s)?; - } - Expr::Now => write!(out, "now64()")?, - Expr::DurationLiteral(d) => write!(out, "{}", d.to_clickhouse_interval())?, - Expr::BinaryExpr(be) => { - write!(out, "(")?; - if matches!(be.op, BinaryOp::RegexMatch | BinaryOp::RegexNotMatch) { - let pattern = match (&be.left, &be.right) { - (_, Expr::Regex(p)) => p.clone(), - _ => { - return Err(HyperbytedbError::QueryParse( - "regex operator =~ / !~ requires identifier and regex".to_string(), - )); - } - }; - let col = regex_match_column_name(&be.left, mapping)?; - let escaped = pattern.replace('\\', "\\\\").replace('\'', "\\'"); - if be.op == BinaryOp::RegexMatch { - write!(out, "match({}, '{}')", col, escaped)?; - } else { - write!(out, "NOT match({}, '{}')", col, escaped)?; - } - } else { - let is_logical = matches!(be.op, BinaryOp::And | BinaryOp::Or); - if is_logical { - translate_expr(&be.left, out, in_where, mapping)?; - let op_str = match be.op { - BinaryOp::And => "AND", - BinaryOp::Or => "OR", - _ => { - return Err(HyperbytedbError::QueryParse( - "internal: expected AND/OR in logical binary expression" - .to_string(), - )); - } - }; - write!(out, " {} ", op_str)?; - translate_expr(&be.right, out, in_where, mapping)?; - } else if in_where && is_time_epoch_comparison(be) { - translate_time_epoch_comparison(be, out)?; - } else { - let handled = if let Some(m) = mapping { - if in_where { - try_translate_where_binary_expr(be, out, m)? - } else { - false - } - } else { - false - }; - if !handled { - translate_expr(&be.left, out, in_where, mapping)?; - write!(out, " {} ", binary_op_to_clickhouse(&be.op))?; - translate_expr(&be.right, out, in_where, mapping)?; - } - } - } - write!(out, ")")?; - } - Expr::StringLiteral(s) => write!(out, "{}", quote_string(s))?, - Expr::IntegerLiteral(n) => write!(out, "{}", n)?, - Expr::FloatLiteral(f) => write!(out, "{}", format_float(*f))?, - Expr::BooleanLiteral(b) => write!(out, "{}", if *b { "true" } else { "false" })?, - Expr::TimeLiteral(s) => write!(out, "{}", quote_string(s))?, - Expr::Regex(r) => write!(out, "'{}'", r.replace('\\', "\\\\").replace('\'', "\\'"))?, - Expr::UnaryExpr(UnaryOp::Not, e) => { - write!(out, "NOT ")?; - translate_expr(e, out, in_where, mapping)?; - } - Expr::UnaryExpr(UnaryOp::Neg, e) => { - write!(out, "-")?; - translate_expr(e, out, in_where, mapping)?; - } - _ => { - return Err(HyperbytedbError::QueryParse(format!( - "unsupported expression in WHERE: {:?}", - expr - ))); - } - } - Ok(()) -} - -fn is_time_identifier(expr: &Expr) -> bool { - matches!(expr, Expr::Identifier(n) if n.to_lowercase() == "time") -} - -/// Detect `time ` where epoch_value is a DurationLiteral -/// (e.g., `1772462462777ms`) or a bare IntegerLiteral (nanosecond epoch). -fn is_time_epoch_comparison(be: &BinaryExpr) -> bool { - if !matches!( - be.op, - BinaryOp::Eq | BinaryOp::Neq | BinaryOp::Lt | BinaryOp::Lte | BinaryOp::Gt | BinaryOp::Gte - ) { - return false; - } - - let (is_left_time, rhs) = if is_time_identifier(&be.left) { - (true, &be.right) - } else if is_time_identifier(&be.right) { - (true, &be.left) - } else { - (false, &be.right) - }; - - if !is_left_time { - return false; - } - - matches!(rhs, Expr::DurationLiteral(_) | Expr::IntegerLiteral(_)) -} - -/// Translate `time >= 1772462462777ms` → `time >= fromUnixTimestamp64Milli(1772462462777)` -fn translate_time_epoch_comparison( - be: &BinaryExpr, - out: &mut String, -) -> Result<(), HyperbytedbError> { - let (time_side_is_left, epoch_expr) = if is_time_identifier(&be.left) { - (true, &be.right) - } else { - (false, &be.left) - }; - - let ts_sql = match epoch_expr { - Expr::DurationLiteral(d) => epoch_duration_to_timestamp(d), - Expr::IntegerLiteral(n) => format!("fromUnixTimestamp64Nano({})", n), - _ => { - return Err(HyperbytedbError::QueryParse( - "expected duration or integer epoch beside time in comparison".to_string(), - )); - } - }; - - if time_side_is_left { - write!(out, "time {} {}", binary_op_to_clickhouse(&be.op), ts_sql)?; - } else { - write!(out, "{} {} time", ts_sql, binary_op_to_clickhouse(&be.op))?; - } - Ok(()) -} - -fn epoch_duration_to_timestamp(d: &Duration) -> String { - match d.unit { - DurationUnit::Second => format!("fromUnixTimestamp({})", d.value), - DurationUnit::Millisecond => format!("fromUnixTimestamp64Milli({})", d.value), - DurationUnit::Microsecond => format!("fromUnixTimestamp64Micro({})", d.value), - DurationUnit::Nanosecond => format!("fromUnixTimestamp64Nano({})", d.value), - _ => { - let nanos = d.to_nanos(); - nanos_to_ch_timestamp(nanos) - } - } -} - -fn nanos_to_ch_timestamp(nanos: i64) -> String { - format!("fromUnixTimestamp64Nano({nanos})") -} - -fn binary_op_to_clickhouse(op: &BinaryOp) -> &'static str { - match op { - BinaryOp::Add => "+", - BinaryOp::Sub => "-", - BinaryOp::Mul => "*", - BinaryOp::Div => "/", - BinaryOp::Mod => "%", - BinaryOp::Eq => "=", - BinaryOp::Neq => "!=", - BinaryOp::Lt => "<", - BinaryOp::Lte => "<=", - BinaryOp::Gt => ">", - BinaryOp::Gte => ">=", - BinaryOp::And => "AND", - BinaryOp::Or => "OR", - BinaryOp::RegexMatch => "~", - BinaryOp::RegexNotMatch => "!~", - } -} - -fn quote_identifier(name: &str) -> Result { - if name.chars().any(char::is_control) { - return Err(HyperbytedbError::QueryParse(format!( - "identifier contains control characters: {name:?}" - ))); - } - Ok(format!( - "\"{}\"", - name.replace('\\', "\\\\").replace('"', "\\\"") - )) -} - -/// Quote a physical column name from [`crate::domain::chdb_naming`] (already sanitized). -fn quote_phys_identifier(name: &str) -> String { - format!("\"{}\"", name.replace('\\', "\\\\").replace('"', "\\\"")) -} - -fn quote_string(s: &str) -> String { - format!("'{}'", s.replace('\\', "\\\\").replace('\'', "\\'")) -} - -fn format_float(f: f64) -> String { - if f.fract() == 0.0 && f.is_finite() { - format!("{}", f as i64) - } else { - format!("{}", f) - } -} - -/// Remove user-supplied `time` comparisons from a WHERE clause. InfluxDB CQs -/// ignore user time ranges and inject their own window each run. -pub fn strip_time_predicates(condition: Option) -> Option { - condition.and_then(strip_time_predicates_expr) -} - -fn strip_time_predicates_expr(expr: Expr) -> Option { - match expr { - Expr::BinaryExpr(be) if matches!(be.op, BinaryOp::And) => { - let left = strip_time_predicates_expr(be.left); - let right = strip_time_predicates_expr(be.right); - match (left, right) { - (None, None) => None, - (Some(l), None) => Some(l), - (None, Some(r)) => Some(r), - (Some(l), Some(r)) => Some(Expr::BinaryExpr(Box::new(BinaryExpr { - op: BinaryOp::And, - left: l, - right: r, - }))), - } - } - Expr::BinaryExpr(be) if is_time_epoch_comparison(&be) => None, - other => Some(other), - } -} - -/// Build a WHERE clause for CQ coverage `[start, end)` in nanoseconds. -pub fn cq_time_window_condition(start_nanos: i64, end_nanos: i64) -> Expr { - Expr::BinaryExpr(Box::new(BinaryExpr { - op: BinaryOp::And, - left: Expr::BinaryExpr(Box::new(BinaryExpr { - op: BinaryOp::Gte, - left: Expr::Identifier("time".to_string()), - right: Expr::IntegerLiteral(start_nanos), - })), - right: Expr::BinaryExpr(Box::new(BinaryExpr { - op: BinaryOp::Lt, - left: Expr::Identifier("time".to_string()), - right: Expr::IntegerLiteral(end_nanos), - })), - })) -} - -/// Prepare a CQ inner SELECT for execution: strip user time bounds, inject the -/// computed coverage window, and optionally strip `fill()` (basic syntax). -pub fn prepare_cq_select( - stmt: &SelectStatement, - start_nanos: i64, - end_nanos: i64, - strip_fill: bool, -) -> SelectStatement { - let mut prepared = stmt.clone(); - let window = cq_time_window_condition(start_nanos, end_nanos); - let remaining = strip_time_predicates(prepared.condition.take()); - prepared.condition = Some(match remaining { - Some(existing) => Expr::BinaryExpr(Box::new(BinaryExpr { - op: BinaryOp::And, - left: existing, - right: window, - })), - None => window, - }); - if strip_fill { - prepared.fill = None; - } - prepared -} - -/// `INSERT INTO SELECT ...` for a bounded CQ run against native tables. -pub fn translate_bounded_cq_into( - stmt: &SelectStatement, - dest_table: &QuotedTableName, - source_table: &QuotedTableName, - mapping: Option<&ColumnMapping>, - series: Option>, - start_nanos: i64, - end_nanos: i64, -) -> Result { - validate_select_into(stmt)?; - let prepared = prepare_cq_select(stmt, start_nanos, end_nanos, false); - let select_sql = translate_inner( - &prepared, - source_table.as_str(), - mapping, - series, - Some((Some(start_nanos), Some(end_nanos))), - )?; - let select_sql = rename_time_bucket_alias(&select_sql); - Ok(format!("INSERT INTO {dest_table}\n{select_sql}")) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::domain::chdb_naming::QuotedTableName; - use crate::timeseriesql::parser; - - fn test_table() -> QuotedTableName { - QuotedTableName::new_quoted("`mydb_autogen_cpu`".to_string()) - } - - fn test_series_table() -> QuotedTableName { - QuotedTableName::new_quoted("`mydb_autogen_cpu_series`".to_string()) - } - - fn qname(s: &str) -> QuotedTableName { - QuotedTableName::new_quoted(s.to_string()) - } - - fn translate_test(stmt: &SelectStatement) -> String { - translate_native_table(stmt, test_table().as_str(), None, None, None).unwrap() - } - - /// Mapping with `host` as a tag and `usage_idle` as a field (no collision). - fn cpu_mapping() -> ColumnMapping { - ColumnMapping { - tag_keys: ["host", "region"].into_iter().map(String::from).collect(), - field_names: ["usage_idle"].into_iter().map(String::from).collect(), - ..Default::default() - } - } - - fn translate_series(stmt: &SelectStatement, m: &ColumnMapping) -> String { - let table = test_table(); - let series = test_series_table(); - translate_native_table( - stmt, - table.as_str(), - Some(m), - Some(SeriesJoin { - table: &series, - force: false, - tag_columns: &[], - }), - None, - ) - .unwrap() - } - - fn parse_select(q: &str) -> SelectStatement { - let stmts = parser::parse_query(q).unwrap(); - match stmts.into_iter().next().unwrap() { - Statement::Select(s) => s, - _ => panic!("expected SELECT statement"), - } - } - - #[test] - fn group_by_tag_uses_physical_column_name() { - let mut map = ColumnMapping::default(); - map.tag_keys.insert("host-name".into()); - map.field_names.insert("v".into()); - let stmt = parse_select(r#"SELECT mean("v") FROM m GROUP BY time(1m), "host-name""#); - let sql = translate_series(&stmt, &map); - assert!( - sql.contains("\"host_name\""), - "tag with punctuation must map to sanitized physical column, got: {sql}" - ); - } - - #[test] - fn quote_identifier_rejects_control_characters() { - assert!(quote_identifier("host\ninject").is_err()); - assert!(quote_identifier("ok_name").is_ok()); - } - - #[test] - fn test_select_star() { - let stmt = parse_select("SELECT * FROM cpu"); - let sql = translate_test(&stmt); - assert!(sql.contains("SELECT *")); - assert!(sql.contains("FROM `mydb_autogen_cpu`")); - } - - #[test] - fn test_mean() { - let stmt = parse_select(r#"SELECT mean("value") FROM cpu"#); - let sql = translate_test(&stmt); - assert!(sql.contains("avg(\"value\")")); - } - - #[test] - fn test_median_count_sum_min_max() { - let stmt = - parse_select(r#"SELECT median("x"), count("x"), sum("x"), min("x"), max("x") FROM m"#); - let sql = translate_test(&stmt); - // InfluxQL median averages the two middle samples on even counts. - assert!(sql.contains("quantileExactInclusive(0.5)(\"x\")")); - assert!(sql.contains("count(\"x\")")); - assert!(sql.contains("sum(\"x\")")); - assert!(sql.contains("min(\"x\")")); - assert!(sql.contains("max(\"x\")")); - } - - #[test] - fn test_first_last() { - let stmt = parse_select(r#"SELECT first("v"), last("v") FROM m"#); - let sql = translate_test(&stmt); - assert!(sql.contains("argMin(\"v\", time)")); - assert!(sql.contains("argMax(\"v\", time)")); - } - - #[test] - fn test_percentile() { - let stmt = parse_select(r#"SELECT percentile("value", 95) FROM m"#); - let sql = translate_test(&stmt); - // Nearest-rank sample percentile, matching InfluxQL. - assert!(sql.contains("quantileExactLow(0.95)(\"value\")")); - } - - #[test] - fn test_spread_stddev_mode_distinct() { - let stmt = - parse_select(r#"SELECT spread("v"), stddev("v"), mode("v"), distinct("v") FROM m"#); - let sql = translate_test(&stmt); - assert!(sql.contains("(max(\"v\") - min(\"v\"))")); - // InfluxQL stddev is sample stddev. - assert!(sql.contains("stddevSamp(\"v\")")); - // mode() must be a scalar, not a one-element Array. - assert!(sql.contains("arrayElement(topKWeighted(1)(\"v\", 1), 1)")); - // distinct() must stay valid inside GROUP BY time(); SELECT DISTINCT is not. - assert!(sql.contains("arrayJoin(groupUniqArray(\"v\"))")); - assert!(!sql.contains("DISTINCT \"v\"")); - } - - #[test] - fn test_count_distinct() { - let stmt = parse_select(r#"SELECT count(distinct("v")) FROM m GROUP BY time(1m)"#); - let sql = translate_test(&stmt); - assert!( - sql.contains("uniqExact(\"v\")"), - "count(distinct(v)) should translate to uniqExact, got: {sql}" - ); - } - - #[test] - fn test_distinct_with_group_by_time_is_valid_expression() { - let stmt = parse_select(r#"SELECT distinct("v") FROM m GROUP BY time(1m)"#); - let sql = translate_test(&stmt); - assert!( - sql.contains("arrayJoin(groupUniqArray(\"v\"))"), - "distinct(v) must be an expression usable with GROUP BY time, got: {sql}" - ); - assert!(!sql.contains("DISTINCT "), "got: {sql}"); - } - - #[test] - fn test_where_time_and_tag() { - let stmt = - parse_select(r#"SELECT * FROM cpu WHERE "host" = 'server01' AND time > now() - 1h"#); - let sql = translate_test(&stmt); - assert!(sql.contains("WHERE")); - assert!(sql.contains("host")); - assert!(sql.contains("server01")); - assert!(sql.contains("time")); - assert!(sql.contains("now64()")); - assert!(sql.contains("INTERVAL 1 HOUR")); - } - - #[test] - fn test_where_regex() { - let stmt = parse_select(r#"SELECT * FROM m WHERE "region" =~ /us-.*/"#); - let sql = translate_test(&stmt); - assert!(sql.contains("match")); - assert!(sql.contains("us-.*")); - } - - #[test] - fn test_group_by_time() { - let stmt = parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(5m)"#); - let sql = translate_test(&stmt); - assert!(sql.contains("GROUP BY")); - assert!(sql.contains("toStartOfInterval(time, INTERVAL 5 MINUTE)")); - } - - #[test] - fn test_group_by_time_with_offset() { - let stmt = parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(1h, 15m)"#); - let sql = translate_test(&stmt); - assert!(sql.contains( - "toStartOfInterval(time - INTERVAL 15 MINUTE, INTERVAL 1 HOUR) + INTERVAL 15 MINUTE" - )); - } - - #[test] - fn test_group_by_time_and_tags() { - let stmt = - parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(5m), "host", "region""#); - let sql = translate_test(&stmt); - assert!(sql.contains("toStartOfInterval(time, INTERVAL 5 MINUTE)")); - assert!(sql.contains("\"host\"")); - assert!(sql.contains("\"region\"")); - // Tag columns must appear in SELECT for result splitting - let select_line = sql.lines().next().unwrap(); - assert!( - select_line.contains("\"host\""), - "SELECT must include tag columns, got: {}", - select_line - ); - assert!( - select_line.contains("\"region\""), - "SELECT must include tag columns, got: {}", - select_line - ); - } - - #[test] - fn test_fill_null() { - let stmt = parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(5m) fill(null)"#); - let sql = translate_test(&stmt); - assert!( - !sql.contains("ifNull"), - "fill(null) must not coerce NULL to 0, got: {sql}" - ); - assert!(sql.contains("avg(\"value\")")); - assert!(sql.contains("WITH FILL STEP INTERVAL 5 MINUTE")); - } - - #[test] - fn test_fill_null_with_time_bounds_uses_from_to() { - let stmt = parse_select( - r#"SELECT mean("load1") FROM "system" WHERE time >= 1781541739132ms AND time <= 1781552539132ms GROUP BY time(10s) fill(null)"#, - ); - let min = 1_781_541_739_132_000_000i64; - let max = 1_781_552_539_132_000_000i64; - let sql = translate_native_table( - &stmt, - test_table().as_str(), - None, - None, - Some((Some(min), Some(max))), - ) - .unwrap(); - assert!( - sql.contains("WITH FILL FROM toStartOfInterval(fromUnixTimestamp64Nano(1781541739132000000), INTERVAL 10 SECOND)"), - "expected FROM bound aligned to bucket, got: {sql}" - ); - // WITH FILL ... TO is exclusive: the anchor extends one step past the - // bucket containing the upper bound so the final bucket is generated. - assert!( - sql.contains("TO toStartOfInterval(fromUnixTimestamp64Nano(1781552539132000000), INTERVAL 10 SECOND) + INTERVAL 10 SECOND"), - "expected TO bound one step past the last bucket, got: {sql}" - ); - assert!( - sql.contains("STEP INTERVAL 10 SECOND"), - "expected STEP after FROM/TO, got: {sql}" - ); - } - - #[test] - fn test_fill_grid_anchors_use_group_by_time_offset() { - // `time(1m, 30s)` bucket expression is `toStartOfInterval(t - 30s, 1m) + 30s`; - // the WITH FILL FROM/TO anchors must use the same shape or the grid - // interleaves phantom buckets between real ones. - let stmt = parse_select( - r#"SELECT mean("v") FROM m WHERE time >= 1781541730000ms AND time <= 1781541790000ms GROUP BY time(1m, 30s) fill(null)"#, - ); - let min = 1_781_541_730_000_000_000i64; - let max = 1_781_541_790_000_000_000i64; - let sql = translate_native_table( - &stmt, - test_table().as_str(), - None, - None, - Some((Some(min), Some(max))), - ) - .unwrap(); - assert!( - sql.contains( - "WITH FILL FROM toStartOfInterval(fromUnixTimestamp64Nano(1781541730000000000) - INTERVAL 30 SECOND, INTERVAL 1 MINUTE) + INTERVAL 30 SECOND" - ), - "FROM anchor must apply the GROUP BY time offset, got: {sql}" - ); - assert!( - sql.contains( - "TO toStartOfInterval(fromUnixTimestamp64Nano(1781541790000000000) - INTERVAL 30 SECOND, INTERVAL 1 MINUTE) + INTERVAL 30 SECOND + INTERVAL 1 MINUTE" - ), - "TO anchor must apply the GROUP BY time offset and extend one step, got: {sql}" - ); - } - - #[test] - fn test_fill_with_group_by_tag_orders_tag_before_time() { - // fill() + GROUP BY tag must order the tag column *before* the - // time-fill column so ClickHouse fills each tag group independently. - // Otherwise WITH FILL emits gap rows with an empty tag value (a phantom - // all-NULL series) and never fills the real per-tag series. - let stmt = parse_select( - r#"SELECT mean("usage_idle") FROM cpu GROUP BY time(10s), "host" fill(null)"#, - ); - let sql = translate_series(&stmt, &cpu_mapping()); - assert!( - sql.contains( - "ORDER BY \"host\" ASC, toStartOfInterval(time, INTERVAL 10 SECOND) ASC WITH FILL" - ), - "tag must precede the time-fill column in ORDER BY, got: {sql}" - ); - } - - #[test] - fn test_raw_select_projects_time_and_orders_ascending() { - // Raw (non-aggregate) selects must carry `time` and default to time ASC, - // matching InfluxDB. Without this, points come back in storage order. - let stmt = parse_select(r#"SELECT "load1", "load5" FROM system"#); - let sql = translate_test(&stmt); - assert!( - sql.starts_with("SELECT \"time\","), - "raw select must project time first, got: {sql}" - ); - assert!( - sql.contains("ORDER BY time ASC"), - "raw select defaults to time ASC, got: {sql}" - ); - } - - #[test] - fn test_group_by_time_defaults_to_order_by_time_ascending() { - let stmt = parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(5m)"#); - let sql = translate_test(&stmt); - assert!( - sql.contains("ORDER BY toStartOfInterval(time, INTERVAL 5 MINUTE) ASC"), - "GROUP BY time defaults to time ASC, got: {sql}" - ); - } - - #[test] - fn test_aggregate_without_group_by_time_has_no_order_by() { - // Collapses to a single row — no ORDER BY (and no raw `time` column). - let stmt = parse_select(r#"SELECT mean("value") FROM cpu"#); - let sql = translate_test(&stmt); - assert!(!sql.contains("ORDER BY"), "got: {sql}"); - assert!(!sql.contains("\"time\""), "no raw time column, got: {sql}"); - } - - #[test] - fn test_select_star_orders_by_time_without_duplicate_time() { - let stmt = parse_select("SELECT * FROM cpu"); - let sql = translate_test(&stmt); - assert!(sql.starts_with("SELECT *"), "got: {sql}"); - assert!(sql.contains("ORDER BY time ASC"), "got: {sql}"); - } - - #[test] - fn test_fill_value() { - let stmt = parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(5m) fill(0)"#); - let sql = translate_test(&stmt); - assert!(sql.contains("ifNull(avg(\"value\"), 0)")); - assert!(sql.contains("WITH FILL")); - // ifNull only reaches existing rows; WITH FILL-generated rows need a - // constant INTERPOLATE or they surface as column defaults, not the value. - assert!( - sql.contains("INTERPOLATE (\"mean_value\" AS 0)"), - "fill(N) must INTERPOLATE generated rows with N, got: {sql}" - ); - } - - #[test] - fn test_fill_value_interpolates_every_field_alias() { - let stmt = parse_select( - r#"SELECT mean("a") AS x, max("b") AS y FROM m GROUP BY time(1m) fill(100)"#, - ); - let sql = translate_test(&stmt); - assert!( - sql.contains("INTERPOLATE (\"x\" AS 100, \"y\" AS 100)"), - "fill(100) must INTERPOLATE all field aliases, got: {sql}" - ); - } - - #[test] - fn test_missing_fill_defaults_to_fill_null() { - // InfluxQL: a GROUP BY time() query without fill() behaves as fill(null). - let stmt = parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(5m)"#); - let sql = translate_test(&stmt); - assert!( - sql.contains("WITH FILL STEP INTERVAL 5 MINUTE"), - "absent fill() must default to fill(null), got: {sql}" - ); - assert!( - !sql.contains("ifNull"), - "default fill must leave NULL aggregates as NULL, got: {sql}" - ); - assert!( - !sql.contains("INTERPOLATE"), - "default fill must not interpolate, got: {sql}" - ); - } - - #[test] - fn test_select_into_does_not_default_fill() { - // Writes must not insert synthetic NULL grid rows. - let stmt = parse_select(r#"SELECT mean("value") INTO "dest" FROM "cpu" GROUP BY time(5m)"#); - let sql = - translate_select_into(&stmt, &qname("`dest`"), test_table().as_str(), None).unwrap(); - assert!( - !sql.contains("WITH FILL"), - "SELECT INTO without fill() must not emit WITH FILL, got: {sql}" - ); - } - - #[test] - fn test_order_by_time_desc_with_fill_wraps_ascending_fill() { - // WITH FILL on a DESC column generates nothing against ascending - // FROM/TO anchors; the fill happens ascending in an inner SELECT and an - // outer SELECT re-orders descending. - let stmt = parse_select( - r#"SELECT mean("value") FROM cpu GROUP BY time(5m) fill(null) ORDER BY time DESC"#, - ); - let sql = translate_test(&stmt); - assert!( - sql.starts_with("SELECT * FROM (\n"), - "DESC + fill must wrap, got: {sql}" - ); - assert!( - sql.contains(" ASC WITH FILL"), - "inner fill must be ascending, got: {sql}" - ); - assert!( - sql.contains(") ORDER BY __time DESC"), - "outer must re-order descending, got: {sql}" - ); - } - - #[test] - fn test_order_by_time_desc_with_fill_and_tags_orders_tags_first() { - let stmt = parse_select( - r#"SELECT mean("usage_idle") FROM cpu GROUP BY time(10s), "host" fill(null) ORDER BY time DESC"#, - ); - let sql = translate_series(&stmt, &cpu_mapping()); - assert!( - sql.contains(") ORDER BY \"host\" ASC, __time DESC"), - "outer ordering must keep tags first, got: {sql}" - ); - } - - #[test] - fn test_fill_none() { - let stmt = parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(5m) fill(none)"#); - let sql = translate_test(&stmt); - assert!(!sql.contains("ifNull")); - assert!(!sql.contains("WITH FILL")); - } - - #[test] - fn test_limit_offset() { - let stmt = parse_select("SELECT * FROM cpu LIMIT 10 OFFSET 5"); - let sql = translate_test(&stmt); - assert!(sql.contains("LIMIT 10")); - assert!(sql.contains("OFFSET 5")); - } - - #[test] - fn test_order_by_desc() { - let stmt = - parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(5m) ORDER BY time DESC"#); - let sql = translate_test(&stmt); - assert!(sql.contains("ORDER BY")); - assert!(sql.contains("DESC")); - } - - #[test] - fn test_derivative() { - let stmt = parse_select(r#"SELECT derivative("value", 1s) FROM cpu"#); - let sql = translate_test(&stmt); - assert!( - sql.contains("lagInFrame"), - "expected lagInFrame, got: {sql}" - ); - assert!( - sql.contains("toFloat64"), - "expected toFloat64 time conversion, got: {sql}" - ); - assert!( - !sql.contains("PARTITION BY"), - "no tags = no PARTITION BY, got: {sql}" - ); - } - - #[test] - fn test_non_negative_derivative() { - let stmt = parse_select(r#"SELECT non_negative_derivative("value", 1s) FROM cpu"#); - let sql = translate_test(&stmt); - assert!( - sql.contains("if("), - "expected if() for non-negative check, got: {sql}" - ); - assert!(sql.contains(">= 0"), "expected >= 0 check, got: {sql}"); - assert!( - sql.contains("lagInFrame"), - "expected lagInFrame, got: {sql}" - ); - assert!( - sql.contains("toFloat64"), - "expected toFloat64 time conversion, got: {sql}" - ); - } - - #[test] - fn test_difference() { - let stmt = parse_select(r#"SELECT difference("value") FROM cpu"#); - let sql = translate_test(&stmt); - assert!(sql.contains("lagInFrame")); - assert!(!sql.contains("if(")); - } - - #[test] - fn test_nested_aggregate_in_derivative() { - let stmt = parse_select( - r#"SELECT non_negative_derivative(mean("reads"), 1s) FROM "diskio" WHERE time >= 1000ms GROUP BY time(10s), "host" fill(null)"#, - ); - let sql = translate_test(&stmt); - assert!( - sql.contains("avg(\"reads\")"), - "expected avg(reads), got: {sql}" - ); - assert!( - sql.contains("ORDER BY __time"), - "expected ORDER BY __time, got: {sql}" - ); - assert!( - sql.contains("lagInFrame"), - "expected lagInFrame, got: {sql}" - ); - assert!( - sql.contains(">= 0"), - "expected non-negative check, got: {sql}" - ); - assert!( - sql.contains("PARTITION BY \"host\""), - "GROUP BY tag must produce PARTITION BY in window clause, got: {sql}" - ); - assert!( - sql.contains("toFloat64"), - "expected toFloat64 time conversion, got: {sql}" - ); - let select_line = sql.lines().next().unwrap(); - assert!( - select_line.contains("\"host\""), - "expected host in SELECT, got: {select_line}" - ); - } - - #[test] - fn test_derivative_with_nested_first() { - let stmt = parse_select( - r#"SELECT derivative(first("bytes_recv"), 1s) * 8 FROM net GROUP BY time(10s) fill(null)"#, - ); - let sql = translate_test(&stmt); - // first() → argMin(field, time) - assert!( - sql.contains("argMin(\"bytes_recv\", time)"), - "expected argMin, got: {sql}" - ); - assert!( - sql.contains("ORDER BY __time"), - "expected ORDER BY __time, got: {sql}" - ); - } - - #[test] - fn test_moving_average() { - let stmt = parse_select(r#"SELECT moving_average("value", 5) FROM cpu"#); - let sql = translate_test(&stmt); - assert!(sql.contains("avg(\"value\") OVER")); - assert!(sql.contains("ROWS BETWEEN 4 PRECEDING AND CURRENT ROW")); - // InfluxQL emits values only once the window holds N points. - assert!( - sql.contains("if(count(\"value\") OVER"), - "moving_average must gate on a full window, got: {sql}" - ); - assert!(sql.contains(">= 5"), "window-full check, got: {sql}"); - } - - #[test] - fn test_cumulative_sum() { - let stmt = parse_select(r#"SELECT cumulative_sum("value") FROM cpu"#); - let sql = translate_test(&stmt); - assert!(sql.contains("sum(\"value\") OVER")); - assert!(sql.contains("ROWS UNBOUNDED PRECEDING")); - } - - #[test] - fn test_elapsed() { - let stmt = parse_select(r#"SELECT elapsed("value", 1s) FROM cpu"#); - let sql = translate_test(&stmt); - assert!( - sql.contains("lagInFrame(toNullable(time), 1)"), - "expected NULL-defaulting lagInFrame so the first row is omitted, got: {sql}" - ); - assert!( - sql.contains("toFloat64"), - "expected toFloat64 time conversion, got: {sql}" - ); - } - - #[test] - fn test_fill_previous() { - let stmt = parse_select( - r#"SELECT mean("value") AS avg_val FROM cpu GROUP BY time(5m) fill(previous)"#, - ); - let sql = translate_test(&stmt); - assert!(sql.contains("WITH FILL STEP INTERVAL 5 MINUTE")); - assert!(sql.contains("INTERPOLATE")); - assert!(sql.contains("\"avg_val\"")); - assert!(!sql.contains("ifNull")); - } - - #[test] - fn test_fill_linear() { - let stmt = parse_select( - r#"SELECT mean("value") AS avg_val FROM cpu GROUP BY time(5m) fill(linear)"#, - ); - let sql = translate_test(&stmt); - assert!(sql.contains("WITH FILL STEP INTERVAL 5 MINUTE")); - assert!(sql.contains("INTERPOLATE")); - assert!(sql.contains("\"avg_val\" AS \"avg_val\"")); - assert!(!sql.contains("ifNull")); - } - - #[test] - fn test_grafana_tag_annotation() { - let stmt = parse_select( - r#"SELECT mean("usage_idle") FROM cpu WHERE time >= 1000ms AND time <= 2000ms GROUP BY time(1s), "host"::tag"#, - ); - let sql = translate_test(&stmt); - assert!(sql.contains("GROUP BY")); - assert!( - sql.contains("\"host\""), - "should strip ::tag suffix, got: {sql}" - ); - assert!( - !sql.contains("::tag"), - "should not contain ::tag, got: {sql}" - ); - } - - #[test] - fn test_epoch_ms_time_comparison() { - let stmt = parse_select( - r#"SELECT * FROM cpu WHERE time >= 1772462462777ms AND time <= 1772466062777ms"#, - ); - let sql = translate_test(&stmt); - assert!( - sql.contains("fromUnixTimestamp64Milli(1772462462777)"), - "should convert ms epoch to timestamp, got: {sql}" - ); - assert!( - sql.contains("fromUnixTimestamp64Milli(1772466062777)"), - "should convert ms epoch to timestamp, got: {sql}" - ); - assert!( - !sql.contains("INTERVAL"), - "should not use INTERVAL for epoch timestamps, got: {sql}" - ); - } - - #[test] - fn test_epoch_ns_time_comparison() { - let stmt = parse_select(r#"SELECT * FROM cpu WHERE time >= 1772462462777000000"#); - let sql = translate_test(&stmt); - assert!( - sql.contains("fromUnixTimestamp64Nano(1772462462777000000)"), - "bare integer should become nanosecond timestamp, got: {sql}" - ); - } - - #[test] - fn test_non_negative_derivative_with_multiple_tags() { - let stmt = parse_select( - r#"SELECT non_negative_derivative(mean("read_bytes"), 1s) AS "Reads", non_negative_derivative(mean("write_bytes"), 1s) AS "Writes" FROM "diskio" WHERE "host" =~ /^(8a8b7bfef1c0)$/ AND time >= 1772542183541ms AND time <= 1772542483541ms GROUP BY time(1s), "host", "name" fill(null)"#, - ); - let sql = translate_test(&stmt); - assert!( - sql.contains(r#"PARTITION BY "host", "name""#), - "window must PARTITION BY all GROUP BY tags to avoid cross-series derivative, got: {sql}" - ); - assert!( - sql.contains("toFloat64"), - "time diff must use toFloat64 for correct arithmetic, got: {sql}" - ); - assert!( - sql.contains("avg(\"read_bytes\")"), - "expected avg(read_bytes), got: {sql}" - ); - assert!( - sql.contains("avg(\"write_bytes\")"), - "expected avg(write_bytes), got: {sql}" - ); - assert!( - sql.contains(">= 0"), - "expected non-negative check, got: {sql}" - ); - assert!( - sql.contains("AS \"Reads\""), - "expected Reads alias, got: {sql}" - ); - assert!( - sql.contains("AS \"Writes\""), - "expected Writes alias, got: {sql}" - ); - } - - #[test] - fn test_difference_with_tags_has_partition_by() { - let stmt = parse_select( - r#"SELECT difference(mean("value")) FROM cpu GROUP BY time(10s), "host", "region""#, - ); - let sql = translate_test(&stmt); - assert!( - sql.contains(r#"PARTITION BY "host", "region""#), - "difference window must PARTITION BY tags, got: {sql}" - ); - } - - #[test] - fn test_moving_average_with_tags_has_partition_by() { - let stmt = parse_select( - r#"SELECT moving_average(mean("value"), 5) FROM cpu GROUP BY time(10s), "host""#, - ); - let sql = translate_test(&stmt); - assert!( - sql.contains(r#"PARTITION BY "host""#), - "moving_average window must PARTITION BY tags, got: {sql}" - ); - } - - #[test] - fn test_cumulative_sum_with_tags_has_partition_by() { - let stmt = parse_select( - r#"SELECT cumulative_sum(mean("value")) FROM cpu GROUP BY time(10s), "host""#, - ); - let sql = translate_test(&stmt); - assert!( - sql.contains(r#"PARTITION BY "host""#), - "cumulative_sum window must PARTITION BY tags, got: {sql}" - ); - } - - #[test] - fn test_non_negative_difference_divided_by_constant() { - let stmt = parse_select( - r#"SELECT NON_NEGATIVE_DIFFERENCE(mean("packets_recv"))/10 AS "in", NON_NEGATIVE_DIFFERENCE(mean("packets_sent"))/10 AS "out" FROM "net" WHERE "host" =~ /^(telegraf-664c6bf94-pgt7t)$/ AND "interface" =~ /(vlan|eth|bond).*/ AND time >= 1772706604176ms AND time <= 1772706904176ms GROUP BY time(1s), "host", "interface" fill(null)"#, - ); - let sql = translate_test(&stmt); - assert!( - sql.contains("lagInFrame"), - "expected lagInFrame for difference, got: {sql}" - ); - assert!(sql.contains("/ 10"), "expected division by 10, got: {sql}"); - assert!(sql.contains("AS \"in\""), "expected alias 'in', got: {sql}"); - assert!( - sql.contains("AS \"out\""), - "expected alias 'out', got: {sql}" - ); - assert!( - !sql.contains("NON_NEGATIVE_DIFFERENCE"), - "should not contain raw TimeseriesQL function name in output SQL, got: {sql}" - ); - } - - #[test] - fn test_derivative_unit_conversion() { - let stmt = parse_select(r#"SELECT derivative("value", 1ms) FROM cpu GROUP BY time(10s)"#); - let sql = translate_test(&stmt); - assert!( - sql.contains("/ 0.001"), - "1ms unit should divide time diff by 0.001 seconds, got: {sql}" - ); - } - - #[test] - fn test_relative_time_still_uses_interval() { - let stmt = parse_select(r#"SELECT * FROM cpu WHERE time > now() - 1h"#); - let sql = translate_test(&stmt); - assert!(sql.contains("now64()"), "should keep now64(), got: {sql}"); - assert!( - sql.contains("INTERVAL 1 HOUR"), - "relative duration should stay as interval, got: {sql}" - ); - } - - #[test] - fn test_translate_select_into() { - let q = r#"SELECT mean("value") INTO "cpu_1h" FROM "cpu" WHERE "host" = 'server01' GROUP BY time(1h), "host""#; - let stmt = parse_select(q); - let sql = translate_select_into( - &stmt, - &qname("`mydb_autogen_cpu_1h`"), - test_table().as_str(), - None, - ) - .unwrap(); - assert!(sql.starts_with("INSERT INTO `mydb_autogen_cpu_1h`")); - assert!(sql.contains("SELECT ")); - assert!(sql.contains("time")); - assert!(!sql.contains("__time")); - assert!(sql.contains("avg(\"value\")")); - assert!(sql.contains("GROUP BY")); - assert!(sql.contains("toStartOfInterval(time, INTERVAL 1 HOUR)")); - } - - #[test] - fn test_select_into_requires_group_by_time() { - let q = r#"SELECT mean("value") INTO "cpu_1h" FROM "cpu""#; - let stmt = parse_select(q); - assert!( - translate_select_into(&stmt, &qname("`dest`"), test_table().as_str(), None).is_err() - ); - } - - #[test] - fn test_translate_materialized_view_select() { - let q = r#"SELECT mean("value") INTO "cpu_5m" FROM "cpu" GROUP BY time(5m), "host""#; - let stmt = parse_select(q); - let map = cpu_mapping(); - let sql = translate_materialized_view_select( - &stmt, - &test_table(), - &test_series_table(), - "cpu_5m", - &map, - ) - .unwrap(); - assert!(sql.starts_with("SELECT ")); - assert!(sql.contains("toStartOfInterval(t.time, INTERVAL 5 MINUTE) AS time")); - assert!(sql.contains("any(t.`_mv_src_origin_node_id`) AS origin_node_id")); - assert!(sql.contains("max(t.`_mv_src_ingest_seq`) AS ingest_seq")); - assert!(sql.contains("sipHash64(")); - assert!(sql.contains("AS \"count_value\"")); - assert!(sql.contains("AS \"sum_value\"")); - assert!(!sql.contains("avg(\"value\")")); - assert!( - sql.contains("argMax(\"value\", `ingest_seq`)"), - "MV source should coalesce duplicate raw rows before aggregating" - ); - assert!( - sql.contains( - "FROM (SELECT `series_id`, `time`, max(`ingest_seq`) AS `_mv_src_ingest_seq`" - ), - "MV should read from coalesced source subquery, got: {sql}" - ); - assert!( - sql.contains("AS t ANY LEFT JOIN `mydb_autogen_cpu_series` AS s"), - "MV must not drop fact rows whose series row hasn't landed, got: {sql}" - ); - assert!(sql.contains("GROUP BY toStartOfInterval(t.time, INTERVAL 5 MINUTE)")); - assert!(sql.contains("s.\"host\"")); - assert!(!sql.contains("INSERT INTO")); - // Field columns must appear in sorted-by-name order (count < sum). - let count_pos = sql.find("AS \"count_value\"").unwrap(); - let sum_pos = sql.find("AS \"sum_value\"").unwrap(); - assert!( - count_pos < sum_pos, - "fields should be sorted: count_value before sum_value, got: {}..{}", - count_pos, - sum_pos - ); - } - - #[test] - fn materialized_view_backfill_orders_insert_columns_by_physical_name() { - let q = r#"SELECT sum("players") AS "players", sum("max_players") AS "maxplayers", sum("cpu") AS "cpu" INTO "server_stats_1m" FROM "server_stats" GROUP BY time(1m), "host""#; - let stmt = parse_select(q); - let map = cpu_mapping(); - let sql = translate_materialized_view_backfill( - &stmt, - &qname("`dest`"), - &qname("`source`"), - &qname("`source_series`"), - "server_stats_1m", - &map, - ) - .unwrap(); - assert!( - sql.starts_with( - "INSERT INTO `dest` (\"time\", \"origin_node_id\", \"ingest_seq\", \"series_id\", \"cpu\", \"maxplayers\", \"players\")" - ), - "backfill must name columns in DDL order, got: {sql}" - ); - assert!( - sql.contains("SELECT \"time\", \"origin_node_id\", \"ingest_seq\", \"series_id\", \"cpu\", \"maxplayers\", \"players\"\nFROM ("), - "backfill outer SELECT must match INSERT column order, got: {sql}" - ); - } - - #[test] - fn rollup_fact_view_uses_sum_for_additive_fields() { - use crate::domain::rollup::RollupCombine; - - let mut map = cpu_mapping(); - map.field_rollups - .insert("usage_idle".to_string(), RollupCombine::Sum); - let sql = build_coalesced_fact_view(&test_table(), &map); - assert!( - sql.contains("sum(\"usage_idle\") AS \"usage_idle\""), - "rollup fields should merge with sum(), got: {sql}" - ); - assert!( - !sql.contains("argMax(\"usage_idle\""), - "rollup sum fields must not use argMax, got: {sql}" - ); - } - - #[test] - fn raw_fact_view_still_uses_argmax_without_rollups() { - let map = cpu_mapping(); - let sql = build_coalesced_fact_view(&test_table(), &map); - assert!( - sql.contains("argMax(\"usage_idle\", `ingest_seq`)"), - "raw measurements should keep argMax coalesce, got: {sql}" - ); - } - - #[test] - fn mean_on_rollup_measurement_rewrites_to_sum_over_count() { - use crate::domain::rollup::{MeanRollupField, RollupCombine}; - - let mut map = cpu_mapping(); - map.mean_fields.insert( - "value".to_string(), - MeanRollupField { - sum_col: "sum_value".to_string(), - count_col: "count_value".to_string(), - }, - ); - map.field_rollups - .insert("sum_value".to_string(), RollupCombine::Sum); - map.field_rollups - .insert("count_value".to_string(), RollupCombine::Sum); - - let stmt = parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(5m), "host""#); - let table = test_table(); - let series = test_series_table(); - let sql = translate_native_table( - &stmt, - table.as_str(), - Some(&map), - Some(SeriesJoin { - table: &series, - force: false, - tag_columns: &[], - }), - None, - ) - .unwrap(); - assert!( - sql.contains("sum(\"sum_value\") / nullIf(sum(\"count_value\"), 0)"), - "expected weighted mean rewrite, got: {sql}" - ); - } - - #[test] - fn test_tag_field_collision_uses_column_mapping() { - let stmt = parse_select(r#"SELECT mean("cpu") FROM m GROUP BY cpu"#); - let mut map = ColumnMapping::default(); - map.tag_keys.insert("cpu".into()); - map.field_names.insert("cpu".into()); - let table = test_table(); - let series = test_series_table(); - let sql = translate_native_table( - &stmt, - table.as_str(), - Some(&map), - Some(SeriesJoin { - table: &series, - force: false, - tag_columns: &[], - }), - None, - ) - .unwrap(); - assert!( - sql.contains("__tag__cpu"), - "tag column should be prefixed when it collides with a field, got: {sql}" - ); - assert!( - sql.contains("avg(\"cpu\")"), - "aggregate should use field column name, got: {sql}" - ); - assert!( - sql.contains("GROUP BY \"__tag__cpu\""), - "GROUP BY must use the physical tag column to match SELECT, got: {sql}" - ); - } - - // --- series_id layout: tag resolution via the dimension-table inline view --- - - #[test] - fn series_field_only_query_has_no_join() { - // No tag referenced → coalesced fact view, no series dimension join. - let stmt = parse_select(r#"SELECT mean("usage_idle") FROM cpu WHERE time > 0"#); - let sql = translate_series(&stmt, &cpu_mapping()); - assert!( - !sql.contains("JOIN") && !sql.contains("_series"), - "field-only query should not join the series table, got: {sql}" - ); - assert!( - sql.contains("argMax(\"usage_idle\", `ingest_seq`)"), - "field-only query should collapse duplicate rows by ingest_seq, got: {sql}" - ); - assert!(sql.contains("FROM `mydb_autogen_cpu`"), "got: {sql}"); - } - - #[test] - fn telegraf_cpu_multi_field_query_coalesces_partial_rows() { - let stmt = parse_select( - r#"SELECT mean("usage_guest") AS "Usage Guest", mean("usage_idle") AS "Usage Idle", mean("usage_user") AS "Usage User" FROM "cpu" WHERE "host" =~ /^(d2ddee27a9f4)$/ AND "cpu" = 'cpu-total' AND time >= 1780922276152ms and time <= 1780925876152ms GROUP BY time(2s), "host" fill(null)"#, - ); - let mut map = ColumnMapping::default(); - map.tag_keys.insert("host".into()); - map.tag_keys.insert("cpu".into()); - for f in [ - "usage_guest", - "usage_idle", - "usage_user", - "usage_system", - "usage_iowait", - ] { - map.field_names.insert(f.into()); - } - let table = test_table(); - let series = test_series_table(); - let sql = translate_native_table( - &stmt, - table.as_str(), - Some(&map), - Some(SeriesJoin { - table: &series, - force: false, - tag_columns: &[], - }), - None, - ) - .unwrap(); - assert!( - sql.contains("argMax(\"usage_idle\", `ingest_seq`)"), - "expected coalesced fact view, got: {sql}" - ); - assert!( - sql.contains("ANY LEFT JOIN `mydb_autogen_cpu_series` AS s"), - "tag filter should join series table, got: {sql}" - ); - assert!(sql.contains("avg(\"usage_idle\")"), "got: {sql}"); - assert!( - sql.contains("toStartOfInterval(time, INTERVAL 2 SECOND)"), - "got: {sql}" - ); - } - - #[test] - fn series_where_tag_filter_joins_dimension() { - let stmt = parse_select(r#"SELECT mean("usage_idle") FROM cpu WHERE "host" = 'h1'"#); - let sql = translate_series(&stmt, &cpu_mapping()); - assert!( - sql.contains("ANY LEFT JOIN `mydb_autogen_cpu_series` AS s"), - "tag filter should join the series table, got: {sql}" - ); - assert!( - sql.contains("t.`series_id` = s.`series_id`"), - "join key should be series_id, got: {sql}" - ); - // The tag predicate resolves against the joined view's tag column. - assert!(sql.contains("\"host\" = 'h1'"), "got: {sql}"); - } - - #[test] - fn series_group_by_all_tags_expands_to_measurement_tags() { - let mut stmt = parse_select(r#"SELECT mean("usage_idle") FROM cpu GROUP BY time(5m), *"#); - let gb = stmt.group_by.as_ref().unwrap().clone(); - let (expanded_gb, tags) = gb.expand_all_tags(&["host".to_string(), "region".to_string()]); - stmt.group_by = Some(expanded_gb); - assert_eq!(tags, vec!["host", "region"]); - let sql = translate_series(&stmt, &cpu_mapping()); - assert!(sql.contains("ANY LEFT JOIN"), "got: {sql}"); - assert!(sql.contains("\"host\""), "got: {sql}"); - assert!(sql.contains("\"region\""), "got: {sql}"); - assert!(!sql.contains("`*`"), "got: {sql}"); - } - - #[test] - fn series_group_by_tag_projects_and_groups_physical() { - let stmt = parse_select(r#"SELECT mean("usage_idle") FROM cpu GROUP BY time(5m), "host""#); - let sql = translate_series(&stmt, &cpu_mapping()); - assert!(sql.contains("ANY LEFT JOIN"), "got: {sql}"); - // host is non-colliding, so physical == logical. - assert!( - sql.contains("\"host\""), - "tag projected/grouped, got: {sql}" - ); - assert!(sql.contains("GROUP BY"), "got: {sql}"); - assert!(sql.contains("avg(\"usage_idle\")"), "got: {sql}"); - } - - #[test] - fn series_view_exposes_only_tag_columns_from_dimension() { - let stmt = parse_select(r#"SELECT mean("usage_idle") FROM cpu GROUP BY "host""#); - let sql = translate_series(&stmt, &cpu_mapping()); - // Inline view selects t.* plus the dimension's tag columns (sorted). - assert!( - sql.contains("SELECT t.*, s.\"host\", s.\"region\""), - "view should re-attach tag columns, got: {sql}" - ); - } - - #[test] - fn series_force_join_without_tag_reference() { - // force=true (e.g. a tombstone references a tag) joins even a field-only body. - let stmt = parse_select(r#"SELECT mean("usage_idle") FROM cpu WHERE time > 0"#); - let m = cpu_mapping(); - let table = test_table(); - let series = test_series_table(); - let sql = translate_native_table( - &stmt, - table.as_str(), - Some(&m), - Some(SeriesJoin { - table: &series, - force: true, - tag_columns: &[], - }), - None, - ) - .unwrap(); - assert!( - sql.contains("ANY LEFT JOIN"), - "force should join, got: {sql}" - ); - } - - #[test] - fn mv_series_select_uses_dest_field_names_for_tag_prefix() { - // Tag "host" collides with a field only in the destination, not the source. - // Source mapping treats "host" as non-colliding (source field_names is - // {"usage_idle"}), so tag_column_name("host") returns "host". - // Destination has field "host", so dest_field_names = {"host", "usage_idle"}, - // and tag_column_name("host") should return "__tag__host". - let stmt = parse_select( - r#"SELECT mean("usage_idle") INTO "dest" FROM "cpu" GROUP BY time(5m), "host""#, - ); - let mut src_mapping = cpu_mapping(); - src_mapping.tag_keys.insert("host".to_string()); - - let dest_field_names: std::collections::HashSet = - ["host".to_string(), "usage_idle".to_string()].into(); - - let sql = translate_materialized_view_series_select( - &stmt, - &qname("`source_series`"), - "dest", - &src_mapping, - Some(&dest_field_names), - ) - .unwrap(); - - assert!( - sql.contains("__tag__host"), - "tag 'host' should be prefixed when dest has colliding field, got: {sql}" - ); - } - - #[test] - fn mv_series_select_uses_source_names_when_no_dest_field_names() { - let stmt = parse_select( - r#"SELECT mean("usage_idle") INTO "dest" FROM "cpu" GROUP BY time(5m), "host""#, - ); - let mut src_mapping = cpu_mapping(); - src_mapping.tag_keys.insert("host".to_string()); - - let sql = translate_materialized_view_series_select( - &stmt, - &qname("`source_series`"), - "dest", - &src_mapping, - None, - ) - .unwrap(); - - // Without dest field names, source mapping says "host" doesn't collide - // (cpu_mapping has only "usage_idle" as field). - assert!( - sql.contains("\"host\""), - "tag 'host' should NOT be prefixed when dest_field_names is None, got: {sql}" - ); - assert!( - !sql.contains("__tag__host"), - "tag 'host' should NOT be prefixed without dest_field_names, got: {sql}" - ); - } - - // --- per-series LIMIT/OFFSET (InfluxQL points-per-series semantics) --- - - #[test] - fn test_limit_with_group_by_tag_uses_limit_by() { - let stmt = - parse_select(r#"SELECT mean("usage_idle") FROM cpu GROUP BY time(1m), "host" LIMIT 3"#); - let sql = translate_series(&stmt, &cpu_mapping()); - assert!( - sql.contains("LIMIT 3 BY (\"host\")"), - "LIMIT with tag grouping must be per series, got: {sql}" - ); - assert!( - !sql.contains("\nLIMIT 3\n") && !sql.ends_with("\nLIMIT 3"), - "no global LIMIT alongside LIMIT BY, got: {sql}" - ); - } - - #[test] - fn test_limit_offset_with_group_by_tags_uses_limit_by() { - let stmt = parse_select( - r#"SELECT mean("usage_idle") FROM cpu GROUP BY time(1m), "host", "region" LIMIT 3 OFFSET 2"#, - ); - let sql = translate_series(&stmt, &cpu_mapping()); - assert!( - sql.contains("LIMIT 2, 3 BY (\"host\", \"region\")"), - "OFFSET with tag grouping must be per series, got: {sql}" - ); - assert!(!sql.contains("\nOFFSET"), "got: {sql}"); - } - - #[test] - fn test_limit_without_tags_stays_global() { - let stmt = parse_select(r#"SELECT mean("v") FROM m GROUP BY time(1m) LIMIT 4 OFFSET 1"#); - let sql = translate_test(&stmt); - assert!(sql.contains("\nLIMIT 4"), "got: {sql}"); - assert!(sql.contains("\nOFFSET 1"), "got: {sql}"); - assert!(!sql.contains(" BY ("), "got: {sql}"); - } - - // --- raw (non-aggregate) SELECT with GROUP BY tag --- - - #[test] - fn test_raw_select_with_group_by_tag_has_no_sql_group_by() { - let stmt = parse_select(r#"SELECT "usage_idle" FROM cpu GROUP BY "host""#); - let sql = translate_series(&stmt, &cpu_mapping()); - assert!( - !sql.contains("\nGROUP BY"), - "raw select must not GROUP BY tags in SQL (NOT_AN_AGGREGATE), got: {sql}" - ); - // Tag stays projected so the result parser can split per-series. - let select_line = sql.lines().next().unwrap(); - assert!( - select_line.contains("\"host\""), - "tag must be projected for series splitting, got: {select_line}" - ); - assert!( - select_line.starts_with("SELECT \"time\""), - "raw select keeps time first, got: {select_line}" - ); - assert!(sql.contains("ORDER BY time ASC"), "got: {sql}"); - } - - // --- per-point window transforms without GROUP BY time --- - - #[test] - fn test_difference_without_group_by_time_projects_time_and_orders() { - let stmt = parse_select(r#"SELECT difference("value") FROM cpu"#); - let sql = translate_test(&stmt); - assert!( - sql.contains("SELECT \"time\","), - "transform must project the point time, got: {sql}" - ); - assert!( - sql.contains("ORDER BY \"time\" ASC"), - "transform output must be time-ordered, got: {sql}" - ); - // InfluxQL omits the first point (no previous value): NULL outputs are - // filtered by an outer SELECT. - assert!( - sql.starts_with("SELECT * FROM (\n"), - "transform must wrap to filter NULL rows, got: {sql}" - ); - assert!( - sql.contains(") WHERE \"difference_value\" IS NOT NULL"), - "leading NULL transform rows must be filtered, got: {sql}" - ); - } - - #[test] - fn test_transform_with_group_by_tag_partitions_without_sql_group_by() { - let stmt = parse_select(r#"SELECT difference("usage_idle") FROM cpu GROUP BY "host""#); - let sql = translate_series(&stmt, &cpu_mapping()); - assert!( - !sql.contains("\nGROUP BY"), - "bare transform must not GROUP BY tags in SQL, got: {sql}" - ); - assert!( - sql.contains("PARTITION BY \"host\""), - "transform must still partition per series, got: {sql}" - ); - } - - #[test] - fn test_transform_with_group_by_time_keeps_grid_nulls() { - // GROUP BY time + fill keeps the filled grid (Grafana relies on the - // NULL rows); no NULL-filtering wrapper. - let stmt = - parse_select(r#"SELECT difference(mean("v")) FROM m GROUP BY time(1m) fill(null)"#); - let sql = translate_test(&stmt); - assert!(!sql.starts_with("SELECT * FROM (\n"), "got: {sql}"); - } - - // --- tag compared to numeric literal --- - - #[test] - fn test_tag_numeric_comparison_is_constant_false() { - let stmt = parse_select(r#"SELECT mean("usage_idle") FROM cpu WHERE "host" = 3"#); - let sql = translate_series(&stmt, &cpu_mapping()); - assert!( - sql.contains("WHERE (1 = 0)"), - "tag vs numeric literal must be constant-false, got: {sql}" - ); - assert!( - !sql.contains("\"host\" = 3"), - "must not emit a string/number comparison, got: {sql}" - ); - } - - #[test] - fn test_tag_string_comparison_is_unaffected() { - let stmt = parse_select(r#"SELECT mean("usage_idle") FROM cpu WHERE "host" = '3'"#); - let sql = translate_series(&stmt, &cpu_mapping()); - assert!(sql.contains("\"host\" = '3'"), "got: {sql}"); - assert!(!sql.contains("1 = 0"), "got: {sql}"); - } - - #[test] - fn test_field_numeric_comparison_is_unaffected() { - let stmt = parse_select(r#"SELECT mean("usage_idle") FROM cpu WHERE "usage_idle" > 3"#); - let sql = translate_series(&stmt, &cpu_mapping()); - assert!(sql.contains("\"usage_idle\" > 3"), "got: {sql}"); - assert!(!sql.contains("1 = 0"), "got: {sql}"); - } - - // --- extract_time_bounds keeps the intersection of ANDed bounds --- - - #[test] - fn test_extract_time_bounds_intersects_anded_bounds() { - let stmt = parse_select( - "SELECT * FROM m WHERE time >= 1000000000 AND time >= 3000000000 \ - AND time <= 9000000000 AND time <= 7000000000", - ); - let (min, max) = extract_time_bounds(stmt.condition.as_ref()); - assert_eq!(min, Some(3_000_000_000), "lower bounds keep the MAX"); - assert_eq!(max, Some(7_000_000_000), "upper bounds keep the MIN"); - } - - // --- precise __time renaming --- - - #[test] - fn test_rename_time_bucket_alias_is_token_precise() { - assert_eq!( - rename_time_bucket_alias("toStartOfInterval(time, INTERVAL 1 MINUTE) AS __time"), - "toStartOfInterval(time, INTERVAL 1 MINUTE) AS time" - ); - assert_eq!( - rename_time_bucket_alias("ORDER BY __time DESC"), - "ORDER BY time DESC" - ); - assert_eq!(rename_time_bucket_alias("\"__time\""), "\"time\""); - assert_eq!(rename_time_bucket_alias("`__time`"), "`time`"); - // Identifiers merely containing the substring survive. - assert_eq!( - rename_time_bucket_alias("\"cpu__time\" AS __time"), - "\"cpu__time\" AS time" - ); - assert_eq!(rename_time_bucket_alias("lag__timer"), "lag__timer"); - } - - // --- subquery source: inner GROUP BY time must expose `time` --- - - #[test] - fn test_subquery_source_bucket_column_composes() { - // Built directly (the parser can't produce subqueries yet): the inner - // statement is translated, its `__time` alias renamed to `time`, and - // used as the outer FROM source. - let minute = Duration { - value: 1, - unit: DurationUnit::Minute, - }; - let five_minutes = Duration { - value: 5, - unit: DurationUnit::Minute, - }; - let inner = SelectStatement { - fields: vec![Field { - expr: Expr::Call(FunctionCall { - name: "mean".to_string(), - args: vec![Expr::Identifier("v".to_string())], - }), - alias: Some("x".to_string()), - }], - into: None, - from: vec![], - condition: None, - group_by: Some(GroupBy { - dimensions: vec![Dimension::Time { - interval: minute, - offset: None, - }], - }), - order_by: None, - limit: None, - offset: None, - slimit: None, - soffset: None, - fill: None, - timezone: None, - }; - let inner_sql = - translate_native_table(&inner, test_table().as_str(), None, None, None).unwrap(); - let inner_sql = rename_time_bucket_alias(&inner_sql); - assert!( - inner_sql.contains("AS time"), - "inner bucket must be exposed as `time`, got: {inner_sql}" - ); - assert!(!inner_sql.contains("__time"), "got: {inner_sql}"); - - let outer = SelectStatement { - fields: vec![Field { - expr: Expr::Call(FunctionCall { - name: "max".to_string(), - args: vec![Expr::Identifier("x".to_string())], - }), - alias: None, - }], - into: None, - from: vec![], - condition: None, - group_by: Some(GroupBy { - dimensions: vec![Dimension::Time { - interval: five_minutes, - offset: None, - }], - }), - order_by: None, - limit: None, - offset: None, - slimit: None, - soffset: None, - fill: None, - timezone: None, - }; - let outer_sql = translate_with_source(&outer, &format!("({inner_sql})")).unwrap(); - assert!( - outer_sql.contains("toStartOfInterval(time, INTERVAL 5 MINUTE) AS __time"), - "outer buckets the inner `time` column, got: {outer_sql}" - ); - assert!(outer_sql.contains("max(\"x\")"), "got: {outer_sql}"); - } - - // --- tz() flows into bucketing and fill anchors --- - - #[test] - fn test_timezone_in_bucket_expr_and_fill_anchors() { - let mut stmt = parse_select( - r#"SELECT mean("v") FROM m WHERE time >= 1000000000 AND time <= 3000000000 GROUP BY time(1d) fill(null)"#, - ); - stmt.timezone = Some("America/New_York".to_string()); - let sql = translate_native_table( - &stmt, - test_table().as_str(), - None, - None, - Some((Some(1_000_000_000), Some(3_000_000_000))), - ) - .unwrap(); - assert!( - sql.contains("toStartOfInterval(time, INTERVAL 1 DAY, 'America/New_York') AS __time"), - "bucket expression must carry the timezone, got: {sql}" - ); - assert!( - sql.contains( - "WITH FILL FROM toStartOfInterval(fromUnixTimestamp64Nano(1000000000), INTERVAL 1 DAY, 'America/New_York')" - ), - "fill anchors must bucket in the same timezone, got: {sql}" - ); - assert!( - sql.contains("GROUP BY toStartOfInterval(time, INTERVAL 1 DAY, 'America/New_York')"), - "GROUP BY must match the SELECT bucket expression, got: {sql}" - ); - } - - #[test] - fn test_timezone_string_is_escaped() { - let mut stmt = parse_select(r#"SELECT mean("v") FROM m GROUP BY time(1h)"#); - stmt.timezone = Some("bad'zone".to_string()); - let sql = translate_test_tz(&stmt); - assert!( - sql.contains("'bad\\'zone'"), - "timezone must go through quote_string escaping, got: {sql}" - ); - } - - fn translate_test_tz(stmt: &SelectStatement) -> String { - translate_native_table(stmt, test_table().as_str(), None, None, None).unwrap() - } -} diff --git a/hyperbytedb/src/timeseriesql/to_clickhouse/aggregates.rs b/hyperbytedb/src/timeseriesql/to_clickhouse/aggregates.rs new file mode 100644 index 0000000..ba92f54 --- /dev/null +++ b/hyperbytedb/src/timeseriesql/to_clickhouse/aggregates.rs @@ -0,0 +1,417 @@ +use crate::domain::column_mapping::ColumnMapping; +use crate::error::HyperbytedbError; +use crate::timeseriesql::ast::*; + +use super::conditions::{binary_op_to_clickhouse, format_float, quote_phys_identifier}; + +use super::select::translate_field_expr; + +pub(super) fn is_window_transform_call(name: &str) -> bool { + matches!( + name.to_ascii_uppercase().as_str(), + "DERIVATIVE" + | "NON_NEGATIVE_DERIVATIVE" + | "DIFFERENCE" + | "NON_NEGATIVE_DIFFERENCE" + | "MOVING_AVERAGE" + | "CUMULATIVE_SUM" + | "ELAPSED" + ) +} + +/// Whether an expression contains a row-collapsing aggregate. Window transforms +/// only count when they wrap a nested aggregate (e.g. `difference(mean(v))`). +pub(super) fn expr_contains_aggregate(expr: &Expr) -> bool { + match expr { + Expr::Call(fc) if is_window_transform_call(&fc.name) => { + fc.args.first().is_some_and(|a| matches!(a, Expr::Call(_))) + } + Expr::Call(_) => true, + Expr::BinaryExpr(be) => { + expr_contains_aggregate(&be.left) || expr_contains_aggregate(&be.right) + } + Expr::UnaryExpr(_, e) => expr_contains_aggregate(e), + _ => false, + } +} + +/// Whether an expression contains a window transform applied directly to a raw +/// field (no nested aggregate) — a per-point transform. +pub(super) fn expr_contains_raw_transform(expr: &Expr) -> bool { + match expr { + Expr::Call(fc) if is_window_transform_call(&fc.name) => { + !fc.args.first().is_some_and(|a| matches!(a, Expr::Call(_))) + } + Expr::Call(_) => false, + Expr::BinaryExpr(be) => { + expr_contains_raw_transform(&be.left) || expr_contains_raw_transform(&be.right) + } + Expr::UnaryExpr(_, e) => expr_contains_raw_transform(e), + _ => false, + } +} +pub(super) fn translate_binary_expr( + be: &BinaryExpr, + use_fill: bool, + fill_value: f64, + group_by: Option<&GroupBy>, + mapping: Option<&ColumnMapping>, +) -> Result { + let left = translate_field_expr(&be.left, use_fill, fill_value, group_by, mapping)?; + let right = translate_field_expr(&be.right, use_fill, fill_value, group_by, mapping)?; + Ok(format!( + "({} {} {})", + left, + binary_op_to_clickhouse(&be.op), + right + )) +} + +pub(super) fn translate_aggregate_call( + func: &FunctionCall, + use_fill: bool, + fill_value: f64, + group_by: Option<&GroupBy>, + mapping: Option<&ColumnMapping>, +) -> Result { + let name_upper = func.name.to_uppercase(); + let wrap_fill = |s: String| -> String { + if use_fill && group_by.is_some() { + format!("ifNull({}, {})", s, format_float(fill_value)) + } else { + s + } + }; + + let result = match name_upper.as_str() { + "MEAN" => { + let arg = get_single_arg(func, "MEAN")?; + if let Some(m) = mapping + && let Expr::Identifier(name) | Expr::FieldRef { name, .. } = arg + && let Some(mean_def) = m.mean_fields.get(name) + { + let sum_q = quote_phys_identifier(&mean_def.sum_col); + let count_q = quote_phys_identifier(&mean_def.count_col); + return Ok(wrap_fill(format!( + "(sum({sum_q}) / nullIf(sum({count_q}), 0))" + ))); + } + let f = translate_aggregate_arg(arg, mapping)?; + wrap_fill(format!("avg({})", f)) + } + "MEDIAN" => { + let arg = get_single_arg(func, "MEDIAN")?; + let f = translate_aggregate_arg(arg, mapping)?; + // InfluxQL median averages the two middle values on even counts; + // quantileExactInclusive(0.5) matches that exactly (ClickHouse + // `median` is sampling-based and approximate). + wrap_fill(format!("quantileExactInclusive(0.5)({})", f)) + } + "COUNT" => { + let arg = get_single_arg(func, "COUNT")?; + // count(distinct("v")) → exact distinct count. + if let Expr::Call(inner) = arg + && inner.name.eq_ignore_ascii_case("distinct") + { + let inner_arg = get_single_arg(inner, "DISTINCT")?; + let f = translate_aggregate_arg(inner_arg, mapping)?; + wrap_fill(format!("uniqExact({})", f)) + } else { + let f = translate_aggregate_arg(arg, mapping)?; + wrap_fill(format!("count({})", f)) + } + } + "SUM" => { + let arg = get_single_arg(func, "SUM")?; + let f = translate_aggregate_arg(arg, mapping)?; + wrap_fill(format!("sum({})", f)) + } + "MIN" => { + let arg = get_single_arg(func, "MIN")?; + let f = translate_aggregate_arg(arg, mapping)?; + wrap_fill(format!("min({})", f)) + } + "MAX" => { + let arg = get_single_arg(func, "MAX")?; + let f = translate_aggregate_arg(arg, mapping)?; + wrap_fill(format!("max({})", f)) + } + "FIRST" => { + let arg = get_single_arg(func, "FIRST")?; + let f = translate_aggregate_arg(arg, mapping)?; + wrap_fill(format!("argMin({}, time)", f)) + } + "LAST" => { + let arg = get_single_arg(func, "LAST")?; + let f = translate_aggregate_arg(arg, mapping)?; + wrap_fill(format!("argMax({}, time)", f)) + } + "PERCENTILE" => { + let (field_arg, pct_arg) = get_two_args(func, "PERCENTILE")?; + let f = translate_aggregate_arg(field_arg, mapping)?; + let pct = match &pct_arg { + Expr::IntegerLiteral(n) => (*n as f64) / 100.0, + Expr::FloatLiteral(f) => *f / 100.0, + _ => { + return Err(HyperbytedbError::QueryParse(format!( + "PERCENTILE second argument must be numeric, got {:?}", + pct_arg + ))); + } + }; + // InfluxQL percentile is nearest-rank and returns an actual sample + // (for [10,20,30,40] p50 = 20); quantileExactLow matches that. + wrap_fill(format!("quantileExactLow({})({})", format_float(pct), f)) + } + "SPREAD" => { + let arg = get_single_arg(func, "SPREAD")?; + let f = translate_aggregate_arg(arg, mapping)?; + wrap_fill(format!("(max({}) - min({}))", f, f)) + } + "STDDEV" => { + let arg = get_single_arg(func, "STDDEV")?; + let f = translate_aggregate_arg(arg, mapping)?; + // InfluxQL stddev is the *sample* standard deviation. + wrap_fill(format!("stddevSamp({})", f)) + } + "MODE" => { + let arg = get_single_arg(func, "MODE")?; + let f = translate_aggregate_arg(arg, mapping)?; + // topKWeighted returns an Array; unwrap to a scalar. Still + // approximate and tie-breaking is unspecified, unlike InfluxQL's + // lowest-value tie-break. + wrap_fill(format!("arrayElement(topKWeighted(1)({}, 1), 1)", f)) + } + "DISTINCT" => { + let arg = get_single_arg(func, "DISTINCT")?; + let f = translate_aggregate_arg(arg, mapping)?; + // arrayJoin(groupUniqArray(...)) yields one row per distinct value + // and — unlike `SELECT DISTINCT` — stays valid inside GROUP BY time(). + format!("arrayJoin(groupUniqArray({}))", f) + } + "DERIVATIVE" | "NON_NEGATIVE_DERIVATIVE" => { + let field_arg = get_single_arg(func, &name_upper)?; + let f = translate_field_or_nested(field_arg, group_by, mapping)?; + let window = build_window_clause(group_by, mapping)?; + let unit_nanos: i64 = if func.args.len() >= 2 { + match &func.args[1] { + Expr::DurationLiteral(d) => d.to_nanos(), + _ => 1_000_000_000, + } + } else { + 1_000_000_000 + }; + let unit_seconds = format_float(unit_nanos as f64 / 1_000_000_000.0); + let delta_value = format!("({f} - lagInFrame({f}, 1) {window})"); + // Use toFloat64() to get Unix timestamps as seconds (Float64) + // for correct arithmetic regardless of DateTime/DateTime64 type. + let time_ref = window_time_ref(group_by); + let delta_time = + format!("(toFloat64({time_ref}) - toFloat64(lagInFrame({time_ref}, 1) {window}))"); + let deriv = format!("{delta_value} / ({delta_time} / {unit_seconds})"); + if name_upper == "NON_NEGATIVE_DERIVATIVE" { + format!("if(({deriv}) >= 0, ({deriv}), NULL)") + } else { + deriv + } + } + "DIFFERENCE" | "NON_NEGATIVE_DIFFERENCE" => { + let arg = get_single_arg(func, &name_upper)?; + let f = translate_field_or_nested(arg, group_by, mapping)?; + let window = build_window_clause(group_by, mapping)?; + let diff = format!("({f} - lagInFrame({f}, 1) {window})"); + if name_upper == "NON_NEGATIVE_DIFFERENCE" { + format!("if({diff} >= 0, {diff}, NULL)") + } else { + diff + } + } + "MOVING_AVERAGE" => { + let (field_arg, n_arg) = get_two_args(func, "MOVING_AVERAGE")?; + let f = translate_field_or_nested(field_arg, group_by, mapping)?; + let time_ref = window_time_ref(group_by); + let n = match &n_arg { + Expr::IntegerLiteral(n) => *n, + _ => { + return Err(HyperbytedbError::QueryParse( + "MOVING_AVERAGE second argument must be integer".to_string(), + )); + } + }; + let partition_tags: Vec<&str> = + group_by.map(|gb| gb.tag_dimensions()).unwrap_or_default(); + let partition_clause = if partition_tags.is_empty() { + String::new() + } else { + let p = partition_tags + .iter() + .map(|t| { + let phys = mapping + .map(|m| m.physical_tag_column_name(t)) + .unwrap_or_else(|| t.to_string()); + Ok(quote_phys_identifier(&phys)) + }) + .collect::, HyperbytedbError>>()? + .join(", "); + format!("PARTITION BY {p} ") + }; + // InfluxQL emits moving_average values only once the window holds N + // points; gate on the frame's non-null count so shorter leading + // frames yield NULL (filtered for per-point transforms). + let frame = format!( + "({partition_clause}ORDER BY {time_ref} ROWS BETWEEN {preceding} PRECEDING AND CURRENT ROW)", + preceding = n - 1 + ); + format!("if(count({f}) OVER {frame} >= {n}, avg({f}) OVER {frame}, NULL)") + } + "CUMULATIVE_SUM" => { + let arg = get_single_arg(func, "CUMULATIVE_SUM")?; + let f = translate_field_or_nested(arg, group_by, mapping)?; + let time_ref = window_time_ref(group_by); + let partition_tags: Vec<&str> = + group_by.map(|gb| gb.tag_dimensions()).unwrap_or_default(); + let partition_clause = if partition_tags.is_empty() { + String::new() + } else { + let p = partition_tags + .iter() + .map(|t| { + let phys = mapping + .map(|m| m.physical_tag_column_name(t)) + .unwrap_or_else(|| t.to_string()); + Ok(quote_phys_identifier(&phys)) + }) + .collect::, HyperbytedbError>>()? + .join(", "); + format!("PARTITION BY {p} ") + }; + format!( + "sum({f}) OVER ({partition_clause}ORDER BY {time_ref} ROWS UNBOUNDED PRECEDING)" + ) + } + "ELAPSED" => { + let _field_arg = get_single_arg(func, "ELAPSED")?; + let time_ref = window_time_ref(group_by); + let window = build_window_clause(group_by, mapping)?; + let unit_nanos: i64 = if func.args.len() >= 2 { + match &func.args[1] { + Expr::DurationLiteral(d) => d.to_nanos(), + _ => 1_000_000_000, + } + } else { + 1_000_000_000 + }; + let unit_seconds = format_float(unit_nanos as f64 / 1_000_000_000.0); + // toNullable: lagInFrame on the non-Nullable time column would + // default to epoch 0 out-of-frame, making the first row a huge + // elapsed value instead of NULL (InfluxQL omits the first point). + format!( + "((toFloat64({time_ref}) - toFloat64(lagInFrame(toNullable({time_ref}), 1) {window})) / {unit_seconds})" + ) + } + _ => { + return Err(HyperbytedbError::QueryParse(format!( + "unsupported aggregate function: {}", + func.name + ))); + } + }; + + Ok(result) +} + +pub(super) fn translate_aggregate_arg( + expr: &Expr, + mapping: Option<&ColumnMapping>, +) -> Result { + match expr { + Expr::Identifier(name) | Expr::FieldRef { name, .. } => { + let col = mapping + .map(|m| m.physical_select_identifier(name)) + .unwrap_or_else(|| name.clone()); + Ok(quote_phys_identifier(&col)) + } + Expr::Star => Ok("*".to_string()), + _ => Err(HyperbytedbError::QueryParse(format!( + "aggregate argument must be identifier or *, got {:?}", + expr + ))), + } +} + +/// Translate the first argument of a transform function (derivative, difference, etc.). +/// Accepts either a plain identifier or a nested aggregate like mean("reads"). +pub(super) fn translate_field_or_nested( + expr: &Expr, + group_by: Option<&GroupBy>, + mapping: Option<&ColumnMapping>, +) -> Result { + match expr { + Expr::Call(inner_func) => { + translate_aggregate_call(inner_func, false, 0.0, group_by, mapping) + } + _ => translate_aggregate_arg(expr, mapping), + } +} + +/// Return the time column reference for window function ORDER BY clauses. +/// Uses `__time` (the time bucket alias) when GROUP BY time() is present, +/// raw `time` otherwise. +pub(super) fn window_time_ref(group_by: Option<&GroupBy>) -> &'static str { + if group_by.and_then(|gb| gb.time_dimension()).is_some() { + "__time" + } else { + "time" + } +} + +/// Build the OVER (...) window clause for transform functions. +/// Includes PARTITION BY for GROUP BY tag dimensions so that window +/// functions (lagInFrame, etc.) operate within each series independently. +pub(super) fn build_window_clause( + group_by: Option<&GroupBy>, + mapping: Option<&ColumnMapping>, +) -> Result { + let time_ref = window_time_ref(group_by); + let partition_tags: Vec<&str> = group_by.map(|gb| gb.tag_dimensions()).unwrap_or_default(); + + if partition_tags.is_empty() { + Ok(format!("OVER (ORDER BY {time_ref})")) + } else { + let partition = partition_tags + .iter() + .map(|t| { + let phys = mapping + .map(|m| m.physical_tag_column_name(t)) + .unwrap_or_else(|| t.to_string()); + Ok(quote_phys_identifier(&phys)) + }) + .collect::, HyperbytedbError>>()? + .join(", "); + Ok(format!( + "OVER (PARTITION BY {partition} ORDER BY {time_ref})" + )) + } +} + +pub(super) fn get_single_arg<'a>( + func: &'a FunctionCall, + name: &str, +) -> Result<&'a Expr, HyperbytedbError> { + func.args.first().ok_or_else(|| { + HyperbytedbError::QueryParse(format!("{} requires exactly one argument", name)) + }) +} + +pub(super) fn get_two_args<'a>( + func: &'a FunctionCall, + name: &str, +) -> Result<(&'a Expr, &'a Expr), HyperbytedbError> { + if func.args.len() < 2 { + return Err(HyperbytedbError::QueryParse(format!( + "{} requires exactly two arguments", + name + ))); + } + Ok((&func.args[0], &func.args[1])) +} diff --git a/hyperbytedb/src/timeseriesql/to_clickhouse/coalesce.rs b/hyperbytedb/src/timeseriesql/to_clickhouse/coalesce.rs new file mode 100644 index 0000000..a844021 --- /dev/null +++ b/hyperbytedb/src/timeseriesql/to_clickhouse/coalesce.rs @@ -0,0 +1,54 @@ +use crate::domain::chdb_naming::QuotedTableName; +use crate::domain::column_mapping::ColumnMapping; +use crate::domain::rollup::RollupCombine; + +use super::conditions::quote_phys_identifier; + +pub fn build_coalesced_fact_view(fact_table: &QuotedTableName, mapping: &ColumnMapping) -> String { + build_coalesced_fact_view_impl(fact_table, mapping, false) +} + +/// Like [`build_coalesced_fact_view`], but preserves `ingest_seq` / `origin_node_id` for +/// downstream aggregates (materialized view source dedup). +pub fn build_coalesced_fact_view_with_row_meta( + fact_table: &QuotedTableName, + mapping: &ColumnMapping, +) -> String { + build_coalesced_fact_view_impl(fact_table, mapping, true) +} + +pub(super) fn build_coalesced_fact_view_impl( + fact_table: &QuotedTableName, + mapping: &ColumnMapping, + include_row_metadata: bool, +) -> String { + let mut field_cols: Vec<&String> = mapping.field_names.iter().collect(); + field_cols.sort(); + let field_aggs: Vec = field_cols + .iter() + .map(|f| { + let q = quote_phys_identifier(f); + let agg = match mapping.field_rollups.get(*f) { + Some(RollupCombine::Sum) => format!("sum({q})"), + Some(RollupCombine::Min) => format!("min({q})"), + Some(RollupCombine::Max) => format!("max({q})"), + Some(RollupCombine::First) => format!("argMin({q}, `time`)"), + Some(RollupCombine::Last) | None => format!("argMax({q}, `ingest_seq`)"), + }; + format!("{agg} AS {q}") + }) + .collect(); + let select_fields = if field_aggs.is_empty() { + String::new() + } else { + format!(", {}", field_aggs.join(", ")) + }; + let row_meta = if include_row_metadata { + ", max(`ingest_seq`) AS `_mv_src_ingest_seq`, any(`origin_node_id`) AS `_mv_src_origin_node_id`" + } else { + "" + }; + format!( + "(SELECT `series_id`, `time`{row_meta}{select_fields} FROM {fact_table} GROUP BY `series_id`, `time`)" + ) +} diff --git a/hyperbytedb/src/timeseriesql/to_clickhouse/conditions.rs b/hyperbytedb/src/timeseriesql/to_clickhouse/conditions.rs new file mode 100644 index 0000000..a7c6cbb --- /dev/null +++ b/hyperbytedb/src/timeseriesql/to_clickhouse/conditions.rs @@ -0,0 +1,373 @@ +use crate::domain::column_mapping::ColumnMapping; +use crate::error::HyperbytedbError; +use crate::timeseriesql::ast::*; +use std::fmt::Write; + +use super::time_bounds::{is_time_epoch_comparison, is_time_identifier}; + +pub fn translate_condition( + expr: &Expr, + mapping: &ColumnMapping, + out: &mut String, +) -> Result<(), HyperbytedbError> { + translate_expr(expr, out, true, Some(mapping)) +} + +pub(super) fn tag_field_collision(m: &ColumnMapping, name: &str) -> bool { + m.tag_keys.contains(name) && m.field_names.contains(name) +} + +pub(super) fn is_where_literal(e: &Expr) -> bool { + matches!( + e, + Expr::IntegerLiteral(_) + | Expr::FloatLiteral(_) + | Expr::StringLiteral(_) + | Expr::BooleanLiteral(_) + ) +} + +pub(super) fn where_identifier_physical_name( + m: &ColumnMapping, + name: &str, + other: &Expr, +) -> Result { + if !tag_field_collision(m, name) { + return quote_identifier(name); + } + match other { + Expr::IntegerLiteral(_) | Expr::FloatLiteral(_) | Expr::BooleanLiteral(_) => { + quote_identifier(name) + } + Expr::StringLiteral(_) | Expr::Regex(_) => { + Ok(quote_phys_identifier(&m.physical_tag_column_name(name))) + } + _ => Ok(quote_phys_identifier(&m.physical_tag_column_name(name))), + } +} + +pub(super) fn regex_match_column_name( + left: &Expr, + mapping: Option<&ColumnMapping>, +) -> Result { + match left { + Expr::FieldRef { + name, + typ: Some(FieldType::Tag), + } => { + let col = mapping + .map(|m| m.physical_tag_column_name(name)) + .unwrap_or_else(|| name.clone()); + Ok(quote_phys_identifier(&col)) + } + Expr::FieldRef { + name, + typ: Some(FieldType::Field), + } => quote_identifier(name), + Expr::FieldRef { name, typ: None } => { + let col = mapping + .map(|m| m.physical_tag_column_name(name)) + .unwrap_or_else(|| name.clone()); + Ok(quote_phys_identifier(&col)) + } + Expr::Identifier(n) => { + let col = if let Some(m) = mapping { + if tag_field_collision(m, n) { + m.physical_tag_column_name(n) + } else { + m.physical_select_identifier(n) + } + } else { + n.clone() + }; + Ok(quote_phys_identifier(&col)) + } + _ => Err(HyperbytedbError::QueryParse( + "regex operator =~ / !~ requires identifier and regex".to_string(), + )), + } +} + +pub(super) fn try_translate_where_binary_expr( + be: &BinaryExpr, + out: &mut String, + m: &ColumnMapping, +) -> Result { + let (name, lit, id_on_left, explicit_tag) = match (&be.left, &be.right) { + (Expr::Identifier(n), rhs) if is_where_literal(rhs) => (n.as_str(), rhs, true, false), + (Expr::FieldRef { name, typ: None }, rhs) if is_where_literal(rhs) => { + (name.as_str(), rhs, true, false) + } + ( + Expr::FieldRef { + name, + typ: Some(FieldType::Tag), + }, + rhs, + ) if is_where_literal(rhs) => (name.as_str(), rhs, true, true), + (lhs, Expr::Identifier(n)) if is_where_literal(lhs) => (n.as_str(), lhs, false, false), + (lhs, Expr::FieldRef { name, typ: None }) if is_where_literal(lhs) => { + (name.as_str(), lhs, false, false) + } + ( + lhs, + Expr::FieldRef { + name, + typ: Some(FieldType::Tag), + }, + ) if is_where_literal(lhs) => (name.as_str(), lhs, false, true), + _ => return Ok(false), + }; + if matches!(be.op, BinaryOp::And | BinaryOp::Or) { + return Ok(false); + } + // Tags are strings; comparing one to a numeric literal never matches in + // InfluxQL (and would be a type error in ClickHouse). Emit constant-false + // so the query runs and returns an empty result. + let is_pure_tag = explicit_tag || (m.tag_keys.contains(name) && !m.field_names.contains(name)); + if is_pure_tag && matches!(lit, Expr::IntegerLiteral(_) | Expr::FloatLiteral(_)) { + write!(out, "1 = 0")?; + return Ok(true); + } + if !tag_field_collision(m, name) { + return Ok(false); + } + let col = where_identifier_physical_name(m, name, lit)?; + if id_on_left { + write!(out, "{}", col)?; + write!(out, " {} ", binary_op_to_clickhouse(&be.op))?; + translate_expr(lit, out, true, Some(m))?; + } else { + translate_expr(lit, out, true, Some(m))?; + write!(out, " {} ", binary_op_to_clickhouse(&be.op))?; + write!(out, "{}", col)?; + } + Ok(true) +} + +pub(super) fn translate_expr( + expr: &Expr, + out: &mut String, + in_where: bool, + mapping: Option<&ColumnMapping>, +) -> Result<(), HyperbytedbError> { + match expr { + Expr::Identifier(name) => { + if in_where && name.to_lowercase() == "time" { + write!(out, "time")?; + } else if in_where { + if let Some(m) = mapping { + if tag_field_collision(m, name) { + write!( + out, + "{}", + quote_phys_identifier(&m.physical_tag_column_name(name)) + )?; + } else { + write!(out, "{}", quote_identifier(name)?)?; + } + } else { + write!(out, "{}", quote_identifier(name)?)?; + } + } else { + write!(out, "{}", quote_identifier(name)?)?; + } + } + Expr::FieldRef { name, typ } => { + let s = match typ { + Some(FieldType::Tag) => { + if let Some(m) = mapping { + quote_phys_identifier(&m.physical_tag_column_name(name)) + } else { + quote_identifier(name)? + } + } + Some(FieldType::Field) => quote_identifier(name)?, + None => { + if let Some(m) = mapping { + if tag_field_collision(m, name) { + quote_phys_identifier(&m.physical_tag_column_name(name)) + } else { + quote_identifier(name)? + } + } else { + quote_identifier(name)? + } + } + }; + write!(out, "{}", s)?; + } + Expr::Now => write!(out, "now64()")?, + Expr::DurationLiteral(d) => write!(out, "{}", d.to_clickhouse_interval())?, + Expr::BinaryExpr(be) => { + write!(out, "(")?; + if matches!(be.op, BinaryOp::RegexMatch | BinaryOp::RegexNotMatch) { + let pattern = match (&be.left, &be.right) { + (_, Expr::Regex(p)) => p.clone(), + _ => { + return Err(HyperbytedbError::QueryParse( + "regex operator =~ / !~ requires identifier and regex".to_string(), + )); + } + }; + let col = regex_match_column_name(&be.left, mapping)?; + let escaped = pattern.replace('\\', "\\\\").replace('\'', "\\'"); + if be.op == BinaryOp::RegexMatch { + write!(out, "match({}, '{}')", col, escaped)?; + } else { + write!(out, "NOT match({}, '{}')", col, escaped)?; + } + } else { + let is_logical = matches!(be.op, BinaryOp::And | BinaryOp::Or); + if is_logical { + translate_expr(&be.left, out, in_where, mapping)?; + let op_str = match be.op { + BinaryOp::And => "AND", + BinaryOp::Or => "OR", + _ => { + return Err(HyperbytedbError::QueryParse( + "internal: expected AND/OR in logical binary expression" + .to_string(), + )); + } + }; + write!(out, " {} ", op_str)?; + translate_expr(&be.right, out, in_where, mapping)?; + } else if in_where && is_time_epoch_comparison(be) { + translate_time_epoch_comparison(be, out)?; + } else { + let handled = if let Some(m) = mapping { + if in_where { + try_translate_where_binary_expr(be, out, m)? + } else { + false + } + } else { + false + }; + if !handled { + translate_expr(&be.left, out, in_where, mapping)?; + write!(out, " {} ", binary_op_to_clickhouse(&be.op))?; + translate_expr(&be.right, out, in_where, mapping)?; + } + } + } + write!(out, ")")?; + } + Expr::StringLiteral(s) => write!(out, "{}", quote_string(s))?, + Expr::IntegerLiteral(n) => write!(out, "{}", n)?, + Expr::FloatLiteral(f) => write!(out, "{}", format_float(*f))?, + Expr::BooleanLiteral(b) => write!(out, "{}", if *b { "true" } else { "false" })?, + Expr::TimeLiteral(s) => write!(out, "{}", quote_string(s))?, + Expr::Regex(r) => write!(out, "'{}'", r.replace('\\', "\\\\").replace('\'', "\\'"))?, + Expr::UnaryExpr(UnaryOp::Not, e) => { + write!(out, "NOT ")?; + translate_expr(e, out, in_where, mapping)?; + } + Expr::UnaryExpr(UnaryOp::Neg, e) => { + write!(out, "-")?; + translate_expr(e, out, in_where, mapping)?; + } + _ => { + return Err(HyperbytedbError::QueryParse(format!( + "unsupported expression in WHERE: {:?}", + expr + ))); + } + } + Ok(()) +} + +pub(super) fn translate_time_epoch_comparison( + be: &BinaryExpr, + out: &mut String, +) -> Result<(), HyperbytedbError> { + let (time_side_is_left, epoch_expr) = if is_time_identifier(&be.left) { + (true, &be.right) + } else { + (false, &be.left) + }; + + let ts_sql = match epoch_expr { + Expr::DurationLiteral(d) => epoch_duration_to_timestamp(d), + Expr::IntegerLiteral(n) => format!("fromUnixTimestamp64Nano({})", n), + _ => { + return Err(HyperbytedbError::QueryParse( + "expected duration or integer epoch beside time in comparison".to_string(), + )); + } + }; + + if time_side_is_left { + write!(out, "time {} {}", binary_op_to_clickhouse(&be.op), ts_sql)?; + } else { + write!(out, "{} {} time", ts_sql, binary_op_to_clickhouse(&be.op))?; + } + Ok(()) +} + +pub(super) fn epoch_duration_to_timestamp(d: &Duration) -> String { + match d.unit { + DurationUnit::Second => format!("fromUnixTimestamp({})", d.value), + DurationUnit::Millisecond => format!("fromUnixTimestamp64Milli({})", d.value), + DurationUnit::Microsecond => format!("fromUnixTimestamp64Micro({})", d.value), + DurationUnit::Nanosecond => format!("fromUnixTimestamp64Nano({})", d.value), + _ => { + let nanos = d.to_nanos(); + nanos_to_ch_timestamp(nanos) + } + } +} + +pub(super) fn nanos_to_ch_timestamp(nanos: i64) -> String { + format!("fromUnixTimestamp64Nano({nanos})") +} + +pub(super) fn binary_op_to_clickhouse(op: &BinaryOp) -> &'static str { + match op { + BinaryOp::Add => "+", + BinaryOp::Sub => "-", + BinaryOp::Mul => "*", + BinaryOp::Div => "/", + BinaryOp::Mod => "%", + BinaryOp::Eq => "=", + BinaryOp::Neq => "!=", + BinaryOp::Lt => "<", + BinaryOp::Lte => "<=", + BinaryOp::Gt => ">", + BinaryOp::Gte => ">=", + BinaryOp::And => "AND", + BinaryOp::Or => "OR", + BinaryOp::RegexMatch => "~", + BinaryOp::RegexNotMatch => "!~", + } +} + +pub(super) fn quote_identifier(name: &str) -> Result { + if name.chars().any(char::is_control) { + return Err(HyperbytedbError::QueryParse(format!( + "identifier contains control characters: {name:?}" + ))); + } + Ok(format!( + "\"{}\"", + name.replace('\\', "\\\\").replace('"', "\\\"") + )) +} + +/// Quote a physical column name from [`crate::domain::chdb_naming`] (already sanitized). +pub(super) fn quote_phys_identifier(name: &str) -> String { + format!("\"{}\"", name.replace('\\', "\\\\").replace('"', "\\\"")) +} + +pub(super) fn quote_string(s: &str) -> String { + format!("'{}'", s.replace('\\', "\\\\").replace('\'', "\\'")) +} + +pub(super) fn format_float(f: f64) -> String { + if f.fract() == 0.0 && f.is_finite() { + format!("{}", f as i64) + } else { + format!("{}", f) + } +} diff --git a/hyperbytedb/src/timeseriesql/to_clickhouse/materialized_view.rs b/hyperbytedb/src/timeseriesql/to_clickhouse/materialized_view.rs new file mode 100644 index 0000000..f94b9e3 --- /dev/null +++ b/hyperbytedb/src/timeseriesql/to_clickhouse/materialized_view.rs @@ -0,0 +1,436 @@ +use crate::domain::chdb_naming::QuotedTableName; +use crate::domain::column_mapping::ColumnMapping; +use crate::domain::rollup::{aggregate_source_field_name, mean_rollup_column_names}; +use crate::error::HyperbytedbError; +use crate::timeseriesql::ast::*; +use std::fmt::Write; + +use super::coalesce::build_coalesced_fact_view_with_row_meta; +use super::conditions::{quote_phys_identifier, translate_expr}; +use super::rename::rename_time_bucket_alias; +use super::select::{ + select_output_field_name, time_bucket_expr_on, translate_field, translate_inner, +}; +use super::time_bounds::is_time_epoch_comparison; +use super::{SeriesJoin, validate_select_into}; + +/// Wrap a translated SELECT as `INSERT INTO SELECT ...`, renaming `__time` to `time` +/// for the destination measurement schema. +pub fn translate_select_into( + stmt: &SelectStatement, + dest_table: &QuotedTableName, + source: &str, + mapping: Option<&ColumnMapping>, +) -> Result { + validate_select_into(stmt)?; + let select_sql = translate_inner(stmt, source, mapping, None, None)?; + let select_sql = rename_time_bucket_alias(&select_sql); + Ok(format!("INSERT INTO {dest_table}\n{select_sql}")) +} + +pub(super) fn translate_materialized_view_field( + field: &Field, + group_by: Option<&GroupBy>, + mapping: &ColumnMapping, +) -> Result { + if let Expr::Call(func) = &field.expr + && func.name.eq_ignore_ascii_case("mean") + { + let source = aggregate_source_field_name(func)?; + let col = mapping.physical_select_identifier(&source); + let col_q = quote_phys_identifier(&col); + let (sum_col, count_col) = mean_rollup_column_names(&source); + return Ok(format!( + "sum({col_q}) AS {}, count({col_q}) AS {}", + quote_phys_identifier(&sum_col), + quote_phys_identifier(&count_col) + )); + } + translate_field(field, false, 0.0, group_by, Some(mapping)) +} + +/// Ensure coalesced MV source rows expose every field referenced in the SELECT. +pub(super) fn mapping_with_mv_aggregate_fields( + mapping: &ColumnMapping, + fields: &[Field], +) -> ColumnMapping { + let mut expanded = mapping.clone(); + for field in fields { + if let Expr::Call(func) = &field.expr + && let Ok(source) = aggregate_source_field_name(func) + { + expanded + .field_names + .insert(mapping.physical_select_identifier(&source)); + } + } + expanded +} + +/// ClickHouse `SELECT` body for a fact-table materialized view. Joins the source +/// series dimension, groups by the MV's `GROUP BY time(...)` bucket and tag +/// dimensions (dropping tags omitted from the GROUP BY, e.g. `server_id`), and +/// assigns a destination `series_id` via [`crate::domain::series::series_id_ch_sql`]. +pub fn translate_materialized_view_select( + stmt: &SelectStatement, + source_fact: &QuotedTableName, + source_series: &QuotedTableName, + dest_measurement: &str, + mapping: &ColumnMapping, +) -> Result { + validate_select_into(stmt)?; + let gb = stmt + .group_by + .as_ref() + .ok_or_else(|| HyperbytedbError::QueryParse("MV requires GROUP BY".to_string()))?; + let Some(Dimension::Time { interval, offset }) = gb.time_dimension() else { + return Err(HyperbytedbError::QueryParse( + "MV requires GROUP BY time(...)".to_string(), + )); + }; + let time_bucket = time_bucket_expr_on( + "t.time", + interval, + offset.as_ref(), + stmt.timezone.as_deref(), + ); + + let mut grouped_tags: Vec = gb + .tag_dimensions() + .iter() + .map(|s| (*s).to_string()) + .collect(); + grouped_tags.sort(); + + let series_id_expr = crate::domain::series::series_id_ch_sql_for_tags( + dest_measurement, + &grouped_tags, + |tag| quote_phys_identifier(&mapping.physical_tag_column_name(tag)), + "s", + ); + + // Field columns must appear in sorted-by-name order to match the + // destination fact table's DDL column order (build_create_table_sql + // sorts fields by physical name). ClickHouse INSERT matches by position + // when no explicit column list is given in the TO clause. + // mean() expands to two columns (sum_col, count_col) — flatten them + // individually so the interleaved sort is correct. + let mut field_expr_by_name: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + for field in &stmt.fields { + if let Expr::Call(func) = &field.expr + && func.name.eq_ignore_ascii_case("mean") + { + let source = aggregate_source_field_name(func)?; + let col = mapping.physical_select_identifier(&source); + let col_q = quote_phys_identifier(&col); + let (sum_col, count_col) = mean_rollup_column_names(&source); + let sum_expr = format!("sum({col_q}) AS {}", quote_phys_identifier(&sum_col)); + let count_expr = format!("count({col_q}) AS {}", quote_phys_identifier(&count_col)); + field_expr_by_name.insert(sum_col.clone(), sum_expr); + field_expr_by_name.insert(count_col.clone(), count_expr); + } else { + let expr = translate_materialized_view_field(field, stmt.group_by.as_ref(), mapping)?; + let name = select_output_field_name(field).ok_or_else(|| { + HyperbytedbError::QueryParse( + "materialized view field requires a name or alias".to_string(), + ) + })?; + field_expr_by_name.insert(name, expr); + } + } + let sorted_field_strs: Vec = field_expr_by_name.into_values().collect(); + + let mut select_parts = vec![ + format!("{time_bucket} AS time"), + "any(t.`_mv_src_origin_node_id`) AS origin_node_id".to_string(), + "max(t.`_mv_src_ingest_seq`) AS ingest_seq".to_string(), + format!("min({series_id_expr}) AS series_id"), + ]; + select_parts.extend(sorted_field_strs); + + let mut group_parts = vec![time_bucket.clone()]; + for tag in &grouped_tags { + group_parts.push(format!( + "s.{}", + quote_phys_identifier(&mapping.physical_tag_column_name(tag)) + )); + } + + let mut out = String::new(); + write!(out, "SELECT {}", select_parts.join(", "))?; + let source_mapping = mapping_with_mv_aggregate_fields(mapping, &stmt.fields); + let coalesced_source = build_coalesced_fact_view_with_row_meta(source_fact, &source_mapping); + // ANY LEFT JOIN for consistency with the query path: fact rows whose series + // row hasn't landed yet must not be silently dropped from the rollup. + write!( + out, + "\nFROM {coalesced_source} AS t ANY LEFT JOIN {source_series} AS s ON t.`series_id` = s.`series_id`" + )?; + + if let Some(ref cond) = stmt.condition { + write!(out, "\nWHERE ")?; + translate_expr(cond, &mut out, true, Some(mapping))?; + } + + write!(out, "\nGROUP BY {}", group_parts.join(", "))?; + Ok(out) +} + +/// ClickHouse `SELECT` for the destination series-dimension MV: one row per +/// rolled-up tag combination (tags not listed in the MV GROUP BY are dropped). +/// +/// `tag_name_mapping` controls how logical tag keys map to physical column +/// names (tag-field collision prefix). The source mapping uses the *source* +/// measurement's field names for collision detection, but the *destination* +/// series table may have a different set of field columns (MV aliases rename +/// fields), so callers should pass a dedicated mapping (or set of field names) +/// that reflects the destination schema for correct physical column naming. +pub fn translate_materialized_view_series_select( + stmt: &SelectStatement, + source_series: &QuotedTableName, + dest_measurement: &str, + mapping: &ColumnMapping, + dest_field_names: Option<&std::collections::HashSet>, +) -> Result { + let gb = stmt + .group_by + .as_ref() + .ok_or_else(|| HyperbytedbError::QueryParse("MV requires GROUP BY".to_string()))?; + let mut grouped_tags: Vec = gb + .tag_dimensions() + .iter() + .map(|s| (*s).to_string()) + .collect(); + grouped_tags.sort(); + + if grouped_tags.is_empty() { + return Ok(format!( + "SELECT min({}) AS series_id FROM {source_series} AS s GROUP BY tuple()", + crate::domain::series::series_id_ch_sql(dest_measurement, &[] as &[String]) + )); + } + + // Resolve physical tag column names: use destination field names when + // provided (the destination series table's column naming depends on the + // destination's field set, not the source's). + let tag_phys_name = |tag: &str| -> String { + match dest_field_names { + Some(dfn) => { + let fields: std::collections::HashSet<&str> = + dfn.iter().map(|s| s.as_str()).collect(); + crate::domain::chdb_naming::tag_column_name(tag, &fields) + } + None => mapping.physical_tag_column_name(tag), + } + }; + + let series_id_expr = crate::domain::series::series_id_ch_sql_for_tags( + dest_measurement, + &grouped_tags, + |tag| quote_phys_identifier(&tag_phys_name(tag)), + "s", + ); + + let tag_cols: Vec = grouped_tags + .iter() + .map(|tag| format!("s.{}", quote_phys_identifier(&tag_phys_name(tag)))) + .collect(); + + let mut select_parts = vec![format!("min({series_id_expr}) AS series_id")]; + select_parts.extend(tag_cols.iter().cloned()); + + let mut out = String::new(); + write!(out, "SELECT {}", select_parts.join(", "))?; + write!(out, "\nFROM {source_series} AS s")?; + write!(out, "\nGROUP BY {}", tag_cols.join(", "))?; + Ok(out) +} + +/// `INSERT INTO SELECT ...` for one-time MV backfill of historical data. +pub fn translate_materialized_view_backfill( + stmt: &SelectStatement, + dest_table: &QuotedTableName, + source_fact: &QuotedTableName, + source_series: &QuotedTableName, + dest_measurement: &str, + mapping: &ColumnMapping, +) -> Result { + let select_sql = translate_materialized_view_select( + stmt, + source_fact, + source_series, + dest_measurement, + mapping, + )?; + let insert_cols = materialized_view_dest_insert_columns(stmt)?; + Ok(format!( + "INSERT INTO {dest_table} ({insert_cols})\nSELECT {insert_cols}\nFROM (\n{select_sql}\n)" + )) +} + +/// Destination fact columns in physical DDL order (matches [`build_create_table_sql`]). +pub(super) fn materialized_view_dest_insert_columns( + stmt: &SelectStatement, +) -> Result { + let mut cols = vec![ + quote_phys_identifier("time"), + quote_phys_identifier("origin_node_id"), + quote_phys_identifier("ingest_seq"), + quote_phys_identifier("series_id"), + ]; + let mut field_names = materialized_view_dest_field_names(stmt)?; + field_names.sort(); + cols.extend(field_names.into_iter().map(|n| quote_phys_identifier(&n))); + Ok(cols.join(", ")) +} + +/// Output column names for MV destination fields (expands `mean()` to sum/count pairs). +pub(super) fn materialized_view_dest_field_names( + stmt: &SelectStatement, +) -> Result, HyperbytedbError> { + let mut names = Vec::new(); + for field in &stmt.fields { + if let Expr::Call(func) = &field.expr + && func.name.eq_ignore_ascii_case("mean") + { + let source = aggregate_source_field_name(func)?; + let (sum_col, count_col) = mean_rollup_column_names(&source); + names.push(sum_col); + names.push(count_col); + continue; + } + names.push(select_output_field_name(field).ok_or_else(|| { + HyperbytedbError::QueryParse( + "materialized view field requires a name or alias".to_string(), + ) + })?); + } + Ok(names) +} + +/// Full `CREATE MATERIALIZED VIEW ... TO ... AS SELECT ...` DDL for the fact MV. +pub fn build_create_fact_materialized_view( + mv_name: &QuotedTableName, + dest_table: &QuotedTableName, + select_sql: &str, +) -> String { + format!("CREATE MATERIALIZED VIEW {mv_name} TO {dest_table} AS\n{select_sql}") +} + +/// Full `CREATE MATERIALIZED VIEW ... TO ... AS SELECT ...` for the series MV. +pub fn build_create_series_materialized_view( + mv_name: &QuotedTableName, + dest_series: &QuotedTableName, + select_sql: &str, +) -> String { + format!("CREATE MATERIALIZED VIEW {mv_name} TO {dest_series} AS\n{select_sql}") +} + +/// Like [`translate_select_into`], targeting a native MergeTree table source. +/// `series` lets a tag-grouped continuous query resolve tags from the source +/// measurement's dimension table. +pub fn translate_select_into_native( + stmt: &SelectStatement, + dest_table: &QuotedTableName, + source_table: &QuotedTableName, + mapping: Option<&ColumnMapping>, + series: Option>, +) -> Result { + validate_select_into(stmt)?; + let select_sql = translate_inner(stmt, source_table.as_str(), mapping, series, None)?; + let select_sql = rename_time_bucket_alias(&select_sql); + Ok(format!("INSERT INTO {dest_table}\n{select_sql}")) +} + +/// Remove user-supplied `time` comparisons from a WHERE clause. InfluxDB CQs +/// ignore user time ranges and inject their own window each run. +pub fn strip_time_predicates(condition: Option) -> Option { + condition.and_then(strip_time_predicates_expr) +} + +pub(super) fn strip_time_predicates_expr(expr: Expr) -> Option { + match expr { + Expr::BinaryExpr(be) if matches!(be.op, BinaryOp::And) => { + let left = strip_time_predicates_expr(be.left); + let right = strip_time_predicates_expr(be.right); + match (left, right) { + (None, None) => None, + (Some(l), None) => Some(l), + (None, Some(r)) => Some(r), + (Some(l), Some(r)) => Some(Expr::BinaryExpr(Box::new(BinaryExpr { + op: BinaryOp::And, + left: l, + right: r, + }))), + } + } + Expr::BinaryExpr(be) if is_time_epoch_comparison(&be) => None, + other => Some(other), + } +} + +/// Build a WHERE clause for CQ coverage `[start, end)` in nanoseconds. +pub fn cq_time_window_condition(start_nanos: i64, end_nanos: i64) -> Expr { + Expr::BinaryExpr(Box::new(BinaryExpr { + op: BinaryOp::And, + left: Expr::BinaryExpr(Box::new(BinaryExpr { + op: BinaryOp::Gte, + left: Expr::Identifier("time".to_string()), + right: Expr::IntegerLiteral(start_nanos), + })), + right: Expr::BinaryExpr(Box::new(BinaryExpr { + op: BinaryOp::Lt, + left: Expr::Identifier("time".to_string()), + right: Expr::IntegerLiteral(end_nanos), + })), + })) +} + +/// Prepare a CQ inner SELECT for execution: strip user time bounds, inject the +/// computed coverage window, and optionally strip `fill()` (basic syntax). +pub fn prepare_cq_select( + stmt: &SelectStatement, + start_nanos: i64, + end_nanos: i64, + strip_fill: bool, +) -> SelectStatement { + let mut prepared = stmt.clone(); + let window = cq_time_window_condition(start_nanos, end_nanos); + let remaining = strip_time_predicates(prepared.condition.take()); + prepared.condition = Some(match remaining { + Some(existing) => Expr::BinaryExpr(Box::new(BinaryExpr { + op: BinaryOp::And, + left: existing, + right: window, + })), + None => window, + }); + if strip_fill { + prepared.fill = None; + } + prepared +} + +/// `INSERT INTO SELECT ...` for a bounded CQ run against native tables. +pub fn translate_bounded_cq_into( + stmt: &SelectStatement, + dest_table: &QuotedTableName, + source_table: &QuotedTableName, + mapping: Option<&ColumnMapping>, + series: Option>, + start_nanos: i64, + end_nanos: i64, +) -> Result { + validate_select_into(stmt)?; + let prepared = prepare_cq_select(stmt, start_nanos, end_nanos, false); + let select_sql = translate_inner( + &prepared, + source_table.as_str(), + mapping, + series, + Some((Some(start_nanos), Some(end_nanos))), + )?; + let select_sql = rename_time_bucket_alias(&select_sql); + Ok(format!("INSERT INTO {dest_table}\n{select_sql}")) +} diff --git a/hyperbytedb/src/timeseriesql/to_clickhouse/mod.rs b/hyperbytedb/src/timeseriesql/to_clickhouse/mod.rs new file mode 100644 index 0000000..ed9c665 --- /dev/null +++ b/hyperbytedb/src/timeseriesql/to_clickhouse/mod.rs @@ -0,0 +1,65 @@ +use crate::domain::chdb_naming::QuotedTableName; +use crate::error::HyperbytedbError; +use crate::timeseriesql::ast::*; + +mod aggregates; +mod coalesce; +mod conditions; +mod materialized_view; +mod rename; +mod select; +mod time_bounds; + +#[cfg(test)] +mod tests; + +pub use rename::rename_time_bucket_alias; +pub use time_bounds::extract_time_bounds; + +pub use coalesce::{build_coalesced_fact_view, build_coalesced_fact_view_with_row_meta}; +pub use conditions::translate_condition; +pub use materialized_view::{ + build_create_fact_materialized_view, build_create_series_materialized_view, + cq_time_window_condition, prepare_cq_select, strip_time_predicates, translate_bounded_cq_into, + translate_materialized_view_backfill, translate_materialized_view_select, + translate_materialized_view_series_select, translate_select_into, translate_select_into_native, +}; +pub use select::{ + select_has_true_aggregate, select_output_field_name, translate_native_table, + translate_with_source, +}; + +/// `SELECT ... INTO` requires `GROUP BY time()` so results are bucketed +/// before writing to the destination measurement. +pub fn validate_select_into(stmt: &SelectStatement) -> Result<(), HyperbytedbError> { + if stmt.into.is_none() { + return Ok(()); + } + let Some(gb) = stmt.group_by.as_ref() else { + return Err(HyperbytedbError::QueryParse( + "SELECT INTO requires GROUP BY time()".to_string(), + )); + }; + if gb.time_dimension().is_none() { + return Err(HyperbytedbError::QueryParse( + "SELECT INTO requires GROUP BY time()".to_string(), + )); + } + Ok(()) +} + +/// The per-measurement series (tag dimension) table to join for tag resolution. +/// In the `series_id` layout the fact table no longer stores tag columns; when a +/// query references a tag we re-attach the tag columns from this table. +#[derive(Debug, Clone, Copy)] +pub struct SeriesJoin<'a> { + /// Backtick-quoted `___series` table name. + pub table: &'a QuotedTableName, + /// Force the inline tag-rejoin view even when the query body references no + /// tag. Set when tombstone predicates (spliced into WHERE post-translation) + /// reference tag columns that must be present in the FROM source. + pub force: bool, + /// Physical column names that actually exist in the series table. + /// When empty, all tags from the ColumnMapping are projected (backward-compat). + pub tag_columns: &'a [String], +} diff --git a/hyperbytedb/src/timeseriesql/to_clickhouse/rename.rs b/hyperbytedb/src/timeseriesql/to_clickhouse/rename.rs new file mode 100644 index 0000000..dea72a4 --- /dev/null +++ b/hyperbytedb/src/timeseriesql/to_clickhouse/rename.rs @@ -0,0 +1,55 @@ +/// Rename the internal `__time` bucket alias to `time`, for INSERT ... SELECT +/// destinations and subquery FROM sources. Only standalone `__time` tokens are +/// rewritten (bare, `"__time"`, or `` `__time` ``); identifiers that merely +/// contain the substring (e.g. `"cpu__time"`) are preserved. +#[must_use] +pub fn rename_time_bucket_alias(sql: &str) -> String { + let bytes = sql.as_bytes(); + let is_ident = |c: u8| c.is_ascii_alphanumeric() || c == b'_'; + let mut out = String::with_capacity(sql.len()); + let mut last = 0usize; + for (pos, _) in sql.match_indices("__time") { + if pos < last { + continue; + } + let prev = if pos == 0 { None } else { Some(bytes[pos - 1]) }; + let next = bytes.get(pos + "__time".len()).copied(); + let exact_quoted = matches!( + (prev, next), + (Some(b'"'), Some(b'"')) | (Some(b'`'), Some(b'`')) + ); + let bare = prev.is_none_or(|c| !is_ident(c) && c != b'"' && c != b'`') + && next.is_none_or(|c| !is_ident(c) && c != b'"' && c != b'`'); + if exact_quoted || bare { + out.push_str(&sql[last..pos]); + out.push_str("time"); + last = pos + "__time".len(); + } + } + out.push_str(&sql[last..]); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rename_time_bucket_alias_is_token_precise() { + assert_eq!( + rename_time_bucket_alias("toStartOfInterval(time, INTERVAL 1 MINUTE) AS __time"), + "toStartOfInterval(time, INTERVAL 1 MINUTE) AS time" + ); + assert_eq!( + rename_time_bucket_alias("ORDER BY __time DESC"), + "ORDER BY time DESC" + ); + assert_eq!(rename_time_bucket_alias("\"__time\""), "\"time\""); + assert_eq!(rename_time_bucket_alias("`__time`"), "`time`"); + assert_eq!( + rename_time_bucket_alias("\"cpu__time\" AS __time"), + "\"cpu__time\" AS time" + ); + assert_eq!(rename_time_bucket_alias("lag__timer"), "lag__timer"); + } +} diff --git a/hyperbytedb/src/timeseriesql/to_clickhouse/select.rs b/hyperbytedb/src/timeseriesql/to_clickhouse/select.rs new file mode 100644 index 0000000..f627f0d --- /dev/null +++ b/hyperbytedb/src/timeseriesql/to_clickhouse/select.rs @@ -0,0 +1,618 @@ +use crate::domain::chdb_naming::QuotedTableName; +use crate::domain::column_mapping::ColumnMapping; +use crate::error::HyperbytedbError; +use crate::timeseriesql::ast::*; +use std::fmt::Write; + +use super::SeriesJoin; +use super::aggregates::{ + expr_contains_aggregate, expr_contains_raw_transform, translate_aggregate_call, + translate_binary_expr, +}; +use super::coalesce::build_coalesced_fact_view; +use super::conditions::{ + format_float, nanos_to_ch_timestamp, quote_identifier, quote_phys_identifier, quote_string, + translate_expr, +}; + +pub fn translate_native_table( + stmt: &SelectStatement, + table_source: &str, + mapping: Option<&ColumnMapping>, + series: Option>, + time_bounds: Option<(Option, Option)>, +) -> Result { + translate_inner(stmt, table_source, mapping, series, time_bounds) +} + +pub(super) fn translate_inner( + stmt: &SelectStatement, + from_source: &str, + mapping: Option<&ColumnMapping>, + series: Option>, + time_bounds: Option<(Option, Option)>, +) -> Result { + let mut out = String::new(); + + // InfluxQL treats a GROUP BY time() query without an explicit fill() as + // fill(null): every bucket in the queried range is emitted, with NULL + // aggregates for empty buckets. Writes (`SELECT ... INTO` / CQ runs) keep + // the absent-fill case as "no fill" so synthetic NULL rows are never + // inserted into the destination. + let effective_fill = match (&stmt.fill, &stmt.into) { + (Some(f), _) => f.clone(), + (None, Some(_)) => FillOption::None, + (None, None) => FillOption::Null, + }; + + // Only `fill()` coerces NULL aggregates to a numeric default in SQL. + // `fill(null)` must leave NULL so JSON shows null, not 0. + let use_ifnull_fill = matches!(effective_fill, FillOption::Value(_)); + let needs_with_fill = !matches!(effective_fill, FillOption::None); + let fill_value = match &effective_fill { + FillOption::Value(v) => *v, + _ => 0.0, + }; + + // tz() flows into every bucketing expression (SELECT / GROUP BY / ORDER BY + // and the WITH FILL grid anchors) so buckets align on local-time boundaries, + // including 23/25-hour DST days. + let tz = stmt.timezone.as_deref(); + + // Collect field alias names for the INTERPOLATE clause. These must match the + // output column names emitted by `translate_field` exactly — otherwise + // `fill(previous)`/`fill(linear)` reference a non-existent identifier (e.g. + // `INTERPOLATE (MEAN)` while the column is `mean_value`) and chDB errors out. + let field_aliases: Vec = stmt + .fields + .iter() + .filter_map(select_output_field_name) + .collect(); + + // SELECT - prepend the time bucket column when GROUP BY time() is present + write!(out, "SELECT ")?; + let mut select_parts: Vec = Vec::new(); + + let has_group_by_time = stmt + .group_by + .as_ref() + .and_then(|gb| gb.time_dimension()) + .is_some(); + + if let Some(ref gb) = stmt.group_by { + if let Some(Dimension::Time { interval, offset }) = gb.time_dimension() { + let time_expr = time_bucket_expr(interval, offset.as_ref(), tz); + // Use __time alias to avoid collision with the raw `time` column, + // then rename back to `time` in the result parser. + select_parts.push(format!("{} AS __time", time_expr)); + } + + // Include GROUP BY tag columns in SELECT so they appear in the result + // and can be used to split rows into separate InfluxDB series. + for tag in gb.tag_dimensions() { + select_parts.push(select_tag_column_sql(tag, mapping)?); + } + } + + let has_aggregate = stmt.fields.iter().any(|f| expr_contains_call(&f.expr)); + let has_star = stmt + .fields + .iter() + .any(|f| matches!(f.expr, Expr::Star | Expr::Wildcard)); + // True aggregates collapse rows; bare window transforms (difference("v"), + // moving_average("v", n), ...) stay per-point and must keep the raw `time` + // column and per-point ordering like raw selects. + let has_true_aggregate = select_has_true_aggregate(stmt); + let has_raw_transform = stmt + .fields + .iter() + .any(|f| expr_contains_raw_transform(&f.expr)); + + // Raw (non-aggregate) selects return one row per point and must carry the + // point's `time` column, like InfluxDB. `SELECT *` already projects `time`, + // and GROUP BY time() / aggregate queries get their time column elsewhere. + let is_raw_select = !has_group_by_time && !has_star && !has_aggregate; + let projects_point_time = + is_raw_select || (has_raw_transform && !has_group_by_time && !has_star); + if projects_point_time { + select_parts.insert(0, quote_phys_identifier("time")); + } + + let field_strs: Vec = stmt + .fields + .iter() + .map(|f| { + translate_field( + f, + use_ifnull_fill, + fill_value, + stmt.group_by.as_ref(), + mapping, + ) + }) + .collect::, HyperbytedbError>>()?; + select_parts.extend(field_strs); + write!(out, "{}", select_parts.join(", "))?; + + // FROM — wrapped in the tag-rejoin inline view when needed. + let from = build_from_source(from_source, series, mapping, stmt); + write!(out, "\nFROM {}", from)?; + + // WHERE + if let Some(ref cond) = stmt.condition { + write!(out, "\nWHERE ")?; + translate_expr(cond, &mut out, true, mapping)?; + } + + // GROUP BY + if let Some(ref gb) = stmt.group_by { + let mut gb_parts = Vec::new(); + + if let Some(Dimension::Time { interval, offset }) = gb.time_dimension() { + gb_parts.push(time_bucket_expr(interval, offset.as_ref(), tz)); + } + + // Tag dimensions only group the SQL when a true aggregate is present. + // Raw selects / bare window transforms keep one row per point: their + // tag columns stay projected (for per-series splitting in the result + // parser and PARTITION BY in window clauses) but grouping by them + // would be NOT_AN_AGGREGATE in ClickHouse. + if has_true_aggregate { + for tag in gb.tag_dimensions() { + // Must match the SELECT expression: physical column name (handles the + // `__tag__` collision prefix). Previously emitted the logical name, + // which is wrong for collision-renamed tags. + gb_parts.push(group_by_tag_sql(tag, mapping)?); + } + } + + if !gb_parts.is_empty() { + write!(out, "\nGROUP BY {}", gb_parts.join(", "))?; + } + } + + // Compute time column expression for ORDER BY + let time_col = stmt.group_by.as_ref().and_then(|gb| { + if let Some(Dimension::Time { interval, offset }) = gb.time_dimension() { + Some(time_bucket_expr(interval, offset.as_ref(), tz)) + } else { + None + } + }); + + // InfluxDB orders every result by time ascending by default; an explicit + // ORDER BY only changes the direction. Order whenever there is a time column + // to sort on: GROUP BY time() buckets, raw per-point selects (incl. `*`), or + // bare window transforms (which are per-point and project raw `time`). + // Aggregates without GROUP BY time() collapse to one row and need no ordering. + let has_orderable_time = + time_col.is_some() || (!has_aggregate && !has_group_by_time) || projects_point_time; + let time_desc = stmt.order_by.as_ref().is_some_and(|o| o.time_desc); + let do_fill = needs_with_fill && time_col.is_some(); + // ClickHouse WITH FILL on a DESC-ordered column never matches the ascending + // FROM/TO anchors we emit, so no fill rows are generated. Fill ascending in + // this (inner) SELECT and re-order descending in a wrapper below. + let wrap_desc_fill = time_desc && do_fill; + + if has_orderable_time { + write!(out, "\nORDER BY ")?; + + // When filling a tag-grouped query, the tag columns must precede the + // time-fill column in ORDER BY so ClickHouse fills each tag group + // independently. Without this, WITH FILL fills globally: gap buckets are + // emitted with empty tag values (a phantom all-NULL series) and the real + // per-tag series is never filled — which surfaces as "no data" in Grafana. + if do_fill && let Some(ref gb) = stmt.group_by { + for tag in gb.tag_dimensions() { + write!(out, "{} ASC, ", group_by_tag_sql(tag, mapping)?)?; + } + } + + if let Some(ref tc) = time_col { + write!(out, "{}", tc)?; + } else { + write!(out, "time")?; + } + if time_desc && !wrap_desc_fill { + write!(out, " DESC")?; + } else { + write!(out, " ASC")?; + } + + if do_fill + && let Some(ref gb) = stmt.group_by + && let Some(Dimension::Time { interval, offset }) = gb.time_dimension() + { + let step = interval.to_clickhouse_interval(); + write!(out, " WITH FILL")?; + if let Some((min_nanos, max_nanos)) = time_bounds + && let (Some(min), Some(max)) = (min_nanos, max_nanos) + { + // The grid anchors must use the same bucket shape (offset + + // timezone) as the bucket expression, or the generated grid + // interleaves phantom buckets. `WITH FILL ... TO` is exclusive, + // so extend one step past the bucket containing the upper WHERE + // bound to emit the final bucket. + let from_anchor = + time_bucket_expr_on(&nanos_to_ch_timestamp(min), interval, offset.as_ref(), tz); + let to_anchor = + time_bucket_expr_on(&nanos_to_ch_timestamp(max), interval, offset.as_ref(), tz); + write!(out, " FROM {from_anchor} TO {to_anchor} + {step}")?; + } + write!(out, " STEP {}", step)?; + + match effective_fill { + // fill(previous): use INTERPOLATE to carry forward last known value + FillOption::Previous if !field_aliases.is_empty() => { + let interp_cols: Vec = field_aliases + .iter() + .map(|a| quote_identifier(a)) + .collect::, HyperbytedbError>>()?; + write!(out, " INTERPOLATE ({})", interp_cols.join(", "))?; + } + // fill(linear): use INTERPOLATE with linear expressions + FillOption::Linear if !field_aliases.is_empty() => { + let interp_cols: Vec = field_aliases + .iter() + .map(|a| -> Result { + let q = quote_identifier(a)?; + Ok(format!("{q} AS {q}")) + }) + .collect::, HyperbytedbError>>()?; + write!(out, " INTERPOLATE ({})", interp_cols.join(", "))?; + } + // fill(): WITH FILL-generated rows get column defaults + // (NULL) that the ifNull() around the aggregate can't reach; a + // constant INTERPOLATE expression sets generated rows — + // including leading gaps — to the fill value. + FillOption::Value(v) if !field_aliases.is_empty() => { + let interp_cols: Vec = field_aliases + .iter() + .map(|a| Ok(format!("{} AS {}", quote_identifier(a)?, format_float(v)))) + .collect::, HyperbytedbError>>()?; + write!(out, " INTERPOLATE ({})", interp_cols.join(", "))?; + } + _ => {} + } + } + } + + // GROUP BY tag dimensions carry InfluxQL per-series LIMIT semantics and + // outer ordering. These are the logical (output) column names. + let tag_dims: Vec<&str> = stmt + .group_by + .as_ref() + .map(|gb| gb.tag_dimensions()) + .unwrap_or_default(); + + if wrap_desc_fill { + // Re-order the ascending filled grid descending, tags first (matching + // the tag-first fill ordering above). Outer clauses stay on the `)` + // line so tombstone WHERE-splicing targets only the inner query. + let mut order_parts: Vec = tag_dims + .iter() + .map(|t| Ok(format!("{} ASC", quote_identifier(t)?))) + .collect::, HyperbytedbError>>()?; + order_parts.push("__time DESC".to_string()); + out = format!( + "SELECT * FROM (\n{out}\n) ORDER BY {}", + order_parts.join(", ") + ); + } else if has_raw_transform && !has_group_by_time { + // InfluxQL omits rows where a per-point window transform has no value + // yet (difference/derivative/elapsed first point, moving_average until + // the window is full). Those surface as NULL transform outputs here; + // filter them in a wrapper. Rows where every named transform output is + // NULL are dropped — in InfluxDB a point with a null input field would + // not exist in that field's series at all. + let transform_aliases: Vec = stmt + .fields + .iter() + .filter(|f| expr_contains_raw_transform(&f.expr)) + .filter_map(select_output_field_name) + .collect(); + if !transform_aliases.is_empty() { + let cond = transform_aliases + .iter() + .map(|a| Ok(format!("{} IS NOT NULL", quote_identifier(a)?))) + .collect::, HyperbytedbError>>()? + .join(" OR "); + let dir = if time_desc { "DESC" } else { "ASC" }; + out = format!( + "SELECT * FROM (\n{out}\n) WHERE {cond} ORDER BY {} {dir}", + quote_phys_identifier("time") + ); + } + } + + // LIMIT / OFFSET — InfluxQL LIMIT/OFFSET paginate points *per series*; with + // tag dimensions in GROUP BY that maps to ClickHouse `LIMIT [m,] n BY tags`. + // Without tag grouping the whole result is one series, so plain LIMIT works. + if !tag_dims.is_empty() && stmt.limit.is_some() { + let by_cols = tag_dims + .iter() + .map(|t| quote_identifier(t)) + .collect::, HyperbytedbError>>()? + .join(", "); + let limit = stmt.limit.unwrap_or(0); + match stmt.offset { + Some(offset) => write!(out, "\nLIMIT {offset}, {limit} BY ({by_cols})")?, + None => write!(out, "\nLIMIT {limit} BY ({by_cols})")?, + } + } else { + if let Some(limit) = stmt.limit { + write!(out, "\nLIMIT {}", limit)?; + } + if let Some(offset) = stmt.offset { + write!(out, "\nOFFSET {}", offset)?; + } + } + + Ok(out) +} + +pub fn translate_with_source( + stmt: &SelectStatement, + source: &str, +) -> Result { + translate_inner(stmt, source, None, None, None) +} +pub(super) fn expr_references_tag(expr: &Expr, m: &ColumnMapping) -> bool { + match expr { + Expr::Identifier(name) => m.tag_keys.contains(name), + Expr::FieldRef { name, typ } => { + matches!(typ, Some(FieldType::Tag)) || m.tag_keys.contains(name) + } + Expr::BinaryExpr(be) => { + expr_references_tag(&be.left, m) || expr_references_tag(&be.right, m) + } + Expr::UnaryExpr(_, e) => expr_references_tag(e, m), + Expr::Call(fc) => fc.args.iter().any(|a| expr_references_tag(a, m)), + _ => false, + } +} + +/// Whether the query references any tag (in SELECT, WHERE, or GROUP BY) — or uses +/// `SELECT *`, which in InfluxDB includes tags. Determines whether the series +/// dimension table must be joined. +pub(super) fn query_references_tag(stmt: &SelectStatement, m: &ColumnMapping) -> bool { + if stmt + .group_by + .as_ref() + .is_some_and(|gb| gb.references_tags()) + { + return true; + } + if stmt + .fields + .iter() + .any(|f| matches!(f.expr, Expr::Star | Expr::Wildcard) || expr_references_tag(&f.expr, m)) + { + return true; + } + stmt.condition + .as_ref() + .is_some_and(|c| expr_references_tag(c, m)) +} + +/// Build the FROM source. When `mapping` is present the fact table is wrapped in +/// a coalesced view so partial-field rows merge before aggregation. When `series` +/// is set and the query references a tag, the coalesced fact table is wrapped in +/// an inline view that re-attaches the tag columns from the dimension table. +/// `ANY LEFT JOIN` takes at most one matching dimension row (so pre-merge duplicate +/// `ReplacingMergeTree` series rows can't fan out fact rows) and preserves fact +/// rows whose series row is briefly missing. Tag columns are exposed under their +/// physical names, so the rest of the translator — which already references tags +/// by physical name — is unchanged. +pub(super) fn build_from_source( + fact_table: &str, + series: Option>, + mapping: Option<&ColumnMapping>, + stmt: &SelectStatement, +) -> String { + let fact = match mapping { + Some(m) => { + build_coalesced_fact_view(&QuotedTableName::new_quoted(fact_table.to_string()), m) + } + None => fact_table.to_string(), + }; + let (Some(sj), Some(m)) = (series, mapping) else { + return fact; + }; + if !sj.force && !query_references_tag(stmt, m) { + return fact; + } + let mut tag_cols: Vec = m + .tag_keys + .iter() + .map(|t| m.physical_tag_column_name(t)) + .collect(); + if tag_cols.is_empty() { + return fact; + } + // Only project tag columns that actually exist in the series table. + // MV destinations may have a subset of source tags (GROUP BY columns only). + if !sj.tag_columns.is_empty() { + tag_cols.retain(|c| sj.tag_columns.contains(c)); + } + if tag_cols.is_empty() { + return fact; + } + tag_cols.sort(); + let projected = tag_cols + .iter() + .map(|c| format!("s.{}", quote_phys_identifier(c))) + .collect::>() + .join(", "); + format!( + "(SELECT t.*, {projected} FROM {fact} AS t ANY LEFT JOIN {series} AS s ON t.`series_id` = s.`series_id`)", + series = sj.table, + ) +} +pub(super) fn group_by_tag_sql( + tag: &str, + mapping: Option<&ColumnMapping>, +) -> Result { + match mapping { + Some(m) => Ok(quote_phys_identifier(&m.physical_tag_column_name(tag))), + None => quote_identifier(tag), + } +} + +pub(super) fn time_bucket_expr( + interval: &Duration, + offset: Option<&Duration>, + tz: Option<&str>, +) -> String { + time_bucket_expr_on("time", interval, offset, tz) +} + +/// Bucketing expression over an arbitrary time expression. `tz` (from `tz()`) +/// makes `toStartOfInterval` bucket on local-time boundaries in that zone, +/// which is what keeps `GROUP BY time(1d)` correct across 23/25-hour DST days. +pub(super) fn time_bucket_expr_on( + time_col: &str, + interval: &Duration, + offset: Option<&Duration>, + tz: Option<&str>, +) -> String { + let interval_str = interval.to_clickhouse_interval(); + let tz_arg = tz + .map(|t| format!(", {}", quote_string(t))) + .unwrap_or_default(); + if let Some(off) = offset { + let off_str = off.to_clickhouse_interval(); + format!( + "toStartOfInterval({time_col} - {}, {}{tz_arg}) + {}", + off_str, interval_str, off_str + ) + } else { + format!("toStartOfInterval({time_col}, {}{tz_arg})", interval_str) + } +} + +pub(super) fn select_tag_column_sql( + tag: &str, + mapping: Option<&ColumnMapping>, +) -> Result { + let Some(m) = mapping else { + return quote_identifier(tag); + }; + let phys = m.physical_tag_column_name(tag); + if phys == tag { + quote_identifier(tag) + } else { + Ok(format!( + "{} AS {}", + quote_phys_identifier(&phys), + quote_identifier(tag)? + )) + } +} + +pub(super) fn translate_field( + field: &Field, + use_fill: bool, + fill_value: f64, + group_by: Option<&GroupBy>, + mapping: Option<&ColumnMapping>, +) -> Result { + let sql = translate_field_expr(&field.expr, use_fill, fill_value, group_by, mapping)?; + let alias = field + .alias + .clone() + .or_else(|| default_field_alias(&field.expr)); + Ok(match alias { + Some(a) => format!("{} AS {}", sql, quote_identifier(&a)?), + None => sql, + }) +} + +/// Output column name for a SELECT field (explicit alias or Influx-style default). +#[must_use] +pub fn select_output_field_name(field: &Field) -> Option { + field + .alias + .clone() + .or_else(|| default_field_alias(&field.expr)) +} + +/// Generate a default column alias matching InfluxDB conventions. +/// Single-arg aggregates include the field name for uniqueness: +/// `mean("usage_idle")` → `"mean_usage_idle"`, `count("x")` → `"count_x"`. +/// No-arg calls use just the function name: `count()` → `"count"`. +/// Non-call expressions get no alias. +pub(super) fn default_field_alias(expr: &Expr) -> Option { + match expr { + Expr::Call(func) => { + let base = func.name.to_lowercase(); + if let Some(Expr::Identifier(field_name)) = func.args.first() { + Some(format!("{}_{}", base, field_name)) + } else { + Some(base) + } + } + _ => None, + } +} + +/// Whether an expression tree contains a function call (aggregate, selector, or +/// transform). Used to distinguish raw per-point selects from aggregate queries. +fn expr_contains_call(expr: &Expr) -> bool { + match expr { + Expr::Call(_) => true, + Expr::BinaryExpr(be) => expr_contains_call(&be.left) || expr_contains_call(&be.right), + Expr::UnaryExpr(_, e) => expr_contains_call(e), + _ => false, + } +} + +pub(super) fn translate_field_expr( + expr: &Expr, + use_fill: bool, + fill_value: f64, + group_by: Option<&GroupBy>, + mapping: Option<&ColumnMapping>, +) -> Result { + match expr { + Expr::Star => Ok("*".to_string()), + Expr::Identifier(name) => { + let col = mapping + .map(|m| m.physical_select_identifier(name)) + .unwrap_or_else(|| name.clone()); + Ok(quote_phys_identifier(&col)) + } + Expr::FieldRef { name, .. } => { + let col = mapping + .map(|m| m.physical_select_identifier(name)) + .unwrap_or_else(|| name.clone()); + Ok(quote_phys_identifier(&col)) + } + Expr::Call(func) => translate_aggregate_call(func, use_fill, fill_value, group_by, mapping), + Expr::BinaryExpr(be) => translate_binary_expr(be, use_fill, fill_value, group_by, mapping), + Expr::UnaryExpr(op, e) => { + let inner = translate_field_expr(e, use_fill, fill_value, group_by, mapping)?; + Ok(match op { + UnaryOp::Neg => format!("(-{})", inner), + UnaryOp::Not => format!("(NOT {})", inner), + }) + } + Expr::StringLiteral(s) => Ok(quote_string(s)), + Expr::IntegerLiteral(n) => Ok(n.to_string()), + Expr::FloatLiteral(f) => Ok(f.to_string()), + Expr::BooleanLiteral(b) => Ok(if *b { "true" } else { "false" }.to_string()), + Expr::DurationLiteral(d) => Ok(d.to_clickhouse_interval()), + Expr::TimeLiteral(s) => Ok(quote_string(s)), + Expr::Regex(r) => Ok(format!( + "'{}'", + r.replace('\\', "\\\\").replace('\'', "\\'") + )), + Expr::Wildcard => Ok("*".to_string()), + Expr::Now => Ok("now64()".to_string()), + } +} + +/// Whether any SELECT field contains a row-collapsing aggregate (not a bare window transform). +pub fn select_has_true_aggregate(stmt: &SelectStatement) -> bool { + stmt.fields.iter().any(|f| expr_contains_aggregate(&f.expr)) +} diff --git a/hyperbytedb/src/timeseriesql/to_clickhouse/tests.rs b/hyperbytedb/src/timeseriesql/to_clickhouse/tests.rs new file mode 100644 index 0000000..e9bd344 --- /dev/null +++ b/hyperbytedb/src/timeseriesql/to_clickhouse/tests.rs @@ -0,0 +1,1456 @@ +use super::conditions::quote_identifier; +use super::*; +use crate::domain::chdb_naming::QuotedTableName; +use crate::domain::column_mapping::ColumnMapping; +use crate::timeseriesql::parser; + +fn test_table() -> QuotedTableName { + QuotedTableName::new_quoted("`mydb_autogen_cpu`".to_string()) +} + +fn test_series_table() -> QuotedTableName { + QuotedTableName::new_quoted("`mydb_autogen_cpu_series`".to_string()) +} + +fn qname(s: &str) -> QuotedTableName { + QuotedTableName::new_quoted(s.to_string()) +} + +fn translate_test(stmt: &SelectStatement) -> String { + translate_native_table(stmt, test_table().as_str(), None, None, None).unwrap() +} + +/// Mapping with `host` as a tag and `usage_idle` as a field (no collision). +fn cpu_mapping() -> ColumnMapping { + ColumnMapping { + tag_keys: ["host", "region"].into_iter().map(String::from).collect(), + field_names: ["usage_idle"].into_iter().map(String::from).collect(), + ..Default::default() + } +} + +fn translate_series(stmt: &SelectStatement, m: &ColumnMapping) -> String { + let table = test_table(); + let series = test_series_table(); + translate_native_table( + stmt, + table.as_str(), + Some(m), + Some(SeriesJoin { + table: &series, + force: false, + tag_columns: &[], + }), + None, + ) + .unwrap() +} + +fn parse_select(q: &str) -> SelectStatement { + let stmts = parser::parse_query(q).unwrap(); + match stmts.into_iter().next().unwrap() { + Statement::Select(s) => s, + _ => panic!("expected SELECT statement"), + } +} + +#[test] +fn group_by_tag_uses_physical_column_name() { + let mut map = ColumnMapping::default(); + map.tag_keys.insert("host-name".into()); + map.field_names.insert("v".into()); + let stmt = parse_select(r#"SELECT mean("v") FROM m GROUP BY time(1m), "host-name""#); + let sql = translate_series(&stmt, &map); + assert!( + sql.contains("\"host_name\""), + "tag with punctuation must map to sanitized physical column, got: {sql}" + ); +} + +#[test] +fn quote_identifier_rejects_control_characters() { + assert!(quote_identifier("host\ninject").is_err()); + assert!(quote_identifier("ok_name").is_ok()); +} + +#[test] +fn test_select_star() { + let stmt = parse_select("SELECT * FROM cpu"); + let sql = translate_test(&stmt); + assert!(sql.contains("SELECT *")); + assert!(sql.contains("FROM `mydb_autogen_cpu`")); +} + +#[test] +fn test_mean() { + let stmt = parse_select(r#"SELECT mean("value") FROM cpu"#); + let sql = translate_test(&stmt); + assert!(sql.contains("avg(\"value\")")); +} + +#[test] +fn test_median_count_sum_min_max() { + let stmt = + parse_select(r#"SELECT median("x"), count("x"), sum("x"), min("x"), max("x") FROM m"#); + let sql = translate_test(&stmt); + // InfluxQL median averages the two middle samples on even counts. + assert!(sql.contains("quantileExactInclusive(0.5)(\"x\")")); + assert!(sql.contains("count(\"x\")")); + assert!(sql.contains("sum(\"x\")")); + assert!(sql.contains("min(\"x\")")); + assert!(sql.contains("max(\"x\")")); +} + +#[test] +fn test_first_last() { + let stmt = parse_select(r#"SELECT first("v"), last("v") FROM m"#); + let sql = translate_test(&stmt); + assert!(sql.contains("argMin(\"v\", time)")); + assert!(sql.contains("argMax(\"v\", time)")); +} + +#[test] +fn test_percentile() { + let stmt = parse_select(r#"SELECT percentile("value", 95) FROM m"#); + let sql = translate_test(&stmt); + // Nearest-rank sample percentile, matching InfluxQL. + assert!(sql.contains("quantileExactLow(0.95)(\"value\")")); +} + +#[test] +fn test_spread_stddev_mode_distinct() { + let stmt = parse_select(r#"SELECT spread("v"), stddev("v"), mode("v"), distinct("v") FROM m"#); + let sql = translate_test(&stmt); + assert!(sql.contains("(max(\"v\") - min(\"v\"))")); + // InfluxQL stddev is sample stddev. + assert!(sql.contains("stddevSamp(\"v\")")); + // mode() must be a scalar, not a one-element Array. + assert!(sql.contains("arrayElement(topKWeighted(1)(\"v\", 1), 1)")); + // distinct() must stay valid inside GROUP BY time(); SELECT DISTINCT is not. + assert!(sql.contains("arrayJoin(groupUniqArray(\"v\"))")); + assert!(!sql.contains("DISTINCT \"v\"")); +} + +#[test] +fn test_count_distinct() { + let stmt = parse_select(r#"SELECT count(distinct("v")) FROM m GROUP BY time(1m)"#); + let sql = translate_test(&stmt); + assert!( + sql.contains("uniqExact(\"v\")"), + "count(distinct(v)) should translate to uniqExact, got: {sql}" + ); +} + +#[test] +fn test_distinct_with_group_by_time_is_valid_expression() { + let stmt = parse_select(r#"SELECT distinct("v") FROM m GROUP BY time(1m)"#); + let sql = translate_test(&stmt); + assert!( + sql.contains("arrayJoin(groupUniqArray(\"v\"))"), + "distinct(v) must be an expression usable with GROUP BY time, got: {sql}" + ); + assert!(!sql.contains("DISTINCT "), "got: {sql}"); +} + +#[test] +fn test_where_time_and_tag() { + let stmt = parse_select(r#"SELECT * FROM cpu WHERE "host" = 'server01' AND time > now() - 1h"#); + let sql = translate_test(&stmt); + assert!(sql.contains("WHERE")); + assert!(sql.contains("host")); + assert!(sql.contains("server01")); + assert!(sql.contains("time")); + assert!(sql.contains("now64()")); + assert!(sql.contains("INTERVAL 1 HOUR")); +} + +#[test] +fn test_where_regex() { + let stmt = parse_select(r#"SELECT * FROM m WHERE "region" =~ /us-.*/"#); + let sql = translate_test(&stmt); + assert!(sql.contains("match")); + assert!(sql.contains("us-.*")); +} + +#[test] +fn test_group_by_time() { + let stmt = parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(5m)"#); + let sql = translate_test(&stmt); + assert!(sql.contains("GROUP BY")); + assert!(sql.contains("toStartOfInterval(time, INTERVAL 5 MINUTE)")); +} + +#[test] +fn test_group_by_time_with_offset() { + let stmt = parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(1h, 15m)"#); + let sql = translate_test(&stmt); + assert!(sql.contains( + "toStartOfInterval(time - INTERVAL 15 MINUTE, INTERVAL 1 HOUR) + INTERVAL 15 MINUTE" + )); +} + +#[test] +fn test_group_by_time_and_tags() { + let stmt = parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(5m), "host", "region""#); + let sql = translate_test(&stmt); + assert!(sql.contains("toStartOfInterval(time, INTERVAL 5 MINUTE)")); + assert!(sql.contains("\"host\"")); + assert!(sql.contains("\"region\"")); + // Tag columns must appear in SELECT for result splitting + let select_line = sql.lines().next().unwrap(); + assert!( + select_line.contains("\"host\""), + "SELECT must include tag columns, got: {}", + select_line + ); + assert!( + select_line.contains("\"region\""), + "SELECT must include tag columns, got: {}", + select_line + ); +} + +#[test] +fn test_fill_null() { + let stmt = parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(5m) fill(null)"#); + let sql = translate_test(&stmt); + assert!( + !sql.contains("ifNull"), + "fill(null) must not coerce NULL to 0, got: {sql}" + ); + assert!(sql.contains("avg(\"value\")")); + assert!(sql.contains("WITH FILL STEP INTERVAL 5 MINUTE")); +} + +#[test] +fn test_fill_null_with_time_bounds_uses_from_to() { + let stmt = parse_select( + r#"SELECT mean("load1") FROM "system" WHERE time >= 1781541739132ms AND time <= 1781552539132ms GROUP BY time(10s) fill(null)"#, + ); + let min = 1_781_541_739_132_000_000i64; + let max = 1_781_552_539_132_000_000i64; + let sql = translate_native_table( + &stmt, + test_table().as_str(), + None, + None, + Some((Some(min), Some(max))), + ) + .unwrap(); + assert!( + sql.contains("WITH FILL FROM toStartOfInterval(fromUnixTimestamp64Nano(1781541739132000000), INTERVAL 10 SECOND)"), + "expected FROM bound aligned to bucket, got: {sql}" + ); + // WITH FILL ... TO is exclusive: the anchor extends one step past the + // bucket containing the upper bound so the final bucket is generated. + assert!( + sql.contains("TO toStartOfInterval(fromUnixTimestamp64Nano(1781552539132000000), INTERVAL 10 SECOND) + INTERVAL 10 SECOND"), + "expected TO bound one step past the last bucket, got: {sql}" + ); + assert!( + sql.contains("STEP INTERVAL 10 SECOND"), + "expected STEP after FROM/TO, got: {sql}" + ); +} + +#[test] +fn test_fill_grid_anchors_use_group_by_time_offset() { + // `time(1m, 30s)` bucket expression is `toStartOfInterval(t - 30s, 1m) + 30s`; + // the WITH FILL FROM/TO anchors must use the same shape or the grid + // interleaves phantom buckets between real ones. + let stmt = parse_select( + r#"SELECT mean("v") FROM m WHERE time >= 1781541730000ms AND time <= 1781541790000ms GROUP BY time(1m, 30s) fill(null)"#, + ); + let min = 1_781_541_730_000_000_000i64; + let max = 1_781_541_790_000_000_000i64; + let sql = translate_native_table( + &stmt, + test_table().as_str(), + None, + None, + Some((Some(min), Some(max))), + ) + .unwrap(); + assert!( + sql.contains( + "WITH FILL FROM toStartOfInterval(fromUnixTimestamp64Nano(1781541730000000000) - INTERVAL 30 SECOND, INTERVAL 1 MINUTE) + INTERVAL 30 SECOND" + ), + "FROM anchor must apply the GROUP BY time offset, got: {sql}" + ); + assert!( + sql.contains( + "TO toStartOfInterval(fromUnixTimestamp64Nano(1781541790000000000) - INTERVAL 30 SECOND, INTERVAL 1 MINUTE) + INTERVAL 30 SECOND + INTERVAL 1 MINUTE" + ), + "TO anchor must apply the GROUP BY time offset and extend one step, got: {sql}" + ); +} + +#[test] +fn test_fill_with_group_by_tag_orders_tag_before_time() { + // fill() + GROUP BY tag must order the tag column *before* the + // time-fill column so ClickHouse fills each tag group independently. + // Otherwise WITH FILL emits gap rows with an empty tag value (a phantom + // all-NULL series) and never fills the real per-tag series. + let stmt = + parse_select(r#"SELECT mean("usage_idle") FROM cpu GROUP BY time(10s), "host" fill(null)"#); + let sql = translate_series(&stmt, &cpu_mapping()); + assert!( + sql.contains( + "ORDER BY \"host\" ASC, toStartOfInterval(time, INTERVAL 10 SECOND) ASC WITH FILL" + ), + "tag must precede the time-fill column in ORDER BY, got: {sql}" + ); +} + +#[test] +fn test_raw_select_projects_time_and_orders_ascending() { + // Raw (non-aggregate) selects must carry `time` and default to time ASC, + // matching InfluxDB. Without this, points come back in storage order. + let stmt = parse_select(r#"SELECT "load1", "load5" FROM system"#); + let sql = translate_test(&stmt); + assert!( + sql.starts_with("SELECT \"time\","), + "raw select must project time first, got: {sql}" + ); + assert!( + sql.contains("ORDER BY time ASC"), + "raw select defaults to time ASC, got: {sql}" + ); +} + +#[test] +fn test_group_by_time_defaults_to_order_by_time_ascending() { + let stmt = parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(5m)"#); + let sql = translate_test(&stmt); + assert!( + sql.contains("ORDER BY toStartOfInterval(time, INTERVAL 5 MINUTE) ASC"), + "GROUP BY time defaults to time ASC, got: {sql}" + ); +} + +#[test] +fn test_aggregate_without_group_by_time_has_no_order_by() { + // Collapses to a single row — no ORDER BY (and no raw `time` column). + let stmt = parse_select(r#"SELECT mean("value") FROM cpu"#); + let sql = translate_test(&stmt); + assert!(!sql.contains("ORDER BY"), "got: {sql}"); + assert!(!sql.contains("\"time\""), "no raw time column, got: {sql}"); +} + +#[test] +fn test_select_star_orders_by_time_without_duplicate_time() { + let stmt = parse_select("SELECT * FROM cpu"); + let sql = translate_test(&stmt); + assert!(sql.starts_with("SELECT *"), "got: {sql}"); + assert!(sql.contains("ORDER BY time ASC"), "got: {sql}"); +} + +#[test] +fn test_fill_value() { + let stmt = parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(5m) fill(0)"#); + let sql = translate_test(&stmt); + assert!(sql.contains("ifNull(avg(\"value\"), 0)")); + assert!(sql.contains("WITH FILL")); + // ifNull only reaches existing rows; WITH FILL-generated rows need a + // constant INTERPOLATE or they surface as column defaults, not the value. + assert!( + sql.contains("INTERPOLATE (\"mean_value\" AS 0)"), + "fill(N) must INTERPOLATE generated rows with N, got: {sql}" + ); +} + +#[test] +fn test_fill_value_interpolates_every_field_alias() { + let stmt = + parse_select(r#"SELECT mean("a") AS x, max("b") AS y FROM m GROUP BY time(1m) fill(100)"#); + let sql = translate_test(&stmt); + assert!( + sql.contains("INTERPOLATE (\"x\" AS 100, \"y\" AS 100)"), + "fill(100) must INTERPOLATE all field aliases, got: {sql}" + ); +} + +#[test] +fn test_missing_fill_defaults_to_fill_null() { + // InfluxQL: a GROUP BY time() query without fill() behaves as fill(null). + let stmt = parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(5m)"#); + let sql = translate_test(&stmt); + assert!( + sql.contains("WITH FILL STEP INTERVAL 5 MINUTE"), + "absent fill() must default to fill(null), got: {sql}" + ); + assert!( + !sql.contains("ifNull"), + "default fill must leave NULL aggregates as NULL, got: {sql}" + ); + assert!( + !sql.contains("INTERPOLATE"), + "default fill must not interpolate, got: {sql}" + ); +} + +#[test] +fn test_select_into_does_not_default_fill() { + // Writes must not insert synthetic NULL grid rows. + let stmt = parse_select(r#"SELECT mean("value") INTO "dest" FROM "cpu" GROUP BY time(5m)"#); + let sql = translate_select_into(&stmt, &qname("`dest`"), test_table().as_str(), None).unwrap(); + assert!( + !sql.contains("WITH FILL"), + "SELECT INTO without fill() must not emit WITH FILL, got: {sql}" + ); +} + +#[test] +fn test_order_by_time_desc_with_fill_wraps_ascending_fill() { + // WITH FILL on a DESC column generates nothing against ascending + // FROM/TO anchors; the fill happens ascending in an inner SELECT and an + // outer SELECT re-orders descending. + let stmt = parse_select( + r#"SELECT mean("value") FROM cpu GROUP BY time(5m) fill(null) ORDER BY time DESC"#, + ); + let sql = translate_test(&stmt); + assert!( + sql.starts_with("SELECT * FROM (\n"), + "DESC + fill must wrap, got: {sql}" + ); + assert!( + sql.contains(" ASC WITH FILL"), + "inner fill must be ascending, got: {sql}" + ); + assert!( + sql.contains(") ORDER BY __time DESC"), + "outer must re-order descending, got: {sql}" + ); +} + +#[test] +fn test_order_by_time_desc_with_fill_and_tags_orders_tags_first() { + let stmt = parse_select( + r#"SELECT mean("usage_idle") FROM cpu GROUP BY time(10s), "host" fill(null) ORDER BY time DESC"#, + ); + let sql = translate_series(&stmt, &cpu_mapping()); + assert!( + sql.contains(") ORDER BY \"host\" ASC, __time DESC"), + "outer ordering must keep tags first, got: {sql}" + ); +} + +#[test] +fn test_fill_none() { + let stmt = parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(5m) fill(none)"#); + let sql = translate_test(&stmt); + assert!(!sql.contains("ifNull")); + assert!(!sql.contains("WITH FILL")); +} + +#[test] +fn test_limit_offset() { + let stmt = parse_select("SELECT * FROM cpu LIMIT 10 OFFSET 5"); + let sql = translate_test(&stmt); + assert!(sql.contains("LIMIT 10")); + assert!(sql.contains("OFFSET 5")); +} + +#[test] +fn test_order_by_desc() { + let stmt = + parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(5m) ORDER BY time DESC"#); + let sql = translate_test(&stmt); + assert!(sql.contains("ORDER BY")); + assert!(sql.contains("DESC")); +} + +#[test] +fn test_derivative() { + let stmt = parse_select(r#"SELECT derivative("value", 1s) FROM cpu"#); + let sql = translate_test(&stmt); + assert!( + sql.contains("lagInFrame"), + "expected lagInFrame, got: {sql}" + ); + assert!( + sql.contains("toFloat64"), + "expected toFloat64 time conversion, got: {sql}" + ); + assert!( + !sql.contains("PARTITION BY"), + "no tags = no PARTITION BY, got: {sql}" + ); +} + +#[test] +fn test_non_negative_derivative() { + let stmt = parse_select(r#"SELECT non_negative_derivative("value", 1s) FROM cpu"#); + let sql = translate_test(&stmt); + assert!( + sql.contains("if("), + "expected if() for non-negative check, got: {sql}" + ); + assert!(sql.contains(">= 0"), "expected >= 0 check, got: {sql}"); + assert!( + sql.contains("lagInFrame"), + "expected lagInFrame, got: {sql}" + ); + assert!( + sql.contains("toFloat64"), + "expected toFloat64 time conversion, got: {sql}" + ); +} + +#[test] +fn test_difference() { + let stmt = parse_select(r#"SELECT difference("value") FROM cpu"#); + let sql = translate_test(&stmt); + assert!(sql.contains("lagInFrame")); + assert!(!sql.contains("if(")); +} + +#[test] +fn test_nested_aggregate_in_derivative() { + let stmt = parse_select( + r#"SELECT non_negative_derivative(mean("reads"), 1s) FROM "diskio" WHERE time >= 1000ms GROUP BY time(10s), "host" fill(null)"#, + ); + let sql = translate_test(&stmt); + assert!( + sql.contains("avg(\"reads\")"), + "expected avg(reads), got: {sql}" + ); + assert!( + sql.contains("ORDER BY __time"), + "expected ORDER BY __time, got: {sql}" + ); + assert!( + sql.contains("lagInFrame"), + "expected lagInFrame, got: {sql}" + ); + assert!( + sql.contains(">= 0"), + "expected non-negative check, got: {sql}" + ); + assert!( + sql.contains("PARTITION BY \"host\""), + "GROUP BY tag must produce PARTITION BY in window clause, got: {sql}" + ); + assert!( + sql.contains("toFloat64"), + "expected toFloat64 time conversion, got: {sql}" + ); + let select_line = sql.lines().next().unwrap(); + assert!( + select_line.contains("\"host\""), + "expected host in SELECT, got: {select_line}" + ); +} + +#[test] +fn test_derivative_with_nested_first() { + let stmt = parse_select( + r#"SELECT derivative(first("bytes_recv"), 1s) * 8 FROM net GROUP BY time(10s) fill(null)"#, + ); + let sql = translate_test(&stmt); + // first() → argMin(field, time) + assert!( + sql.contains("argMin(\"bytes_recv\", time)"), + "expected argMin, got: {sql}" + ); + assert!( + sql.contains("ORDER BY __time"), + "expected ORDER BY __time, got: {sql}" + ); +} + +#[test] +fn test_moving_average() { + let stmt = parse_select(r#"SELECT moving_average("value", 5) FROM cpu"#); + let sql = translate_test(&stmt); + assert!(sql.contains("avg(\"value\") OVER")); + assert!(sql.contains("ROWS BETWEEN 4 PRECEDING AND CURRENT ROW")); + // InfluxQL emits values only once the window holds N points. + assert!( + sql.contains("if(count(\"value\") OVER"), + "moving_average must gate on a full window, got: {sql}" + ); + assert!(sql.contains(">= 5"), "window-full check, got: {sql}"); +} + +#[test] +fn test_cumulative_sum() { + let stmt = parse_select(r#"SELECT cumulative_sum("value") FROM cpu"#); + let sql = translate_test(&stmt); + assert!(sql.contains("sum(\"value\") OVER")); + assert!(sql.contains("ROWS UNBOUNDED PRECEDING")); +} + +#[test] +fn test_elapsed() { + let stmt = parse_select(r#"SELECT elapsed("value", 1s) FROM cpu"#); + let sql = translate_test(&stmt); + assert!( + sql.contains("lagInFrame(toNullable(time), 1)"), + "expected NULL-defaulting lagInFrame so the first row is omitted, got: {sql}" + ); + assert!( + sql.contains("toFloat64"), + "expected toFloat64 time conversion, got: {sql}" + ); +} + +#[test] +fn test_fill_previous() { + let stmt = parse_select( + r#"SELECT mean("value") AS avg_val FROM cpu GROUP BY time(5m) fill(previous)"#, + ); + let sql = translate_test(&stmt); + assert!(sql.contains("WITH FILL STEP INTERVAL 5 MINUTE")); + assert!(sql.contains("INTERPOLATE")); + assert!(sql.contains("\"avg_val\"")); + assert!(!sql.contains("ifNull")); +} + +#[test] +fn test_fill_linear() { + let stmt = + parse_select(r#"SELECT mean("value") AS avg_val FROM cpu GROUP BY time(5m) fill(linear)"#); + let sql = translate_test(&stmt); + assert!(sql.contains("WITH FILL STEP INTERVAL 5 MINUTE")); + assert!(sql.contains("INTERPOLATE")); + assert!(sql.contains("\"avg_val\" AS \"avg_val\"")); + assert!(!sql.contains("ifNull")); +} + +#[test] +fn test_grafana_tag_annotation() { + let stmt = parse_select( + r#"SELECT mean("usage_idle") FROM cpu WHERE time >= 1000ms AND time <= 2000ms GROUP BY time(1s), "host"::tag"#, + ); + let sql = translate_test(&stmt); + assert!(sql.contains("GROUP BY")); + assert!( + sql.contains("\"host\""), + "should strip ::tag suffix, got: {sql}" + ); + assert!( + !sql.contains("::tag"), + "should not contain ::tag, got: {sql}" + ); +} + +#[test] +fn test_epoch_ms_time_comparison() { + let stmt = parse_select( + r#"SELECT * FROM cpu WHERE time >= 1772462462777ms AND time <= 1772466062777ms"#, + ); + let sql = translate_test(&stmt); + assert!( + sql.contains("fromUnixTimestamp64Milli(1772462462777)"), + "should convert ms epoch to timestamp, got: {sql}" + ); + assert!( + sql.contains("fromUnixTimestamp64Milli(1772466062777)"), + "should convert ms epoch to timestamp, got: {sql}" + ); + assert!( + !sql.contains("INTERVAL"), + "should not use INTERVAL for epoch timestamps, got: {sql}" + ); +} + +#[test] +fn test_epoch_ns_time_comparison() { + let stmt = parse_select(r#"SELECT * FROM cpu WHERE time >= 1772462462777000000"#); + let sql = translate_test(&stmt); + assert!( + sql.contains("fromUnixTimestamp64Nano(1772462462777000000)"), + "bare integer should become nanosecond timestamp, got: {sql}" + ); +} + +#[test] +fn test_non_negative_derivative_with_multiple_tags() { + let stmt = parse_select( + r#"SELECT non_negative_derivative(mean("read_bytes"), 1s) AS "Reads", non_negative_derivative(mean("write_bytes"), 1s) AS "Writes" FROM "diskio" WHERE "host" =~ /^(8a8b7bfef1c0)$/ AND time >= 1772542183541ms AND time <= 1772542483541ms GROUP BY time(1s), "host", "name" fill(null)"#, + ); + let sql = translate_test(&stmt); + assert!( + sql.contains(r#"PARTITION BY "host", "name""#), + "window must PARTITION BY all GROUP BY tags to avoid cross-series derivative, got: {sql}" + ); + assert!( + sql.contains("toFloat64"), + "time diff must use toFloat64 for correct arithmetic, got: {sql}" + ); + assert!( + sql.contains("avg(\"read_bytes\")"), + "expected avg(read_bytes), got: {sql}" + ); + assert!( + sql.contains("avg(\"write_bytes\")"), + "expected avg(write_bytes), got: {sql}" + ); + assert!( + sql.contains(">= 0"), + "expected non-negative check, got: {sql}" + ); + assert!( + sql.contains("AS \"Reads\""), + "expected Reads alias, got: {sql}" + ); + assert!( + sql.contains("AS \"Writes\""), + "expected Writes alias, got: {sql}" + ); +} + +#[test] +fn test_difference_with_tags_has_partition_by() { + let stmt = parse_select( + r#"SELECT difference(mean("value")) FROM cpu GROUP BY time(10s), "host", "region""#, + ); + let sql = translate_test(&stmt); + assert!( + sql.contains(r#"PARTITION BY "host", "region""#), + "difference window must PARTITION BY tags, got: {sql}" + ); +} + +#[test] +fn test_moving_average_with_tags_has_partition_by() { + let stmt = parse_select( + r#"SELECT moving_average(mean("value"), 5) FROM cpu GROUP BY time(10s), "host""#, + ); + let sql = translate_test(&stmt); + assert!( + sql.contains(r#"PARTITION BY "host""#), + "moving_average window must PARTITION BY tags, got: {sql}" + ); +} + +#[test] +fn test_cumulative_sum_with_tags_has_partition_by() { + let stmt = + parse_select(r#"SELECT cumulative_sum(mean("value")) FROM cpu GROUP BY time(10s), "host""#); + let sql = translate_test(&stmt); + assert!( + sql.contains(r#"PARTITION BY "host""#), + "cumulative_sum window must PARTITION BY tags, got: {sql}" + ); +} + +#[test] +fn test_non_negative_difference_divided_by_constant() { + let stmt = parse_select( + r#"SELECT NON_NEGATIVE_DIFFERENCE(mean("packets_recv"))/10 AS "in", NON_NEGATIVE_DIFFERENCE(mean("packets_sent"))/10 AS "out" FROM "net" WHERE "host" =~ /^(telegraf-664c6bf94-pgt7t)$/ AND "interface" =~ /(vlan|eth|bond).*/ AND time >= 1772706604176ms AND time <= 1772706904176ms GROUP BY time(1s), "host", "interface" fill(null)"#, + ); + let sql = translate_test(&stmt); + assert!( + sql.contains("lagInFrame"), + "expected lagInFrame for difference, got: {sql}" + ); + assert!(sql.contains("/ 10"), "expected division by 10, got: {sql}"); + assert!(sql.contains("AS \"in\""), "expected alias 'in', got: {sql}"); + assert!( + sql.contains("AS \"out\""), + "expected alias 'out', got: {sql}" + ); + assert!( + !sql.contains("NON_NEGATIVE_DIFFERENCE"), + "should not contain raw TimeseriesQL function name in output SQL, got: {sql}" + ); +} + +#[test] +fn test_derivative_unit_conversion() { + let stmt = parse_select(r#"SELECT derivative("value", 1ms) FROM cpu GROUP BY time(10s)"#); + let sql = translate_test(&stmt); + assert!( + sql.contains("/ 0.001"), + "1ms unit should divide time diff by 0.001 seconds, got: {sql}" + ); +} + +#[test] +fn test_relative_time_still_uses_interval() { + let stmt = parse_select(r#"SELECT * FROM cpu WHERE time > now() - 1h"#); + let sql = translate_test(&stmt); + assert!(sql.contains("now64()"), "should keep now64(), got: {sql}"); + assert!( + sql.contains("INTERVAL 1 HOUR"), + "relative duration should stay as interval, got: {sql}" + ); +} + +#[test] +fn test_translate_select_into() { + let q = r#"SELECT mean("value") INTO "cpu_1h" FROM "cpu" WHERE "host" = 'server01' GROUP BY time(1h), "host""#; + let stmt = parse_select(q); + let sql = translate_select_into( + &stmt, + &qname("`mydb_autogen_cpu_1h`"), + test_table().as_str(), + None, + ) + .unwrap(); + assert!(sql.starts_with("INSERT INTO `mydb_autogen_cpu_1h`")); + assert!(sql.contains("SELECT ")); + assert!(sql.contains("time")); + assert!(!sql.contains("__time")); + assert!(sql.contains("avg(\"value\")")); + assert!(sql.contains("GROUP BY")); + assert!(sql.contains("toStartOfInterval(time, INTERVAL 1 HOUR)")); +} + +#[test] +fn test_select_into_requires_group_by_time() { + let q = r#"SELECT mean("value") INTO "cpu_1h" FROM "cpu""#; + let stmt = parse_select(q); + assert!(translate_select_into(&stmt, &qname("`dest`"), test_table().as_str(), None).is_err()); +} + +#[test] +fn test_translate_materialized_view_select() { + let q = r#"SELECT mean("value") INTO "cpu_5m" FROM "cpu" GROUP BY time(5m), "host""#; + let stmt = parse_select(q); + let map = cpu_mapping(); + let sql = translate_materialized_view_select( + &stmt, + &test_table(), + &test_series_table(), + "cpu_5m", + &map, + ) + .unwrap(); + assert!(sql.starts_with("SELECT ")); + assert!(sql.contains("toStartOfInterval(t.time, INTERVAL 5 MINUTE) AS time")); + assert!(sql.contains("any(t.`_mv_src_origin_node_id`) AS origin_node_id")); + assert!(sql.contains("max(t.`_mv_src_ingest_seq`) AS ingest_seq")); + assert!(sql.contains("sipHash64(")); + assert!(sql.contains("AS \"count_value\"")); + assert!(sql.contains("AS \"sum_value\"")); + assert!(!sql.contains("avg(\"value\")")); + assert!( + sql.contains("argMax(\"value\", `ingest_seq`)"), + "MV source should coalesce duplicate raw rows before aggregating" + ); + assert!( + sql.contains("FROM (SELECT `series_id`, `time`, max(`ingest_seq`) AS `_mv_src_ingest_seq`"), + "MV should read from coalesced source subquery, got: {sql}" + ); + assert!( + sql.contains("AS t ANY LEFT JOIN `mydb_autogen_cpu_series` AS s"), + "MV must not drop fact rows whose series row hasn't landed, got: {sql}" + ); + assert!(sql.contains("GROUP BY toStartOfInterval(t.time, INTERVAL 5 MINUTE)")); + assert!(sql.contains("s.\"host\"")); + assert!(!sql.contains("INSERT INTO")); + // Field columns must appear in sorted-by-name order (count < sum). + let count_pos = sql.find("AS \"count_value\"").unwrap(); + let sum_pos = sql.find("AS \"sum_value\"").unwrap(); + assert!( + count_pos < sum_pos, + "fields should be sorted: count_value before sum_value, got: {}..{}", + count_pos, + sum_pos + ); +} + +#[test] +fn materialized_view_backfill_orders_insert_columns_by_physical_name() { + let q = r#"SELECT sum("players") AS "players", sum("max_players") AS "maxplayers", sum("cpu") AS "cpu" INTO "server_stats_1m" FROM "server_stats" GROUP BY time(1m), "host""#; + let stmt = parse_select(q); + let map = cpu_mapping(); + let sql = translate_materialized_view_backfill( + &stmt, + &qname("`dest`"), + &qname("`source`"), + &qname("`source_series`"), + "server_stats_1m", + &map, + ) + .unwrap(); + assert!( + sql.starts_with( + "INSERT INTO `dest` (\"time\", \"origin_node_id\", \"ingest_seq\", \"series_id\", \"cpu\", \"maxplayers\", \"players\")" + ), + "backfill must name columns in DDL order, got: {sql}" + ); + assert!( + sql.contains("SELECT \"time\", \"origin_node_id\", \"ingest_seq\", \"series_id\", \"cpu\", \"maxplayers\", \"players\"\nFROM ("), + "backfill outer SELECT must match INSERT column order, got: {sql}" + ); +} + +#[test] +fn rollup_fact_view_uses_sum_for_additive_fields() { + use crate::domain::rollup::RollupCombine; + + let mut map = cpu_mapping(); + map.field_rollups + .insert("usage_idle".to_string(), RollupCombine::Sum); + let sql = build_coalesced_fact_view(&test_table(), &map); + assert!( + sql.contains("sum(\"usage_idle\") AS \"usage_idle\""), + "rollup fields should merge with sum(), got: {sql}" + ); + assert!( + !sql.contains("argMax(\"usage_idle\""), + "rollup sum fields must not use argMax, got: {sql}" + ); +} + +#[test] +fn raw_fact_view_still_uses_argmax_without_rollups() { + let map = cpu_mapping(); + let sql = build_coalesced_fact_view(&test_table(), &map); + assert!( + sql.contains("argMax(\"usage_idle\", `ingest_seq`)"), + "raw measurements should keep argMax coalesce, got: {sql}" + ); +} + +#[test] +fn mean_on_rollup_measurement_rewrites_to_sum_over_count() { + use crate::domain::rollup::{MeanRollupField, RollupCombine}; + + let mut map = cpu_mapping(); + map.mean_fields.insert( + "value".to_string(), + MeanRollupField { + sum_col: "sum_value".to_string(), + count_col: "count_value".to_string(), + }, + ); + map.field_rollups + .insert("sum_value".to_string(), RollupCombine::Sum); + map.field_rollups + .insert("count_value".to_string(), RollupCombine::Sum); + + let stmt = parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(5m), "host""#); + let table = test_table(); + let series = test_series_table(); + let sql = translate_native_table( + &stmt, + table.as_str(), + Some(&map), + Some(SeriesJoin { + table: &series, + force: false, + tag_columns: &[], + }), + None, + ) + .unwrap(); + assert!( + sql.contains("sum(\"sum_value\") / nullIf(sum(\"count_value\"), 0)"), + "expected weighted mean rewrite, got: {sql}" + ); +} + +#[test] +fn test_tag_field_collision_uses_column_mapping() { + let stmt = parse_select(r#"SELECT mean("cpu") FROM m GROUP BY cpu"#); + let mut map = ColumnMapping::default(); + map.tag_keys.insert("cpu".into()); + map.field_names.insert("cpu".into()); + let table = test_table(); + let series = test_series_table(); + let sql = translate_native_table( + &stmt, + table.as_str(), + Some(&map), + Some(SeriesJoin { + table: &series, + force: false, + tag_columns: &[], + }), + None, + ) + .unwrap(); + assert!( + sql.contains("__tag__cpu"), + "tag column should be prefixed when it collides with a field, got: {sql}" + ); + assert!( + sql.contains("avg(\"cpu\")"), + "aggregate should use field column name, got: {sql}" + ); + assert!( + sql.contains("GROUP BY \"__tag__cpu\""), + "GROUP BY must use the physical tag column to match SELECT, got: {sql}" + ); +} + +// --- series_id layout: tag resolution via the dimension-table inline view --- + +#[test] +fn series_field_only_query_has_no_join() { + // No tag referenced → coalesced fact view, no series dimension join. + let stmt = parse_select(r#"SELECT mean("usage_idle") FROM cpu WHERE time > 0"#); + let sql = translate_series(&stmt, &cpu_mapping()); + assert!( + !sql.contains("JOIN") && !sql.contains("_series"), + "field-only query should not join the series table, got: {sql}" + ); + assert!( + sql.contains("argMax(\"usage_idle\", `ingest_seq`)"), + "field-only query should collapse duplicate rows by ingest_seq, got: {sql}" + ); + assert!(sql.contains("FROM `mydb_autogen_cpu`"), "got: {sql}"); +} + +#[test] +fn telegraf_cpu_multi_field_query_coalesces_partial_rows() { + let stmt = parse_select( + r#"SELECT mean("usage_guest") AS "Usage Guest", mean("usage_idle") AS "Usage Idle", mean("usage_user") AS "Usage User" FROM "cpu" WHERE "host" =~ /^(d2ddee27a9f4)$/ AND "cpu" = 'cpu-total' AND time >= 1780922276152ms and time <= 1780925876152ms GROUP BY time(2s), "host" fill(null)"#, + ); + let mut map = ColumnMapping::default(); + map.tag_keys.insert("host".into()); + map.tag_keys.insert("cpu".into()); + for f in [ + "usage_guest", + "usage_idle", + "usage_user", + "usage_system", + "usage_iowait", + ] { + map.field_names.insert(f.into()); + } + let table = test_table(); + let series = test_series_table(); + let sql = translate_native_table( + &stmt, + table.as_str(), + Some(&map), + Some(SeriesJoin { + table: &series, + force: false, + tag_columns: &[], + }), + None, + ) + .unwrap(); + assert!( + sql.contains("argMax(\"usage_idle\", `ingest_seq`)"), + "expected coalesced fact view, got: {sql}" + ); + assert!( + sql.contains("ANY LEFT JOIN `mydb_autogen_cpu_series` AS s"), + "tag filter should join series table, got: {sql}" + ); + assert!(sql.contains("avg(\"usage_idle\")"), "got: {sql}"); + assert!( + sql.contains("toStartOfInterval(time, INTERVAL 2 SECOND)"), + "got: {sql}" + ); +} + +#[test] +fn series_where_tag_filter_joins_dimension() { + let stmt = parse_select(r#"SELECT mean("usage_idle") FROM cpu WHERE "host" = 'h1'"#); + let sql = translate_series(&stmt, &cpu_mapping()); + assert!( + sql.contains("ANY LEFT JOIN `mydb_autogen_cpu_series` AS s"), + "tag filter should join the series table, got: {sql}" + ); + assert!( + sql.contains("t.`series_id` = s.`series_id`"), + "join key should be series_id, got: {sql}" + ); + // The tag predicate resolves against the joined view's tag column. + assert!(sql.contains("\"host\" = 'h1'"), "got: {sql}"); +} + +#[test] +fn series_group_by_all_tags_expands_to_measurement_tags() { + let mut stmt = parse_select(r#"SELECT mean("usage_idle") FROM cpu GROUP BY time(5m), *"#); + let gb = stmt.group_by.as_ref().unwrap().clone(); + let (expanded_gb, tags) = gb.expand_all_tags(&["host".to_string(), "region".to_string()]); + stmt.group_by = Some(expanded_gb); + assert_eq!(tags, vec!["host", "region"]); + let sql = translate_series(&stmt, &cpu_mapping()); + assert!(sql.contains("ANY LEFT JOIN"), "got: {sql}"); + assert!(sql.contains("\"host\""), "got: {sql}"); + assert!(sql.contains("\"region\""), "got: {sql}"); + assert!(!sql.contains("`*`"), "got: {sql}"); +} + +#[test] +fn series_group_by_tag_projects_and_groups_physical() { + let stmt = parse_select(r#"SELECT mean("usage_idle") FROM cpu GROUP BY time(5m), "host""#); + let sql = translate_series(&stmt, &cpu_mapping()); + assert!(sql.contains("ANY LEFT JOIN"), "got: {sql}"); + // host is non-colliding, so physical == logical. + assert!( + sql.contains("\"host\""), + "tag projected/grouped, got: {sql}" + ); + assert!(sql.contains("GROUP BY"), "got: {sql}"); + assert!(sql.contains("avg(\"usage_idle\")"), "got: {sql}"); +} + +#[test] +fn series_view_exposes_only_tag_columns_from_dimension() { + let stmt = parse_select(r#"SELECT mean("usage_idle") FROM cpu GROUP BY "host""#); + let sql = translate_series(&stmt, &cpu_mapping()); + // Inline view selects t.* plus the dimension's tag columns (sorted). + assert!( + sql.contains("SELECT t.*, s.\"host\", s.\"region\""), + "view should re-attach tag columns, got: {sql}" + ); +} + +#[test] +fn series_force_join_without_tag_reference() { + // force=true (e.g. a tombstone references a tag) joins even a field-only body. + let stmt = parse_select(r#"SELECT mean("usage_idle") FROM cpu WHERE time > 0"#); + let m = cpu_mapping(); + let table = test_table(); + let series = test_series_table(); + let sql = translate_native_table( + &stmt, + table.as_str(), + Some(&m), + Some(SeriesJoin { + table: &series, + force: true, + tag_columns: &[], + }), + None, + ) + .unwrap(); + assert!( + sql.contains("ANY LEFT JOIN"), + "force should join, got: {sql}" + ); +} + +#[test] +fn mv_series_select_uses_dest_field_names_for_tag_prefix() { + // Tag "host" collides with a field only in the destination, not the source. + // Source mapping treats "host" as non-colliding (source field_names is + // {"usage_idle"}), so tag_column_name("host") returns "host". + // Destination has field "host", so dest_field_names = {"host", "usage_idle"}, + // and tag_column_name("host") should return "__tag__host". + let stmt = parse_select( + r#"SELECT mean("usage_idle") INTO "dest" FROM "cpu" GROUP BY time(5m), "host""#, + ); + let mut src_mapping = cpu_mapping(); + src_mapping.tag_keys.insert("host".to_string()); + + let dest_field_names: std::collections::HashSet = + ["host".to_string(), "usage_idle".to_string()].into(); + + let sql = translate_materialized_view_series_select( + &stmt, + &qname("`source_series`"), + "dest", + &src_mapping, + Some(&dest_field_names), + ) + .unwrap(); + + assert!( + sql.contains("__tag__host"), + "tag 'host' should be prefixed when dest has colliding field, got: {sql}" + ); +} + +#[test] +fn mv_series_select_uses_source_names_when_no_dest_field_names() { + let stmt = parse_select( + r#"SELECT mean("usage_idle") INTO "dest" FROM "cpu" GROUP BY time(5m), "host""#, + ); + let mut src_mapping = cpu_mapping(); + src_mapping.tag_keys.insert("host".to_string()); + + let sql = translate_materialized_view_series_select( + &stmt, + &qname("`source_series`"), + "dest", + &src_mapping, + None, + ) + .unwrap(); + + // Without dest field names, source mapping says "host" doesn't collide + // (cpu_mapping has only "usage_idle" as field). + assert!( + sql.contains("\"host\""), + "tag 'host' should NOT be prefixed when dest_field_names is None, got: {sql}" + ); + assert!( + !sql.contains("__tag__host"), + "tag 'host' should NOT be prefixed without dest_field_names, got: {sql}" + ); +} + +// --- per-series LIMIT/OFFSET (InfluxQL points-per-series semantics) --- + +#[test] +fn test_limit_with_group_by_tag_uses_limit_by() { + let stmt = + parse_select(r#"SELECT mean("usage_idle") FROM cpu GROUP BY time(1m), "host" LIMIT 3"#); + let sql = translate_series(&stmt, &cpu_mapping()); + assert!( + sql.contains("LIMIT 3 BY (\"host\")"), + "LIMIT with tag grouping must be per series, got: {sql}" + ); + assert!( + !sql.contains("\nLIMIT 3\n") && !sql.ends_with("\nLIMIT 3"), + "no global LIMIT alongside LIMIT BY, got: {sql}" + ); +} + +#[test] +fn test_limit_offset_with_group_by_tags_uses_limit_by() { + let stmt = parse_select( + r#"SELECT mean("usage_idle") FROM cpu GROUP BY time(1m), "host", "region" LIMIT 3 OFFSET 2"#, + ); + let sql = translate_series(&stmt, &cpu_mapping()); + assert!( + sql.contains("LIMIT 2, 3 BY (\"host\", \"region\")"), + "OFFSET with tag grouping must be per series, got: {sql}" + ); + assert!(!sql.contains("\nOFFSET"), "got: {sql}"); +} + +#[test] +fn test_limit_without_tags_stays_global() { + let stmt = parse_select(r#"SELECT mean("v") FROM m GROUP BY time(1m) LIMIT 4 OFFSET 1"#); + let sql = translate_test(&stmt); + assert!(sql.contains("\nLIMIT 4"), "got: {sql}"); + assert!(sql.contains("\nOFFSET 1"), "got: {sql}"); + assert!(!sql.contains(" BY ("), "got: {sql}"); +} + +// --- raw (non-aggregate) SELECT with GROUP BY tag --- + +#[test] +fn test_raw_select_with_group_by_tag_has_no_sql_group_by() { + let stmt = parse_select(r#"SELECT "usage_idle" FROM cpu GROUP BY "host""#); + let sql = translate_series(&stmt, &cpu_mapping()); + assert!( + !sql.contains("\nGROUP BY"), + "raw select must not GROUP BY tags in SQL (NOT_AN_AGGREGATE), got: {sql}" + ); + // Tag stays projected so the result parser can split per-series. + let select_line = sql.lines().next().unwrap(); + assert!( + select_line.contains("\"host\""), + "tag must be projected for series splitting, got: {select_line}" + ); + assert!( + select_line.starts_with("SELECT \"time\""), + "raw select keeps time first, got: {select_line}" + ); + assert!(sql.contains("ORDER BY time ASC"), "got: {sql}"); +} + +// --- per-point window transforms without GROUP BY time --- + +#[test] +fn test_difference_without_group_by_time_projects_time_and_orders() { + let stmt = parse_select(r#"SELECT difference("value") FROM cpu"#); + let sql = translate_test(&stmt); + assert!( + sql.contains("SELECT \"time\","), + "transform must project the point time, got: {sql}" + ); + assert!( + sql.contains("ORDER BY \"time\" ASC"), + "transform output must be time-ordered, got: {sql}" + ); + // InfluxQL omits the first point (no previous value): NULL outputs are + // filtered by an outer SELECT. + assert!( + sql.starts_with("SELECT * FROM (\n"), + "transform must wrap to filter NULL rows, got: {sql}" + ); + assert!( + sql.contains(") WHERE \"difference_value\" IS NOT NULL"), + "leading NULL transform rows must be filtered, got: {sql}" + ); +} + +#[test] +fn test_transform_with_group_by_tag_partitions_without_sql_group_by() { + let stmt = parse_select(r#"SELECT difference("usage_idle") FROM cpu GROUP BY "host""#); + let sql = translate_series(&stmt, &cpu_mapping()); + assert!( + !sql.contains("\nGROUP BY"), + "bare transform must not GROUP BY tags in SQL, got: {sql}" + ); + assert!( + sql.contains("PARTITION BY \"host\""), + "transform must still partition per series, got: {sql}" + ); +} + +#[test] +fn test_transform_with_group_by_time_keeps_grid_nulls() { + // GROUP BY time + fill keeps the filled grid (Grafana relies on the + // NULL rows); no NULL-filtering wrapper. + let stmt = parse_select(r#"SELECT difference(mean("v")) FROM m GROUP BY time(1m) fill(null)"#); + let sql = translate_test(&stmt); + assert!(!sql.starts_with("SELECT * FROM (\n"), "got: {sql}"); +} + +// --- tag compared to numeric literal --- + +#[test] +fn test_tag_numeric_comparison_is_constant_false() { + let stmt = parse_select(r#"SELECT mean("usage_idle") FROM cpu WHERE "host" = 3"#); + let sql = translate_series(&stmt, &cpu_mapping()); + assert!( + sql.contains("WHERE (1 = 0)"), + "tag vs numeric literal must be constant-false, got: {sql}" + ); + assert!( + !sql.contains("\"host\" = 3"), + "must not emit a string/number comparison, got: {sql}" + ); +} + +#[test] +fn test_tag_string_comparison_is_unaffected() { + let stmt = parse_select(r#"SELECT mean("usage_idle") FROM cpu WHERE "host" = '3'"#); + let sql = translate_series(&stmt, &cpu_mapping()); + assert!(sql.contains("\"host\" = '3'"), "got: {sql}"); + assert!(!sql.contains("1 = 0"), "got: {sql}"); +} + +#[test] +fn test_field_numeric_comparison_is_unaffected() { + let stmt = parse_select(r#"SELECT mean("usage_idle") FROM cpu WHERE "usage_idle" > 3"#); + let sql = translate_series(&stmt, &cpu_mapping()); + assert!(sql.contains("\"usage_idle\" > 3"), "got: {sql}"); + assert!(!sql.contains("1 = 0"), "got: {sql}"); +} + +// --- subquery source: inner GROUP BY time must expose `time` --- + +#[test] +fn test_subquery_source_bucket_column_composes() { + // Built directly (the parser can't produce subqueries yet): the inner + // statement is translated, its `__time` alias renamed to `time`, and + // used as the outer FROM source. + let minute = Duration { + value: 1, + unit: DurationUnit::Minute, + }; + let five_minutes = Duration { + value: 5, + unit: DurationUnit::Minute, + }; + let inner = SelectStatement { + fields: vec![Field { + expr: Expr::Call(FunctionCall { + name: "mean".to_string(), + args: vec![Expr::Identifier("v".to_string())], + }), + alias: Some("x".to_string()), + }], + into: None, + from: vec![], + condition: None, + group_by: Some(GroupBy { + dimensions: vec![Dimension::Time { + interval: minute, + offset: None, + }], + }), + order_by: None, + limit: None, + offset: None, + slimit: None, + soffset: None, + fill: None, + timezone: None, + }; + let inner_sql = + translate_native_table(&inner, test_table().as_str(), None, None, None).unwrap(); + let inner_sql = rename_time_bucket_alias(&inner_sql); + assert!( + inner_sql.contains("AS time"), + "inner bucket must be exposed as `time`, got: {inner_sql}" + ); + assert!(!inner_sql.contains("__time"), "got: {inner_sql}"); + + let outer = SelectStatement { + fields: vec![Field { + expr: Expr::Call(FunctionCall { + name: "max".to_string(), + args: vec![Expr::Identifier("x".to_string())], + }), + alias: None, + }], + into: None, + from: vec![], + condition: None, + group_by: Some(GroupBy { + dimensions: vec![Dimension::Time { + interval: five_minutes, + offset: None, + }], + }), + order_by: None, + limit: None, + offset: None, + slimit: None, + soffset: None, + fill: None, + timezone: None, + }; + let outer_sql = translate_with_source(&outer, &format!("({inner_sql})")).unwrap(); + assert!( + outer_sql.contains("toStartOfInterval(time, INTERVAL 5 MINUTE) AS __time"), + "outer buckets the inner `time` column, got: {outer_sql}" + ); + assert!(outer_sql.contains("max(\"x\")"), "got: {outer_sql}"); +} + +// --- tz() flows into bucketing and fill anchors --- + +#[test] +fn test_timezone_in_bucket_expr_and_fill_anchors() { + let mut stmt = parse_select( + r#"SELECT mean("v") FROM m WHERE time >= 1000000000 AND time <= 3000000000 GROUP BY time(1d) fill(null)"#, + ); + stmt.timezone = Some("America/New_York".to_string()); + let sql = translate_native_table( + &stmt, + test_table().as_str(), + None, + None, + Some((Some(1_000_000_000), Some(3_000_000_000))), + ) + .unwrap(); + assert!( + sql.contains("toStartOfInterval(time, INTERVAL 1 DAY, 'America/New_York') AS __time"), + "bucket expression must carry the timezone, got: {sql}" + ); + assert!( + sql.contains( + "WITH FILL FROM toStartOfInterval(fromUnixTimestamp64Nano(1000000000), INTERVAL 1 DAY, 'America/New_York')" + ), + "fill anchors must bucket in the same timezone, got: {sql}" + ); + assert!( + sql.contains("GROUP BY toStartOfInterval(time, INTERVAL 1 DAY, 'America/New_York')"), + "GROUP BY must match the SELECT bucket expression, got: {sql}" + ); +} + +#[test] +fn test_timezone_string_is_escaped() { + let mut stmt = parse_select(r#"SELECT mean("v") FROM m GROUP BY time(1h)"#); + stmt.timezone = Some("bad'zone".to_string()); + let sql = translate_test_tz(&stmt); + assert!( + sql.contains("'bad\\'zone'"), + "timezone must go through quote_string escaping, got: {sql}" + ); +} + +fn translate_test_tz(stmt: &SelectStatement) -> String { + translate_native_table(stmt, test_table().as_str(), None, None, None).unwrap() +} diff --git a/hyperbytedb/src/timeseriesql/to_clickhouse/time_bounds.rs b/hyperbytedb/src/timeseriesql/to_clickhouse/time_bounds.rs new file mode 100644 index 0000000..66f02be --- /dev/null +++ b/hyperbytedb/src/timeseriesql/to_clickhouse/time_bounds.rs @@ -0,0 +1,221 @@ +use crate::timeseriesql::ast::{BinaryExpr, BinaryOp, Expr}; + +/// Extract `(min_time_nanos, max_time_nanos)` from a WHERE clause for WITH FILL anchoring. +pub fn extract_time_bounds(condition: Option<&Expr>) -> (Option, Option) { + condition.map_or((None, None), bounds_for_expr) +} + +/// Bounds for a single OR-free expression tree (AND of time comparisons). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct TimeBounds { + min: Option, + max: Option, +} + +impl TimeBounds { + fn intersect(self, other: Self) -> Self { + Self { + min: match (self.min, other.min) { + (Some(a), Some(b)) => Some(a.max(b)), + (Some(a), None) => Some(a), + (None, Some(b)) => Some(b), + (None, None) => None, + }, + max: match (self.max, other.max) { + (Some(a), Some(b)) => Some(a.min(b)), + (Some(a), None) => Some(a), + (None, Some(b)) => Some(b), + (None, None) => None, + }, + } + } + + fn has_time_constraint(self) -> bool { + self.min.is_some() || self.max.is_some() + } +} + +fn bounds_for_expr(expr: &Expr) -> (Option, Option) { + let disjuncts = collect_or_disjuncts(expr); + let mut disjunct_bounds = Vec::with_capacity(disjuncts.len()); + + for disjunct in disjuncts { + let b = bounds_for_and_tree(&disjunct); + if !b.has_time_constraint() { + // e.g. `host = 'x'` with no time predicate — time is unbounded on this branch. + return (None, None); + } + disjunct_bounds.push(b); + } + + if disjunct_bounds.is_empty() { + return (None, None); + } + + // OR envelope: only anchor a bound when every disjunct defines that side. + // A disjunct with `time >= N` but no upper cap must not tighten max from + // another disjunct; likewise for missing lower caps. + let min = if disjunct_bounds.iter().all(|b| b.min.is_some()) { + disjunct_bounds.iter().filter_map(|b| b.min).min() + } else { + None + }; + let max = if disjunct_bounds.iter().all(|b| b.max.is_some()) { + disjunct_bounds.iter().filter_map(|b| b.max).max() + } else { + None + }; + + (min, max) +} + +fn collect_or_disjuncts(expr: &Expr) -> Vec { + match expr { + Expr::BinaryExpr(be) if be.op == BinaryOp::Or => { + let mut out = collect_or_disjuncts(&be.left); + out.extend(collect_or_disjuncts(&be.right)); + out + } + other => vec![other.clone()], + } +} + +fn bounds_for_and_tree(expr: &Expr) -> TimeBounds { + match expr { + Expr::BinaryExpr(be) if be.op == BinaryOp::And => { + bounds_for_and_tree(&be.left).intersect(bounds_for_and_tree(&be.right)) + } + Expr::BinaryExpr(be) if is_time_epoch_comparison(be) => bounds_from_comparison(be), + _ => TimeBounds::default(), + } +} + +fn bounds_from_comparison(be: &BinaryExpr) -> TimeBounds { + let (time_is_left, epoch_expr) = if is_time_identifier(&be.left) { + (true, &be.right) + } else { + (false, &be.left) + }; + + let nanos = match epoch_expr { + Expr::DurationLiteral(d) => d.to_nanos(), + Expr::IntegerLiteral(n) => *n, + _ => return TimeBounds::default(), + }; + + let effective_op = if time_is_left { + be.op.clone() + } else { + match be.op { + BinaryOp::Gt => BinaryOp::Lt, + BinaryOp::Gte => BinaryOp::Lte, + BinaryOp::Lt => BinaryOp::Gt, + BinaryOp::Lte => BinaryOp::Gte, + ref other => other.clone(), + } + }; + + let mut bounds = TimeBounds::default(); + match effective_op { + BinaryOp::Gte | BinaryOp::Gt | BinaryOp::Eq => bounds.min = Some(nanos), + _ => {} + } + match effective_op { + BinaryOp::Lte | BinaryOp::Lt | BinaryOp::Eq => bounds.max = Some(nanos), + _ => {} + } + bounds +} + +pub(crate) fn is_time_identifier(expr: &Expr) -> bool { + matches!(expr, Expr::Identifier(n) if n.to_lowercase() == "time") +} + +/// Detect `time ` where epoch_value is a duration or integer epoch. +pub(crate) fn is_time_epoch_comparison(be: &BinaryExpr) -> bool { + if !matches!( + be.op, + BinaryOp::Eq | BinaryOp::Neq | BinaryOp::Lt | BinaryOp::Lte | BinaryOp::Gt | BinaryOp::Gte + ) { + return false; + } + + let (is_left_time, rhs) = if is_time_identifier(&be.left) { + (true, &be.right) + } else if is_time_identifier(&be.right) { + (true, &be.left) + } else { + (false, &be.right) + }; + + if !is_left_time { + return false; + } + + matches!(rhs, Expr::DurationLiteral(_) | Expr::IntegerLiteral(_)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::timeseriesql::ast::Statement; + use crate::timeseriesql::parser::parse_query; + + fn bounds(q: &str) -> (Option, Option) { + let stmt = match parse_query(q).unwrap().remove(0) { + Statement::Select(s) => s, + _ => panic!("expected SELECT"), + }; + extract_time_bounds(stmt.condition.as_ref()) + } + + #[test] + fn intersects_anded_bounds() { + let (min, max) = bounds( + "SELECT * FROM m WHERE time >= 1000000000 AND time >= 3000000000 \ + AND time <= 9000000000 AND time <= 7000000000", + ); + assert_eq!(min, Some(3_000_000_000)); + assert_eq!(max, Some(7_000_000_000)); + } + + #[test] + fn or_of_bounded_ranges_uses_envelope() { + let (min, max) = bounds( + "SELECT * FROM m WHERE (time >= 100 AND time <= 500) OR (time >= 300 AND time <= 700)", + ); + assert_eq!(min, Some(100)); + assert_eq!(max, Some(700)); + } + + #[test] + fn or_with_time_unconstrained_branch_is_conservative() { + let (min, max) = bounds("SELECT * FROM m WHERE time >= 100 OR \"host\" = 'x'"); + assert_eq!(min, None); + assert_eq!(max, None); + } + + #[test] + fn or_of_lower_bounds_only() { + let (min, max) = bounds("SELECT * FROM m WHERE time >= 100 OR time >= 300"); + assert_eq!(min, Some(100)); + assert_eq!(max, None); + } + + #[test] + fn or_partial_max_omitted_when_one_disjunct_unbounded_above() { + let (min, max) = bounds( + "SELECT * FROM m WHERE (time >= 100 AND \"host\" = 'x') \ + OR (time >= 300 AND time <= 700)", + ); + assert_eq!(min, Some(100)); + assert_eq!(max, None); + } + + #[test] + fn or_partial_bounds_omitted_when_disjuncts_lack_opposite_side() { + let (min, max) = bounds("SELECT * FROM m WHERE time <= 500 OR time >= 300"); + assert_eq!(min, None); + assert_eq!(max, None); + } +}