refactor(timeseriesql): shared scan module and split to_clickhouse

.gitignore

@@ -9,6 +9,9 @@ Cargo.lock.bk
 # Python virtual environment
 .venv/
 
+# Git worktrees
+.worktrees/
+
 # IDE/editor/devtool files
 /.vscode/
 /.idea/

deploy/kind/kind-config.yaml

@@ -15,6 +15,9 @@ nodes:
       - containerPort: 30000
         hostPort: 13000
         protocol: TCP
+      - containerPort: 30001
+        hostPort: 13001
+        protocol: TCP
   - role: worker
     extraMounts:
       - hostPath: /tmp/hyperbytedb-data/worker-0

docs/developer-guide/system-architecture.md

@@ -449,7 +449,16 @@ All queries use `OutputFormat::JSONEachRow` — one JSON object per result row.
 
 The query language module is `src/timeseriesql/` (Influx-compatible TimeseriesQL).
 
-The parser is a **hand-rolled recursive descent parser** (no parser generator). It lives in `src/timeseriesql/parser.rs`.
+Parsing uses **two statement grammars** on top of a shared masking scanner (`scan.rs`):
+
+| Path | Module | Mechanism |
+|------|--------|-----------|
+| `SELECT` | `parser.rs` | Clause scanner finds `FROM`/`WHERE`/… keywords; expression parser for fields and predicates |
+| DDL/SHOW | `ddl_parser.rs` + `lexer.rs` | Token stream + `TokenCursor` for keyword-sequence statements |
+
+Both paths share `scan.rs` for quote/regex/paren masking and `lexer::split_statements` for multi-statement input. DDL `WHERE` clauses and CQ/MV inner queries delegate to `parser::parse_expr`.
+
+The SELECT expression parser is **hand-rolled recursive descent** (no parser generator).
 
 ### Parse flow
 
@@ -528,6 +537,10 @@ Key AST nodes (in `src/timeseriesql/ast.rs`):
 - `FillOption` — Null, None, Previous, Linear, Value(f64)
 - `Measurement` — optional database, optional RP, name or regex
 
+**SLIMIT/SOFFSET:** Parsed into the AST but not translated to ClickHouse SQL. The query service applies series-level pagination after merging result series (`query_service.rs`).
+
+**Time bounds for fill:** `extract_time_bounds` (in `to_clickhouse/time_bounds.rs`) derives min/max epoch nanoseconds from WHERE clauses for WITH FILL grid anchoring. AND predicates intersect bounds; OR predicates use the envelope of disjuncts only when every branch defines that side (min and/or max); if any OR branch lacks a time predicate, or any branch is missing a lower or upper cap, the corresponding bound is omitted (conservative).
+
 ---
 
 ## 13. ClickHouse SQL Translator

hyperbytedb/src/timeseriesql/lexer.rs

@@ -5,6 +5,7 @@
 
 use crate::error::HyperbytedbError;
 use crate::timeseriesql::ast::{Duration, DurationUnit};
+use crate::timeseriesql::scan::{ScannedChar, is_regex_start_at, scan_chars};
 
 /// Lexer token with source span.
 #[derive(Debug, Clone, PartialEq)]
@@ -71,93 +72,30 @@ pub fn tokenize(input: &str) -> Result<Vec<Token>, HyperbytedbError> {
     Ok(tokens)
 }
 
-/// Split multi-statement input on `;` outside quotes and BEGIN…END blocks.
+/// Split multi-statement input on `;` outside quotes, regex literals, and
+/// BEGIN…END blocks.
 pub fn split_statements(input: &str) -> Result<Vec<String>, HyperbytedbError> {
+    let scan = scan_chars(input)?;
     let mut statements = Vec::new();
     let mut start = 0usize;
-    let mut i = 0usize;
-    let bytes = input.as_bytes();
-    let mut in_single = false;
-    let mut in_double = false;
-    let mut in_regex = false;
     let mut begin_depth = 0i32;
-    // Last significant (non-whitespace) char outside string/regex literals,
-    // used to decide whether a `/` is in operand position (regex start).
-    let mut prev_sig: Option<char> = None;
-
-    while i < bytes.len() {
-        let c = bytes[i] as char;
 
-        if in_regex {
-            if c == '\\' {
-                i += 2;
-                continue;
-            }
-            if c == '/' {
-                in_regex = false;
-                prev_sig = Some('/');
-            }
-            i += 1;
-            continue;
-        }
-        if in_single {
-            if c == '\'' {
-                if i + 1 < bytes.len() && bytes[i + 1] == b'\'' {
-                    i += 2;
-                    continue;
-                }
-                in_single = false;
-                prev_sig = Some('\'');
+    for (si, sc) in scan.iter().enumerate() {
+        if !sc.masked && sc.depth == 0 {
+            if matches_unmasked_keyword_at(input, &scan, si, "BEGIN") {
+                begin_depth += 1;
+            } else if begin_depth > 0 && matches_unmasked_keyword_at(input, &scan, si, "END") {
+                begin_depth -= 1;
             }
-            i += 1;
-            continue;
         }
-        if in_double {
-            if c == '"' {
-                if i + 1 < bytes.len() && bytes[i + 1] == b'"' {
-                    i += 2;
-                    continue;
-                }
-                in_double = false;
-                prev_sig = Some('"');
-            }
-            i += 1;
-            continue;
-        }
-
-        let at_word_boundary = i == 0 || !is_ident_continue(bytes[i - 1] as char);
 
-        match c {
-            '\'' => in_single = true,
-            '"' => in_double = true,
-            // `/` in operand position (after `=~`, `!~`, `(`, `,` or `=`)
-            // starts a regex literal; a `;` inside it must not split.
-            '/' if matches!(prev_sig, Some('~') | Some('(') | Some(',') | Some('=')) => {
-                in_regex = true;
-            }
-            ';' if begin_depth == 0 => {
-                let slice = input[start..i].trim();
-                if !slice.is_empty() {
-                    statements.push(slice.to_string());
-                }
-                start = i + 1;
-            }
-            _ if is_ident_start(c) && at_word_boundary && matches_keyword_at(input, i, "BEGIN") => {
-                begin_depth += 1
+        if sc.ch == ';' && begin_depth == 0 && !sc.masked && sc.depth == 0 {
+            let slice = input[start..sc.idx].trim();
+            if !slice.is_empty() {
+                statements.push(slice.to_string());
             }
-            _ if is_ident_start(c)
-                && at_word_boundary
-                && begin_depth > 0
-                && matches_keyword_at(input, i, "END") =>
-            {
-                begin_depth -= 1;
-            }
-            _ => {}
-        }
-        if !c.is_whitespace() {
-            prev_sig = Some(c);
+            start = sc.idx + sc.ch.len_utf8();
         }
-        i += 1;
     }
 
     let tail = input[start..].trim();
@@ -167,15 +105,34 @@ pub fn split_statements(input: &str) -> Result<Vec<String>, HyperbytedbError> {
     Ok(statements)
 }
 
-fn matches_keyword_at(input: &str, start: usize, kw: &str) -> bool {
-    // Byte-wise compare: slicing `rest[..kw.len()]` panics when a multibyte
-    // char straddles the boundary (e.g. an identifier containing `fi`).
-    let rest = input.as_bytes().get(start..);
-    let Some(rest) = rest else { return false };
-    if rest.len() < kw.len() || !rest[..kw.len()].eq_ignore_ascii_case(kw.as_bytes()) {
+/// Match an ASCII keyword at scan index `si` when it is not inside a masked span.
+fn matches_unmasked_keyword_at(input: &str, scan: &[ScannedChar], si: usize, kw: &str) -> bool {
+    let sc = &scan[si];
+    if sc.masked || sc.depth != 0 {
         return false;
     }
-    !matches!(rest.get(kw.len()), Some(b) if is_ident_continue(*b as char))
+    if si > 0 {
+        let prev = scan[si - 1].ch;
+        if prev.is_alphanumeric() || prev == '_' {
+            return false;
+        }
+    }
+
+    let start = sc.idx;
+    let bytes = input.as_bytes();
+    if start + kw.len() > bytes.len()
+        || !bytes[start..start + kw.len()].eq_ignore_ascii_case(kw.as_bytes())
+    {
+        return false;
+    }
+
+    if start + kw.len() < bytes.len() {
+        let next = bytes[start + kw.len()] as char;
+        if next.is_alphanumeric() || next == '_' {
+            return false;
+        }
+    }
+    true
 }
 
 /// Sum compound duration text (e.g. `1h30m`, `0`, `INF`) to nanoseconds.
@@ -401,9 +358,6 @@ struct Lexer<'a> {
     input: &'a str,
     chars: Vec<(usize, char)>,
     pos: usize,
-    /// Kind of the previously emitted token, used to decide whether a `/`
-    /// begins a regex literal (operand position after `=~`/`!~`) or division.
-    last_kind: Option<TokenKind>,
 }
 
 impl<'a> Lexer<'a> {
@@ -412,7 +366,6 @@ impl<'a> Lexer<'a> {
             input,
             chars: input.char_indices().collect(),
             pos: 0,
-            last_kind: None,
         }
     }
 
@@ -439,9 +392,7 @@ impl<'a> Lexer<'a> {
     }
 
     fn next_token(&mut self) -> Result<Token, HyperbytedbError> {
-        let tok = self.scan_token()?;
-        self.last_kind = Some(tok.kind.clone());
-        Ok(tok)
+        self.scan_token()
     }
 
     fn scan_token(&mut self) -> Result<Token, HyperbytedbError> {
@@ -592,15 +543,7 @@ impl<'a> Lexer<'a> {
             '\'' => self.read_string_lit(start),
             '"' => self.read_ident_quoted(start),
             '/' => {
-                // A `/` only starts a regex literal in operand position, i.e.
-                // immediately after a regex-match operator. Anywhere else it is
-                // division; treating every `/` as a regex made a lone `/` (e.g.
-                // arithmetic in a WHERE clause) abort tokenization of the whole
-                // statement.
-                if matches!(
-                    self.last_kind,
-                    Some(TokenKind::MatchRegex) | Some(TokenKind::NotMatchRegex)
-                ) {
+                if is_regex_start_at(self.input, start) {
                     self.read_regex(start)
                 } else {
                     self.bump_char();
@@ -992,6 +935,28 @@ mod tests {
         );
     }
 
+    #[test]
+    fn regex_measurement_after_from_keyword() {
+        let toks = tokenize("SHOW SERIES FROM /^cpu/").unwrap();
+        assert!(
+            toks.iter()
+                .any(|t| matches!(&t.kind, TokenKind::Regex(r) if r == "^cpu")),
+            "FROM /^cpu/ must tokenize as regex measurement: {toks:?}"
+        );
+    }
+
+    #[test]
+    fn split_and_tokenize_agree_on_regex_measurement() {
+        let input = "SHOW SERIES FROM /^cpu/; SHOW DATABASES";
+        let stmts = split_statements(input).unwrap();
+        assert_eq!(stmts.len(), 2);
+        let toks = tokenize(&stmts[0]).unwrap();
+        assert!(
+            toks.iter()
+                .any(|t| matches!(&t.kind, TokenKind::Regex(r) if r == "^cpu"))
+        );
+    }
+
     #[test]
     fn regex_swallows_keyword_like_content() {
         // The inner LIMIT belongs to the regex; only the trailing one is a
@@ -1030,4 +995,48 @@ mod tests {
         assert_eq!(stmts.len(), 2);
         assert!(stmts[0].ends_with("END"));
     }
+
+    #[test]
+    fn split_statements_ignores_semicolon_inside_parens() {
+        let stmts = split_statements("SELECT * FROM t WHERE x IN (1; 2); SHOW DATABASES").unwrap();
+        assert_eq!(stmts.len(), 2);
+        assert_eq!(stmts[0], "SELECT * FROM t WHERE x IN (1; 2)");
+        assert_eq!(stmts[1], "SHOW DATABASES");
+    }
+
+    #[test]
+    fn split_statements_fails_on_unclosed_paren_in_batch() {
+        let err = split_statements("SELECT ( FROM cpu; SHOW DATABASES").unwrap_err();
+        assert!(
+            err.to_string().contains("unclosed"),
+            "expected unclosed paren error, got: {err}"
+        );
+    }
+
+    #[test]
+    fn split_statements_ignores_begin_end_inside_string_literals() {
+        let stmts = split_statements(
+            "CREATE CONTINUOUS QUERY cq ON db BEGIN SELECT * FROM m WHERE msg = 'BEGIN; END'; END; SHOW DATABASES",
+        )
+        .unwrap();
+        assert_eq!(
+            stmts.len(),
+            2,
+            "BEGIN/END substrings inside quoted literals must not affect block depth: {stmts:?}"
+        );
+        assert!(stmts[0].contains("'BEGIN; END'"));
+        assert_eq!(stmts[1], "SHOW DATABASES");
+    }
+
+    #[test]
+    fn split_statements_does_not_treat_quoted_begin_as_block_open() {
+        let stmts = split_statements("SELECT 'BEGIN'; SHOW DATABASES").unwrap();
+        assert_eq!(
+            stmts.len(),
+            2,
+            "quoted BEGIN must not suppress splitting: {stmts:?}"
+        );
+        assert_eq!(stmts[0], "SELECT 'BEGIN'");
+        assert_eq!(stmts[1], "SHOW DATABASES");
+    }
 }

hyperbytedb/src/timeseriesql/mod.rs

@@ -3,6 +3,7 @@ pub mod ddl_parser;
 pub mod digest;
 pub mod lexer;
 pub mod parser;
+pub(crate) mod scan;
 pub mod to_clickhouse;
 
 use crate::error::HyperbytedbError;