diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 0000000..0c82311 --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,23 @@ +name: Deploy docs + +on: + workflow_dispatch: + +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Deploy + uses: cloudflare/wrangler-action@ebbaa1584979971c8614a24965b4405ff95890e0 # v4.0.0 + with: + apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} + accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + command: pages deploy docs --project-name=mq-db diff --git a/.gitignore b/.gitignore index 967a11b..cc48c84 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ target .cargo *.mq-db .DS_Store +/work diff --git a/README.md b/README.md index 2f6e563..cbe3ba4 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![ci](https://img.shields.io/github/actions/workflow/status/harehare/mq-db/ci.yml?logo=github-actions&label=ci)](https://github.com/harehare/mq-db/actions/workflows/ci.yml) [![audit](https://img.shields.io/github/actions/workflow/status/harehare/mq-db/audit.yml?logo=shield&label=audit)](https://github.com/harehare/mq-db/actions/workflows/audit.yml) -[![license](https://img.shields.io/badge/license-MIT-blue)](LICENSE) +[![license](https://img.shields.io/badge/license-MIT-b3402c)](LICENSE) ![demo](./assets/demo.gif) @@ -16,6 +16,7 @@ `mq-db` treats Markdown documents as **structured, hierarchical databases** rather than plain text. It parses Markdown into a flat block list with an **interval index** (Nested Set / Pre-Post Order), enabling O(1) section hierarchy queries. Documents can be queried with **SQL** or **[mq](https://github.com/harehare/mq)** and persisted to a compact custom page-file format. ```mermaid +%%{init: {'theme':'base', 'themeVariables': {'primaryColor':'#f2ebdb','primaryTextColor':'#2a2420','primaryBorderColor':'#b3402c','lineColor':'#b3402c','secondaryColor':'#e3c3b7','tertiaryColor':'#faf6ef','background':'#faf6ef','fontFamily':'JetBrains Mono, monospace'}}}%% flowchart TD A["Markdown File(s)"] -->|"CST Parser (mq-markdown)"| B["Block Tree\n(heading · paragraph · code · list …)"] B -->|"Interval Index + Secondary Indexes"| C["Flat Block Vector\n(pre/post integers)"] @@ -525,6 +526,7 @@ struct Block { mq-db applies three complementary index layers, cheapest-first. ```mermaid +%%{init: {'theme':'base', 'themeVariables': {'primaryColor':'#f2ebdb','primaryTextColor':'#2a2420','primaryBorderColor':'#b3402c','lineColor':'#b3402c','secondaryColor':'#e3c3b7','tertiaryColor':'#faf6ef','background':'#faf6ef','fontFamily':'JetBrains Mono, monospace'}}}%% flowchart LR Q["SQL Query"] --> ZM["Layer 1\nZone Maps\n(document skip)"] ZM -->|"relevant docs"| II["Layer 2\nInterval Index\n(section scope)"] @@ -553,6 +555,7 @@ Built once per document and stored in the `.mq-db` file. Checked before any bloc Heading hierarchy encoded as `(pre, post)` pairs via Pre-Post Order (Nested Set) traversal: ```mermaid +%%{init: {'theme':'base', 'themeVariables': {'primaryColor':'#f2ebdb','primaryTextColor':'#2a2420','primaryBorderColor':'#b3402c','lineColor':'#b3402c','secondaryColor':'#e3c3b7','tertiaryColor':'#faf6ef','background':'#faf6ef','fontFamily':'JetBrains Mono, monospace'}}}%% graph TD doc["# Doc\npre=0 · post=11"] secA["## Section A\npre=2 · post=7"] @@ -581,6 +584,7 @@ graph TD SQL predicate pushdown picks an `IndexHint`: ```mermaid +%%{init: {'theme':'base', 'themeVariables': {'primaryColor':'#f2ebdb','primaryTextColor':'#2a2420','primaryBorderColor':'#b3402c','lineColor':'#b3402c','secondaryColor':'#e3c3b7','tertiaryColor':'#faf6ef','background':'#faf6ef','fontFamily':'JetBrains Mono, monospace'}}}%% flowchart TD P["SQL WHERE predicate"] P -->|"block_type = '...'"| B["BitmapIndex"] @@ -597,6 +601,7 @@ flowchart TD Custom 8 KB page file: ```mermaid +%%{init: {'theme':'base', 'themeVariables': {'primaryColor':'#f2ebdb','primaryTextColor':'#2a2420','primaryBorderColor':'#b3402c','lineColor':'#b3402c','secondaryColor':'#e3c3b7','tertiaryColor':'#faf6ef','background':'#faf6ef','fontFamily':'JetBrains Mono, monospace'}}}%% graph TD P0["Page 0 — File Header\nmagic 0x4D514442 · version · page count"] P1["Page 1 — Catalog\ndoc_id → first_block_page · num_blocks · ZoneMaps"] diff --git a/docs/assets/logo.svg b/docs/assets/logo.svg new file mode 100644 index 0000000..c6c196e --- /dev/null +++ b/docs/assets/logo.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + diff --git a/docs/assets/ogp.png b/docs/assets/ogp.png new file mode 100644 index 0000000..2168465 Binary files /dev/null and b/docs/assets/ogp.png differ diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..3a23bc1 --- /dev/null +++ b/docs/index.html @@ -0,0 +1,589 @@ + + + + + + + mq-db – a Markdown-specialized embedded database in Rust + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+
+

+ Your Markdown
+ is already a database. +

+

+ mq-db parses Markdown into a flat block list with an interval + index (Nested Set / Pre-Post Order), turning heading hierarchy + into O(1) integer comparisons. Query it with + SQL or + mq, + persist it to a compact custom page-file format — no SQLite + dependency. +

+
+ install mq-db → + view source +
+ + +
+

the source, annotated at index time

+
+
+ README.md + block_type · pre / post +
+
+
+ # mq-db + heading · 1 / 18 +
+
+ mq-db parses Markdown into a flat block list with an interval index. + paragraph · 2 / 3 +
+
+ ## Architecture + heading · 4 / 17 +
+
+ Every heading opens an interval; blocks inherit pre/post bounds. + paragraph · 5 / 6 +
+
+ - Bitmap index on block_type + list_item · 8 / 9 +
+
+ - BTree index on (pre, post) + list_item · 10 / 11 +
+
+ - Hash index on content, lang, depth + list_item · 12 / 13 +
+
+ ```sql SELECT block_type, count(*) FROM blocks GROUP BY 1; ``` + code · 15 / 16 +
+
+
+

1 every block gets a pre/post pair at index time — "is X inside Y?" becomes one integer range comparison, not a tree walk.

+

2 the query planner picks a BitmapIndex, BTreeIndex, or HashIndex automatically, based on the predicate.

+
+
+
+
+
+ + +
+
+
+

+ Hierarchy checks
without a tree walk. +

+

+ Most Markdown tooling answers "what's inside this section?" by + walking a tree. mq-db assigns every block a + pre/post + pair (Nested Set encoding) at index time, so an ancestor + check becomes a single interval comparison — + under(pre, post, anc_pre, anc_post). +

+

+ Three secondary index layers push SQL predicates down before a + single block is materialized: a BitmapIndex + on block type, a BTreeIndex on pre/post, and a + HashIndex on content/lang/depth. +

+
+
+
+
+ sql> RAG extraction under a section + under() +
+ 
-- all text/code nested under the "Architecture" heading
+SELECT b.block_type, b.content
+FROM blocks b
+WHERE under(b.pre, b.post,
+  (SELECT pre  FROM blocks WHERE block_type = 'heading' AND content = 'Architecture'),
+  (SELECT post FROM blocks WHERE block_type = 'heading' AND content = 'Architecture'))
+  AND b.block_type IN ('paragraph', 'code')
+ORDER BY b.pre;
+
+

fig. 1 — scoping a query to everything nested under a heading, via a single interval comparison.

+
+
+

O(1) ancestry

+

+ Interval index turns "is X inside Y?" into an integer range check. +

+
+
+

Dual engines

+

+ A custom sqlparser-based SQL evaluator, and mq via mq-lang — no SQLite. +

+
+
+

Zone pruning

+

+ Per-document stats skip irrelevant files before any block is scanned. +

+
+
+
+
+
+ + +
+
+

SQL, joins included

+

+ documents and blocks are + plain virtual tables — join them like any relational schema, plus + self-joins over the interval index for structural checks. +

+
+
+
+
+ documents ⋈ blocks + JOIN +
+ 
-- documents that contain Python code
+SELECT DISTINCT d.path
+FROM documents d
+JOIN blocks b ON b.document_id = d.id
+WHERE b.block_type = 'code' AND b.lang = 'python';
+
+

fig. 2 — a plain relational join across documents and blocks.

+
+
+
+
+ self-join over pre/post + JOIN ×2 +
+ 
-- H2 headings immediately followed by a list
+SELECT d.path, h.content AS heading
+FROM blocks h
+JOIN blocks nxt ON nxt.document_id = h.document_id
+                 AND nxt.pre = h.pre + 1
+JOIN documents d ON d.id = h.document_id
+WHERE h.block_type = 'heading' AND h.depth = 2
+  AND nxt.block_type = 'list';
+
+

fig. 3 — a structural self-join, one row apart in pre-order.

+
+
+
+
+ mq() scalar function + inline mq +
+ 
-- run an mq program against Markdown content, in SQL
+SELECT mq('.h1 | to_text', content) AS title
+FROM blocks
+WHERE block_type = 'code' AND lang = 'markdown';
+
+

fig. 4 — mq embedded as a scalar function inside a SQL projection.

+
+
+
+
+ CASE + aggregates + GROUP BY +
+ 
-- bucket headings by depth, summarize per bucket
+SELECT
+  CASE WHEN depth <= 1 THEN 'top-level' ELSE 'nested' END AS bucket,
+  count(*),
+  group_concat(initcap(trim(content)), ', ') AS headings
+FROM blocks
+WHERE block_type = 'heading'
+GROUP BY CASE WHEN depth <= 1 THEN 'top-level' ELSE 'nested' END;
+
+

fig. 5 — CASE expressions and aggregates over heading depth.

+
+
+
+
+ + +
+
+
+

Index

+ 9 entries +
+
+
+
+ 01 +

Flat block storage

+ +
+

Every Markdown element becomes a typed Block with row-polymorphic properties.

+
+
+
+ 02 +

O(1) hierarchy queries

+ +
+

Interval index (pre/post) makes ancestor/descendant checks a single comparison.

+
+
+
+ 03 +

Three-layer indexes

+ +
+

Bitmap, BTree, and Hash indexes for fast SQL predicate pushdown.

+
+
+
+ 04 +

Zone maps

+ +
+

Per-document statistics skip irrelevant files before scanning any blocks.

+
+
+
+ 05 +

Dual query engines

+ +
+

SQL via a custom sqlparser-based evaluator, and mq via mq-lang.

+
+
+
+ 06 +

DDL support

+ +
+

CREATE TABLE, INSERT INTO, DROP TABLE for in-memory custom tables.

+
+
+
+ 07 +

Function library

+ +
+

String, numeric, null-handling, CASE, and aggregate functions comparable to a general-purpose RDBMS.

+
+
+
+ 08 +

Custom page storage

+ +
+

8 KB fixed pages, checksums, atomic writes to a single-file format.

+
+
+
+ 09 +

CLI + REPL + TUI

+ +
+

Full terminal experience, plus an HTTP server for SQL and mq over JSON.

+
+
+
+
+ + +
+
+

Install

+
+
+ + + +
+
+
+ +

# downloads, verifies (SHA256), and installs to ~/.local/bin

+

curl -fsSL https://raw.githubusercontent.com/harehare/mq-db/main/bin/install.sh | bash

+
+ + +
+
+

+ mq-db index docs/ --recursive && + mq-db sql "SELECT block_type, count(*) FROM blocks GROUP BY block_type" +

+
+
+
+ + + + + + + diff --git a/src/bin/mq-db.rs b/src/bin/mq-db.rs index 7f12ecd..de05125 100644 --- a/src/bin/mq-db.rs +++ b/src/bin/mq-db.rs @@ -14,7 +14,7 @@ use axum::{ routing::{get, post}, }; use clap::{Parser, Subcommand, ValueEnum}; -use mq_db::{DocumentStore, MqEngine, SqlEngine, block::BlockType, sql::html_escape}; +use mq_db::{DocumentStore, MqEngine, MqdbError, SqlEngine, block::BlockType, sql::html_escape}; use serde::Deserialize; // ───────────────────────────────────────────────────────────────────────────── @@ -231,6 +231,39 @@ fn is_markdown(path: &Path) -> bool { ) } +/// Reads every file in `files`, in order, using a small worker-thread pool — +/// indexing many small files is I/O-latency bound, not CPU bound. +fn read_files_parallel(files: &[PathBuf]) -> Vec> { + let worker_count = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + .min(files.len().max(1)); + if worker_count <= 1 { + return files + .iter() + .map(|p| std::fs::read_to_string(p).map_err(MqdbError::from)) + .collect(); + } + + let chunk_size = files.len().div_ceil(worker_count); + std::thread::scope(|scope| { + files + .chunks(chunk_size) + .map(|chunk| { + scope.spawn(move || { + chunk + .iter() + .map(|p| std::fs::read_to_string(p).map_err(MqdbError::from)) + .collect::>() + }) + }) + .collect::>() + .into_iter() + .flat_map(|handle| handle.join().expect("file-read worker thread panicked")) + .collect() + }) +} + fn load_store(db: &Path) -> anyhow::Result { if !db.exists() { anyhow::bail!( @@ -340,8 +373,9 @@ async fn main() -> anyhow::Result<()> { if no_spans { store.set_store_spans(false); } - for path in &files { - match store.add_file(path) { + let contents = read_files_parallel(&files); + for (path, content) in files.iter().zip(contents) { + match content.and_then(|c| store.add_str_with_path(&c, Some(path.clone()))) { Ok(_) => eprintln!(" ✓ {}", path.display()), Err(e) => { eprintln!(" ✗ {}: {}", path.display(), e); diff --git a/src/indexes.rs b/src/indexes.rs index 7f17a11..e81806c 100644 --- a/src/indexes.rs +++ b/src/indexes.rs @@ -270,7 +270,10 @@ impl DocumentIndex { out.extend_from_slice(&(content_entries.len() as u32).to_le_bytes()); for (key, indices) in &content_entries { let kb = key.as_bytes(); - out.extend_from_slice(&(kb.len() as u16).to_le_bytes()); + // Block content is unbounded (e.g. a large code block or table cell can + // exceed 64KB), so the length prefix must be u32 — a u16 here would + // silently wrap and desync the rest of the index stream. + out.extend_from_slice(&(kb.len() as u32).to_le_bytes()); out.extend_from_slice(kb); out.extend_from_slice(&(indices.len() as u32).to_le_bytes()); for &idx in indices.iter() { @@ -284,7 +287,7 @@ impl DocumentIndex { out.extend_from_slice(&(lang_entries.len() as u32).to_le_bytes()); for (key, indices) in &lang_entries { let kb = key.as_bytes(); - out.extend_from_slice(&(kb.len() as u16).to_le_bytes()); + out.extend_from_slice(&(kb.len() as u32).to_le_bytes()); out.extend_from_slice(kb); out.extend_from_slice(&(indices.len() as u32).to_le_bytes()); for &idx in indices.iter() { @@ -321,17 +324,6 @@ impl DocumentIndex { v }}; } - macro_rules! read_u16 { - () => {{ - let end = pos + 2; - if end > data.len() { - return Err(MqdbError::Storage("unexpected end of index data".into())); - } - let v = u16::from_le_bytes(data[pos..end].try_into().unwrap()); - pos = end; - v - }}; - } macro_rules! read_u32 { () => {{ let end = pos + 4; @@ -391,7 +383,7 @@ impl DocumentIndex { let num_content = read_u32!() as usize; let mut by_content: HashMap> = HashMap::new(); for _ in 0..num_content { - let key_len = read_u16!() as usize; + let key_len = read_u32!() as usize; let key = read_str!(key_len); let count = read_u32!() as usize; let mut indices = Vec::with_capacity(count); @@ -405,7 +397,7 @@ impl DocumentIndex { let num_lang = read_u32!() as usize; let mut by_lang: HashMap> = HashMap::new(); for _ in 0..num_lang { - let key_len = read_u16!() as usize; + let key_len = read_u32!() as usize; let key = read_str!(key_len); let count = read_u32!() as usize; let mut indices = Vec::with_capacity(count); diff --git a/src/sql.rs b/src/sql.rs index d7e3b06..fd675e9 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -130,6 +130,31 @@ impl Value { } } +/// Hashable projection of [`Value`], mirroring its derived `PartialEq` (no +/// cross-variant coercion, `NULL` equals `NULL`, `NaN` matches nothing). +#[derive(PartialEq, Eq, Hash)] +enum JoinKey { + Str(String), + Int(i64), + Bool(bool), + FloatBits(u64), + Null, +} + +fn value_join_key(v: &Value) -> Option { + match v { + Value::Str(s) => Some(JoinKey::Str(s.clone())), + Value::Int(i) => Some(JoinKey::Int(*i)), + Value::Bool(b) => Some(JoinKey::Bool(*b)), + Value::Null => Some(JoinKey::Null), + Value::Float(f) if f.is_nan() => None, // NaN matches nothing + Value::Float(f) => { + let normalized = if *f == 0.0 { 0.0 } else { *f }; + Some(JoinKey::FloatBits(normalized.to_bits())) + } + } +} + #[derive(Debug, Clone)] struct Row { columns: Vec, @@ -514,6 +539,50 @@ fn cross_join(left: Vec, right: Vec) -> Vec { out } +/// Equi-join fast path: hashes `right` by `right_key_expr` and probes it with +/// `left_key_expr` per left row instead of the full `left * right` cross +/// product. `full_predicate` is still checked per candidate pair, so results +/// match `cross_join` + `.retain(full_predicate)` exactly. +fn hash_equi_join( + left: Vec, + right: Vec, + left_key_expr: &Expr, + right_key_expr: &Expr, + full_predicate: &Expr, +) -> Vec { + let mut buckets: HashMap> = HashMap::new(); + for (i, r) in right.iter().enumerate() { + if let Some(key) = value_join_key(&eval_expr(right_key_expr, r)) { + buckets.entry(key).or_default().push(i); + } + } + + let mut out = Vec::new(); + for l in &left { + let Some(key) = value_join_key(&eval_expr(left_key_expr, l)) else { + continue; + }; + let Some(candidates) = buckets.get(&key) else { + continue; + }; + for &i in candidates { + let r = &right[i]; + let mut cols = l.columns.clone(); + cols.extend(r.columns.iter().cloned()); + let mut vals = l.values.clone(); + vals.extend(r.values.iter().cloned()); + let combined = Row { + columns: cols, + values: vals, + }; + if eval_expr(full_predicate, &combined).is_truthy() { + out.push(combined); + } + } + } + out +} + fn eval_sql_value(v: &SqlValue) -> Value { match v { SqlValue::Number(n, _) => { @@ -1807,16 +1876,28 @@ impl<'a> SqlEngine<'a> { for join in &from[0].joins { // Joined tables always full-scan (join partner) let right = self.table_rows_with_hint(&join.relation, &IndexHint::FullScan, None)?; - rows = cross_join(rows, right); match &join.join_operator { JoinOperator::Inner(JoinConstraint::On(on)) | JoinOperator::Join(JoinConstraint::On(on)) | JoinOperator::Left(JoinConstraint::On(on)) | JoinOperator::LeftOuter(JoinConstraint::On(on)) => { let resolved = self.resolve_subqueries(on)?; - rows.retain(|row| eval_expr(&resolved, row).is_truthy()); + let left_cols = rows.first().map(|r| r.columns.clone()).unwrap_or_default(); + let right_cols = right.first().map(|r| r.columns.clone()).unwrap_or_default(); + rows = match find_equi_join_exprs(&resolved, &left_cols, &right_cols) { + Some((left_key, right_key)) => { + hash_equi_join(rows, right, left_key, right_key, &resolved) + } + None => { + let mut combined = cross_join(rows, right); + combined.retain(|row| eval_expr(&resolved, row).is_truthy()); + combined + } + }; + } + _ => { + rows = cross_join(rows, right); } - _ => {} } } for twj in from.iter().skip(1) { @@ -2311,6 +2392,46 @@ fn flatten_and_conjuncts(expr: &Expr) -> Vec<&Expr> { } } +/// Whether `schema` has a column matching `short` (an already-lowercased, +/// unqualified name from [`expr_col_name`]). Mirrors `Row::get`'s fallback. +fn schema_has_short_col(schema: &[String], short: &str) -> bool { + schema.iter().any(|c| { + let cl = c.to_lowercase(); + cl == short || cl.split('.').next_back().unwrap_or(&cl) == short + }) +} + +/// First top-level `AND`-conjunct of `on` that is a plain `column = column` +/// equality across `left_cols`/`right_cols`, as `(left_key_expr, +/// right_key_expr)`. `None` if there's no such conjunct (e.g. only a +/// computed key like `nxt.pre = h.pre + 1`) — caller falls back to cross-join. +fn find_equi_join_exprs<'a>( + on: &'a Expr, + left_cols: &[String], + right_cols: &[String], +) -> Option<(&'a Expr, &'a Expr)> { + for conjunct in flatten_and_conjuncts(on) { + let Expr::BinaryOp { + left, + op: BinaryOperator::Eq, + right, + } = conjunct + else { + continue; + }; + let (Some(lname), Some(rname)) = (expr_col_name(left), expr_col_name(right)) else { + continue; + }; + if schema_has_short_col(left_cols, &lname) && schema_has_short_col(right_cols, &rname) { + return Some((left, right)); + } + if schema_has_short_col(right_cols, &lname) && schema_has_short_col(left_cols, &rname) { + return Some((right, left)); + } + } + None +} + /// Decides whether a whole document can be skipped using [`ZoneMaps`], /// without reading any of its blocks. Unlike [`IndexHint`], a wrong skip /// here silently drops matching rows, so this only returns `true` when it diff --git a/src/storage.rs b/src/storage.rs index edbc14a..7a8db27 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -156,7 +156,8 @@ impl Storage { entries: &[CatalogEntry], custom_tables: &[CustomTableEntry], ) -> Result<(), MqdbError> { - write_catalog(&mut self.page_file, entries, custom_tables) + write_catalog(&mut self.page_file, entries, custom_tables)?; + self.page_file.sync_header() } /// Read the catalog. @@ -640,6 +641,34 @@ mod tests { cleanup(&path); } + #[test] + fn persisted_index_round_trip_large_block_content() { + use crate::indexes::DocumentIndex; + + // A single block whose content exceeds 64KB must not desync the + // index's by_content length prefix (was u16, truncating/wrapping). + let path = test_file_path("index-round-trip-large-block"); + cleanup(&path); + + let big_code = "x".repeat(70_000); + let content = format!("# Title\n\n```text\n{big_code}\n```\n"); + + let mut store = DocumentStore::new(); + store.add_str(&content).unwrap(); + store.save(&path).unwrap(); + + let mut opened = DocumentStore::open(&path).unwrap(); + opened.load_all_blocks().unwrap(); + opened.load_all_indexes().unwrap(); + + let doc = &opened.documents()[0]; + let from_file = opened.get_doc_index(0).unwrap().clone(); + let from_blocks = DocumentIndex::build(&doc.blocks); + assert_eq!(from_file.to_bytes(), from_blocks.to_bytes()); + + cleanup(&path); + } + #[rstest] #[case(BlockType::Heading)] #[case(BlockType::Paragraph)] diff --git a/src/storage/page.rs b/src/storage/page.rs index c07e4d1..bc53cb1 100644 --- a/src/storage/page.rs +++ b/src/storage/page.rs @@ -19,7 +19,7 @@ pub(crate) const PAGE_TYPE_INDEX: u32 = 5; pub(crate) const PAGE_TYPE_TABLE_DATA: u32 = 6; const FILE_MAGIC: u32 = 0x4D51_4442; -const FILE_VERSION: u32 = 3; +const FILE_VERSION: u32 = 4; const CATALOG_START_PAGE: u32 = 1; fn invalid_data(message: impl Into) -> MqdbError { @@ -77,6 +77,9 @@ pub fn make_page(page_type: u32, page_id: u32, next_page: u32, body: &[u8]) -> [ pub struct PageFile { file: File, pub num_pages: u32, + /// `true` if `num_pages` has advanced since the header page was last + /// written to disk; `append_page` no longer writes it eagerly. + header_dirty: bool, } impl PageFile { @@ -87,7 +90,11 @@ impl PageFile { .create(true) .truncate(true) .open(path)?; - let mut page_file = Self { file, num_pages: 1 }; + let mut page_file = Self { + file, + num_pages: 1, + header_dirty: false, + }; page_file.write_file_header()?; Ok(page_file) } @@ -146,7 +153,11 @@ impl PageFile { )); } - Ok(Self { file, num_pages }) + Ok(Self { + file, + num_pages, + header_dirty: false, + }) } pub fn read_page(&mut self, page_id: u32) -> Result<[u8; PAGE_SIZE], MqdbError> { @@ -195,10 +206,20 @@ impl PageFile { .num_pages .checked_add(1) .ok_or_else(|| invalid_data("page count overflow"))?; - self.write_file_header()?; + self.header_dirty = true; Ok(page_id) } + /// Persists `num_pages` if it changed since the last write. Must run + /// before the file is reopened from disk (see `Storage::flush_catalog`). + pub fn sync_header(&mut self) -> Result<(), MqdbError> { + if self.header_dirty { + self.write_file_header()?; + self.header_dirty = false; + } + Ok(()) + } + fn write_file_header(&mut self) -> Result<(), MqdbError> { let body = file_header_body(self.num_pages); let page = make_page(PAGE_TYPE_FILE_HEADER, 0, 0, &body); diff --git a/src/store.rs b/src/store.rs index 5ce6dd2..7481c76 100644 --- a/src/store.rs +++ b/src/store.rs @@ -180,7 +180,10 @@ impl DocumentStore { self.add_str_with_path(content, None) } - fn add_str_with_path( + /// Parses and adds already-read Markdown content, attributing it to + /// `path`. For callers that read files concurrently and want to skip + /// [`add_file`](Self::add_file)'s own read. + pub fn add_str_with_path( &mut self, content: &str, path: Option, diff --git a/src/tui.rs b/src/tui.rs index 215b828..3f0377f 100644 --- a/src/tui.rs +++ b/src/tui.rs @@ -526,7 +526,7 @@ fn render_doc_list(f: &mut Frame, app: &mut App, area: Rect) { if title.is_empty() { String::new() } else { - format!(" {}", &title[..title.len().min(18)]) + format!(" {}", title.chars().take(18).collect::()) } ), Style::default().fg(Color::DarkGray),