diff --git a/README.md b/README.md index 14499e9..4f0a982 100644 --- a/README.md +++ b/README.md @@ -32,9 +32,29 @@ The interesting parts: - **Complete `.pyi` stubs inside the wheel**, checked against the runtime in CI, so mypy and pyright and your editor all work with no extra install. - **Graph values are real classes.** `Node`, `Rel`, and `Path` have `.labels`, `.id`, `.properties`, and an HTML repr. Not dicts, because a dict cannot tell a property named `labels` apart from the label set. +## Building a graph + +A statement writes one row at a time, which is the wrong shape for loading data and cannot make a rel table at all. `load` is the other shape: a table's columns whole, the edges between them whole, one file written once. + +```python +zudb.load( + "social.zu1", + nodes="person", + rels="knows", + columns={"uid": [1, 2, 3], "name": ["ada", "grace", "kay"]}, + edges=[(0, 1), (1, 2)], +) + +with zudb.connect("social.zu1", read_only=True) as conn: + for a, b in conn.execute("MATCH (a:person)-[:knows]->(b:person) RETURN a.name AS a, b.name AS b"): + print(a, "knows", b) +``` + +Edges name rows by position, counting from zero, because at load time a row has no other name. Columns may hold booleans, integers, floats, strings, dates, times, datetimes or durations, one kind to a column, and the GIL is released for the write. + ## What works today -The list above is what this client is for. What it does so far is the core of it: `connect`, `execute` and `sql` with named parameters, results that iterate and fetch, values as Python objects both ways including dates, times, datetimes and durations, `Node`, `Rel` and `Path` as classes, every condition as an exception class carrying its code, its position and its documentation link, and the GIL released around every statement. Arrow, `register`, the stubs and the interrupt are next, and each one lands with the tests that say it works. +The list above is what this client is for. What it does so far is the core of it: `connect`, `execute` and `sql` with named parameters, results that iterate and fetch, values as Python objects both ways including dates, times, datetimes and durations, `Node`, `Rel` and `Path` as classes, `load` for building a graph with edges in it, every condition as an exception class carrying its code, its position and its documentation link, and the GIL released around every statement and every load. Arrow, `register`, the stubs and the interrupt are next, and each one lands with the tests that say it works. ## Wheels diff --git a/python/zudb/__init__.py b/python/zudb/__init__.py index 6a41e80..9c79e37 100644 --- a/python/zudb/__init__.py +++ b/python/zudb/__init__.py @@ -14,7 +14,17 @@ from __future__ import annotations -from ._zudb import Connection, Duration, Node, Path, Rel, Result, __abi_version__, connect +from ._zudb import ( + Connection, + Duration, + Node, + Path, + Rel, + Result, + __abi_version__, + connect, + load, +) from .errors import ( ConnectionError, DataError, @@ -30,6 +40,7 @@ __all__ = [ "connect", + "load", "Connection", "Result", "Node", diff --git a/src/lib.rs b/src/lib.rs index 54c33e7..217dcae 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,6 +14,7 @@ mod conn; mod error; +mod load; mod value; use std::path::PathBuf; @@ -46,6 +47,7 @@ fn _zudb(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add("__engine_version__", env!("CARGO_PKG_VERSION"))?; module.add("__abi_version__", zudb::C_ABI_VERSION)?; module.add_function(wrap_pyfunction!(connect, module)?)?; + module.add_function(wrap_pyfunction!(load::load, module)?)?; module.add_class::()?; module.add_class::()?; module.add_class::()?; diff --git a/src/load.rs b/src/load.rs new file mode 100644 index 0000000..cc7ef0f --- /dev/null +++ b/src/load.rs @@ -0,0 +1,385 @@ +//! Building a database out of columns and an edge list. +//! +//! A row at a time through `INSERT` is the wrong shape for loading +//! data and the wrong shape for making a graph: every row is parsed, +//! bound and committed, and a rel table cannot be made that way at +//! all, because the statement that would make one says which two +//! tables it joins only for the edge it is writing. This is the other +//! shape, and it is the one the C ABI's loader has: a table's columns +//! whole, an edge list whole, one file written once. +//! +//! What it writes is what a bulk load writes: a node table with a row +//! per element of every column, a rel table holding the edges between +//! those rows, and a primary-key index over the rows so a lookup by +//! key does not scan. Edges name rows by position, counting from zero, +//! because at load time a row has no other name. + +use std::path::PathBuf; + +use pyo3::prelude::*; +use pyo3::types::{PyBool, PyDate, PyDateTime, PyDelta, PyDict, PyTime}; +use zu_common::DurationKind; +use zu_common::temporal::days_from_civil; +use zudb::zu1::file::Zu1File; +use zudb::zu1::graph::bulk_load_keyed; +use zudb::zu1::props::{PropValues, store_props}; + +use crate::error::to_py_err; +use crate::value::{Duration, clock_nanos}; + +/// One column, reduced to the vector the property store keeps it in. +/// +/// Owned rather than borrowed from the caller's lists, because a +/// Python list holds objects and the store holds numbers: there is +/// nothing here to borrow. Which arm a column is comes from its first +/// value, and every value after it has to be that arm too. +enum Column { + Int(Vec), + Float(Vec), + Bool(Vec), + Str(Vec>), + Date(Vec), + LocalTime(Vec), + LocalDatetime(Vec), + Duration(DurationKind, Vec), +} + +impl Column { + /// How many values are in it, which is how many rows the table has + /// if this is the first column and a refusal if it is not. + fn len(&self) -> usize { + match self { + Column::Int(v) => v.len(), + Column::Float(v) => v.len(), + Column::Bool(v) => v.len(), + Column::Str(v) => v.len(), + Column::Date(v) => v.len(), + Column::LocalTime(v) => v.len(), + Column::LocalDatetime(v) => v.len(), + Column::Duration(_, v) => v.len(), + } + } +} + +/// Writes a new database at `path` and answers what went into it. +/// +/// The path must not exist. A bulk load builds a database rather than +/// adding to one, so a path that already holds one is a caller who +/// meant a different path, and overwriting it would be the worst +/// possible reading of the call. +/// +/// `columns` is a dictionary of column name to a list of values, all +/// of them the same length, which is the number of rows the node table +/// gets. `edges` is a sequence of pairs of row numbers. Either may be +/// left out: a graph with no properties is a graph, and so is one with +/// no edges. +#[pyfunction] +#[pyo3(signature = (path, *, nodes, rels = "rel", columns = None, edges = None, rows = None))] +pub fn load( + py: Python<'_>, + path: PathBuf, + nodes: &str, + rels: &str, + columns: Option<&Bound<'_, PyDict>>, + edges: Option<&Bound<'_, PyAny>>, + rows: Option, +) -> PyResult> { + if nodes.is_empty() || rels.is_empty() { + return Err(pyo3::exceptions::PyValueError::new_err( + "a table has a name, and a load names both the node table and the rel table", + )); + } + let built = build(columns)?; + let rows = match (rows, built.first()) { + (Some(rows), Some((name, column))) if column.len() as u64 != rows => { + return Err(pyo3::exceptions::PyValueError::new_err(format!( + "column '{name}' holds {} values against the {rows} rows this load asks for", + column.len() + ))); + } + (Some(rows), _) => rows, + (None, Some((_, column))) => column.len() as u64, + (None, None) => { + return Err(pyo3::exceptions::PyValueError::new_err( + "a load with no columns has no rows to count, so it has to be told how many", + )); + } + }; + let pairs = pairs(edges, rows)?; + + // Released for the write, which is the whole cost of a load: the + // edges are sorted, the graph is built, and every column is + // encoded and written to disk. Everything above this line was + // reading Python objects, which needs the GIL and cannot be done + // without it. + let written = py.detach(|| -> zudb::Result<(usize, usize)> { + let mut db = Zu1File::create(&path)?; + let mut pairs = pairs; + pairs.sort_unstable(); + pairs.dedup(); + bulk_load_keyed(&mut db, nodes, rels, rows, &pairs, None)?; + if !built.is_empty() { + // The store wants a slice of slices for a string column, + // which a `Vec>` is not, so the row borrows are + // built first and handed over after. + let strings: Vec> = built + .iter() + .map(|(_, column)| match column { + Column::Str(v) => v.iter().map(Vec::as_slice).collect(), + _ => Vec::new(), + }) + .collect(); + let props: Vec<(&str, PropValues<'_>)> = built + .iter() + .zip(&strings) + .map(|((name, column), strings)| { + let values = match column { + Column::Str(_) => PropValues::Str(strings), + Column::Int(v) => PropValues::Int(v), + Column::Float(v) => PropValues::Float(v), + Column::Bool(v) => PropValues::Bool(v), + Column::Date(v) => PropValues::Date(v), + Column::LocalTime(v) => PropValues::LocalTime(v), + Column::LocalDatetime(v) => PropValues::LocalDatetime(v), + Column::Duration(kind, v) => PropValues::Duration(*kind, v), + }; + (name.as_str(), values) + }) + .collect(); + store_props(&mut db, nodes, &props)?; + } + Ok((built.len(), pairs.len())) + }); + let (written_columns, written_edges) = written.map_err(|err| to_py_err(py, err))?; + + let stats = PyDict::new(py); + stats.set_item("nodes", rows)?; + stats.set_item("rels", written_edges)?; + stats.set_item("columns", written_columns)?; + Ok(stats.unbind()) +} + +/// Every column, in the order the dictionary holds them, which is the +/// order they were written. +fn build(columns: Option<&Bound<'_, PyDict>>) -> PyResult> { + let Some(columns) = columns else { + return Ok(Vec::new()); + }; + let mut built: Vec<(String, Column)> = Vec::with_capacity(columns.len()); + for (name, values) in columns.iter() { + let name = name.extract::()?; + if name.is_empty() { + return Err(pyo3::exceptions::PyValueError::new_err( + "a column has a name", + )); + } + let column = column(&name, &values)?; + if let Some((first, had)) = built.first().map(|(name, column)| (name, column.len())) + && column.len() != had + { + return Err(pyo3::exceptions::PyValueError::new_err(format!( + "column '{name}' holds {} values and column '{first}' holds {had}, and a table is as wide as it is long", + column.len() + ))); + } + built.push((name, column)); + } + Ok(built) +} + +/// One column, read out of a sequence of Python objects. +/// +/// The first value settles what the column is and every value after it +/// has to be the same thing. There is no null: a column that holds one +/// cannot be loaded this way, so a null here could only be refused, +/// and refusing it where it is named is better than refusing it at the +/// end of a million rows. +fn column(name: &str, values: &Bound<'_, PyAny>) -> PyResult { + let mut column: Option = None; + for (row, value) in values.try_iter()?.enumerate() { + let value = value?; + let mismatch = |want: &str| { + let got = value + .get_type() + .getattr("__name__") + .and_then(|name| name.extract::()) + .unwrap_or_else(|_| "unknown".to_string()); + pyo3::exceptions::PyTypeError::new_err(format!( + "column '{name}' holds {want} and row {row} is of type '{got}'" + )) + }; + match column.as_mut() { + None => column = Some(started(name, row, &value)?), + // Bool before int, since in Python every bool is an int + // and a column of `True` is not a column of ones. + Some(Column::Bool(v)) => v.push( + value + .cast::() + .map_err(|_| mismatch("booleans"))? + .is_true(), + ), + Some(Column::Int(v)) => { + if value.cast::().is_ok() { + return Err(mismatch("integers")); + } + v.push(value.extract::().map_err(|_| mismatch("integers"))? as u64); + } + Some(Column::Float(v)) => { + v.push(value.extract::().map_err(|_| mismatch("floats"))?) + } + Some(Column::Str(v)) => v.push( + value + .extract::() + .map_err(|_| mismatch("strings"))? + .into_bytes(), + ), + // Datetime before date, since a datetime is a date and + // reading one as the other would throw the time away. + Some(Column::LocalDatetime(v)) => { + let dt = value + .cast::() + .map_err(|_| mismatch("datetimes"))?; + v.push(datetime_nanos(dt)?); + } + Some(Column::Date(v)) => { + if value.cast::().is_ok() { + return Err(mismatch("dates")); + } + let date = value.cast::().map_err(|_| mismatch("dates"))?; + v.push(date_days(date)?); + } + Some(Column::LocalTime(v)) => { + let time = value.cast::().map_err(|_| mismatch("times"))?; + v.push(clock_nanos(time.as_any())?); + } + Some(Column::Duration(kind, v)) => v.push(duration_count(*kind, &value, mismatch)?), + } + } + column.ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err(format!( + "column '{name}' is empty, and an empty column says nothing about what it would hold" + )) + }) +} + +/// The column a first value starts, with that value already in it. +fn started(name: &str, row: usize, value: &Bound<'_, PyAny>) -> PyResult { + if let Ok(b) = value.cast::() { + return Ok(Column::Bool(vec![b.is_true()])); + } + if let Ok(s) = value.extract::() { + return Ok(Column::Str(vec![s.into_bytes()])); + } + if let Ok(n) = value.extract::() { + return Ok(Column::Int(vec![n as u64])); + } + if let Ok(f) = value.extract::() { + return Ok(Column::Float(vec![f])); + } + if let Ok(d) = value.extract::() { + return Ok(if d.months == 0 { + Column::Duration(DurationKind::DayTime, vec![d.nanoseconds]) + } else { + Column::Duration(DurationKind::YearMonth, vec![d.months]) + }); + } + if let Ok(delta) = value.cast::() { + return Ok(Column::Duration( + DurationKind::DayTime, + vec![delta_nanos(delta)?], + )); + } + if let Ok(dt) = value.cast::() { + return Ok(Column::LocalDatetime(vec![datetime_nanos(dt)?])); + } + if let Ok(date) = value.cast::() { + return Ok(Column::Date(vec![date_days(date)?])); + } + if let Ok(time) = value.cast::() { + return Ok(Column::LocalTime(vec![clock_nanos(time.as_any())?])); + } + let got = value + .get_type() + .getattr("__name__") + .and_then(|name| name.extract::()) + .unwrap_or_else(|_| "unknown".to_string()); + Err(pyo3::exceptions::PyTypeError::new_err(format!( + "column '{name}' starts at row {row} with a value of type '{got}', and a loaded column holds booleans, integers, floats, strings, dates, times, datetimes or durations" + ))) +} + +/// A duration for a column that is already one of the two kinds, which +/// is the one place the two do not mix: a column of months has no room +/// for a count of nanoseconds and the other way about. +fn duration_count( + kind: DurationKind, + value: &Bound<'_, PyAny>, + mismatch: impl Fn(&str) -> PyErr, +) -> PyResult { + let want = match kind { + DurationKind::YearMonth => "year-month durations", + DurationKind::DayTime => "day-time durations", + }; + if let Ok(d) = value.extract::() { + return match kind { + DurationKind::YearMonth if d.months != 0 || d.nanoseconds == 0 => Ok(d.months), + DurationKind::DayTime if d.months == 0 => Ok(d.nanoseconds), + _ => Err(mismatch(want)), + }; + } + match (kind, value.cast::()) { + (DurationKind::DayTime, Ok(delta)) => delta_nanos(delta), + _ => Err(mismatch(want)), + } +} + +fn date_days(date: &Bound<'_, PyDate>) -> PyResult { + Ok(days_from_civil( + date.getattr("year")?.extract()?, + date.getattr("month")?.extract()?, + date.getattr("day")?.extract()?, + )) +} + +fn datetime_nanos(dt: &Bound<'_, PyDateTime>) -> PyResult { + const NANOS_PER_DAY: i64 = 86_400 * 1_000_000_000; + let days = date_days(dt.as_any().cast::()?)?; + Ok(i64::from(days) * NANOS_PER_DAY + clock_nanos(dt.as_any())?) +} + +fn delta_nanos(delta: &Bound<'_, PyDelta>) -> PyResult { + let days: i64 = delta.getattr("days")?.extract()?; + let seconds: i64 = delta.getattr("seconds")?.extract()?; + let micros: i64 = delta.getattr("microseconds")?.extract()?; + Ok(days * 86_400 * 1_000_000_000 + seconds * 1_000_000_000 + micros * 1_000) +} + +/// The edge list, as the pairs of row numbers it is. +/// +/// An edge naming a row the table has not got is refused here rather +/// than written, because a graph builder handed one would either +/// invent the row or lose the edge and neither is what the caller +/// meant. +fn pairs(edges: Option<&Bound<'_, PyAny>>, rows: u64) -> PyResult> { + let Some(edges) = edges else { + return Ok(Vec::new()); + }; + let mut pairs = Vec::new(); + for (at, edge) in edges.try_iter()?.enumerate() { + let edge = edge?; + let (from, to): (i64, i64) = edge.extract().map_err(|_| { + pyo3::exceptions::PyTypeError::new_err(format!( + "edge {at} is not a pair of row numbers" + )) + })?; + for end in [from, to] { + if end < 0 || end as u64 >= rows { + return Err(pyo3::exceptions::PyValueError::new_err(format!( + "edge {at} joins row {end} of a table with {rows} rows in it" + ))); + } + } + pairs.push((from as u32, to as u32)); + } + Ok(pairs) +} diff --git a/src/value.rs b/src/value.rs index 22eb7bd..327999a 100644 --- a/src/value.rs +++ b/src/value.rs @@ -97,10 +97,11 @@ impl Node { /// One edge of the graph. /// -/// `ord` is which edge this is of the ones running from `src` to -/// `dst`, since a pair of endpoints can be joined more than once and -/// each of those edges carries its own properties. It is the field a -/// caller usually ignores and the one nothing else can replace. +/// `ord` is where the edge's properties sit, which is its place in the +/// order the table was loaded in. That is what names an edge: a pair of +/// endpoints does not, since the same pair may run more than once and +/// each of those edges carries its own values. It is the field a caller +/// usually ignores and the one nothing else can replace. #[pyclass(module = "zudb", frozen, eq, hash, skip_from_py_object)] #[derive(PartialEq, Eq, Hash, Clone)] pub struct Rel { @@ -161,6 +162,20 @@ impl Path { self.every(py, 1) } + fn __len__(&self, py: Python<'_>) -> usize { + self.elements.bind(py).len() / 2 + } + + fn __repr__(&self, py: Python<'_>) -> String { + format!("Path({} hops)", self.__len__(py)) + } +} + +impl Path { + /// Every other element starting at `from`, which is the nodes when + /// that is zero and the edges when it is one. Outside `pymethods` + /// because it is how the two getters are written and not a third + /// thing to call from Python. fn every<'py>(&self, py: Python<'py>, from: usize) -> PyResult> { let all = self.elements.bind(py); let picked = (from..all.len()) @@ -169,14 +184,6 @@ impl Path { .collect::>>()?; PyList::new(py, picked) } - - fn __len__(&self, py: Python<'_>) -> usize { - self.elements.bind(py).len() / 2 - } - - fn __repr__(&self, py: Python<'_>) -> String { - format!("Path({} hops)", self.__len__(py)) - } } /// A duration, which Python has no type for. @@ -535,7 +542,7 @@ fn time_from_py(t: &Bound<'_, PyTime>) -> PyResult { /// The clock reading of a `time` or a `datetime`, in nanoseconds since /// midnight. -fn clock_nanos(value: &Bound<'_, PyAny>) -> PyResult { +pub fn clock_nanos(value: &Bound<'_, PyAny>) -> PyResult { let hour: i64 = value.getattr("hour")?.extract()?; let minute: i64 = value.getattr("minute")?.extract()?; let second: i64 = value.getattr("second")?.extract()?; diff --git a/tests/conftest.py b/tests/conftest.py index 3c8177b..ebcc1e5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,6 +49,33 @@ def social(tmp_path: Path) -> zudb.Connection: conn.close() +@pytest.fixture +def loaded(tmp_path: Path) -> zudb.Connection: + """The same three people, with a rel table joining them. + + Built by a load rather than by statements because a load is the only + thing that makes a rel table: `INSERT` names the two tables an edge + joins but cannot declare the table that holds it. Ada knows Grace + and Grace knows Kay, so there is a two-hop path through the middle + of it and a node at each end with one edge. + """ + path = tmp_path / "loaded.zu1" + zudb.load( + path, + nodes="person", + rels="knows", + columns={ + "uid": [uid for uid, _, _ in PEOPLE], + "name": [name for _, name, _ in PEOPLE], + "score": [score for _, _, score in PEOPLE], + }, + edges=[(0, 1), (1, 2)], + ) + conn = zudb.connect(path, read_only=True) + yield conn + conn.close() + + @pytest.fixture def crowd(tmp_path: Path) -> zudb.Connection: """Enough people that a statement over the pairs of them takes long diff --git a/tests/test_load.py b/tests/test_load.py new file mode 100644 index 0000000..c392615 --- /dev/null +++ b/tests/test_load.py @@ -0,0 +1,269 @@ +"""Building a database out of columns and an edge list. + +A load is the only way a Python program makes a graph with edges in it, +so these check both halves: that what went in comes back out through +statements, and that a load which cannot mean anything is refused where +the mistake is rather than written to disk and found later. +""" + +from __future__ import annotations + +import datetime +import threading +from pathlib import Path + +import pytest +import zudb + + +def test_a_load_says_what_it_wrote(tmp_path: Path) -> None: + stats = zudb.load( + tmp_path / "g.zu1", + nodes="person", + rels="knows", + columns={"uid": [1, 2, 3], "name": ["ada", "grace", "kay"]}, + edges=[(0, 1), (1, 2)], + ) + assert stats == {"nodes": 3, "rels": 2, "columns": 2} + + +def test_the_rows_read_back_in_the_order_they_went_in(loaded: zudb.Connection) -> None: + rows = loaded.execute("MATCH (p:person) RETURN p.uid AS uid, p.name AS name") + assert list(rows) == [(10, "ada"), (20, "grace"), (30, "kay")] + + +def test_the_edges_are_a_table_a_pattern_can_walk(loaded: zudb.Connection) -> None: + rows = loaded.execute("MATCH (a:person)-[:knows]->(b:person) RETURN a.name AS a, b.name AS b") + assert list(rows) == [("ada", "grace"), ("grace", "kay")] + + +def test_an_edge_comes_back_as_a_rel(loaded: zudb.Connection) -> None: + rels = [rel for (rel,) in loaded.execute("MATCH ()-[r:knows]->() RETURN r")] + assert [rel.table for rel in rels] == ["knows", "knows"] + # The ordinal is the edge's place in the load, which is where its + # properties sit, so the second edge loaded is the second ordinal. + assert [(rel.src, rel.dst, rel.ord) for rel in rels] == [(0, 1, 0), (1, 2, 1)] + assert repr(rels[0]) == "Rel(knows, 0 -> 1)" + + +def test_two_reads_of_one_edge_are_equal_and_hash_alike(loaded: zudb.Connection) -> None: + statement = "MATCH (a:person)-[r:knows]->(b:person) WHERE a.uid = 10 RETURN r" + one = loaded.execute(statement).fetchone()[0] + two = loaded.execute(statement).fetchone()[0] + assert one == two + assert len({one, two}) == 1 + + +def test_a_walk_comes_back_as_a_path(loaded: zudb.Connection) -> None: + walk = loaded.execute( + "MATCH q = (a:person)-[:knows]->(b:person) WHERE a.uid = 10 RETURN q" + ).fetchone()[0] + assert len(walk) == 1 + assert [node.offset for node in walk.nodes] == [0, 1] + assert [rel.dst for rel in walk.rels] == [1] + assert walk.elements == [*walk.nodes[:1], *walk.rels, *walk.nodes[1:]] + assert repr(walk) == "Path(1 hops)" + + +def test_a_two_hop_walk_alternates_nodes_and_edges(loaded: zudb.Connection) -> None: + walk = loaded.execute( + "MATCH q = (a:person)-[:knows]->()-[:knows]->(c:person) RETURN q" + ).fetchone()[0] + assert len(walk) == 2 + assert [node.offset for node in walk.nodes] == [0, 1, 2] + assert [(rel.src, rel.dst) for rel in walk.rels] == [(0, 1), (1, 2)] + + +def test_a_load_with_no_edges_is_a_graph_with_none(tmp_path: Path) -> None: + stats = zudb.load(tmp_path / "g.zu1", nodes="person", rels="knows", columns={"uid": [1, 2]}) + assert stats["rels"] == 0 + with zudb.connect(tmp_path / "g.zu1", read_only=True) as conn: + assert conn.execute("MATCH ()-[r:knows]->() RETURN count(r) AS n").fetchone() == (0,) + + +def test_a_load_with_no_columns_still_has_rows(tmp_path: Path) -> None: + stats = zudb.load(tmp_path / "g.zu1", nodes="person", rels="knows", rows=4, edges=[(0, 3)]) + assert stats == {"nodes": 4, "rels": 1, "columns": 0} + with zudb.connect(tmp_path / "g.zu1", read_only=True) as conn: + assert conn.execute("MATCH (p:person) RETURN count(p) AS n").fetchone() == (4,) + + +def test_the_rel_table_is_called_rel_when_it_is_not_named(tmp_path: Path) -> None: + zudb.load(tmp_path / "g.zu1", nodes="person", columns={"uid": [1, 2]}, edges=[(0, 1)]) + with zudb.connect(tmp_path / "g.zu1", read_only=True) as conn: + rel = conn.execute("MATCH ()-[r]->() RETURN r").fetchone()[0] + assert rel.table == "rel" + + +def test_the_same_edge_twice_is_one_edge(tmp_path: Path) -> None: + stats = zudb.load( + tmp_path / "g.zu1", + nodes="person", + rels="knows", + columns={"uid": [1, 2]}, + edges=[(0, 1), (0, 1), (0, 1)], + ) + assert stats["rels"] == 1 + + +def test_a_column_of_every_kind_reads_back_as_what_it_was(tmp_path: Path) -> None: + columns = { + "count": [1, -2], + "ratio": [1.5, -0.25], + "flag": [True, False], + "name": ["ada", "grace"], + "born": [datetime.date(1815, 12, 10), datetime.date(1906, 12, 9)], + "woke": [datetime.time(6, 30), datetime.time(23, 59, 59)], + "seen": [datetime.datetime(2024, 1, 2, 3, 4, 5, 6), datetime.datetime(1900, 1, 1)], + "took": [datetime.timedelta(days=1, seconds=2), datetime.timedelta(0)], + "aged": [zudb.Duration(months=14), zudb.Duration(months=-1)], + } + zudb.load(tmp_path / "g.zu1", nodes="person", rels="knows", columns=columns) + with zudb.connect(tmp_path / "g.zu1", read_only=True) as conn: + rows = conn.execute( + "MATCH (p:person) RETURN " + ", ".join(f"p.{name} AS {name}" for name in columns) + ) + got = list(rows) + assert got[0] == ( + 1, + 1.5, + True, + "ada", + datetime.date(1815, 12, 10), + datetime.time(6, 30), + datetime.datetime(2024, 1, 2, 3, 4, 5, 6), + zudb.Duration(nanoseconds=86_402_000_000_000), + zudb.Duration(months=14), + ) + assert got[1][:4] == (-2, -0.25, False, "grace") + + +def test_a_load_never_writes_over_a_database_that_is_there(tmp_path: Path) -> None: + path = tmp_path / "g.zu1" + zudb.load(path, nodes="person", rels="knows", columns={"uid": [1]}) + with pytest.raises(zudb.ConnectionError): + zudb.load(path, nodes="person", rels="knows", columns={"uid": [2]}) + with zudb.connect(path, read_only=True) as conn: + assert conn.execute("MATCH (p:person) RETURN p.uid AS uid").fetchone() == (1,) + + +@pytest.mark.parametrize( + "kwargs,message", + [ + ({"nodes": "", "rels": "knows", "rows": 1}, "a table has a name"), + ({"nodes": "person", "rels": "", "rows": 1}, "a table has a name"), + ({"nodes": "person", "rels": "knows"}, "has to be told how many"), + ( + {"nodes": "person", "rels": "knows", "columns": {"a": [1, 2], "b": [3]}}, + "as wide as it is long", + ), + ({"nodes": "person", "rels": "knows", "columns": {"a": []}}, "is empty"), + ( + {"nodes": "person", "rels": "knows", "columns": {"a": [1, 2]}, "rows": 3}, + "against the 3 rows", + ), + ( + {"nodes": "person", "rels": "knows", "columns": {"a": [1, 2]}, "edges": [(0, 5)]}, + "row 5 of a table with 2 rows", + ), + ( + {"nodes": "person", "rels": "knows", "columns": {"a": [1, 2]}, "edges": [(0, -1)]}, + "row -1 of a table", + ), + ], +) +def test_a_load_that_cannot_mean_anything_is_refused( + tmp_path: Path, kwargs: dict, message: str +) -> None: + with pytest.raises(ValueError, match=message): + zudb.load(tmp_path / "g.zu1", **kwargs) + assert not (tmp_path / "g.zu1").exists() + + +@pytest.mark.parametrize( + "values,message", + [ + ([1, True], "holds integers and row 1 is of type 'bool'"), + ([True, 1], "holds booleans and row 1 is of type 'int'"), + ([1, "ada"], "holds integers and row 1 is of type 'str'"), + ([1.5, "ada"], "holds floats and row 1 is of type 'str'"), + (["ada", 1], "holds strings and row 1 is of type 'int'"), + ( + [datetime.date(2020, 1, 1), datetime.datetime(2020, 1, 1)], + "holds dates and row 1 is of type 'datetime'", + ), + ( + [datetime.datetime(2020, 1, 1), datetime.date(2020, 1, 1)], + "holds datetimes and row 1 is of type 'date'", + ), + ( + [zudb.Duration(months=1), datetime.timedelta(days=1)], + "holds year-month durations and row 1 is of type 'timedelta'", + ), + ( + [datetime.timedelta(days=1), zudb.Duration(months=1)], + "holds day-time durations and row 1 is of type 'Duration'", + ), + ([object()], "starts at row 0 with a value of type 'object'"), + ], +) +def test_a_column_holds_one_kind_of_value(tmp_path: Path, values: list, message: str) -> None: + with pytest.raises(TypeError, match=message): + zudb.load(tmp_path / "g.zu1", nodes="person", rels="knows", columns={"a": values}) + + +def test_an_edge_that_is_not_a_pair_is_refused(tmp_path: Path) -> None: + with pytest.raises(TypeError, match="edge 1 is not a pair of row numbers"): + zudb.load( + tmp_path / "g.zu1", + nodes="person", + rels="knows", + columns={"uid": [1, 2]}, + edges=[(0, 1), 7], + ) + + +def test_columns_and_edges_may_be_any_iterable(tmp_path: Path) -> None: + stats = zudb.load( + tmp_path / "g.zu1", + nodes="person", + rels="knows", + columns={"uid": range(4), "name": (f"p{i}" for i in range(4))}, + edges=((i, i + 1) for i in range(3)), + ) + assert stats == {"nodes": 4, "rels": 3, "columns": 2} + + +def test_python_keeps_running_while_a_load_does(tmp_path: Path) -> None: + # Big enough that the write takes long enough to watch, and shaped + # so the edges are out of order and have to be sorted, which is the + # other half of the work the GIL is released for. + rows = 200_000 + columns = {"uid": list(range(rows)), "name": [f"p{uid}" for uid in range(rows)]} + edges = [(uid, (uid * 7 + 1) % rows) for uid in range(rows)] + ticks = 0 + done = threading.Event() + stats: list[dict] = [] + + def run() -> None: + stats.append( + zudb.load( + tmp_path / "big.zu1", + nodes="person", + rels="knows", + columns=columns, + edges=edges, + ) + ) + done.set() + + worker = threading.Thread(target=run) + worker.start() + while not done.is_set(): + ticks += 1 + worker.join(timeout=120) + assert not worker.is_alive() + assert stats[0]["nodes"] == rows + # A GIL held for the length of the write would leave the main thread + # nothing but the switch interval, which is a handful of turns. + assert ticks > 1000, f"the main thread only got {ticks} turns"