diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e890d1..973357a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,3 +61,38 @@ jobs: # checkout it was built from. - run: pip install pytest griffe - run: pytest + + # The shared corpus, which is the same 945 cases the engine runs + # against itself and the eight other clients run against theirs. It is + # a job of its own because it needs a second checkout, and it runs on + # one platform because what it is asking about is this client's value + # mapping rather than anything the operating system decides. + corpus: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 + with: + python-version: "3.14" + - uses: Swatinem/rust-cache@v2 + # The cases are versioned with the engine, so the revision comes + # out of the pin this client already builds against rather than + # being written down a second time and drifting. + - id: pin + run: | + rev=$(sed -n 's/^zudb = .*rev = "\([0-9a-f]*\)".*/\1/p' Cargo.toml) + test -n "$rev" + echo "rev=$rev" >> "$GITHUB_OUTPUT" + - uses: actions/checkout@v7 + with: + repository: tamnd/zu + ref: ${{ steps.pin.outputs.rev }} + path: engine + - run: pip install . pytest + # The runner first, because its summary is the line a person + # reads, and the suite second, because it is the one that knows + # which cases this client is allowed to leave unanswered. + - run: python -m conformance engine/conformance/cases + - run: pytest tests/test_conformance.py + env: + ZU_CASES: engine/conformance/cases diff --git a/Cargo.lock b/Cargo.lock index cd2fc5e..5ed99c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1647,7 +1647,7 @@ dependencies = [ [[package]] name = "zu" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=67afd055032932eec36f4e373c2b82bdc4b188c9#67afd055032932eec36f4e373c2b82bdc4b188c9" +source = "git+https://github.com/tamnd/zu?rev=6ceb6d50abe20cfbef97c3d0d033d051c0649521#6ceb6d50abe20cfbef97c3d0d033d051c0649521" dependencies = [ "zu-common", "zu-encoding", @@ -1663,7 +1663,7 @@ dependencies = [ [[package]] name = "zu-common" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=67afd055032932eec36f4e373c2b82bdc4b188c9#67afd055032932eec36f4e373c2b82bdc4b188c9" +source = "git+https://github.com/tamnd/zu?rev=6ceb6d50abe20cfbef97c3d0d033d051c0649521#6ceb6d50abe20cfbef97c3d0d033d051c0649521" dependencies = [ "thiserror", ] @@ -1671,7 +1671,7 @@ dependencies = [ [[package]] name = "zu-encoding" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=67afd055032932eec36f4e373c2b82bdc4b188c9#67afd055032932eec36f4e373c2b82bdc4b188c9" +source = "git+https://github.com/tamnd/zu?rev=6ceb6d50abe20cfbef97c3d0d033d051c0649521#6ceb6d50abe20cfbef97c3d0d033d051c0649521" dependencies = [ "ruzstd", "zu-common", @@ -1680,7 +1680,7 @@ dependencies = [ [[package]] name = "zu-exec" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=67afd055032932eec36f4e373c2b82bdc4b188c9#67afd055032932eec36f4e373c2b82bdc4b188c9" +source = "git+https://github.com/tamnd/zu?rev=6ceb6d50abe20cfbef97c3d0d033d051c0649521#6ceb6d50abe20cfbef97c3d0d033d051c0649521" dependencies = [ "zu-common", "zu-query", @@ -1690,7 +1690,7 @@ dependencies = [ [[package]] name = "zu-query" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=67afd055032932eec36f4e373c2b82bdc4b188c9#67afd055032932eec36f4e373c2b82bdc4b188c9" +source = "git+https://github.com/tamnd/zu?rev=6ceb6d50abe20cfbef97c3d0d033d051c0649521#6ceb6d50abe20cfbef97c3d0d033d051c0649521" dependencies = [ "crossbeam-deque", "zu-common", @@ -1701,7 +1701,7 @@ dependencies = [ [[package]] name = "zu-s3" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=67afd055032932eec36f4e373c2b82bdc4b188c9#67afd055032932eec36f4e373c2b82bdc4b188c9" +source = "git+https://github.com/tamnd/zu?rev=6ceb6d50abe20cfbef97c3d0d033d051c0649521#6ceb6d50abe20cfbef97c3d0d033d051c0649521" dependencies = [ "crc32c", "object_store", @@ -1712,7 +1712,7 @@ dependencies = [ [[package]] name = "zu-sqlite" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=67afd055032932eec36f4e373c2b82bdc4b188c9#67afd055032932eec36f4e373c2b82bdc4b188c9" +source = "git+https://github.com/tamnd/zu?rev=6ceb6d50abe20cfbef97c3d0d033d051c0649521#6ceb6d50abe20cfbef97c3d0d033d051c0649521" dependencies = [ "rusqlite", "zu-common", @@ -1722,7 +1722,7 @@ dependencies = [ [[package]] name = "zu-storage" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=67afd055032932eec36f4e373c2b82bdc4b188c9#67afd055032932eec36f4e373c2b82bdc4b188c9" +source = "git+https://github.com/tamnd/zu?rev=6ceb6d50abe20cfbef97c3d0d033d051c0649521#6ceb6d50abe20cfbef97c3d0d033d051c0649521" dependencies = [ "zu-common", "zu-encoding", @@ -1731,7 +1731,7 @@ dependencies = [ [[package]] name = "zu-vector" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=67afd055032932eec36f4e373c2b82bdc4b188c9#67afd055032932eec36f4e373c2b82bdc4b188c9" +source = "git+https://github.com/tamnd/zu?rev=6ceb6d50abe20cfbef97c3d0d033d051c0649521#6ceb6d50abe20cfbef97c3d0d033d051c0649521" dependencies = [ "zu-common", ] @@ -1739,7 +1739,7 @@ dependencies = [ [[package]] name = "zu-zu1" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=67afd055032932eec36f4e373c2b82bdc4b188c9#67afd055032932eec36f4e373c2b82bdc4b188c9" +source = "git+https://github.com/tamnd/zu?rev=6ceb6d50abe20cfbef97c3d0d033d051c0649521#6ceb6d50abe20cfbef97c3d0d033d051c0649521" dependencies = [ "crc32c", "loom", diff --git a/Cargo.toml b/Cargo.toml index 656a815..fa13847 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,8 @@ crate-type = ["cdylib"] # with (ADR 0002), so a revision is the honest way to say which one. # A local checkout is used instead with a `paths` override in # `.cargo/config.toml`, which is untracked on purpose. -zudb = { package = "zu", git = "https://github.com/tamnd/zu", rev = "67afd055032932eec36f4e373c2b82bdc4b188c9" } -zu-common = { git = "https://github.com/tamnd/zu", rev = "67afd055032932eec36f4e373c2b82bdc4b188c9" } +zudb = { package = "zu", git = "https://github.com/tamnd/zu", rev = "6ceb6d50abe20cfbef97c3d0d033d051c0649521" } +zu-common = { git = "https://github.com/tamnd/zu", rev = "6ceb6d50abe20cfbef97c3d0d033d051c0649521" } # `extension-module` is asked for by maturin, in pyproject.toml, and # not here. Only the build backend knows how an extension is linked on # the platform it is building for, and a crate that turns the feature diff --git a/conformance/README.md b/conformance/README.md new file mode 100644 index 0000000..4760d7d --- /dev/null +++ b/conformance/README.md @@ -0,0 +1,63 @@ +# The shared corpus, run through this client + +The corpus is one set of hand written YAML files in the engine's repository, under `conformance/cases`. Every client reads the same files and runs the same statements, so a value that survives one binding and not another is a diff rather than an argument. This directory is the Python end of that: a reader for the subset of YAML the cases are written in, a decoder for the value encoding, and a runner that reports what happened in the form the reference runner reports it. + +It is a development tool and not part of the wheel. `pyproject.toml` packages `python/` and nothing else, so `conformance` is on the path when the repository is checked out and absent when `zudb` is installed. Nothing under `zudb` imports it. + +## Running it + +The cases live in the engine's repository, pinned in `Cargo.toml` to the same revision this client builds against: + +``` +git clone https://github.com/tamnd/zu /tmp/zu +git -C /tmp/zu checkout 6ceb6d50abe20cfbef97c3d0d033d051c0649521 +python -m conformance /tmp/zu/conformance/cases +``` + +which prints every case that did not pass and then one line saying what the run came to: + +``` +945 cases, 940 passed, 0 failed, 5 unsupported +``` + +`--strict` turns an unsupported case into a failed run, `--quiet` prints the summary alone, and `--work DIR` keeps the databases the cases were run against instead of removing them, which is what to reach for when a failure wants opening. + +The exit code is 0 when nothing failed, and 1 when something did or when the corpus will not read. One and not two for a corpus that will not read, because the reference runner exits one and a report compared line for line is worth less if the two runners disagree about what the run came to. + +## What it is checking + +Three things, and the third is the one that matters. + +A client can decode a value and pass every case, because the case and the answer both went through the same decoder. So the reader here is a third implementation of the corpus format rather than a consumer of one: it refuses what `crates/zu-corpus/src/yaml.rs` refuses, with the same words and the same line numbers, and the tests in `tests/test_conformance.py` are that file's own tables ported case for case. A reader that grew a hole would pass its own tests and fail those. + +The value encoding is the same again. An INT64 written bare is refused, a value wider than the type it claims is refused, a float is exact or it is not a float, and a temporal is written the way the engine prints it. What a report prints is the encoding's own spelling, so a failure can be pasted back into a case. + +And the report itself is compared. Two mutation sweeps of the corpus, one that corrupts every third payload so the readers refuse and one that rewrites every row value so the reports compare, produce output this runner and the Rust one agree on line for line, with the single exception below. Five defects in this client were found that way and none of them by its own tests: a helper shadowed by a loop variable so two refusals would have raised, a float printed `1e+16` where the engine prints `1e16`, a time printed with six fractional digits where the engine prints nine, a zero offset printed `+00:00` where the engine prints `Z`, and a duration printed with the fields that are zero left in. + +## What this client cannot answer + +Five cases, all of them a time written finer than a microsecond: + +- `temporal/local-time-nanoseconds` +- `temporal/a-time-carries-a-single-nanosecond` +- `param/localtime-to-the-nanosecond` +- `stored/a-localtime-column-keeps-every-digit` +- `stored/the-columns-of-one-row-belong-to-that-row` + +The engine keeps a time to the nanosecond and Python's `datetime` keeps one to the microsecond, so `LOCAL TIME '12:34:56.123456789'` comes back as `datetime.time(12, 34, 56, 123456)`. That is the value mapping this client documents rather than a defect in it, and it is what `temporal/local-time-nanoseconds` says out loud it was written to catch. + +The trap is that a runner catches it only if it tries to. Decoding the case's own expectation into a `datetime` truncates it too, and the case then passes by comparing one truncated value against another. So a temporal payload with a digit past the sixth decodes to a value that equals nothing, including itself, and a case holding one is reported unsupported with the value named. Unsupported and not failed, because nothing went wrong: the engine answered, and Python's `datetime` is where the digits went. + +A load is the one place a value this client cannot hold still goes in, truncated. A column has to be in the file for the suite to have a graph at all, and refusing it would take out the thirty six cases of `stored` rather than the two that read that column back. Those two say so on their own, because their own expectation is a value nothing equals. + +## The one check that is weaker here than in the engine + +A case naming a condition writes its GQLSTATUS, and the reference reader checks that code against the table the standard defines, which lives in `zu-common`. This reader checks the shape, five characters of digits and capitals, because the table is not something the client has. A code of the right shape that no standard defines is caught by the reference runner and not by this one. The C reader in `conformance/c` takes the same position. + +## The files + +`reader.py` is the YAML subset: block mappings, block sequences, scalars, and a refusal with a line number for everything else. It does not use PyYAML, which would read these files and a good deal more besides, and would hand back `9223372036854775807` as a float on the way. + +`values.py` is the `{type, value}` encoding, both directions, and the comparison. The comparison is not `==`: `NaN` matches `NaN`, `-0.0` does not match `0.0`, and a `bool` does not match an `int`, which is a rule Python needs and the reference runner does not. + +`cases.py` is what a case is, and `runner.py` runs them, one database per case with a fresh copy of the suite's load, so a case that leaked a table into the next one would be a failure that moves when the file is reordered. diff --git a/conformance/__init__.py b/conformance/__init__.py new file mode 100644 index 0000000..64b396c --- /dev/null +++ b/conformance/__init__.py @@ -0,0 +1,20 @@ +"""The shared conformance corpus, run through this client. + +The cases live in the engine repository, versioned with the engine and +shipped as a release artifact every client consumes. Nothing here holds +a case: a case that lived beside one runner would be a case that runner +cannot fail. + +What is here is the third implementation of the reader, the second of +the value encoding, and the second runner, all of which exist so that a +value put in through this client and taken out through another means the +same thing. +""" + +from __future__ import annotations + +from .cases import Case, Suite, read_dir +from .reader import CorpusError +from .runner import Report, run + +__all__ = ["Case", "CorpusError", "Report", "Suite", "read_dir", "run"] diff --git a/conformance/__main__.py b/conformance/__main__.py new file mode 100644 index 0000000..4b93674 --- /dev/null +++ b/conformance/__main__.py @@ -0,0 +1,7 @@ +"""``python -m conformance ``, which is how CI runs it.""" + +from __future__ import annotations + +from .runner import main + +raise SystemExit(main()) diff --git a/conformance/cases.py b/conformance/cases.py new file mode 100644 index 0000000..45ae31e --- /dev/null +++ b/conformance/cases.py @@ -0,0 +1,395 @@ +"""What a case is, and how a file of them is read. + +A case is a statement and what running it must produce. That is +deliberately the whole of it. Every client in every language can run a +statement and look at the rows that come back, so a corpus written in +those terms is one every client can run, and a corpus written in terms +of a client's own API would be nine corpora. + +The expectation is either rows or a condition. A case expecting a +condition names the GQLSTATUS code, not the message, because the code is +the contract and the message is prose that will improve. + +A statement may take parameters, which is the other direction the same +values travel: a case with ``params:`` writes a value in the encoding, +hands it to this client's own binding call, and asserts what came back. +A client that decodes a date correctly and encodes it a day early passes +every case that has no parameters in it. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +from . import values +from .reader import CorpusError, Node, parse, quote + +__all__ = ["SCHEMA", "Case", "Suite", "Column", "Load", "read_dir"] + +#: The schema version a file declares. It exists so that a corpus +#: unpacked from an old release tells a new runner what it is instead of +#: failing in the middle. +SCHEMA = 3 + +_SUITE_KEYS = ("schema", "suite", "doc", "load", "cases") +_CASE_KEYS = ("name", "doc", "setup", "params", "query", "columns", "rows", "raises") +_LOAD_KEYS = ("nodes", "edges", "count", "columns", "pairs") + + +@dataclass +class Case: + """One statement and what it owes. + + ``columns`` and ``rows`` are set together or neither is, and + ``raises`` is set when neither is: a case says what it produces one + way or the other.""" + + name: str + doc: str + query: str + line: int + setup: list[str] = field(default_factory=list) + params: list[tuple[str, object]] = field(default_factory=list) + columns: list[str] | None = None + rows: list[list[object]] | None = None + raises: str | None = None + + +@dataclass +class Column: + """One column of a load: a name, the type every value in it has, and + the values in row order.""" + + name: str + ty: str + values: list[object] + + +@dataclass +class Load: + """One node table, its columns, and the edges between its rows. + + Everything else in the corpus is an expression, and an expression + says what a value means on the way out and nothing about how it got + in. A load is the other half, and every runner puts it in through + its own bulk load path, which for this client is ``zudb.load``.""" + + nodes: str + edges: str + count: int + columns: list[Column] + pairs: list[tuple[int, int]] + + +@dataclass +class Suite: + """One file of cases.""" + + name: str + doc: str + load: Load | None + cases: list[Case] + + +def read_dir(directory: Path) -> list[Suite]: + """Every suite in a directory, in the order a sorted listing gives, + which is the order the reference runner walks them in.""" + suites = [] + for path in sorted(Path(directory).glob("*.yaml")): + try: + suite = read(path.read_text(encoding="utf-8")) + except CorpusError as e: + raise CorpusError(f"{path}: {e}") from None + if suite.name != path.stem: + raise CorpusError( + f"{path}: the suite calls itself {quote(suite.name)} and the file calls it " + f"{quote(path.stem)}" + ) + suites.append(suite) + if not suites: + raise CorpusError(f"{directory}: no case files") + return suites + + +def read(text: str) -> Suite: + """A suite, or the first thing in the file that is not one.""" + doc = parse(text) + unknown = doc.unknown(_SUITE_KEYS) + if unknown: + raise CorpusError(f"line {doc.line}: a suite has no key {quote(unknown[0])}") + schema_node = doc.get("schema") + schema = schema_node.str_() if schema_node is not None else None + if schema is None: + raise CorpusError("the file does not open with `schema:`") + try: + version = int(schema) + except ValueError: + raise CorpusError(f"{quote(schema)} is not a schema version") from None + if version != SCHEMA: + raise CorpusError(f"this is schema {version} and the runner reads schema {SCHEMA}") + + name = _field(doc, "suite") + doc_text = _field(doc, "doc") + load_node = doc.get("load") + load = _load(load_node) if load_node is not None else None + + cases_node = doc.get("cases") + if cases_node is None: + raise CorpusError("a suite with no `cases:`") + items = cases_node.seq() + if items is None: + raise CorpusError("`cases:` is a sequence") + if not items: + raise CorpusError("a suite with no cases in it") + + cases = [_case(item) for item in items] + # Names are what a report cites and what a binding's skip list + # names, so two cases sharing one is a report that says less than it + # looks like it does. + seen: set[str] = set() + for case in cases: + if case.name in seen: + raise CorpusError(f"two cases are called {quote(case.name)}") + seen.add(case.name) + return Suite(name=name, doc=doc_text, load=load, cases=cases) + + +def _field(node: Node, key: str) -> str: + value = node.get(key) + if value is None: + raise CorpusError(f"line {node.line}: no `{key}:`") + text = value.str_() + if text is None: + raise CorpusError(f"line {node.line}: `{key}:` is one line of text") + return text + + +def _case(node: Node) -> Case: + line = node.line + if node.map() is None: + raise CorpusError(f"line {line}: a case is a mapping, and this is {node.what()}") + unknown = node.unknown(_CASE_KEYS) + if unknown: + raise CorpusError(f"line {line}: a case has no key {quote(unknown[0])}") + + name = _field(node, "name") + spelled = name.isascii() and all(c.islower() or c.isdigit() or c == "-" for c in name) + if not name or not spelled: + raise CorpusError( + f"line {line}: {quote(name)} is a case name, which is lower case words joined by dashes" + ) + doc = _field(node, "doc") + query = _field(node, "query") + + setup: list[str] = [] + setup_node = node.get("setup") + if setup_node is not None: + items = setup_node.seq() + if items is None: + raise CorpusError(f"line {line}: `setup:` is a sequence of statements") + for item in items: + text = item.str_() + if text is None: + raise CorpusError(f"line {item.line}: a setup statement is one line") + setup.append(text) + + params = _params(node) + + raises_node = node.get("raises") + columns_node = node.get("columns") + if raises_node is not None and columns_node is not None: + raise CorpusError( + f"line {line}: a case that raises has no rows, and one that returns rows does not raise" + ) + if raises_node is not None: + code = raises_node.str_() + if code is None: + raise CorpusError(f"line {line}: `raises:` is a GQLSTATUS code") + if len(code) != 5 or not all(c.isdigit() or (c.isupper() and c.isascii()) for c in code): + raise CorpusError( + f"line {raises_node.line}: {quote(code)} is not the shape of a GQLSTATUS, which is " + "five characters of digits and capitals" + ) + return Case(name, doc, query, line, setup, params, raises=code) + if columns_node is None: + raise CorpusError( + f"line {line}: a case says what it produces, with `columns:` and `rows:` or with " + "`raises:`" + ) + names = columns_node.seq() + if names is None: + raise CorpusError(f"line {line}: `columns:` is a sequence of names") + columns = [] + for item in names: + text = item.str_() + if text is None: + raise CorpusError(f"line {item.line}: a column name is one word") + columns.append(text) + rows = _rows(node) + for row in rows: + if len(row) != len(columns): + raise CorpusError(f"line {line}: a row of {len(row)} against {len(columns)} columns") + return Case(name, doc, query, line, setup, params, columns=columns, rows=rows) + + +def _params(node: Node) -> list[tuple[str, object]]: + """The parameters a case binds, which is the value encoding with a + name beside it. + + A name is what the statement spells after the ``$``, so it is checked + against what a statement may spell: a case whose name is ``n one`` is + one no client can bind.""" + params_node = node.get("params") + if params_node is None: + return [] + items = params_node.seq() + if items is None: + raise CorpusError(f"line {params_node.line}: `params:` is a sequence") + out: list[tuple[str, object]] = [] + for item in items: + line = item.line + if item.map() is None: + raise CorpusError( + f"line {line}: a parameter is a mapping of `name`, `type` and `value`, and this " + f"is {item.what()}" + ) + unknown = item.unknown(("name", "type", "value")) + if unknown: + raise CorpusError(f"line {line}: a parameter has no key {quote(unknown[0])}") + name = _field(item, "name") + if not name or not all(c.isascii() and (c.isalnum() or c == "_") for c in name): + raise CorpusError( + f"line {line}: {quote(name)} is a parameter name, which is what a statement writes " + "after the `$`" + ) + if any(n == name for n, _ in out): + raise CorpusError(f"line {line}: two parameters are called {quote(name)}") + out.append((name, values.typed(item))) + return out + + +def _rows(node: Node) -> list[list[object]]: + rows_node = node.get("rows") + if rows_node is None: + # A statement that returns no rows is a case worth having, and + # writing it as an absent `rows:` would make it the same shape as + # one somebody forgot to finish. + raise CorpusError( + f"line {node.line}: `columns:` with no `rows:`. A case expecting nothing back writes " + "`rows:` with an empty sequence under it." + ) + items = rows_node.seq_or_empty() + if items is None: + raise CorpusError(f"line {rows_node.line}: `rows:` is a sequence of rows") + out: list[list[object]] = [] + for item in items: + unknown = item.unknown(("values",)) + if unknown: + raise CorpusError(f"line {item.line}: a row has no key {quote(unknown[0])}") + cells_node = item.get("values") + if cells_node is None: + raise CorpusError(f"line {item.line}: a row is a `values:` and the values under it") + cells = cells_node.seq_or_empty() + if cells is None: + raise CorpusError(f"line {cells_node.line}: `values:` is a sequence of values") + out.append([values.decode(cell) for cell in cells]) + return out + + +def _load(node: Node) -> Load: + line = node.line + if node.map() is None: + raise CorpusError(f"line {line}: a load is a mapping, and this is {node.what()}") + unknown = node.unknown(_LOAD_KEYS) + if unknown: + raise CorpusError(f"line {line}: a load has no key {quote(unknown[0])}") + nodes = _name(node, "nodes") + edges = _name(node, "edges") + count_node = node.get("count") + count_text = count_node.str_() if count_node is not None else None + if count_text is None: + raise CorpusError(f"line {line}: a load says how many rows it has, with `count:`") + try: + count = int(count_text) + except ValueError: + raise CorpusError(f"line {line}: `count:` is a number of rows") from None + if count == 0: + raise CorpusError(f"line {line}: a load of no rows is a load nothing can be read back from") + + columns_node = node.get("columns") + if columns_node is None: + raise CorpusError(f"line {line}: a load has `columns:`") + items = columns_node.seq() + if items is None: + raise CorpusError(f"line {line}: `columns:` is a sequence") + columns = [_column(item, count) for item in items] + if not columns: + raise CorpusError(f"line {line}: a load with no columns holds no values") + seen: set[str] = set() + for column in columns: + if column.name in seen: + raise CorpusError(f"line {line}: two columns are called {quote(column.name)}") + seen.add(column.name) + + pairs: list[tuple[int, int]] = [] + pairs_node = node.get("pairs") + if pairs_node is not None: + items = pairs_node.seq_or_empty() + if items is None: + raise CorpusError(f"line {line}: `pairs:` is a sequence of edges") + for item in items: + pairs.append(_edge(item, count)) + return Load(nodes=nodes, edges=edges, count=count, columns=columns, pairs=pairs) + + +def _name(node: Node, key: str) -> str: + text = _field(node, key) + if not text or not all(c.isascii() and (c.isalnum() or c == "_") for c in text): + raise CorpusError(f"line {node.line}: {quote(text)} is not a table name") + return text + + +def _column(node: Node, count: int) -> Column: + line = node.line + unknown = node.unknown(("name", "type", "values")) + if unknown: + raise CorpusError(f"line {line}: a column has no key {quote(unknown[0])}") + name = _name(node, "name") + ty = _field(node, "type") + if values.form(ty) is None: + raise CorpusError(f"line {line}: {ty} is not a type this encoding knows") + values_node = node.get("values") + items = values_node.seq() if values_node is not None else None + if items is None: + raise CorpusError(f"line {line}: a column holds `values:` in row order") + if len(items) != count: + raise CorpusError( + f"line {line}: column {quote(name)} holds {len(items)} values against the {count} " + "rows the load declares" + ) + return Column(name=name, ty=ty, values=[values.payload(ty, item) for item in items]) + + +def _edge(node: Node, count: int) -> tuple[int, int]: + line = node.line + unknown = node.unknown(("from", "to")) + if unknown: + raise CorpusError(f"line {line}: an edge has no key {quote(unknown[0])}") + ends = [] + for key in ("from", "to"): + value = node.get(key) + text = value.str_() if value is not None else None + if text is None: + raise CorpusError(f"line {line}: an edge has a `{key}:` row number") + try: + end = int(text) + except ValueError: + raise CorpusError(f"line {line}: `{key}:` is a row number") from None + if not 0 <= end < count: + raise CorpusError( + f"line {line}: `{key}: {end}` against a table of {count} rows, which are numbered " + f"0 to {count - 1}" + ) + ends.append(end) + return ends[0], ends[1] diff --git a/conformance/reader.py b/conformance/reader.py new file mode 100644 index 0000000..70ff294 --- /dev/null +++ b/conformance/reader.py @@ -0,0 +1,399 @@ +"""The subset of YAML the corpus is written in. + +YAML is a large language and the corpus needs a small corner of it: +block mappings, block sequences, and scalars. Everything else is refused +with a line number. The files are hand written and are read by people in +nine repositories who did not write them, so a construct a reader +quietly reinterpreted would be a case that says one thing to a reviewer +and another to the runner. + +So: two space indentation and no tabs, ``- `` with exactly one space, +plain, single quoted and double quoted scalars on one line, and +comments. No flow collections, no block scalars, no anchors, no aliases, +no tags, no document markers, no multi document streams. + +This is the third implementation of that subset, after +``crates/zu-corpus/src/yaml.rs`` in the engine and ``conformance/c/yaml.c`` +beside it. PyYAML would read these files, and would read a good deal +more besides: it would take a flow sequence, a block scalar and an +anchor, none of which a case may use, and it would hand back ``9223372036854775807`` +as a float on the way. What the corpus needs is a reader that refuses, +and the cheapest way to have one is to write it. + +Whether a scalar was quoted survives parsing, because the value encoding +turns on it. An INT64 written bare is a number some reader in some +language will round, and refusing it is the whole point of the encoding. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +__all__ = ["Node", "CorpusError", "parse", "quote"] + + +def quote(text: str) -> str: + """A string the way Rust's ``{:?}`` writes one. + + Every refusal in the corpus is written in three languages and diffed + across them, so a value quoted one way here and another way there + would be a difference in the report that is not a difference in the + answer. Python's ``repr`` reaches for single quotes and Rust never + does, so the quoting is written out rather than borrowed.""" + out = ['"'] + for c in text: + if c in '"\\': + out.append("\\" + c) + elif c == "\n": + out.append("\\n") + elif c == "\r": + out.append("\\r") + elif c == "\t": + out.append("\\t") + else: + out.append(c) + out.append('"') + return "".join(out) + + +class CorpusError(Exception): + """A file the corpus will not read, with the line it gave up on.""" + + +@dataclass(frozen=True) +class Node: + """One node of a document, with the line it started on. + + Four kinds, told apart by which field is set. ``EMPTY`` is a key + with nothing under it: it is a node rather than an error because a + case that expects no rows back writes ``rows:`` and stops, and that + is a real expectation which needs a spelling. Every accessor says no + to it, so a ``name:`` left blank is still caught by whoever wanted a + name. + """ + + kind: str + line: int + text: str = "" + quoted: bool = False + items: tuple[Node, ...] = () + pairs: tuple[tuple[str, Node], ...] = () + + def what(self) -> str: + """What kind of node this is, for an error that has to say what + it found instead of what it wanted.""" + return { + "scalar": "a scalar", + "seq": "a sequence", + "map": "a mapping", + "empty": "nothing", + }[self.kind] + + def scalar(self) -> tuple[str, bool] | None: + return (self.text, self.quoted) if self.kind == "scalar" else None + + def str_(self) -> str | None: + return self.text if self.kind == "scalar" else None + + def seq(self) -> tuple[Node, ...] | None: + return self.items if self.kind == "seq" else None + + def seq_or_empty(self) -> tuple[Node, ...] | None: + """A sequence, counting a key with nothing under it as the empty + one. Only a caller for whom empty is a meaningful answer should + reach for this; the rest want :meth:`seq`, so that a list + somebody left unfinished is refused rather than read as none.""" + if self.kind == "empty": + return () + return self.seq() + + def map(self) -> tuple[tuple[str, Node], ...] | None: + return self.pairs if self.kind == "map" else None + + def get(self, key: str) -> Node | None: + if self.kind != "map": + return None + for k, v in self.pairs: + if k == key: + return v + return None + + def unknown(self, known: tuple[str, ...]) -> list[str]: + """The keys that are not in ``known``, so a caller can refuse a + typo rather than drop the field on the floor.""" + if self.kind != "map": + return [] + return [k for k, _ in self.pairs if k not in known] + + +@dataclass +class Line: + """One meaningful line: its indent, whether a ``- `` opened it, what + is left after that, and where it was.""" + + indent: int + dash: bool + text: str + no: int + + +def parse(text: str) -> Node: + """A document, or the first thing in it this reader will not read.""" + lines = _lex(text) + if not lines: + raise CorpusError("the file has nothing in it") + if lines[0].indent != 0: + raise CorpusError(f"line {lines[0].no}: the first line is indented") + at = _Cursor(lines) + node = _node(at, 0) + if at.i < len(lines): + raise CorpusError(f"line {lines[at.i].no}: this belongs to nothing above it") + return node + + +@dataclass +class _Cursor: + """Where the parser is, which the recursive calls share.""" + + lines: list[Line] + i: int = field(default=0) + + def at(self, offset: int = 0) -> Line | None: + j = self.i + offset + return self.lines[j] if j < len(self.lines) else None + + +def _lex(text: str) -> list[Line]: + """Lines, with blanks and comments dropped and every ``- `` split + into the item it opens and the content that followed it on the same + line. Splitting here rather than in the parser is what lets + ``- name: x`` and a ``name: x`` on its own line be the same shape by + the time anything looks at them.""" + out: list[Line] = [] + for n, raw in enumerate(text.split("\n")): + no = n + 1 + tab = raw.find("\t") + if tab >= 0: + raise CorpusError( + f"line {no}: a tab at column {tab + 1}, and indentation here is spaces" + ) + content = _strip_comment(raw).rstrip() + indent = len(content) - len(content.lstrip()) + rest = content.lstrip() + if not rest: + continue + if rest in ("---", "..."): + raise CorpusError( + f"line {no}: {quote(rest)} opens or closes a document, and a file here holds one" + ) + if indent % 2: + raise CorpusError( + f"line {no}: indented {indent}, and indentation here goes two spaces at a time" + ) + + if not (rest == "-" or rest.startswith("- ")): + out.append(Line(indent, False, rest, no)) + continue + rest = rest[1:] + if rest.startswith(" "): + raise CorpusError( + f"line {no}: a `- ` takes exactly one space, so that what follows it lines up " + "with the lines under it" + ) + rest = rest.lstrip() + if rest.startswith("- "): + raise CorpusError( + f"line {no}: a sequence opening straight into another one, which nothing here needs" + ) + out.append(Line(indent, True, "", no)) + if rest: + out.append(Line(indent + 2, False, rest, no)) + return out + + +def _strip_comment(line: str) -> str: + """Everything from an unquoted ``` #``` on is a comment. + + Three rules keep this from eating content. A ``#`` starts a comment + only with whitespace before it, because one inside a word is part of + the word. A quote opens a quoted run only with whitespace before it, + because a quote inside a word is part of the word too, which is what + lets a ``doc:`` say "it's" without opening a run that never closes. + And a quote that opens nothing that closes was not a run at all, + which is what lets a ``query:`` hold ``cast(' 42 ' AS INT64)``. + """ + i = 0 + n = len(line) + while i < n: + c = line[i] + opens = i == 0 or line[i - 1].isspace() + if c == "#" and opens: + return line[:i] + if c in "\"'" and opens: + end = _closing_quote(line[i + 1 :], c) + if end is not None: + i += 1 + end + i += 1 + return line + + +def _closing_quote(rest: str, mark: str) -> int | None: + """The offset of the quote that closes a run whose opening quote has + already been passed, or ``None`` if the line ends first. + + The two styles hide a quote differently: a double quoted run escapes + with a backslash, and a single quoted run doubles the quote, which + is the only escape it has.""" + i = 0 + n = len(rest) + while i < n: + if rest[i] == "\\" and mark == '"': + i += 2 + continue + if rest[i] == mark: + if mark == "'" and rest[i + 1 : i + 2] == "'": + i += 2 + continue + return i + i += 1 + return None + + +def _node(at: _Cursor, indent: int) -> Node: + """The node that starts where the cursor is and is indented + ``indent``, leaving the cursor on the first line that is not part of + it.""" + line = at.at() + assert line is not None + if line.dash: + return _seq(at, indent) + # A mapping key is a bare word and a `:`. Anything else at this + # position is a scalar standing on its own, which is what the items + # of a sequence of scalars are. + if _split_key(line.text) is not None: + return _map(at, indent) + at.i += 1 + return _scalar(line.text, line.no) + + +def _seq(at: _Cursor, indent: int) -> Node: + line = at.at() + assert line is not None + start = line.no + items: list[Node] = [] + while True: + here = at.at() + if here is None or not here.dash or here.indent != indent: + break + opened = here.no + at.i += 1 + nxt = at.at() + if nxt is not None and nxt.indent == indent + 2: + items.append(_node(at, indent + 2)) + elif nxt is not None and nxt.indent > indent: + raise CorpusError( + f"line {nxt.no}: indented {nxt.indent}, where an item of the sequence on line " + f"{opened} is indented {indent + 2}" + ) + else: + raise CorpusError(f"line {opened}: a `-` with nothing after it") + return Node("seq", start, items=tuple(items)) + + +def _map(at: _Cursor, indent: int) -> Node: + line = at.at() + assert line is not None + start = line.no + pairs: list[tuple[str, Node]] = [] + while True: + here = at.at() + if here is None or here.dash or here.indent != indent: + break + split = _split_key(here.text) + if split is None: + break + key, rest = split + opened = here.no + at.i += 1 + if rest: + value = _scalar(rest, opened) + else: + nxt = at.at() + if nxt is not None and nxt.indent == indent + 2: + value = _node(at, indent + 2) + elif nxt is not None and nxt.indent > indent: + raise CorpusError( + f"line {nxt.no}: indented {nxt.indent}, where what is under `{key}:` on line " + f"{opened} is indented {indent + 2}" + ) + else: + value = Node("empty", opened) + if any(k == key for k, _ in pairs): + raise CorpusError(f"line {opened}: {key} is set twice in one mapping") + pairs.append((key, value)) + return Node("map", start, pairs=tuple(pairs)) + + +def _split_key(text: str) -> tuple[str, str] | None: + """The key and the rest of the line, when the line opens a mapping + entry. A key is a bare word, and the ``:`` after it ends the line or + has a space after it, so that a plain scalar holding a colon is still + a scalar.""" + key, sep, rest = text.partition(": ") + if sep: + rest = rest.lstrip() + else: + if not text.endswith(":"): + return None + key, rest = text[:-1], "" + if not key or not all(c.isascii() and (c.isalnum() or c in "_-") for c in key): + return None + return key, rest + + +def _scalar(text: str, line: int) -> Node: + for mark in "\"'": + if not text.startswith(mark): + continue + body = text[1:] + # The closing quote is found by scanning rather than by taking + # the last one on the line, so that `"a" and "b"` is refused + # instead of read as one scalar with quotes in the middle. + end = _closing_quote(body, mark) + if end is None: + raise CorpusError(f"line {line}: a {mark} that opens and does not close on its line") + if end + 1 != len(body): + raise CorpusError(f"line {line}: {quote(body[end + 1 :])} after the scalar ends") + inner = body[:end] + # A single quoted run has one escape, the doubled quote, and a + # backslash in it is a backslash. + value = _unescape(inner, line) if mark == '"' else inner.replace("''", "'") + return Node("scalar", line, text=value, quoted=True) + if text and text[0] in "[]{}&*!|>%@`": + raise CorpusError( + f"line {line}: a plain scalar opening with '{text[0]}', which is a construct this " + "reader does not read" + ) + return Node("scalar", line, text=text, quoted=False) + + +_ESCAPES = {'"': '"', "\\": "\\", "n": "\n", "r": "\r", "t": "\t", "0": "\0"} + + +def _unescape(body: str, line: int) -> str: + out: list[str] = [] + i = 0 + while i < len(body): + c = body[i] + if c != "\\": + out.append(c) + i += 1 + continue + if i + 1 >= len(body): + raise CorpusError(f"line {line}: a scalar ending in a backslash") + nxt = body[i + 1] + if nxt not in _ESCAPES: + raise CorpusError(f"line {line}: \\{nxt} is not an escape") + out.append(_ESCAPES[nxt]) + i += 2 + return "".join(out) diff --git a/conformance/runner.py b/conformance/runner.py new file mode 100644 index 0000000..d267e8b --- /dev/null +++ b/conformance/runner.py @@ -0,0 +1,304 @@ +"""Running the corpus through this client, and saying what happened in +the form the other eight runners are compared against. + +Each case gets a database of its own. Cases in a suite are written as if +nothing came before them, and the cheapest way to keep that true is to +make it true: a case that leaked a table into the next one would be a +failure that moves when the file is reordered, which is the worst kind to +be handed. + +An outcome is one of three things and not two. Passed and failed are +obvious. Unsupported is the third, and it exists because the corpus is +versioned with the engine and shipped to nine clients that will not all +implement the same subset at the same time: a client that cannot yet +parse a statement should say so, and a report should be able to tell +that apart from an answer that came back wrong. + +What this prints is what the Rust runner prints, line for line, so that +a disagreement between two clients is a diff and not a reading exercise. +""" + +from __future__ import annotations + +import argparse +import sys +import tempfile +from dataclasses import dataclass, field +from pathlib import Path + +import zudb + +from .cases import Case, Load, Suite, read_dir +from .reader import CorpusError +from .values import same, show, too_fine, truncated + +__all__ = ["Ran", "Report", "run", "main"] + +PASSED = "passed" +FAILED = "failed" +UNSUPPORTED = "unsupported" + +_MARK = {PASSED: "ok", FAILED: "FAILED", UNSUPPORTED: "unsupported"} + + +@dataclass +class Ran: + """What one case did, with the account of why when it did not pass.""" + + suite: str + case: str + line: int + outcome: str + #: What went wrong, in enough detail to fix the case or the engine + #: without running it again. + detail: str = "" + + def __str__(self) -> str: + head = f"{self.suite}/{self.case} line {self.line} {_MARK[self.outcome]}" + return f"{head}: {self.detail}" if self.detail else head + + +@dataclass +class Report: + """What a whole run did.""" + + ran: list[Ran] = field(default_factory=list) + + def count(self, outcome: str) -> int: + return sum(1 for r in self.ran if r.outcome == outcome) + + def failures(self) -> list[Ran]: + return [r for r in self.ran if r.outcome == FAILED] + + def summary(self) -> str: + """One line saying what the run came to, which is what a CI log + keeps and what two runs are compared by.""" + return ( + f"{len(self.ran)} cases, {self.count(PASSED)} passed, {self.count(FAILED)} failed, " + f"{self.count(UNSUPPORTED)} unsupported" + ) + + +def run(suites: list[Suite], directory: Path) -> Report: + """Runs every case of every suite, in the order they were written. + + ``directory`` is one the runner may make databases under. Each case + gets its own file in it, named after the case, so that a failure + leaves something to open.""" + report = Report() + for suite in suites: + for case in suite.cases: + report.ran.append(_one(suite, case, directory)) + return report + + +def _one(suite: Suite, case: Case, directory: Path) -> Ran: + def ran(outcome: str, detail: str = "") -> Ran: + return Ran(suite.name, case.name, case.line, outcome, detail) + + unheld = _unheld(case) + if unheld is not None: + return ran(UNSUPPORTED, unheld) + + path = directory / f"{suite.name}-{case.name}.zu" + # The load goes in before the connection opens, because it is bulk + # load and bulk load is the path that builds the file rather than one + # that goes through a statement. Every case of the suite gets its own + # copy of it for the same reason every case gets its own database. + if suite.load is not None: + try: + _apply(suite.load, path) + except zudb.Error as e: + return ran(FAILED, f"the suite's load: {e}") + try: + conn = zudb.connect(path) + except zudb.Error as e: + return ran(FAILED, f"opening {path}: {e}") + + try: + for i, statement in enumerate(case.setup): + try: + conn.execute(statement) + except zudb.Error as e: + # A setup that fails is not a result about the statement + # under test, so it is never a pass and never a quiet + # skip. + if _unsupported(e): + return ran(UNSUPPORTED, f"setup {i + 1}: {_said(e)}") + return ran(FAILED, f"setup {i + 1} failed: {_said(e)}") + + params = dict(case.params) if case.params else None + try: + result = conn.execute(case.query, params) + rows = result.fetchall() + except zudb.Error as e: + if case.raises is not None: + if e.code is None: + return ran( + FAILED, + f"failed with no GQLSTATUS where the case wants {case.raises}: {_said(e)}", + ) + if e.code == case.raises: + return ran(PASSED) + return ran( + FAILED, + f"raised {e.code} where the case wants {case.raises}: {_said(e)}", + ) + if _unsupported(e): + return ran(UNSUPPORTED, _said(e)) + return ran(FAILED, _said(e)) + + if case.raises is not None: + return ran(FAILED, f"returned rows where the case wants {case.raises}") + detail = _compare(case.columns or [], case.rows or [], result.columns, rows) + return ran(PASSED) if detail is None else ran(FAILED, detail) + finally: + conn.close() + + +def _apply(load: Load, path: Path) -> None: + """The suite's load, through this client's own bulk load path, which + is the strongest form of the corpus question: the value crosses the + boundary twice and by two different mechanisms. + + A column holding a value finer than this client can carry goes in + truncated rather than not at all, so that the cases reading the + other columns of the same suite still run. The cases that read that + column back say what happened, because their own expectation is a + value nothing equals.""" + zudb.load( + path, + nodes=load.nodes, + rels=load.edges, + columns={column.name: truncated(column.values) for column in load.columns}, + edges=load.pairs, + rows=load.count, + ) + + +def _unheld(case: Case) -> str | None: + """Why this client cannot run the case at all, if it cannot. + + The one reason so far is a temporal written finer than a microsecond, + which the engine keeps and Python's ``datetime`` does not. It is + unsupported rather than failed because nothing went wrong: the + engine answered, and the value mapping this client documents is + where the digits went. Reported before anything runs, because the + value would otherwise be handed to a binding that has no idea what it + is. The suite's load is not looked at, because a column this client + truncates only matters to the cases that read it back, and those + cases say so on their own.""" + where = [value for _, value in case.params] + where += [value for row in case.rows or [] for value in row] + for value in where: + found = too_fine(value) + if found is not None: + return ( + f"this client holds a time to the microsecond and the case writes " + f"{show(found)}, which is finer" + ) + return None + + +def _unsupported(e: zudb.Error) -> bool: + """Whether a condition means the engine does not implement the + statement rather than that the statement is wrong. + + The two GQL classes that say so are 42, syntax error or access rule + violation, and 0A, feature not supported. A case landing on either is + a case ahead of the engine, which the corpus allows on purpose: the + cases are the contract and the engine catches up to them.""" + return e.code is not None and (e.code.startswith("42") or e.code.startswith("0A")) + + +def _said(e: zudb.Error) -> str: + """What the engine said, which is what the Rust runner prints for the + same failure. The message a condition carries already opens with its + own code, so nothing is added to it here and the two reports diff + clean.""" + return str(e) + + +def _compare( + want_columns: list[str], + want_rows: list[list[object]], + got_columns: list[str], + got_rows: list[tuple[object, ...]], +) -> str | None: + """What differs between what a case wants and what came back, or + ``None`` if nothing does. + + It reports the first difference rather than all of them, because the + first is nearly always the cause of the rest, and a report that + prints a hundred rows is one nobody reads to the end. The order the + checks run in is the reference runner's, so that two runners looking + at the same wrong answer say the same thing about it.""" + if want_columns != got_columns: + return f"columns {_names(got_columns)} where the case wants {_names(want_columns)}" + for i, (want, got) in enumerate(zip(want_rows, got_rows, strict=False)): + for j, (a, b) in enumerate(zip(want, got, strict=False)): + if not same(a, b): + name = want_columns[j] if j < len(want_columns) else "?" + return f"row {i + 1} column {name} is {show(b)} where the case wants {show(a)}" + if len(want_rows) != len(got_rows): + return f"{len(got_rows)} rows where the case wants {len(want_rows)}" + return None + + +def _names(columns: list[str]) -> str: + """A list of column names the way Rust's ``{:?}`` writes one.""" + quoted = ", ".join(f'"{name}"' for name in columns) + return f"[{quoted}]" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="python -m conformance", + description="run the shared corpus cases against this client", + ) + parser.add_argument("dir", type=Path, help="the directory of case files") + parser.add_argument( + "--strict", + action="store_true", + help="an unsupported case fails the run, which is what a release branch wants", + ) + parser.add_argument("--quiet", action="store_true", help="print the summary and nothing else") + parser.add_argument( + "--work", + type=Path, + default=None, + help="a directory to make the case databases under, kept rather than removed", + ) + args = parser.parse_args(argv) + + try: + suites = read_dir(args.dir) + except CorpusError as e: + # One rather than two, because the reference runner exits one for + # a corpus it cannot read and a report that is compared line for + # line is worth less if the two disagree about what the run came + # to. + print(f"zu corpus: {e}", file=sys.stderr) + return 1 + + if args.work is not None: + args.work.mkdir(parents=True, exist_ok=True) + report = run(suites, args.work) + else: + with tempfile.TemporaryDirectory(prefix="zudb-corpus-") as work: + report = run(suites, Path(work)) + + if not args.quiet: + for ran in report.ran: + if ran.outcome != PASSED: + print(ran) + print(report.summary()) + if report.count(FAILED): + return 1 + if args.strict and report.count(UNSUPPORTED): + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/conformance/values.py b/conformance/values.py new file mode 100644 index 0000000..34395ce --- /dev/null +++ b/conformance/values.py @@ -0,0 +1,596 @@ +"""The ``{type, value}`` encoding a case writes its values in. + +Every value in the corpus is a mapping with a ``type`` naming the GQL +type and a ``value`` holding the payload. The type is written down +rather than inferred because the corpus is read by nine languages and +inference is where they differ: a bare ``1`` is an integer in YAML, and +which integer it becomes is a decision each host language makes on its +own. + +The payload is a YAML scalar where a YAML scalar is exact, and a string +where it is not. An integer wider than 53 bits is a string, because most +YAML readers hand a number to a double. A float is a string, for that +reason and for ``NaN``, ``inf`` and ``-0.0``. A temporal value is a +string, because YAML has no type that keeps an offset. + +Refusing the wrong form is half the point, and refusing it here is what +makes this a second reader of the corpus rather than a consumer of it. + +What a decoded value becomes is a Python value, because that is what a +statement gives back through this client and what a comparison has to be +against. The declared width is dropped in the process, so a case that +says INT8 and one that says INT64 both become ``int``: that is a fact +about Python rather than about the corpus, and the reference runner in +Rust drops it too, since the engine's own value is one signed 64 bit +integer either way. +""" + +from __future__ import annotations + +import datetime +import math +from dataclasses import dataclass + +from zudb import Duration, Node, Path, Rel + +from .reader import CorpusError, quote +from .reader import Node as YamlNode + +__all__ = ["decode", "typed", "payload", "same", "show", "form", "TooFine", "too_fine"] + +#: Whether a type's payload is written as a quoted string. ``False`` is +#: a type a YAML scalar carries without loss, ``True`` is one it does +#: not. +_TYPES: dict[str, bool] = { + "NULL": False, + "BOOL": False, + "INT8": False, + "INT16": False, + "INT32": False, + "INT64": True, + "UINT8": False, + "UINT16": False, + "UINT32": False, + "UINT64": True, + "FLOAT32": True, + "FLOAT64": True, + "STRING": False, + "DATE": True, + "LOCALTIME": True, + "ZONEDTIME": True, + "LOCALDATETIME": True, + "ZONEDDATETIME": True, + "DURATION": True, + "LIST": False, +} + +#: The types the encoding reserves a name for and the engine has no +#: runtime value for yet, kept apart from an outright typo so that the +#: error says which of the two it is. +_RESERVED = ("DECIMAL", "BYTES", "NODE", "EDGE", "PATH") + +#: The range each integer width holds, so that a case writing a value +#: its own type cannot carry is refused rather than stored wider than it +#: says. UINT64 stops at the signed maximum because the engine's integer +#: is signed and 64 bits wide, and wrapping the top half into a negative +#: would be a case that passes while meaning the opposite of what it says. +_RANGES: dict[str, tuple[int, int]] = { + "INT8": (-(2**7), 2**7 - 1), + "INT16": (-(2**15), 2**15 - 1), + "INT32": (-(2**31), 2**31 - 1), + "INT64": (-(2**63), 2**63 - 1), + "UINT8": (0, 2**8 - 1), + "UINT16": (0, 2**16 - 1), + "UINT32": (0, 2**32 - 1), + "UINT64": (0, 2**63 - 1), +} + + +@dataclass(frozen=True) +class TooFine: + """A temporal value written finer than this client can hold. + + The engine keeps a time to the nanosecond and Python's ``datetime`` + keeps one to the microsecond, so a case asserting nine digits is a + case this client's value mapping cannot answer. Decoding it to a + ``datetime`` would truncate it, and the case would then pass by + comparing one truncated value against another, which is the exact + defect the case was written to catch. + + So it decodes to this instead. Nothing equals it, it prints as the + text that was written, and the runner turns a case holding one into + an unsupported rather than a failure, because a value mapping that + cannot carry the value is a limit of the client and not a wrong + answer from the engine.""" + + ty: str + text: str + + +def truncated(value: object) -> object: + """The nearest thing Python has to a value it cannot hold exactly. + + Only the load needs this. A column has to go into the file for the + suite to have a graph at all, and refusing to load it would take out + every case of the suite rather than the ones that read the column. + So the column goes in truncated, and the cases that read it back get + a truncated answer against an expectation nothing equals, which is + the report those cases should give.""" + if isinstance(value, TooFine): + head, _, rest = value.text.partition(".") + digits = "" + while rest[len(digits) : len(digits) + 1].isdigit(): + digits += rest[len(digits)] + # The offset is after the fraction and belongs to the value, so + # what is cut is the digits and not the tail behind them. + out = _scalar(value.ty, f"{head}.{digits[:6]}{rest[len(digits) :]}") + if out is _NOT_ONE: + raise CorpusError(f"{value.ty} {value.text} does not truncate to a value") + return out + if isinstance(value, list): + return [truncated(item) for item in value] + return value + + +def too_fine(value: object) -> TooFine | None: + """The first value inside this one that this client cannot hold, if + there is one. Recursive, because a list of times is a list.""" + if isinstance(value, TooFine): + return value + if isinstance(value, list): + for item in value: + found = too_fine(item) + if found is not None: + return found + return None + + +def form(ty: str) -> bool | None: + """Whether a type is written quoted, or ``None`` if it is not a + type.""" + return _TYPES.get(ty) + + +def _unknown(ty: str) -> str: + if ty in _RESERVED: + return f"{ty} is a type the encoding reserves and the engine has no value for" + return f"{ty} is not a type this encoding knows" + + +def decode(node: YamlNode) -> object: + """The value a ``{type, value}`` mapping describes.""" + if node.map() is None: + raise CorpusError( + f"line {node.line}: a value is a mapping of `type` and `value`, and this is " + f"{node.what()}" + ) + unknown = node.unknown(("type", "value")) + if unknown: + raise CorpusError(f"line {node.line}: a value has no key {quote(unknown[0])}") + return typed(node) + + +def typed(node: YamlNode) -> object: + """The ``type`` and ``value`` of a mapping that carries more than + those two, which is a parameter: it is a value with a name, and the + name belongs to the case rather than to the encoding.""" + line = node.line + ty_node = node.get("type") + if ty_node is None: + raise CorpusError(f"line {line}: a value with no `type`") + ty = ty_node.str_() + if ty is None: + raise CorpusError(f"line {line}: a `type` that is not a name") + + # Checked here as well as in `payload`, because a value whose type is + # not a type and which also has no `value` under it should be told + # about the type first: that is the mistake, and the missing payload + # is a consequence of it. + if form(ty) is None: + raise CorpusError(f"line {line}: {_unknown(ty)}") + + if ty == "NULL": + if node.get("value") is not None: + raise CorpusError(f"line {line}: NULL carries no `value`") + return None + value = node.get("value") + if value is None: + raise CorpusError(f"line {line}: a {ty} with no `value`") + return payload(ty, value) + + +def payload(ty: str, value: YamlNode) -> object: + """The value a payload spells under a type that has already been + read. + + A row of a case names its type beside every value. A column of a load + names it once at the top and every value under it is a bare payload, + which is the same encoding with the type factored out, so it is the + same function reading it.""" + quoted_form = form(ty) + if quoted_form is None: + raise CorpusError(f"line {value.line}: {_unknown(ty)}") + + if ty == "LIST": + # The empty list is a value worth a case and needs a spelling, + # which is a `value:` with nothing under it. + items = value.seq_or_empty() + if items is None: + raise CorpusError( + f"line {value.line}: a LIST holds a sequence of values, and this is {value.what()}" + ) + return [decode(item) for item in items] + + scalar = value.scalar() + if scalar is None: + raise CorpusError(f"line {value.line}: a {ty} holds one scalar, and this is {value.what()}") + text, quoted = scalar + line = value.line + # The one rule the whole encoding exists for, checked before the text + # is looked at, because a value that parses is exactly the case where + # a silent misread would survive review. + if quoted_form and not quoted: + raise CorpusError( + f"line {line}: {ty} is written in quotes, because a bare {text} is a number and some " + "reader of this file will round it" + ) + if not quoted_form and quoted and ty != "STRING": + raise CorpusError( + f"line {line}: {ty} is written without quotes, so that a reader cannot take it for a " + "string" + ) + out = _scalar(ty, text) + if out is _NOT_ONE: + raise CorpusError(f"line {line}: {quote(text)} is not a {ty}") + return out + + +#: What `_scalar` gives back for text that does not spell a value of the +#: type, which cannot be `None` because `None` is a value NULL spells. +_NOT_ONE = object() + + +def _scalar(ty: str, text: str) -> object: + if ty == "BOOL": + return {"true": True, "false": False}.get(text, _NOT_ONE) + if ty == "STRING": + return text + if ty in _RANGES: + try: + n = int(text) + except ValueError: + return _NOT_ONE + # Python's int is unbounded and the corpus's is not, so the + # width has to be checked rather than trusted, which is the one + # place this reader does work its Rust counterpart gets from the + # type system. + if text != str(n): + return _NOT_ONE + low, high = _RANGES[ty] + return n if low <= n <= high else _NOT_ONE + if ty in ("FLOAT32", "FLOAT64"): + f = _float(text) + if f is _NOT_ONE: + return f + assert isinstance(f, float) + return _to_float32(f) if ty == "FLOAT32" else f + if ty == "DATE": + return _parse(text, _date) + if ty in ("LOCALTIME", "ZONEDTIME", "LOCALDATETIME", "ZONEDDATETIME"): + fn = { + "LOCALTIME": _local_time, + "ZONEDTIME": _zoned_time, + "LOCALDATETIME": _local_datetime, + "ZONEDDATETIME": _zoned_datetime, + }[ty] + out = _parse(text, fn) + # Parsed first, so that text which is not a time of any precision + # is refused as such rather than reported as one this client + # cannot hold. + if out is not _NOT_ONE and _finer_than_a_microsecond(text): + return TooFine(ty, text) + return out + if ty == "DURATION": + return _parse(text, _duration) + return _NOT_ONE + + +def _parse(text: str, fn) -> object: + try: + return fn(text) + except ValueError: + return _NOT_ONE + + +def _float(text: str) -> object: + """A float, including the three spellings YAML has no opinion about. + + They are spelled the way Rust prints them, because that is what the + reference runner writes into a failure report and what a case is + pasted from.""" + if text == "NaN": + return math.nan + if text == "inf": + return math.inf + if text == "-inf": + return -math.inf + # A float is exact here, so `1` is not a FLOAT64 and neither is + # `1e400`. The first is an integer somebody meant to write as `1.0` + # and the second is `inf` under another name. + if not any(c in text for c in ".eE"): + return _NOT_ONE + try: + f = float(text) + except ValueError: + return _NOT_ONE + return f if math.isfinite(f) else _NOT_ONE + + +def _to_float32(f: float) -> float: + """The double a float rounds to when it is stored as one, which is + what the engine gives back for a FLOAT32 column.""" + import struct + + return struct.unpack(" bool: + """Whether a temporal's text carries a digit Python's ``datetime`` + would drop. Read off the text rather than off the parsed value, + because the parse is where the digits go.""" + _, dot, rest = text.partition(".") + if not dot: + return False + digits = "" + for c in rest: + if not c.isdigit(): + break + digits += c + return any(c != "0" for c in digits[6:]) + + +def _date(text: str) -> datetime.date: + return datetime.date.fromisoformat(text) + + +def _local_time(text: str) -> datetime.time: + t = datetime.time.fromisoformat(text) + if t.tzinfo is not None: + raise ValueError("a local time carries no offset") + return t + + +def _zoned_time(text: str) -> datetime.time: + t = datetime.time.fromisoformat(text.replace("Z", "+00:00")) + if t.tzinfo is None: + raise ValueError("a zoned time carries an offset") + return t + + +def _local_datetime(text: str) -> datetime.datetime: + d = datetime.datetime.fromisoformat(text) + if d.tzinfo is not None: + raise ValueError("a local datetime carries no offset") + return d + + +def _zoned_datetime(text: str) -> datetime.datetime: + d = datetime.datetime.fromisoformat(text.replace("Z", "+00:00")) + if d.tzinfo is None: + raise ValueError("a zoned datetime carries an offset") + return d + + +def _duration(text: str) -> Duration: + """An ISO 8601 duration, in the two kinds the engine keeps apart. + + A duration is months or it is nanoseconds and never both, because a + month is not a number of days and adding one to a date is not the + same operation. The text says which: a duration whose only fields + are years and months is a year-month one, and everything else is + day-time.""" + negative = text.startswith("-") + if negative or text.startswith("+"): + text = text[1:] + if not text.startswith("P"): + raise ValueError("a duration starts with P") + body = text[1:] + date_part, _, time_part = body.partition("T") + months = 0 + nanos = 0 + for value, unit in _fields(date_part): + if unit == "Y": + months += int(value * 12) + elif unit == "M": + months += int(value) + elif unit == "W": + nanos += int(value * 7 * 86_400 * 1_000_000_000) + elif unit == "D": + nanos += int(value * 86_400 * 1_000_000_000) + else: + raise ValueError(f"{unit} is not a date field of a duration") + for value, unit in _fields(time_part): + if unit == "H": + nanos += int(value * 3_600 * 1_000_000_000) + elif unit == "M": + nanos += int(value * 60 * 1_000_000_000) + elif unit == "S": + nanos += int(round(value * 1_000_000_000)) + else: + raise ValueError(f"{unit} is not a time field of a duration") + if months and nanos: + raise ValueError("a duration is months or it is nanoseconds, not both") + if not date_part and not time_part: + raise ValueError("a duration with nothing in it") + if negative: + months, nanos = -months, -nanos + return Duration(months=months, nanoseconds=nanos) + + +def _fields(text: str) -> list[tuple[float, str]]: + out: list[tuple[float, str]] = [] + number = "" + for c in text: + if c.isdigit() or c in ".-+": + number += c + continue + if not number: + raise ValueError(f"{c} with no number before it") + out.append((float(number), c)) + number = "" + if number: + raise ValueError(f"{number} with no unit after it") + return out + + +def same(want: object, got: object) -> bool: + """Whether two values are the same value. + + Not ``==``, for one reason: a float. ``NaN`` is not equal to itself + and a case asserting ``NaN`` has to pass, and ``0.0`` equals + ``-0.0`` and a case asserting ``-0.0`` has to fail on ``0.0``, + because the sign of zero is exactly the sort of thing that survives + one binding and not another. + + Python needs one rule its Rust counterpart does not: ``True == 1`` + there is false and here it is true, so a case asserting a boolean + must not be answered with an integer, and the types are compared + before the values are.""" + if isinstance(want, bool) != isinstance(got, bool): + return False + # A value this client cannot hold is equal to nothing, including + # itself, so that a case carrying one never passes quietly. + if isinstance(want, TooFine) or isinstance(got, TooFine): + return False + if isinstance(want, float) and isinstance(got, float): + if math.isnan(want) and math.isnan(got): + return True + return math.copysign(1.0, want) == math.copysign(1.0, got) and want == got + if isinstance(want, list) and isinstance(got, list): + return len(want) == len(got) and all(same(a, b) for a, b in zip(want, got, strict=True)) + if isinstance(want, dict) and isinstance(got, dict): + return list(want) == list(got) and all(same(want[k], got[k]) for k in want) + return type(want) is type(got) and want == got + + +def show(value: object) -> str: + """How a value reads in a failure report, in the encoding's own + spelling so that it can be pasted into a case, and line for line + what the Rust runner prints so that two reports can be diffed.""" + if value is None: + return "NULL" + if isinstance(value, bool): + return f"BOOL {'true' if value else 'false'}" + if isinstance(value, int): + return f'INT64 "{value}"' + if isinstance(value, float): + return f'FLOAT64 "{_show_float(value)}"' + if isinstance(value, str): + return f"STRING {quote(value)}" + if isinstance(value, Duration): + return f'DURATION "{_show_duration(value)}"' + if isinstance(value, TooFine): + return f'{value.ty} "{value.text}"' + if isinstance(value, datetime.datetime): + name = "ZONEDDATETIME" if value.tzinfo else "LOCALDATETIME" + text = f"{_show_date(value)}T{_show_time(value)}{_show_offset(value)}" + return f'{name} "{text}"' + if isinstance(value, datetime.date): + return f'DATE "{_show_date(value)}"' + if isinstance(value, datetime.time): + name = "ZONEDTIME" if value.tzinfo else "LOCALTIME" + return f'{name} "{_show_time(value)}{_show_offset(value)}"' + if isinstance(value, list): + return f"LIST [{', '.join(show(item) for item in value)}]" + if isinstance(value, dict): + fields = ", ".join(f"{name}: {show(v)}" for name, v in value.items()) + return f"RECORD {{{fields}}}" + if isinstance(value, (Node, Rel, Path)): + return repr(value) + return repr(value) + + +def _show_float(f: float) -> str: + if math.isnan(f): + return "NaN" + if f == math.inf: + return "inf" + if f == -math.inf: + return "-inf" + # Rust's `{:?}` is the shortest text that reads back as the same + # double and always carries a point or an exponent, which is what + # `repr` gives here except for two things: a float that is a whole + # number wants the point written, and an exponent is written bare + # rather than with the sign and the padding Python puts on it. + text = repr(f) + if "e" in text: + mantissa, _, exponent = text.partition("e") + return f"{mantissa}e{int(exponent)}" + return text if "." in text else text + ".0" + + +def _show_date(d: datetime.date) -> str: + return f"{d.year:04}-{d.month:02}-{d.day:02}" + + +def _show_time(t: datetime.time | datetime.datetime) -> str: + """A time the way the engine prints one, which is seconds always and + a fraction of nine digits when there is one. Python writes six and + drops them when they are zero, and a report that is diffed against + the reference one cannot do either.""" + text = f"{t.hour:02}:{t.minute:02}:{t.second:02}" + if t.microsecond: + text += f".{t.microsecond * 1000:09d}" + return text + + +def _show_offset(t: datetime.time | datetime.datetime) -> str: + """An offset, which is ``Z`` at zero rather than ``+00:00``.""" + offset = t.utcoffset() + if offset is None: + return "" + minutes = int(offset.total_seconds()) // 60 + if minutes == 0: + return "Z" + sign = "-" if minutes < 0 else "+" + hours, rest = divmod(abs(minutes), 60) + return f"{sign}{hours:02}:{rest:02}" + + +def _show_duration(d: Duration) -> str: + """The ISO 8601 text the engine prints a duration as, which is the + text it parses back: a field that is zero is left out, and a + duration with nothing left in it is written ``PT0S`` or ``P0M``, + because ``P`` on its own is not a value.""" + if d.kind == "year_month": + count = d.months + sign = "-" if count < 0 else "" + years, months = divmod(abs(count), 12) + out = f"{sign}P" + if years: + out += f"{years}Y" + if months or not years: + out += f"{months}M" + return out + nanos = d.nanoseconds + sign = "-" if nanos < 0 else "" + days, rest = divmod(abs(nanos), 86_400 * 1_000_000_000) + out = f"{sign}P" + if days: + out += f"{days}D" + if rest == 0 and days: + return out + out += "T" + hours, rest = divmod(rest, 3_600 * 1_000_000_000) + minutes, rest = divmod(rest, 60 * 1_000_000_000) + seconds, fraction = divmod(rest, 1_000_000_000) + if hours: + out += f"{hours}H" + if minutes: + out += f"{minutes}M" + if seconds or fraction or (not hours and not minutes): + out += str(seconds) + if fraction: + out += f".{fraction:09d}" + out += "S" + return out diff --git a/pyproject.toml b/pyproject.toml index a6ea159..339323f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,8 +65,10 @@ strip = true testpaths = ["tests"] addopts = "-q" # `tools` holds the checks the release job runs, which are tested here -# rather than only on the tag that would fail because of one. -pythonpath = ["tools"] +# rather than only on the tag that would fail because of one, and `.` +# holds `conformance`, which is a development tool rather than something +# the wheel ships. +pythonpath = ["tools", "."] [tool.ruff] target-version = "py311" diff --git a/src/appender.rs b/src/appender.rs index ce385d2..d63e10f 100644 --- a/src/appender.rs +++ b/src/appender.rs @@ -531,7 +531,8 @@ impl Buffer { /// a rel table, which is nothing beside the commit either of them is /// about to do. fn catalog(engine: &mut zudb::Connection) -> Result { - Catalog::load(engine.session_mut().file_mut()).map_err(Snag::Engine) + let file = engine.session_mut().file_mut().map_err(Snag::Engine)?; + Catalog::load(file).map_err(Snag::Engine) } /// The columns of the table an appender was opened on, in the order it @@ -584,7 +585,8 @@ fn shape(engine: &mut zudb::Connection, table: &str) -> Result, Snag "no node table or rel table '{table}'" ))) })?; - let directory = zudb::zu1::props::load_props(engine.session_mut().file_mut(), id) + let file = engine.session_mut().file_mut().map_err(Snag::Engine)?; + let directory = zudb::zu1::props::load_props(file, id) .map_err(Snag::Engine)? .ok_or_else(|| { Snag::Engine(ZuError::InvalidArgument(format!( diff --git a/tests/test_conformance.py b/tests/test_conformance.py new file mode 100644 index 0000000..03027f3 --- /dev/null +++ b/tests/test_conformance.py @@ -0,0 +1,922 @@ +"""The corpus reader and runner, checked against the reference ones. + +The corpus is one set of files read by three readers, and the whole +value of a second reader is that it refuses what the first refuses. So +the tables below are the tables in `crates/zu-corpus/src/*.rs`, case for +case: a document, and the words the refusal has to contain. A reader +that grew a hole would pass its own tests and fail these. + +The run over the cases themselves needs the cases, which live in the +engine's repository rather than this one. `ZU_CASES` points at them and +the run is skipped without it, so a checkout with no engine beside it +still tests everything that does not need one. +""" + +from __future__ import annotations + +import math +import os +import subprocess +import sys +from pathlib import Path + +import pytest +import zudb + +from conformance import cases, reader, runner, values + +CASES = os.environ.get("ZU_CASES") + +needs_cases = pytest.mark.skipif(not CASES, reason="ZU_CASES does not point at the case files") + +HEAD = "schema: 3\nsuite: int\ndoc: the integer tower\n" + + +def rows_of(value: str, ty: str = "INT64", column: str = "n") -> str: + """One column and one row of it, which is the shape most of these + fixtures want and none of them is about.""" + return ( + f" columns:\n - {column}\n rows:\n - values:\n" + f' - type: {ty}\n value: "{value}"\n' + ) + + +def suite(text: str) -> cases.Suite: + return cases.read(f"{HEAD}\ncases:\n{text}") + + +def one(text: str) -> cases.Case: + return suite(text).cases[0] + + +def value(text: str) -> object: + return values.decode(reader.parse(text)) + + +# The YAML subset. + + +def test_a_mapping_of_scalars_is_the_shape_everything_else_is_made_of() -> None: + doc = reader.parse("schema: 1\nsuite: int\n") + assert doc.get("schema").str_() == "1" + assert doc.get("suite").str_() == "int" + assert doc.get("nothing") is None + + +def test_a_sequence_item_carrying_a_mapping_is_the_same_shape_as_one_written_out() -> None: + doc = reader.parse( + "cases:\n - name: a\n query: RETURN 1\n - name: b\n query: RETURN 2\n" + ) + items = doc.get("cases").seq() + assert len(items) == 2 + assert items[0].get("name").str_() == "a" + assert items[0].get("query").str_() == "RETURN 1" + assert items[1].get("name").str_() == "b" + + +def test_a_sequence_of_scalars_is_not_read_as_anything_cleverer() -> None: + doc = reader.parse("columns:\n - n\n - m\n") + assert [c.str_() for c in doc.get("columns").seq()] == ["n", "m"] + + +def test_nesting_goes_as_deep_as_a_list_of_records_needs() -> None: + doc = reader.parse( + "rows:\n - values:\n - type: LIST\n value:\n - type: INT64\n" + ' value: "1"\n' + ) + row = doc.get("rows").seq()[0].get("values").seq()[0] + assert row.get("type").str_() == "LIST" + assert row.get("value").seq()[0].get("value").str_() == "1" + + +def test_whether_a_scalar_was_quoted_survives_because_the_encoding_turns_on_it() -> None: + doc = reader.parse("bare: 42\nquoted: \"42\"\nsingle: '42'\n") + assert doc.get("bare").scalar() == ("42", False) + assert doc.get("quoted").scalar() == ("42", True) + assert doc.get("single").scalar() == ("42", True) + + +def test_a_colon_inside_a_value_is_part_of_it_and_not_another_key() -> None: + doc = reader.parse("query: RETURN datetime('2024-01-01T00:00:00')\n") + assert doc.get("query").str_() == "RETURN datetime('2024-01-01T00:00:00')" + + +def test_a_hash_is_a_comment_only_where_a_comment_can_start() -> None: + doc = reader.parse( + '# the whole line\nname: a # and the end of this one\nhash: "a # b"\nword: c#d\n' + ) + assert doc.get("name").str_() == "a" + assert doc.get("hash").str_() == "a # b" + assert doc.get("word").str_() == "c#d" + + +def test_a_quote_that_never_closes_is_an_ordinary_character_inside_a_plain_scalar() -> None: + """The second quote has a space before it, so it looks like the start + of a run, and there is nothing after it to close one.""" + doc = reader.parse("query: RETURN cast(' 42 ' AS INT64) AS n # a note\n") + assert doc.get("query").str_() == "RETURN cast(' 42 ' AS INT64) AS n" + + +def test_an_escape_and_a_doubled_quote_are_the_two_ways_a_quote_gets_in() -> None: + doc = reader.parse("a: \"say \\\"no\\\"\"\nb: 'say ''no'''\n") + assert doc.get("a").str_() == 'say "no"' + assert doc.get("b").str_() == "say 'no'" + + +def test_the_control_characters_a_query_can_hold_all_have_an_escape() -> None: + doc = reader.parse('a: "one\\ntwo\\rthree\\tfour\\0five"\n') + assert doc.get("a").str_() == "one\ntwo\rthree\tfour\0five" + + +def test_a_negative_number_is_a_scalar_and_not_a_sequence() -> None: + assert reader.parse("value: -1\n").get("value").str_() == "-1" + + +def test_every_node_says_which_line_it_started_on() -> None: + doc = reader.parse("schema: 1\n\ncases:\n - name: a\n query: RETURN 1\n") + case = doc.get("cases").seq()[0] + assert case.line == 4 + assert case.get("query").line == 5 + + +def test_a_key_with_nothing_under_it_is_a_node_and_every_accessor_says_no_to_it() -> None: + rows = reader.parse("rows:\n").get("rows") + assert rows.what() == "nothing" + assert rows.str_() is None + assert rows.seq() is None + # The one caller for whom empty is an answer, which is a case that + # expects no rows back. + assert rows.seq_or_empty() == () + + +def test_a_key_nobody_reads_can_be_asked_for() -> None: + doc = reader.parse("name: a\nqeury: RETURN 1\n") + assert doc.unknown(("name", "query")) == ["qeury"] + + +@pytest.mark.parametrize( + ("text", "want"), + [ + ("", "nothing in it"), + (" name: a\n", "line 1"), + ("name:\ta\n", "line 1"), + ("name: a\n doc: b\n", "line 2"), + ("a: 1\n b: 2\n", "line 2"), + ("name: a\nname: b\n", "line 2"), + ("cases:\n -\n", "line 2"), + ("cases:\n - name: a\n", "line 2"), + ("cases:\n - - a\n", "line 2"), + ("columns: [n, m]\n", "line 1"), + ("doc: >\n folded\n", "line 1"), + ("doc: |\n literal\n", "line 1"), + ("anchor: &a 1\n", "line 1"), + ("---\nname: a\n", "line 1"), + ('a: "unterminated\n', "line 1"), + ('a: "bad \\q escape"\n', "line 1"), + ("a: 'unterminated\n", "line 1"), + ("a: 1\ncases:\n - b\n", "line 3"), + ], +) +def test_what_the_reader_does_not_read_it_refuses_and_says_where(text: str, want: str) -> None: + with pytest.raises(reader.CorpusError) as raised: + reader.parse(text) + assert want in str(raised.value) + + +@pytest.mark.parametrize( + ("text", "want"), + [ + ('a: "one" and "two"\n', "after the scalar ends"), + ("a: [1]\n", "a plain scalar opening with '['"), + ("...\na: 1\n", '"..." opens or closes'), + ], +) +def test_a_refusal_quotes_what_it_found_the_way_the_reference_reader_does( + text: str, want: str +) -> None: + """Rust writes a string with double quotes and a character with + single ones, and a report that is diffed against the reference one + has to write them the same way.""" + with pytest.raises(reader.CorpusError) as raised: + reader.parse(text) + assert want in str(raised.value) + + +def test_the_quoting_helper_writes_what_rust_writes() -> None: + assert reader.quote("a") == '"a"' + assert reader.quote('say "no"') == '"say \\"no\\""' + assert reader.quote("one\ntwo") == '"one\\ntwo"' + assert reader.quote("a\\b") == '"a\\\\b"' + + +# The value encoding. + + +def test_the_widths_a_yaml_number_carries_are_written_as_numbers() -> None: + assert value("type: INT8\nvalue: -128\n") == -128 + assert value("type: INT32\nvalue: 2147483647\n") == 2147483647 + assert value("type: BOOL\nvalue: true\n") is True + assert value("type: NULL\n") is None + + +def test_the_widths_it_does_not_carry_are_written_as_strings() -> None: + assert value('type: INT64\nvalue: "9223372036854775807"\n') == 2**63 - 1 + assert value('type: INT64\nvalue: "-9223372036854775808"\n') == -(2**63) + + +def test_an_int64_written_bare_is_the_defect_the_encoding_exists_to_stop() -> None: + with pytest.raises(reader.CorpusError) as raised: + value("type: INT64\nvalue: 9223372036854775807\n") + assert "written in quotes" in str(raised.value) + assert "will round it" in str(raised.value) + + +def test_a_number_written_in_quotes_is_refused_the_other_way_round() -> None: + with pytest.raises(reader.CorpusError) as raised: + value('type: INT8\nvalue: "42"\n') + assert "without quotes" in str(raised.value) + # A string is the one type whose payload is quoted or not as YAML + # pleases, because either way it is the same text. + assert value('type: STRING\nvalue: "42"\n') == "42" + assert value("type: STRING\nvalue: 42\n") == "42" + + +@pytest.mark.parametrize( + "text", + [ + "type: INT8\nvalue: 128\n", + "type: UINT8\nvalue: -1\n", + "type: INT32\nvalue: 2147483648\n", + 'type: UINT64\nvalue: "18446744073709551615"\n', + ], +) +def test_a_value_too_wide_for_the_type_it_claims_is_not_quietly_widened(text: str) -> None: + with pytest.raises(reader.CorpusError) as raised: + value(text) + assert "is not a" in str(raised.value) + + +def test_a_float_says_which_float_because_the_three_awkward_ones_have_no_yaml_spelling() -> None: + assert value('type: FLOAT64\nvalue: "1.5"\n') == 1.5 + assert math.isnan(value('type: FLOAT64\nvalue: "NaN"\n')) + assert value('type: FLOAT64\nvalue: "inf"\n') == math.inf + assert value('type: FLOAT64\nvalue: "-inf"\n') == -math.inf + # A negative zero is a different value from a zero, and saying so is + # why `same` compares bits. + minus = value('type: FLOAT64\nvalue: "-0.0"\n') + assert not values.same(minus, 0.0) + assert values.same(minus, -0.0) + + +def test_a_nan_matches_a_nan_whatever_the_hardware_put_in_its_sign_and_payload() -> None: + want = value('type: FLOAT64\nvalue: "NaN"\n') + assert values.same(want, math.nan) + assert values.same(want, -math.nan) + assert values.same(want, float("nan")) + assert not values.same(want, 0.0) + + +def test_a_float32_is_narrowed_so_a_case_asserts_what_the_narrower_type_can_hold() -> None: + narrowed = value('type: FLOAT32\nvalue: "0.1"\n') + assert not values.same(narrowed, 0.1) + assert values.same(narrowed, narrowed) + + +def test_an_integer_written_as_a_float_is_refused_rather_than_promoted() -> None: + with pytest.raises(reader.CorpusError) as raised: + value('type: FLOAT64\nvalue: "1"\n') + assert "is not a FLOAT64" in str(raised.value) + + +@pytest.mark.parametrize( + ("text", "want"), + [ + ('type: DATE\nvalue: "2024-02-29"\n', "2024-02-29"), + ('type: LOCALTIME\nvalue: "12:34:56"\n', "12:34:56"), + ('type: ZONEDDATETIME\nvalue: "2024-01-01T00:00:00+07:00"\n', "2024-01-01T00:00:00+07:00"), + ('type: DURATION\nvalue: "P1Y2M"\n', "P1Y2M"), + ], +) +def test_a_temporal_is_written_the_way_the_engine_prints_it(text: str, want: str) -> None: + got = values.show(value(text)) + assert got.split(" ", 1)[1] == f'"{want}"' + + +def test_a_duration_carries_its_sign_because_a_difference_can_go_either_way() -> None: + assert value('type: DURATION\nvalue: "-PT1H"\n') == zudb.Duration(nanoseconds=-3600_000_000_000) + + +def test_a_list_holds_encoded_values_and_not_bare_ones() -> None: + assert value( + "type: LIST\nvalue:\n - type: INT8\n value: 1\n - type: STRING\n value: two\n" + ) == [1, "two"] + with pytest.raises(reader.CorpusError) as raised: + value("type: LIST\nvalue:\n - 1\n") + assert "a value is a mapping" in str(raised.value) + # The empty list is a value, and a `value:` with nothing under it is + # how it is written. + assert value("type: LIST\nvalue:\n") == [] + + +def test_a_type_the_engine_cannot_hold_yet_says_so_rather_than_looking_like_a_typo() -> None: + with pytest.raises(reader.CorpusError) as raised: + value('type: DECIMAL\nvalue: "1.00"\n') + assert "reserves" in str(raised.value) + with pytest.raises(reader.CorpusError) as raised: + value('type: INT65\nvalue: "1"\n') + assert "not a type this encoding knows" in str(raised.value) + + +@pytest.mark.parametrize( + ("text", "want"), + [ + ("type: INT8\n", "with no `value`"), + ("value: 1\n", "with no `type`"), + ("type: NULL\nvalue: 1\n", "carries no `value`"), + ("type: INT8\nvalue: 1\nnote: hi\n", 'no key "note"'), + ("type: INT8\nvalue:\n - 1\n", "holds one scalar"), + ], +) +def test_a_mapping_that_is_not_a_value_is_refused_with_its_line(text: str, want: str) -> None: + with pytest.raises(reader.CorpusError) as raised: + value(text) + assert want in str(raised.value) + + +def test_what_a_report_prints_is_what_a_case_would_be_written_as() -> None: + assert values.show(7) == 'INT64 "7"' + assert values.show(1.0) == 'FLOAT64 "1.0"' + assert values.show(math.nan) == 'FLOAT64 "NaN"' + assert values.show(None) == "NULL" + assert values.show([True, "a"]) == 'LIST [BOOL true, STRING "a"]' + + +@pytest.mark.parametrize( + ("f", "want"), + [ + (1.0, "1.0"), + (0.1, "0.1"), + (-0.0, "-0.0"), + (1e16, "1e16"), + (1e300, "1e300"), + (1e-7, "1e-7"), + (3.4028234663852886e38, "3.4028234663852886e38"), + ], +) +def test_a_float_prints_the_way_rust_prints_one(f: float, want: str) -> None: + """Python writes an exponent with a sign and a padded width and Rust + writes it bare, and a report diffed against the reference one cannot + have either.""" + assert values.show(f) == f'FLOAT64 "{want}"' + + +@pytest.mark.parametrize( + ("text", "want"), + [ + ('type: LOCALTIME\nvalue: "12:34:56"\n', 'LOCALTIME "12:34:56"'), + ('type: LOCALTIME\nvalue: "12:34:56.789"\n', 'LOCALTIME "12:34:56.789000000"'), + ('type: ZONEDTIME\nvalue: "12:34:56Z"\n', 'ZONEDTIME "12:34:56Z"'), + ('type: ZONEDTIME\nvalue: "12:34:56+00:00"\n', 'ZONEDTIME "12:34:56Z"'), + ('type: ZONEDTIME\nvalue: "12:34:56-05:30"\n', 'ZONEDTIME "12:34:56-05:30"'), + ('type: DATE\nvalue: "0001-01-01"\n', 'DATE "0001-01-01"'), + ( + 'type: ZONEDDATETIME\nvalue: "2024-01-01T00:00:00+07:00"\n', + 'ZONEDDATETIME "2024-01-01T00:00:00+07:00"', + ), + ], +) +def test_a_temporal_prints_the_way_the_engine_prints_one(text: str, want: str) -> None: + """Nine digits of fraction and never six, seconds always, and ``Z`` + at a zero offset.""" + assert values.show(value(text)) == want + + +@pytest.mark.parametrize( + ("text", "want"), + [ + ("P1Y2M", "P1Y2M"), + ("P1Y", "P1Y"), + ("P2M", "P2M"), + ("PT1S", "PT1S"), + ("PT1H30M", "PT1H30M"), + ("P1D", "P1D"), + ("PT0S", "PT0S"), + ("-PT1H", "-PT1H"), + ("PT0.5S", "PT0.500000000S"), + ("P1DT2H", "P1DT2H"), + ], +) +def test_a_duration_leaves_out_the_fields_that_are_zero(text: str, want: str) -> None: + """The text a duration prints as is the text it parses back, so a + field that is zero is left out and a duration with nothing left in + it is written out anyway.""" + assert values.show(value(f'type: DURATION\nvalue: "{text}"\n')) == f'DURATION "{want}"' + + +def test_a_zero_duration_is_the_one_the_two_kinds_cannot_be_told_apart_by() -> None: + """The engine keeps which kind a duration is beside the count, and + this client keeps two fields and reads the kind off whichever is set, + so a zero of either kind is a zero of the other. The engine writes + `P0M` for one and `PT0S` for the other and this writes `PT0S` for + both. No case turns on it, and it is written down here rather than + left for somebody to find.""" + assert values.show(value('type: DURATION\nvalue: "P0M"\n')) == 'DURATION "PT0S"' + assert values.show(value('type: DURATION\nvalue: "PT0S"\n')) == 'DURATION "PT0S"' + + +def test_a_time_finer_than_this_client_holds_is_equal_to_nothing() -> None: + """Python's ``datetime`` keeps microseconds and the engine keeps + nanoseconds, so a case asserting nine digits would otherwise pass by + comparing one truncated value against another, which is what it was + written to catch.""" + fine = value('type: LOCALTIME\nvalue: "12:34:56.123456789"\n') + assert isinstance(fine, values.TooFine) + assert values.show(fine) == 'LOCALTIME "12:34:56.123456789"' + assert not values.same(fine, fine) + assert not values.same(fine, value('type: LOCALTIME\nvalue: "12:34:56.123456"\n')) + assert values.too_fine([1, [fine]]) is fine + assert values.too_fine([1, "a"]) is None + + +def test_a_time_the_client_cannot_hold_still_truncates_for_a_load() -> None: + """A column has to go into the file for the suite to have a graph at + all, and the offset behind the fraction is part of the value rather + than part of what is cut.""" + import datetime + + fine = value('type: LOCALTIME\nvalue: "23:59:59.999999999"\n') + assert values.truncated(fine) == datetime.time(23, 59, 59, 999999) + zoned = value('type: ZONEDTIME\nvalue: "12:34:56.123456789+07:00"\n') + assert values.truncated(zoned).utcoffset() == datetime.timedelta(hours=7) + assert values.truncated([fine, 1]) == [datetime.time(23, 59, 59, 999999), 1] + + +def test_a_fraction_of_six_digits_or_fewer_is_a_value_this_client_holds() -> None: + assert not isinstance(value('type: LOCALTIME\nvalue: "12:34:56.123456"\n'), values.TooFine) + assert not isinstance(value('type: LOCALTIME\nvalue: "12:34:56.789000000"\n'), values.TooFine) + + +def test_a_boolean_is_not_an_integer_however_python_stores_it() -> None: + """Python's ``True`` is ``1`` and its ``bool`` is a subclass of + ``int``, which no other client in the corpus has to think about. A + case wanting BOOL true and getting INT64 1 back is a failure, and it + would pass without this.""" + assert not values.same(True, 1) + assert not values.same(1, True) + assert values.same(True, True) + assert values.show(True) == "BOOL true" + assert values.show(1) == 'INT64 "1"' + + +# What a case is. + + +def test_a_case_is_a_statement_and_the_rows_it_owes() -> None: + case = one( + " - name: int64-max\n doc: the largest INT64\n" + " query: RETURN 9223372036854775807 AS n\n" + rows_of("9223372036854775807") + ) + assert case.name == "int64-max" + assert case.doc == "the largest INT64" + assert case.query == "RETURN 9223372036854775807 AS n" + assert case.setup == [] + assert case.columns == ["n"] + assert case.rows == [[2**63 - 1]] + + +def test_a_case_may_load_its_own_data_first() -> None: + case = one( + " - name: with-setup\n doc: a case that needs a graph\n setup:\n" + " - CREATE NODE TABLE Person(name STRING)\n - INSERT (:Person {name: 'a'})\n" + " query: MATCH (p:Person) RETURN p.name AS name\n columns:\n - name\n" + " rows:\n - values:\n - type: STRING\n value: a\n" + ) + assert len(case.setup) == 2 + assert case.setup[0].startswith("CREATE NODE TABLE") + + +def test_a_case_may_bind_parameters_and_they_keep_the_order_they_were_written_in() -> None: + case = one( + " - name: bound\n doc: a statement with two parameters in it\n params:\n" + ' - name: n\n type: INT64\n value: "42"\n' + " - name: s\n type: STRING\n value: ada\n" + " query: RETURN $n AS n, $s AS s\n columns:\n - n\n - s\n" + ' rows:\n - values:\n - type: INT64\n value: "42"\n' + " - type: STRING\n value: ada\n" + ) + assert case.params == [("n", 42), ("s", "ada")] + + +def test_a_parameter_is_a_value_of_the_same_encoding_and_is_read_the_same_way() -> None: + """A NULL parameter carries no `value`, the quoting rule is the one + every other value follows, and a list is a list. The whole point of + `params:` is that it is the row encoding with a name added, so what + is checked here is that it did not become a second encoding.""" + case = one( + " - name: null-param\n doc: a parameter that is nothing\n params:\n" + " - name: n\n type: NULL\n query: RETURN $n AS n\n" + " columns:\n - n\n rows:\n - values:\n - type: NULL\n" + ) + assert case.params == [("n", None)] + case = one( + " - name: list-param\n doc: a parameter holding a list of two\n params:\n" + " - name: xs\n type: LIST\n value:\n" + " - type: INT8\n value: 1\n" + " - type: INT8\n value: 2\n" + " query: RETURN size($xs) AS n\n columns:\n - n\n" + ' rows:\n - values:\n - type: INT64\n value: "2"\n' + ) + assert case.params == [("xs", [1, 2])] + with pytest.raises(reader.CorpusError) as raised: + suite( + " - name: a\n doc: d\n params:\n - name: n\n type: INT64\n" + " value: 42\n query: RETURN $n\n raises: 22012\n" + ) + assert "written in quotes" in str(raised.value) + + +@pytest.mark.parametrize( + ("text", "want"), + [ + ( + " - name: a\n doc: d\n params:\n - type: INT8\n value: 1\n" + " query: RETURN $n\n raises: 22012\n", + "no `name:`", + ), + ( + " - name: a\n doc: d\n params:\n - name: n one\n type: INT8\n" + " value: 1\n query: RETURN $n\n raises: 22012\n", + "is a parameter name", + ), + ( + " - name: a\n doc: d\n params:\n - name: n\n type: INT8\n" + " value: 1\n - name: n\n type: INT8\n value: 2\n" + " query: RETURN $n\n raises: 22012\n", + 'two parameters are called "n"', + ), + ( + " - name: a\n doc: d\n params:\n - name: n\n type: INT8\n" + " value: 1\n note: hi\n query: RETURN $n\n raises: 22012\n", + 'a parameter has no key "note"', + ), + ( + " - name: a\n doc: d\n params:\n - $n\n query: RETURN $n\n" + " raises: 22012\n", + "a parameter is a mapping", + ), + ( + " - name: a\n doc: d\n params: n\n query: RETURN $n\n raises: 22012\n", + "`params:` is a sequence", + ), + ], +) +def test_a_parameter_a_statement_could_not_name_is_refused_where_it_is_written( + text: str, want: str +) -> None: + with pytest.raises(reader.CorpusError) as raised: + suite(text) + assert want in str(raised.value) + + +def test_a_suite_may_load_a_table_every_case_in_it_reads_back() -> None: + read = cases.read( + "schema: 3\nsuite: int\ndoc: d\nload:\n nodes: person\n edges: knows\n count: 1\n" + ' columns:\n - name: age\n type: INT64\n values:\n - "30"\n' + "cases:\n - name: a\n doc: d\n query: MATCH (p:person) RETURN p.age AS n\n" + + rows_of("30") + ) + assert read.load.nodes == "person" + assert read.load.columns[0].values == [30] + + +def test_a_suite_of_expressions_has_no_load() -> None: + assert suite(" - name: a\n doc: d\n query: RETURN 1\n raises: 22012\n").load is None + + +def test_a_case_may_expect_a_condition_instead_of_rows() -> None: + case = one( + " - name: divide-by-zero\n" + " doc: division by zero raises rather than returning inf\n" + " query: RETURN 1 / 0\n raises: 22012\n" + ) + assert case.raises == "22012" + assert case.rows is None + + +def test_a_case_expecting_nothing_back_says_so_out_loud() -> None: + case = one( + " - name: empty\n doc: a filter nothing satisfies gives no rows\n" + " query: UNWIND [1] AS n WHERE false RETURN n\n columns:\n - n\n rows:\n" + ) + assert case.rows == [] + + +def test_a_suite_carries_the_line_of_every_case_so_a_failure_can_be_opened() -> None: + read = suite( + " - name: a\n doc: the first\n query: RETURN 1 AS n\n columns:\n - n\n" + " rows:\n - values:\n - type: INT8\n value: 1\n" + " - name: b\n doc: the second\n query: RETURN 2 AS n\n columns:\n - n\n" + " rows:\n - values:\n - type: INT8\n value: 2\n" + ) + assert [c.line for c in read.cases] == [6, 15] + + +@pytest.mark.parametrize( + ("text", "want"), + [ + (" - name: a\n", "no `doc:`"), + (" - doc: a\n query: RETURN 1\n", "no `name:`"), + (" - name: A\n doc: d\n query: RETURN 1\n raises: 22012\n", "is a case name"), + (" - name: a\n doc: d\n query: RETURN 1\n", "says what it produces"), + ( + " - name: a\n doc: d\n query: RETURN 1\n raises: 22012\n" + " columns:\n - n\n rows:\n", + "has no rows", + ), + ( + " - name: a\n doc: d\n query: RETURN 1\n raises: 999999\n", + "not the shape of a GQLSTATUS", + ), + ( + " - name: a\n doc: d\n query: RETURN 1\n columns:\n - n\n", + "with no `rows:`", + ), + ( + " - name: a\n doc: d\n query: RETURN 1 AS n, 2 AS m\n" + " columns:\n - n\n - m\n" + " rows:\n - values:\n - type: INT8\n value: 1\n", + "a row of 1 against 2 columns", + ), + ( + " - name: a\n doc: d\n qeury: RETURN 1\n raises: 22012\n", + 'no key "qeury"', + ), + ], +) +def test_a_file_the_runner_cannot_read_says_which_line_and_why(text: str, want: str) -> None: + with pytest.raises(reader.CorpusError) as raised: + suite(text) + assert want in str(raised.value) + + +def test_the_same_case_name_twice_is_refused_because_a_report_cites_names() -> None: + with pytest.raises(reader.CorpusError) as raised: + suite( + " - name: a\n doc: d\n query: RETURN 1\n raises: 22012\n" + " - name: a\n doc: e\n query: RETURN 2\n raises: 22012\n" + ) + assert 'two cases are called "a"' in str(raised.value) + + +def test_a_file_from_another_schema_says_so_rather_than_failing_in_the_middle() -> None: + with pytest.raises(reader.CorpusError) as raised: + cases.read("schema: 1\nsuite: int\ndoc: d\ncases:\n - name: a\n") + assert "schema 1 and the runner reads schema 3" in str(raised.value) + + +def test_a_suite_whose_name_is_not_its_file_name_is_refused(tmp_path: Path) -> None: + """A report cites a suite by name and a reader opens it by file name, + so the two disagreeing is a failure nobody can find.""" + (tmp_path / "float.yaml").write_text( + f"{HEAD}\ncases:\n - name: a\n doc: d\n query: RETURN 1\n raises: 22012\n" + ) + with pytest.raises(reader.CorpusError) as raised: + cases.read_dir(tmp_path) + assert 'calls itself "int" and the file calls it "float"' in str(raised.value) + + +def test_a_directory_with_no_case_files_is_refused_rather_than_passing_empty( + tmp_path: Path, +) -> None: + with pytest.raises(reader.CorpusError) as raised: + cases.read_dir(tmp_path) + assert "no case files" in str(raised.value) + + +# Running. + + +def _write(tmp_path: Path, name: str, body: str) -> Path: + directory = tmp_path / "cases" + directory.mkdir(exist_ok=True) + (directory / f"{name}.yaml").write_text( + f"schema: 3\nsuite: {name}\ndoc: a suite written by a test\ncases:\n{body}" + ) + return directory + + +def _run(tmp_path: Path, name: str, body: str) -> runner.Report: + directory = _write(tmp_path, name, body) + work = tmp_path / "work" + work.mkdir(exist_ok=True) + return runner.run(cases.read_dir(directory), work) + + +def test_a_case_that_asks_for_what_the_engine_gives_passes(tmp_path: Path) -> None: + report = _run( + tmp_path, + "small", + " - name: one\n doc: the smallest statement there is\n query: RETURN 1 AS n\n" + + rows_of("1"), + ) + assert report.summary() == "1 cases, 1 passed, 0 failed, 0 unsupported" + + +def test_a_case_that_asks_for_a_condition_passes_on_the_code_and_not_the_message( + tmp_path: Path, +) -> None: + report = _run( + tmp_path, + "small", + " - name: over-zero\n doc: division by zero raises\n query: RETURN 1 / 0 AS n\n" + " raises: 22012\n", + ) + assert report.count(runner.PASSED) == 1 + + +def test_a_wrong_row_is_reported_in_the_encoding_a_case_is_written_in(tmp_path: Path) -> None: + report = _run( + tmp_path, + "small", + " - name: one\n doc: a case that wants the wrong number\n query: RETURN 1 AS n\n" + + rows_of("2"), + ) + assert ( + report.failures()[0].detail == 'row 1 column n is INT64 "1" where the case wants INT64 "2"' + ) + + +def test_a_wrong_column_list_is_reported_before_the_rows_are_looked_at(tmp_path: Path) -> None: + report = _run( + tmp_path, + "small", + " - name: one\n doc: a case that wants another name\n query: RETURN 1 AS n\n" + + rows_of("1", column="m"), + ) + assert report.failures()[0].detail == 'columns ["n"] where the case wants ["m"]' + + +def test_a_row_count_that_differs_is_reported_after_the_rows_that_match(tmp_path: Path) -> None: + report = _run( + tmp_path, + "small", + " - name: two\n doc: a case that wants a row that is not there\n" + " query: UNWIND [1] AS n RETURN n\n columns:\n - n\n" + ' rows:\n - values:\n - type: INT64\n value: "1"\n' + ' - values:\n - type: INT64\n value: "2"\n', + ) + assert report.failures()[0].detail == "1 rows where the case wants 2" + + +def test_a_case_the_engine_cannot_parse_is_unsupported_rather_than_failed(tmp_path: Path) -> None: + report = _run( + tmp_path, + "small", + " - name: ahead\n doc: a statement this engine does not have yet\n" + " query: RETURN nonesuch(1) AS n\n columns:\n - n\n rows:\n", + ) + assert report.count(runner.UNSUPPORTED) == 1 + assert report.count(runner.FAILED) == 0 + + +def test_a_case_that_wanted_a_condition_and_got_rows_is_a_failure(tmp_path: Path) -> None: + report = _run( + tmp_path, + "small", + " - name: quiet\n doc: a statement that was meant to raise\n query: RETURN 1 AS n\n" + " raises: 22012\n", + ) + assert report.failures()[0].detail == "returned rows where the case wants 22012" + + +def test_a_case_that_raised_the_wrong_code_says_both(tmp_path: Path) -> None: + report = _run( + tmp_path, + "small", + " - name: wrong-code\n doc: a case naming the wrong condition\n" + " query: RETURN 1 / 0 AS n\n raises: 22003\n", + ) + assert report.failures()[0].detail.startswith("raised 22012 where the case wants 22003") + + +def test_a_suite_load_reaches_every_case_and_no_case_reaches_another(tmp_path: Path) -> None: + """Each case gets its own database with its own copy of the load, so + what one case inserts is not there for the next one.""" + directory = tmp_path / "cases" + directory.mkdir() + (directory / "graph.yaml").write_text( + "schema: 3\nsuite: graph\ndoc: a loaded suite\nload:\n nodes: person\n edges: knows\n" + " count: 2\n columns:\n - name: name\n type: STRING\n values:\n" + " - ada\n - bob\n pairs:\n - from: 0\n to: 1\n" + "cases:\n" + " - name: reads-the-load\n doc: the load is there\n" + " query: MATCH (p:person) RETURN count(p) AS n\n columns:\n - n\n" + ' rows:\n - values:\n - type: INT64\n value: "2"\n' + " - name: inserts-a-row\n doc: a case that adds one\n" + " query: INSERT (p:person {name: 'cyd'}) RETURN p.name AS n\n columns:\n - n\n" + " rows:\n - values:\n - type: STRING\n value: cyd\n" + " - name: does-not-see-it\n doc: the row the case before inserted is not here\n" + " query: MATCH (p:person) RETURN count(p) AS n\n columns:\n - n\n" + ' rows:\n - values:\n - type: INT64\n value: "2"\n' + ) + work = tmp_path / "work" + work.mkdir() + report = runner.run(cases.read_dir(directory), work) + assert report.count(runner.FAILED) == 0, [str(r) for r in report.failures()] + + +def test_a_case_binds_its_parameters_through_this_client(tmp_path: Path) -> None: + report = _run( + tmp_path, + "small", + " - name: bound\n doc: a parameter crosses the boundary and comes back\n" + ' params:\n - name: n\n type: INT64\n value: "42"\n' + " query: RETURN $n AS n\n columns:\n - n\n" + ' rows:\n - values:\n - type: INT64\n value: "42"\n', + ) + assert report.count(runner.PASSED) == 1 + + +def test_the_command_line_exits_zero_on_a_run_that_passes(tmp_path: Path) -> None: + directory = _write( + tmp_path, + "small", + " - name: one\n doc: the smallest statement there is\n query: RETURN 1 AS n\n" + + rows_of("1"), + ) + assert runner.main([str(directory), "--quiet"]) == 0 + + +def test_the_command_line_exits_one_on_a_corpus_it_cannot_read(tmp_path: Path) -> None: + """One and not two, because the reference runner exits one and a + report compared line for line is worth less if the two runners + disagree about what the run came to.""" + directory = tmp_path / "cases" + directory.mkdir() + (directory / "small.yaml").write_text("schema: 3\nsuite: small\ndoc: d\ncases:\n - name: a\n") + assert runner.main([str(directory), "--quiet"]) == 1 + + +def test_strict_turns_an_unsupported_case_into_a_failed_run(tmp_path: Path) -> None: + directory = _write( + tmp_path, + "small", + " - name: ahead\n doc: a statement this engine does not have yet\n" + " query: RETURN nonesuch(1) AS n\n columns:\n - n\n rows:\n", + ) + assert runner.main([str(directory), "--quiet"]) == 0 + assert runner.main([str(directory), "--quiet", "--strict"]) == 1 + + +def test_the_module_runs_as_a_command(tmp_path: Path) -> None: + directory = _write( + tmp_path, + "small", + " - name: one\n doc: the smallest statement there is\n query: RETURN 1 AS n\n" + + rows_of("1"), + ) + done = subprocess.run( + [sys.executable, "-m", "conformance", str(directory)], + capture_output=True, + text=True, + cwd=Path(__file__).parent.parent, + ) + assert done.returncode == 0, done.stderr + assert done.stdout.strip() == "1 cases, 1 passed, 0 failed, 0 unsupported" + + +#: The cases this client cannot answer, and the only ones it may leave +#: unanswered. Written out rather than counted, so that a sixth one +#: arriving is a failure here and not a number nobody looks at. All five +#: are a time to the nanosecond, which is a digit finer than Python's +#: `datetime` holds. +UNHELD = { + "param/localtime-to-the-nanosecond", + "stored/a-localtime-column-keeps-every-digit", + "stored/the-columns-of-one-row-belong-to-that-row", + "temporal/local-time-nanoseconds", + "temporal/a-time-carries-a-single-nanosecond", +} + + +@needs_cases +def test_every_case_the_engine_ships_passes_through_this_client(tmp_path: Path) -> None: + """The corpus itself, which is the check the other two runners run + and the reason this one exists.""" + report = runner.run(cases.read_dir(Path(CASES)), tmp_path) + assert report.count(runner.FAILED) == 0, [str(r) for r in report.failures()][:10] + unheld = {f"{r.suite}/{r.case}" for r in report.ran if r.outcome == runner.UNSUPPORTED} + assert unheld == UNHELD + assert report.count(runner.PASSED) == len(report.ran) - len(UNHELD) + + +@needs_cases +def test_the_cases_this_client_cannot_hold_are_a_precision_and_not_a_wrong_answer( + tmp_path: Path, +) -> None: + """Every one of them says which value it is and why, because a case + reported as unsupported with no reason is a case nobody revisits.""" + report = runner.run(cases.read_dir(Path(CASES)), tmp_path) + for ran in report.ran: + if ran.outcome == runner.UNSUPPORTED: + assert "finer" in ran.detail, str(ran) diff --git a/tests/test_errors.py b/tests/test_errors.py index 2b7bc29..9f42d29 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -38,12 +38,21 @@ def test_an_excerpt_and_a_caret_point_at_the_token(empty: zudb.Connection) -> No assert pointer.index("^") == err.column - 1 -def test_a_label_nothing_declares_is_a_reference_error(empty: zudb.Connection) -> None: +def test_a_name_nothing_bound_is_a_reference_error(empty: zudb.Connection) -> None: with pytest.raises(zudb.Error) as raised: - empty.execute("MATCH (p:person)-[:follows]->(q:person) RETURN p") + empty.execute("RETURN nobody AS x") assert raised.value.code == "42002" +def test_a_pattern_the_graph_cannot_satisfy_matches_nothing(empty: zudb.Connection) -> None: + """A label nothing declares is not an error, it is a pattern with no + answer, which is the reading a client that composes a query out of + optional parts depends on.""" + result = empty.execute("MATCH (p:person)-[:follows]->(q:person) RETURN p") + assert result.columns == ["p"] + assert result.fetchall() == [] + + def test_every_condition_is_catchable_as_the_base_class(empty: zudb.Connection) -> None: with pytest.raises(zudb.Error): empty.execute("this is not a statement")