Measure rebuild/search cost; make the @UF clear opt-in

Makefile

@@ -1,9 +1,9 @@
 .PHONY: \
 	check nits test python-check python-nits rust-check rust-nits \
-	proof-tests benchmark-smoke nightly nightly-uv nightly-rustup \
+	proof-tests benchmark-smoke nightly nightly-local nightly-uv nightly-rustup \
 	update-snapshots format \
 	python-lock python-format-check python-lint python-typecheck python-test \
-	rust-format-check rust-clippy rust-test
+	rust-format-check rust-clippy rust-doc-links rust-test
 
 BENCHMARK_SMOKE_REPORT ?= /tmp/egglog-encoding-bench-smoke.jsonl
 
@@ -47,7 +47,7 @@ python-test:
 
 rust-check: rust-nits rust-test
 
-rust-nits: rust-format-check rust-clippy
+rust-nits: rust-format-check rust-clippy rust-doc-links
 
 rust-format-check:
 	cargo fmt --all -- --check
@@ -60,6 +60,12 @@ rust-clippy:
 	cargo clippy --workspace --all-targets -- -D warnings
 	cargo clippy -p egglog-experimental --features dd-backend --all-targets -- -D warnings
 
+# Clippy does not resolve doc links, and plain `cargo doc` skips the private
+# items most of this codebase documents, so a rename leaves stale links behind
+# unless rustdoc is run over them too.
+rust-doc-links:
+	RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --document-private-items --workspace
+
 # This is a name-filtered subset of rust-test, useful for proof iteration.
 proof-tests:
 	cargo test --workspace --test files 'proofs/'
@@ -74,12 +80,12 @@ benchmark-smoke:
 		'from pathlib import Path; import sys; from benchmarking.reports.store import ReportStore; assert ReportStore(Path(sys.argv[1])).row_count > 0' \
 		"$(BENCHMARK_SMOKE_REPORT)"
 
-# Benchmark every endpoint on this checkout and on main, then copy eval-live's
-# interactive report to nightly/output/. The egraphs-good nightly service
-# (nightly.cs.washington.edu) runs this target and serves that directory,
-# matching `report=` in the nightly configuration.
+# Benchmark each endpoint in nightly_bench.py's ENDPOINTS on this checkout and on
+# main, then copy eval-live's interactive report to nightly/output/. The
+# egraphs-good nightly service (nightly.cs.washington.edu) runs this target and
+# serves that directory, matching `report=` in the nightly configuration.
 nightly: nightly-uv nightly-rustup
-	$(NIGHTLY_UV) run --locked python scripts/nightly_bench.py
+	CARGO_HOME="$(CARGO_HOME_DIR)" $(NIGHTLY_UV) run --locked python scripts/nightly_bench.py
 
 nightly-uv:
 	@command -v uv >/dev/null || test -x "$(UV_BOOTSTRAP_DIR)/uv" || \
@@ -92,6 +98,11 @@ nightly-rustup:
 			| env CARGO_HOME="$(CARGO_HOME_DIR)" sh -s -- \
 				-y --no-modify-path --default-toolchain none
 
+# The nightly host's run at one round, for trying it locally. nightly/output/ is
+# git-ignored, so this writes it just as the host does.
+nightly-local: nightly-uv nightly-rustup
+	CARGO_HOME="$(CARGO_HOME_DIR)" $(NIGHTLY_UV) run --locked python scripts/nightly_bench.py --rounds 1
+
 update-snapshots:
 	uv run --locked pytest -q --snapshot-update --snapshot-details
 

README.md

@@ -30,6 +30,7 @@ make rust-nits      # rustfmt check and Clippy only
 make proof-tests    # proof-focused subset of the workspace tests
 make benchmark-smoke
 make nightly        # benchmark the nightly endpoints and publish nightly/output/
+make nightly-local  # the same run at one round, for trying it out
 make update-snapshots
 make format         # apply Ruff and rustfmt formatting
 ```
@@ -380,6 +381,9 @@ failing the run, and the output directory is only overwritten after a
 successful run. Edit `TARGETS` and `ENDPOINTS` in `scripts/nightly_bench.py` to
 change what is measured.
 
+`make nightly-local` is the same run at `--rounds 1`, for trying the whole
+pipeline out without waiting for a full nightly.
+
 The [egraphs-good nightly service](https://nightly.cs.washington.edu) checks out
 this repository, runs `make nightly`, and serves `nightly/output/`, matching the
 `report=` entry in the nightly configuration. Two things that runner does not
@@ -390,6 +394,10 @@ the box has none. The runner also leaves rustup's `~/.cargo/bin` off `PATH`,
 which would leave cargo resolving to Ubuntu's — too old for
 `rust-toolchain.toml`'s pin — so `nightly_bench.py` puts those shims first.
 
+Both installers run `curl … | sh`, so on a developer box prefer installing uv
+and rustup yourself: `make nightly` and `make nightly-local` skip each one that
+is already there. Only a bare CI runner needs them.
+
 ### CPU profiling
 
 Benchmark reports answer whether and where performance changed. Use the

egglog-experimental/dd/src/compile.rs

@@ -144,4 +144,15 @@ impl Slot {
 pub struct ReadKey {
     pub func: FunctionId,
     pub mode: ReadMode,
+    /// Zero for an ordinary read. Otherwise this stream is the *occurrence view*
+    /// of `func`: one row per (value, base row) for each value the base row holds
+    /// in a column whose bit is set here, deduplicated per row.
+    pub occurrence_cols: u64,
+}
+
+impl ReadKey {
+    /// The columns the occurrence view reads, empty for an ordinary read.
+    pub fn occurrence_columns(&self) -> impl Iterator<Item = usize> + '_ {
+        (0..u64::BITS as usize).filter(|c| self.occurrence_cols & (1u64 << c) != 0)
+    }
 }

egglog-experimental/dd/src/dd_native.rs

@@ -80,7 +80,7 @@ pub const W: usize = 48;
 /// A fixed-width relation or binding row flowing through the DD dataflow. Input
 /// rows store relation columns in the low slots. Intermediate binding columns
 /// are assigned by the current `ProjectionPlan` stage, and captured outputs
-/// repack surviving variables into the low slots in [`JoinPlan::var_order`].
+/// repack surviving variables into the low slots in [`JoinPlan`]'s variable order.
 ///
 /// A NEWTYPE over `[u32; W]` (rather than the bare array) because timely's
 /// `ExchangeData` bound required by DD joins is
@@ -181,6 +181,50 @@ pub fn plan_join(rule: &RuleSpec) -> Result<JoinPlan, String> {
 
     for atom in &rule.core.body.atoms {
         match atom.head {
+            // An index atom reads the occurrence view of `id`: the leading arg is
+            // the occurring value, then the base row. The relation's own unit
+            // output trails the args and is not part of the view.
+            RuleBodyCall::IndexTable {
+                id,
+                ref any_of,
+                read,
+            } => {
+                if let Some(col) = any_of.iter().find(|c| **c >= u64::BITS as usize) {
+                    return Err(format!(
+                        "index column {col} is beyond the {} a read key can track",
+                        u64::BITS
+                    ));
+                }
+                let occurrence_cols = any_of.iter().fold(0u64, |m, c| m | (1u64 << c));
+                if occurrence_cols == 0 {
+                    return Err("index atom lists no columns".to_string());
+                }
+                let view_args = atom
+                    .args
+                    .split_last()
+                    .map(|(_unit, rest)| rest)
+                    .unwrap_or(&atom.args);
+                if view_args.len() > W {
+                    return Err(format!("atom arity {} > W {}", view_args.len(), W));
+                }
+                let slots = view_args
+                    .iter()
+                    .map(Slot::from_term)
+                    .collect::<Result<Vec<_>, _>>()?;
+                for s in &slots {
+                    if let Slot::Var(v) = s {
+                        body_vars.insert(*v);
+                    }
+                }
+                atoms.push(PlanAtom {
+                    read_key: ReadKey {
+                        func: id,
+                        mode: read,
+                        occurrence_cols,
+                    },
+                    slots,
+                });
+            }
             RuleBodyCall::Table { id, read } => {
                 if atom.args.len() > W {
                     return Err(format!("atom arity {} > W {}", atom.args.len(), W));
@@ -199,6 +243,7 @@ pub fn plan_join(rule: &RuleSpec) -> Result<JoinPlan, String> {
                     read_key: ReadKey {
                         func: id,
                         mode: read,
+                        occurrence_cols: 0,
                     },
                     slots,
                 });
@@ -406,7 +451,7 @@ pub struct FusedDdJoin {
     /// The fused rules in caller-supplied build order. The sorted rule-index list
     /// identifies the ruleset cache entry but does not reorder these outputs.
     rules: Vec<FusedRule>,
-    /// Current epoch (monotonic; advanced once per [`step`]).
+    /// Current epoch (monotonic; advanced once per [`FusedDdJoin::step`]).
     epoch: u32,
 }
 
@@ -816,6 +861,7 @@ mod tests {
 
     fn live(func: FunctionId) -> ReadKey {
         ReadKey {
+            occurrence_cols: 0,
             func,
             mode: ReadMode::Live,
         }

egglog-experimental/dd/src/interpret.rs

@@ -44,6 +44,26 @@ use crate::{EGraph, TableDefault, ViewOp};
 pub(crate) type Env = HashMap<u32, u32>;
 
 type DdDeltaRows = HashMap<ReadKey, Vec<(Vec<u32>, isize)>>;
+
+/// The occurrence view of `base`: for each row, one derived row `(value, row…)`
+/// per distinct value the row holds in `read`'s columns. A value sitting in two
+/// of them yields one derived row, not two — distinct base rows always give
+/// distinct derived rows, so the map keys deduplicate exactly.
+fn occurrence_view(base: &HashMap<Row, u64>, read: ReadKey) -> HashMap<Row, u64> {
+    let mut out = HashMap::new();
+    for (row, version) in base {
+        for col in read.occurrence_columns() {
+            let Some(&val) = row.get(col) else {
+                continue;
+            };
+            let mut derived = Vec::with_capacity(row.len() + 1);
+            derived.push(val);
+            derived.extend_from_slice(row);
+            out.insert(derived.into_boxed_slice(), *version);
+        }
+    }
+    out
+}
 type LookupIndex = HashMap<FunctionId, HashMap<Row, Row>>;
 
 /// Retractions batched per function: the key length plus the set of keys to
@@ -219,7 +239,7 @@ pub fn run_iteration(eg: &mut EGraph, rules: &[(usize, RuleSpec)]) -> Result<Ite
 
 /// Compute every rule's binding envs in ONE fused pass: the whole atom-bearing
 /// ruleset's body joins run on a SINGLE shared timely worker
-/// ([`dd_native::FusedDdJoin`]) clocked once this iteration, then each rule's
+/// ([`crate::dd_native::FusedDdJoin`]) clocked once this iteration, then each rule's
 /// host-side body primitives are re-run over its own bindings. Atom-less rules
 /// (`(rule () …)`) have no input relation to drive the DD dataflow, so they are
 /// fired once host-side. Returns a `Vec<Vec<Env>>` parallel to `rules` (same
@@ -321,13 +341,22 @@ fn fused_bindings(eg: &mut EGraph, rules: &[(usize, RuleSpec)]) -> Result<Vec<Ve
     {
         let fed = eg.dd_fused_fed_versions.entry(key.clone()).or_default();
         for &read in &all_reads {
-            let cur = match read.mode {
+            let base = match read.mode {
                 ReadMode::Live => eg.live_versions.get(&read.func),
                 ReadMode::Subsumed => eg.subsumed_versions.get(&read.func),
                 ReadMode::All => eg.all_versions.get(&read.func),
             };
             let cur_empty: HashMap<Row, u64> = HashMap::new();
-            let cur = cur.unwrap_or(&cur_empty);
+            let base = base.unwrap_or(&cur_empty);
+            // An occurrence read is a view over the same rows, so deriving it
+            // here lets the delta diff below stay exactly the same.
+            let occurrence_rows;
+            let cur: &HashMap<Row, u64> = if read.occurrence_cols == 0 {
+                base
+            } else {
+                occurrence_rows = occurrence_view(base, read);
+                &occurrence_rows
+            };
             let prev = fed.entry(read).or_default();
             if prev == cur {
                 continue;

egglog-experimental/dd/src/lib.rs

@@ -319,6 +319,28 @@ impl EGraph {
 
         for atom in &rule.core.body.atoms {
             match atom.head {
+                RuleBodyCall::IndexTable { id, ref any_of, .. } => {
+                    let info = relation(id, "rule body")?;
+                    // The occurring value, the base row, and the index relation's
+                    // own unit output.
+                    let expected = info.arity + 2;
+                    if atom.args.len() != expected {
+                        bail!(
+                            "DD backend cannot add rule {:?}: index atom over `{}` has {} columns, expected {expected}",
+                            rule.name,
+                            info.name,
+                            atom.args.len()
+                        );
+                    }
+                    if let Some(col) = any_of.iter().find(|c| **c >= info.arity) {
+                        bail!(
+                            "DD backend cannot add rule {:?}: index atom over `{}` reads column {col}, but it has arity {}",
+                            rule.name,
+                            info.name,
+                            info.arity
+                        );
+                    }
+                }
                 RuleBodyCall::Table { id, .. } => {
                     if atom.args.len() > dd_native::W {
                         bail!(

egglog-experimental/src/fresh_macro.rs

@@ -169,7 +169,7 @@ fn collect_fresh_options(actions: Actions) -> Result<Vec<FreshOptions>, Error> {
 }
 
 /// Parse the arguments to unstable-fresh!
-/// Syntax: (unstable-fresh! SortName [:cost N] [:unextractable])
+/// Syntax: `(unstable-fresh! SortName [:cost N] [:unextractable])`
 fn parse_fresh_args(span: &egglog::ast::Span, args: &[Expr]) -> Result<FreshOptions, Error> {
     if args.is_empty() {
         return Err(Error::ParseError(ParseError(

egglog-experimental/tests/snapshots/files__proofs__test_fresh_proof_testing.snap

@@ -20,4 +20,4 @@ expression: snapshot
       (substitution (a (Var "x")) (b (Var "y"))))
     prf0
     (Fiat (= () ())))
-  (substitution (?y (Var "x")) (?x t0)))
+  (substitution (?x t0) (?y (Var "x"))))

egglog-experimental/tests/snapshots/files__proofs__with_ruleset_proof_testing.snap

@@ -15,4 +15,4 @@ expression: snapshot
   (= t0 (Add (Num 3) (Num 2)))
   (name "(rewrite (Add a b) (Add b a) :ruleset optimization)")
   (premises (Fiat (= t0 t0)))
-  (substitution (a (Num 2)) (b (Num 3)) (@rewrite_var__ t0)))
+  (substitution (@rewrite_var__ t0) (a (Num 2)) (b (Num 3))))