Merge upstream egglog: 41 commits, and it is faster everywhere

egglog-experimental/benches/files.rs

@@ -3,7 +3,7 @@ use std::{
     path::{Path, PathBuf},
 };
 
-use egglog_experimental::{EGraph, new_experimental_egraph_with_proofs};
+use egglog_experimental::new_experimental_egraph_with_proofs;
 
 const CODSPEED_FILES: &[&str] = &[
     "egglog/tests/web-demo/rw-analysis.egg",
@@ -53,13 +53,14 @@ fn benchmark_cases() -> Vec<BenchCase> {
 #[divan::bench(args = benchmark_cases())]
 fn files(case: &BenchCase) {
     let mut egraph = new_experimental_egraph_with_proofs();
+    // Benchmark the serial path, independent of the host's core count.
+    egraph.set_num_threads(1);
     egraph
         .parse_and_run_program(Some(case.filename.clone()), &case.program)
         .unwrap_or_else(|err| panic!("{} failed: {err}", case.path.display()));
     std::mem::forget(egraph);
 }
 
 fn main() {
-    EGraph::set_num_threads(1);
     divan::main();
 }

egglog-experimental/src/either.rs

@@ -1,4 +1,4 @@
-use egglog::ast::Expr;
+use egglog::ast::{Expr, Span};
 use egglog::prelude::ContainerSort;
 use egglog::sort::{ContainerValues, Presort, ValueRebuilder};
 use egglog::{
@@ -81,6 +81,7 @@ impl Presort for EitherSort {
         typeinfo: &mut TypeInfo,
         name: String,
         args: &[Expr],
+        span: Span,
     ) -> Result<ArcSort, TypeError> {
         if let [Expr::Var(left_span, left), Expr::Var(right_span, right)] = args {
             let left = typeinfo
@@ -97,7 +98,10 @@ impl Presort for EitherSort {
             }
             .to_arcsort())
         } else {
-            panic!("Either sort requires exactly two arguments")
+            Err(TypeError::BadPresortArguments(
+                Self::presort_name().to_owned(),
+                span,
+            ))
         }
     }
 }

egglog-experimental/src/maybe.rs

@@ -1,4 +1,4 @@
-use egglog::ast::{Expr, Literal};
+use egglog::ast::{Expr, Literal, Span};
 use egglog::prelude::ContainerSort;
 use egglog::sort::{ContainerValues, F, Presort, ValueRebuilder};
 use egglog::{
@@ -64,6 +64,7 @@ impl Presort for MaybeSort {
         typeinfo: &mut TypeInfo,
         name: String,
         args: &[Expr],
+        span: Span,
     ) -> Result<ArcSort, TypeError> {
         if let [Expr::Var(span, element)] = args {
             let element = typeinfo
@@ -76,7 +77,10 @@ impl Presort for MaybeSort {
             }
             .to_arcsort())
         } else {
-            panic!("Maybe sort requires exactly one argument")
+            Err(TypeError::BadPresortArguments(
+                Self::presort_name().to_owned(),
+                span,
+            ))
         }
     }
 }

egglog/.github/workflows/build.yml

@@ -20,6 +20,18 @@ jobs:
           tool: nextest,cargo-insta
       - uses: Swatinem/rust-cache@v2
       - run: make test
+  # Build the library the way downstream crates consume it: no default
+  # features (so `clap`/`clap_derive` are absent). This guards against
+  # relying on another crate to unify a `syn` feature onto ours — the bug
+  # that broke downstream builds when clap_derive moved to syn 3.x.
+  no-default-features:
+    if: github.event_name != 'issue_comment'
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV"
+      - uses: actions/checkout@v3
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo check -p egglog --no-default-features --lib
   coverage:
     if: github.event_name != 'issue_comment'
     runs-on: ubuntu-latest

egglog/CHANGELOG.md

@@ -23,6 +23,11 @@
 - Fix user-defined primitives (registered through the Rust API after construction) being reported as unbound under term encoding / proofs: primitive registration now also reaches the term-encoding typechecker, so the encoder can typecheck the encoded program. Previously callers had to manually register the primitive on `proof_state.original_typechecking` as well.
 - **Pluggable backend SPI.** `EGraph::with_backend(Box<dyn Backend>)` lets a third party drive the egglog frontend with their own backend (see the `egglog-backend-trait` crate and the `egglog-experimental-dd` example). A backend without a native union-find declares `Backend::requires_term_encoding()`; `EGraph::with_term_encoding()` opts such an e-graph into the term-encoding pipeline (congruence and rebuild lower to rules over `@uf` tables), and running a term-encoding-only backend without it now errors with `Error::BackendRequiresTermEncoding` instead of silently dropping `union`s.
 - Route the term/proof encoding's `get-fresh!` (id minting) and `set-if-empty` (view canonicalization) primitives through the backend SPI (`Backend::register_get_fresh` / `register_set_if_empty` / `register_view_column_read`), so a backend can service them against its own storage instead of reaching into core-relations tables. The backend SPI stays proof-agnostic: reading a view's proof column is just a generic view-column read. The differential-dataflow backend implements these over its host-side mirror and now runs eq-sort programs under term/proof encoding.
+- Fix a build failure when egglog is compiled without default features (as a library dependency). The `egglog-add-primitive` proc macro parses full Rust expressions and now declares `syn`'s `full` feature directly, instead of relying on another crate (`clap_derive`, via the `bin` feature) to unify it onto our `syn`. This surfaced when `clap_derive` moved to `syn` 3.x. A CI job now builds `-p egglog --no-default-features` to catch regressions.
+- Convert many reachable `panic!`/`unwrap`/`expect`/`todo!` sites into recoverable errors, so malformed or edge-case programs report an error instead of aborting the process. Examples now returning `Error`/`TypeError`/`ParseError`/`ProveExistsError`: malformed sort-constructor declarations like `(sort S (Vec))` or `(sort S (UnstableFn))`; negative `extract` variant counts; duplicate rule names; `unstable-fn` referencing an unknown, non-literal, or mis-typed target; `(fail ...)` wrapping `include` or an empty expansion; subsuming a non-call rewrite; running `prove`/`prove-exists` without proofs enabled; and missing/unreadable files for `input`, `print-function`, `print-overall-statistics`, and the CLI. Several primitives became partial (returning no result instead of panicking) on out-of-range input: `vec-set`/`vec-remove` indices, `multiset-pick` on an empty multiset, count overflow in `multiset` operations, and the numeric primitives `bigint <<`/`>>`, `bigrat`, and `log2`. A few scheduler edge cases (unknown ruleset, rules with no free variables) no longer panic; variable-free rules now correctly apply their actions when scheduled. Primitive resolution now returns `TypeError::AmbiguousPrimitive`/`TypeError::UnresolvedPrimitive` instead of panicking when duplicate same-signature registrations are indistinguishable or nothing resolves; both direct calls and `unstable-fn` primitive targets report the same variants. `step_rules_with_scheduler` now restores its `rulesets`/`schedulers` on every fallible path, so an error during scheduled rule compilation no longer leaves the `EGraph` in a corrupted state.
+- **Breaking:** `EGraph::print_function` now takes its output sink as `Option<(File, PathBuf)>` plus a `Span`, so write failures return `Error::IoError` instead of panicking.
+- Speed up query evaluation by building on-the-fly per-subset column indexes as sorted arrays (`SortedColumnIndex`) instead of hash maps. These indexes are typically iterated once and probed a bounded number of times over high-cardinality columns, so skipping hash-table construction is a large win (e.g. ~33% faster on the `gemma` benchmark).
+- Share trie roots (and their cached sub-indexes and child nodes) across query plans within a single `run_rule_set` instead of rebuilding a fresh trie per plan. Plans that scan the same table under the same header (fast) constraints reuse one root, so on-the-fly per-subset index builds happen once rather than per plan; only roots that more than one plan uses are shared, so workloads that would not benefit keep the per-plan behavior. Large speedups on transformer workloads (e.g. ~15% faster on `whisper`, ~12% on `gemma`, ~8% on `qwen3_moe`).
 - Add `make nightly` and `scripts/nightly_bench.py`, a hyperfine-based benchmark harness that measures every `tests/**/*.egg` program at 1/2/4/8 threads and (where supported) in proof-testing mode, caps each run at a 2-minute timeout, skips sub-50ms programs, and emits an HTML dashboard (one row per benchmark, one column per configuration) for nightly.cs.washington.edu. The dashboard uses [eval-live](https://github.com/oflatt/eval-live) for interactive filtering and sorting.
 - Rework the term/proof encoding's union-find and congruence maintenance,
   substantially reducing proof-mode time and memory.
@@ -54,9 +59,12 @@
   or as the head of a call expression.
 - Add typed `EGraph` extension state that clones with `EGraph` and is restored by `push`/`pop`.
 - Fix custom scheduler queries so subsumed rows are not offered as fresh matches.
+- Replace the global Rayon thread pool with an `egglog-concurrency` scoped `ThreadPool`; configure parallelism per `EGraph` via `with_num_threads` / `set_num_threads`.
 - Report full source file paths in egglog span and error messages.
 - Fix seminaive matching after nested containers rebuild in place by propagating dirty container ids through parent containers.
+- Fix multi-column secondary index rebuilds so each value's rows come back sorted by row id, and make all rebuild paths (serial, parallel, and bulk) record a row once even when its value repeats across covered columns (#914).
 - Render nullary AST calls without a trailing space, e.g. (foo) instead of (foo ).
+- Escape `"` and `\` when displaying string literals so printed/serialized programs round-trip through the parser.
 - Add a BigRat to-i64 primitive for integral rationals.
 - Add f64 exp, log, and sqrt primitives.
 - Add `RunReport::can_stop` so scheduler progress can be reported separately from database updates.

egglog/Cargo.toml

@@ -69,7 +69,6 @@ egglog-numeric-id = { workspace = true }
 egglog-add-primitive = { workspace = true }
 egglog-reports = { workspace = true }
 egglog-backend-trait = { workspace = true }
-rayon = { workspace = true }
 serde_json = { workspace = true }
 
 [build-dependencies]

egglog/Makefile

@@ -5,6 +5,10 @@ TESTS=$(shell find tests/ -type f -name '*.egg' -not -name '*repro-*')
 
 WWW=${PWD}/target/www
 
+# Keep release-mode test builds checking debug assertions without changing the
+# normal release profile used by CodSpeed benchmarks and production binaries.
+TEST_PROFILE_ENV=CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS=true
+
 all: test nits docs
 
 # Build egglog and benchmark every tests/*.egg file with hyperfine, writing an
@@ -18,14 +22,14 @@ nightly:
 	nightly/.venv/bin/python scripts/nightly_bench.py
 
 test: doctest
-	cargo insta test --test-runner nextest --release --workspace  --unreferenced reject
+	$(TEST_PROFILE_ENV) cargo insta test --test-runner nextest --release --workspace  --unreferenced reject
 
 coverage:
-	cargo llvm-cov nextest --release --workspace --lcov --output-path lcov.info
+	$(TEST_PROFILE_ENV) cargo llvm-cov nextest --release --workspace --lcov --output-path lcov.info
 	# Note: doctests are not included in coverage reports
 
 doctest:
-	cargo test --doc --release --workspace
+	$(TEST_PROFILE_ENV) cargo test --doc --release --workspace
 
 
 nits:

egglog/benches/common.rs

@@ -1,13 +1,11 @@
 #![allow(dead_code)]
 
 use egglog::EGraph;
-use std::{fmt, sync::Once};
+use std::fmt;
 
 #[global_allocator]
 static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
-static CONFIGURE_RAYON: Once = Once::new();
-
 pub fn run_example(filename: &str, program: &str, proof_testing: bool) {
     let mut egraph = if proof_testing {
         EGraph::new_with_proofs().with_proof_testing()
@@ -38,8 +36,6 @@ impl fmt::Display for BenchCase {
 }
 
 pub fn bench_cases(glob: &str) -> Vec<BenchCase> {
-    configure_rayon_once();
-
     let mut cases = Vec::new();
 
     // Add regular test cases
@@ -83,8 +79,6 @@ const PROOF_UNSUPPORTED_FILES: &[&str] = &[
 ];
 
 pub fn bench_cases_proof_testing(glob: &str) -> Vec<BenchCase> {
-    configure_rayon_once();
-
     glob::glob(glob)
         .unwrap()
         .filter_map(Result::ok)
@@ -114,16 +108,5 @@ fn proof_benchmark_supported(path: &std::path::Path) -> bool {
 }
 
 pub fn bench_case(case: &BenchCase) {
-    configure_rayon_once();
-
     run_example(&case.filename, &case.program, case.proof_testing);
 }
-
-pub fn configure_rayon_once() {
-    CONFIGURE_RAYON.call_once(|| {
-        rayon::ThreadPoolBuilder::new()
-            .num_threads(1)
-            .build_global()
-            .unwrap();
-    });
-}

egglog/benches/rust_api_benchmarking.rs

@@ -1,5 +1,3 @@
-mod common;
-
 #[derive(Clone, Copy)]
 struct RustRuleBenchCase {
     n_facts_input: Option<usize>,
@@ -23,8 +21,6 @@ struct RustRuleBenchInput {
 fn match_only_rust_rule_setup(case: RustRuleBenchCase) -> RustRuleBenchInput {
     use egglog::prelude::*;
 
-    common::configure_rayon_once();
-
     let mut program = String::new();
     program.push_str("(relation R (i64))\n");
 
@@ -120,8 +116,6 @@ fn rust_rule_match_with_serialize(bencher: divan::Bencher, case: RustRuleBenchCa
 fn insert_loop_setup(case: RustRuleInsertLoopBenchCase) -> RustRuleBenchInput {
     use egglog::prelude::*;
 
-    common::configure_rayon_once();
-
     let mut program = String::new();
     program.push_str("(relation R (i64))\n");
     program.push_str("(function f (i64) i64 :no-merge)\n");
@@ -162,8 +156,6 @@ fn insert_loop_setup(case: RustRuleInsertLoopBenchCase) -> RustRuleBenchInput {
 fn tableaction_hot_path_setup(case: RustRuleTableActionBenchCase) -> RustRuleBenchInput {
     use egglog::prelude::*;
 
-    common::configure_rayon_once();
-
     let mut program = String::new();
     program.push_str("(relation R (i64))\n");
     program.push_str("(function f (i64) i64 :no-merge)\n");
@@ -299,14 +291,12 @@ impl std::fmt::Display for ReadScanBenchCase {
 fn read_scan_setup(case: ReadScanBenchCase) -> egglog::EGraph {
     use std::fmt::Write;
 
-    common::configure_rayon_once();
-
     let mut program = String::from("(sort Math)\n(constructor Add (i64 i64) Math)\n");
     for i in 0..case.n_enodes {
         let _ = writeln!(&mut program, "(Add {} {})", i as i64, (i + 1) as i64);
     }
 
-    let mut egraph = egglog::EGraph::default();
+    let mut egraph = egglog::EGraph::new(1);
     egraph.parse_and_run_program(None, &program).unwrap();
     egraph
 }
@@ -340,8 +330,6 @@ fn main() {
 
 fn fib_setup() -> RustRuleBenchInput {
     use egglog::prelude::*;
-    common::configure_rayon_once();
-
     let mut program = String::new();
     program.push_str("(function fib (i64) i64 :no-merge)");
     program.push_str("(set (fib 0) 0)\n");

egglog/concurrency/Cargo.toml

@@ -11,7 +11,7 @@ readme = { workspace = true }
 [dependencies]
 arc-swap = { workspace = true}
 bumpalo = { workspace = true}
-rayon = { workspace = true}
+crossbeam = { workspace = true}
 egglog-numeric-id = { workspace = true }
 smallvec = { workspace = true }
 
@@ -23,3 +23,7 @@ rayon = { workspace = true}
 [[bench]]
 name = "simple_read"
 harness = false
+
+[[bench]]
+name = "sum_vector"
+harness = false