From 3074aeed648d1a93bf10c414aeefc6ca4ed16477 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Wed, 9 Sep 2026 16:22:02 -0700 Subject: [PATCH 1/9] refactor(parser): split ParserTrait into parse and metric halves ParserTrait now carries only the Checker / Getter classifiers and the tree accessors; the thirteen per-metric associated types move to a new MetricSuite supertrait with a blanket impl over Parser. The metric walks bound on MetricSuite, everything that only classifies nodes stays on ParserTrait, so the parse layer no longer depends on the metric modules. The language-dispatched carrier behind Ast becomes AnyParser, whose variants exist regardless of the feature set (only the constructors are feature-gated), and a hand-written with_any_parser! macro replaces the per-operation run_* methods. Exhaustive matching makes a forgotten language a compile error and keeps the macro free of cfg attributes. Preparatory step for #1376: no behaviour change, no metric moves. --- src/find.rs | 2 +- src/lib.rs | 23 +- src/macros/mod.rs | 345 +++++++++------------------ src/metric_suite.rs | 86 +++++++ src/metrics/cognitive.rs | 2 +- src/metrics/container_scope_tests.rs | 2 +- src/metrics/halstead.rs | 14 +- src/metrics/nom.rs | 2 +- src/ops.rs | 12 +- src/parser.rs | 90 +------ src/spaces.rs | 2 +- src/spaces/ast.rs | 35 +-- src/spaces/compute.rs | 19 +- src/spaces_tests.rs | 6 +- src/test_support.rs | 15 +- src/traits.rs | 39 +-- 16 files changed, 273 insertions(+), 421 deletions(-) create mode 100644 src/metric_suite.rs diff --git a/src/find.rs b/src/find.rs index 0c609618e..83ee9b7be 100644 --- a/src/find.rs +++ b/src/find.rs @@ -26,7 +26,7 @@ use crate::traits::ParserTrait; /// with the other walk cores so callers can use the `?` operator /// uniformly. // The `Result` is deliberate forward-compat (see doc above) and is -// propagated unchanged through `AstInner::run_find` / `Ast::find`; +// propagated unchanged through `Ast::find`; // `unnecessary_wraps` would have us drop it and break that uniform // `?`-able shape across the walk cores. #[allow(clippy::unnecessary_wraps)] diff --git a/src/lib.rs b/src/lib.rs index 1e8c959ef..b46489074 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -167,7 +167,7 @@ pub(crate) use crate::langs::{ PhpCode, PreprocCode, PythonCode, RubyCode, RustCode, TclCode, TsxCode, TypescriptCode, }; // The `Parser` aliases are the concrete `Parser<Code>` types -// driven by the `AstInner` dispatch in `crate::langs`; at the crate root +// driven by the `AnyParser` dispatch in `crate::langs`; at the crate root // they are reached only from `#[cfg(test)]` modules, so the re-export is // `unused` in a non-test build. #[allow(unused_imports)] @@ -210,13 +210,6 @@ pub(crate) use crate::metrics::{ // --- Core analysis entry points and result types (spaces.rs) --- mod spaces; pub use crate::spaces::{Ast, CodeMetrics, FuncSpace, MetricsOptions, Source, SpaceKind, analyze}; -// `metrics_inner` is the per-`ParserTrait` metric walk core consumed by -// feature-gated arms in `mk_action!` (`AstInner::run_metrics`). With -// `--no-default-features` and no language feature, every arm compiles -// out and the re-export becomes nominally unused; the language-features -// that ship in the default set keep the symbol live in any normal build. -#[allow(unused_imports)] -pub(crate) use crate::spaces::metrics_inner; /// Per-metric implementations. /// @@ -317,7 +310,7 @@ pub use crate::concurrent_files::{ // --- Comment removal --- // // `rm_comments` is the internal walk core reached only through the -// [`Ast::strip_comments`] seam (`AstInner::run_strip_comments`). +// [`Ast::strip_comments`] seam (`with_any_parser!` in `spaces/ast.rs`). mod comment_rm; // --- Per-file node counting / finding (reached via the `Ast` seam) --- @@ -339,11 +332,6 @@ mod recursion; // --- Halstead operator/operand result type --- mod ops; pub use crate::ops::Ops; -// `ops_inner` is the explicit-name walk core consumed by feature-gated -// `mk_action!` arms (`AstInner::run_ops`); mirrors the `metrics_inner` -// re-export above and is nominally unused under `--no-default-features`. -#[allow(unused_imports)] -pub(crate) use crate::ops::ops_inner; // --- Preprocessor handling (C/C++) --- mod preproc; @@ -364,7 +352,7 @@ pub(crate) use crate::alterator::Alterator; // `Parser`, `ParserTrait`, `Filter`, and `LanguageInfo` are the // internal parser machinery driving every metric walk. They are // `pub(crate)` only: the single public analysis seam is [`Ast`], -// which wraps the language-dispatched `AstInner` carrier. See +// which wraps the language-dispatched `AnyParser` carrier. See // STABILITY.md. mod parser; pub(crate) use crate::parser::Parser; @@ -372,6 +360,11 @@ pub(crate) use crate::parser::Parser; mod traits; pub(crate) use crate::traits::{LanguageInfo, ParserTrait, Search}; +// The metric half of the parser contract; `ParserTrait` above is the +// parse half. See `src/metric_suite.rs`. +mod metric_suite; +pub(crate) use crate::metric_suite::MetricSuite; + /// Re-export of the underlying `tree-sitter` crate. /// /// Lets callers build a [`tree_sitter::Tree`] (via diff --git a/src/macros/mod.rs b/src/macros/mod.rs index 856cd6f46..8cfdb3eb4 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -394,268 +394,138 @@ impl ::std::error::Error for ParseLangError {} macro_rules! mk_action { ( $( ($feature:literal, $camel:ident, $parser:ident) ),* ) => { - /// Language-dispatched bundle of a parsed tree plus its - /// source bytes, one variant per Cargo-feature-enabled - /// language. The public seam is [`crate::Ast`]; this enum is - /// the macro-generated internal carrier it wraps. + /// A parsed tree plus its source bytes for a language chosen at + /// runtime — one variant per [`LANG`], each holding that + /// language's `Parser`. The public seam is + /// [`crate::Ast`]; this enum is the language-dispatched carrier + /// it wraps, and the value a caller matches on to reach a + /// concrete parser (`with_any_parser!`). /// - /// With every per-language feature disabled this enum is a - /// 0-variant uninhabited type. Each method below therefore - /// terminates its `match self` with a - /// `#[cfg(not(any(feature = …)))] _ => match *self {}` arm: - /// stable Rust treats `&UninhabitedType` as inhabited (E0004), - /// so the outer match needs a wildcard, and `match *self {}` - /// is exhaustive over the uninhabited dereferenced value — - /// divergent, no panic, no `unsafe`, statically unreachable in - /// safe code because the public seam `crate::Ast` has only - /// fallible constructors that return `Err(LanguageDisabled)` - /// for every `LANG` variant under that build. - /// - /// When a method takes by-value parameters (see - /// [`Self::run_metrics`]), prefix the divergent arm with - /// `let _ = (param1, param2, …);` to silence - /// `unused_variables` under `RUSTFLAGS=-D warnings` — the - /// `match *self {}` body is `!`, so the consumed values are - /// never actually dropped at runtime. - pub(crate) enum AstInner { + /// Every variant exists regardless of the Cargo feature set: a + /// `Parser` *type* needs no grammar crate, only + /// [`Self::parse`] / [`Self::from_tree`] do, and those are the + /// arms that are feature-gated. A disabled language is therefore + /// never *constructed* — the constructors return + /// `Err(LanguageDisabled)` for it — but it can always be *named*, + /// which is what lets `with_any_parser!` be written once without + /// any `cfg` of its own (#1376). + pub(crate) enum AnyParser { $( - #[cfg(feature = $feature)] + #[doc = concat!("The `", stringify!($camel), "` parser.")] $camel($parser), )* } - impl AstInner { - /// Run the metric walker against the held parse. The - /// caller passes `name` and `options` per call so a - /// single `AstInner` can be reused with different metric - /// subsets. - pub(crate) fn run_metrics( - &self, - name: Option, - options: MetricsOptions, - ) -> Result { - match self { - $( - #[cfg(feature = $feature)] - AstInner::$camel(parser) => metrics_inner(parser, name, options), - )* - #[cfg(not(any( $( feature = $feature ),* )))] - _ => { - let _ = (name, options); - match *self {} - }, - } - } - - /// Run the operator/operand walk against the held parse, - /// carrying an explicit `name` end-to-end. Backs - /// [`crate::Ast::ops`]; the ops analogue of [`Self::run_metrics`]. - pub(crate) fn run_ops( - &self, - name: Option, - ) -> Result { - match self { - $( - #[cfg(feature = $feature)] - AstInner::$camel(parser) => ops_inner(parser, name), - )* - #[cfg(not(any( $( feature = $feature ),* )))] - _ => { - let _ = name; - match *self {} - }, - } - } - - /// Strip comments from the held parse. Backs - /// [`crate::Ast::strip_comments`]; the comment-removal analogue - /// of [`Self::run_ops`]. - pub(crate) fn run_strip_comments(&self) -> Option> { - match self { - $( - #[cfg(feature = $feature)] - AstInner::$camel(parser) => crate::comment_rm::rm_comments(parser), - )* - #[cfg(not(any( $( feature = $feature ),* )))] - _ => match *self {}, - } - } - - /// Detect the span of every function in the held parse. Backs - /// [`crate::Ast::functions`]. - pub(crate) fn run_functions(&self) -> Vec { - match self { - $( - #[cfg(feature = $feature)] - AstInner::$camel(parser) => crate::function::function(parser), - )* - #[cfg(not(any( $( feature = $feature ),* )))] - _ => match *self {}, - } - } - - /// Build the AST dump for the held parse under `cfg`. Backs - /// [`crate::Ast::dump`]. - pub(crate) fn run_dump(&self, cfg: crate::AstCfg) -> crate::AstResponse { - match self { - $( - #[cfg(feature = $feature)] - AstInner::$camel(parser) => crate::ast::dump_inner(parser, cfg), - )* - #[cfg(not(any( $( feature = $feature ),* )))] - _ => { - let _ = cfg; - match *self {} - }, - } - } - - /// Count `(matching, total)` nodes for `filters` in the held - /// parse. Backs [`crate::Ast::count`]. - pub(crate) fn run_count(&self, filters: &[String]) -> (usize, usize) { - match self { - $( - #[cfg(feature = $feature)] - AstInner::$camel(parser) => crate::count::count(parser, filters), - )* - #[cfg(not(any( $( feature = $feature ),* )))] - _ => { - let _ = filters; - match *self {} - }, - } - } - - /// Find every node matching `filters` in the held parse. Backs - /// [`crate::Ast::find`]; the returned nodes borrow the held tree. - pub(crate) fn run_find( - &self, - filters: &[String], - ) -> Result>, MetricsError> { - match self { - $( - #[cfg(feature = $feature)] - AstInner::$camel(parser) => crate::find::find(parser, filters), - )* - #[cfg(not(any( $( feature = $feature ),* )))] - _ => { - let _ = filters; - match *self {} - }, - } - } - - /// Collect every in-source suppression marker in the held parse. - /// Backs [`crate::Ast::suppressions`]. - pub(crate) fn run_suppressions(&self) -> Vec { - match self { + impl AnyParser { + /// Parse `source` as `lang`. + /// + /// `Parser::new` keys the C-family macro-expansion lookup off + /// the caller-supplied path; callers analysing in-memory + /// snippets pass `None` and get the empty `Path` (`""`), + /// which the lookup ignores. That path never leaks into a + /// display name — `Ast` carries the name separately. + /// `source` is taken by value so an owned buffer moves + /// straight into the parser instead of being copied. + /// + /// # Errors + /// + /// `MetricsError::LanguageDisabled` when `lang`'s Cargo + /// feature is not enabled in this build. + pub(crate) fn parse( + lang: LANG, + source: Vec, + preproc_path: Option<&Path>, + preproc: Option>, + ) -> Result { + let preproc_path = preproc_path.unwrap_or(Path::new("")); + match lang { $( #[cfg(feature = $feature)] - AstInner::$camel(parser) => crate::suppression::suppression_markers(parser), + LANG::$camel => Ok(AnyParser::$camel($parser::new(source, preproc_path, preproc))), + #[cfg(not(feature = $feature))] + LANG::$camel => { + let _ = (source, preproc_path, preproc); + Err(MetricsError::LanguageDisabled(lang)) + }, )* - #[cfg(not(any( $( feature = $feature ),* )))] - _ => match *self {}, } } - /// Borrow the root [`crate::Node`] of the held parse. Backs - /// [`crate::Ast::root_node`]. - pub(crate) fn root_node(&self) -> crate::Node<'_> { - match self { + /// Adopt a caller-built [`tree_sitter::Tree`] produced from + /// `source` with `lang`'s grammar. + /// + /// # Errors + /// + /// `MetricsError::LanguageDisabled` when `lang`'s Cargo + /// feature is not enabled in this build. + pub(crate) fn from_tree( + lang: LANG, + tree: ::tree_sitter::Tree, + source: Vec, + ) -> Result { + match lang { $( #[cfg(feature = $feature)] - AstInner::$camel(parser) => parser.root(), + LANG::$camel => Ok(AnyParser::$camel($parser::from_tree(tree, source))), + #[cfg(not(feature = $feature))] + LANG::$camel => { + let _ = (tree, source); + Err(MetricsError::LanguageDisabled(lang)) + }, )* - #[cfg(not(any( $( feature = $feature ),* )))] - _ => match *self {}, } } + /// The language this parser was built for. + #[must_use] pub(crate) fn language(&self) -> LANG { match self { - $( - #[cfg(feature = $feature)] - AstInner::$camel(_) => LANG::$camel, - )* - #[cfg(not(any( $( feature = $feature ),* )))] - _ => match *self {}, - } - } - - pub(crate) fn code_bytes(&self) -> &[u8] { - match self { - $( - #[cfg(feature = $feature)] - AstInner::$camel(parser) => parser.code(), - )* - #[cfg(not(any( $( feature = $feature ),* )))] - _ => match *self {}, - } - } - - pub(crate) fn ts_tree(&self) -> &::tree_sitter::Tree { - match self { - $( - #[cfg(feature = $feature)] - AstInner::$camel(parser) => parser.ts_tree(), - )* - #[cfg(not(any( $( feature = $feature ),* )))] - _ => match *self {}, + $( AnyParser::$camel(_) => LANG::$camel, )* } } } + }; +} - /// Internal parse-dispatch shim that backs [`crate::Ast::parse`]. - /// Lives in the `mk_action!` macro so each new language only - /// has to declare its parser tag once. - pub(crate) fn ast_parse_dispatch( - lang: LANG, - source: Vec, - preproc_path: Option<&Path>, - preproc: Option>, - ) -> Result { - // `Parser::new` keys the C++ macro-expansion lookup off the - // caller-supplied path; for callers analysing in-memory - // snippets with no preprocessor path, fall back to an - // empty `Path` ("") which the lookup ignores. The empty - // path is *not* leaked into `FuncSpace::name` — that - // is carried separately on `Ast`. `source` is taken by value - // so an owned `Source` (`Source::from_bytes`) moves its - // buffer straight into the parser instead of copying it. - let preproc_path = preproc_path.unwrap_or(Path::new("")); - match lang { - $( - #[cfg(feature = $feature)] - LANG::$camel => Ok(AstInner::$camel($parser::new(source, preproc_path, preproc))), - #[cfg(not(feature = $feature))] - LANG::$camel => { - let _ = (source, preproc_path, preproc); - Err(MetricsError::LanguageDisabled(lang)) - }, - )* - } - } - - /// Internal tree-adoption dispatch that backs - /// [`crate::Ast::from_tree_sitter`]. - pub(crate) fn ast_from_tree_dispatch( - lang: LANG, - tree: ::tree_sitter::Tree, - source: Vec, - ) -> Result { - match lang { - $( - #[cfg(feature = $feature)] - LANG::$camel => Ok(AstInner::$camel($parser::from_tree(tree, source))), - #[cfg(not(feature = $feature))] - LANG::$camel => { - let _ = (tree, source); - Err(MetricsError::LanguageDisabled(lang)) - }, - )* - } +/// Dispatches over every [`AnyParser`] variant, binding the concrete +/// `Parser` to `$p` and evaluating `$body` once per arm. +/// +/// Written out by hand rather than generated inside `mk_action!` so the +/// arm list stays a plain match: a variant missing here is a +/// non-exhaustive-match compile error, which is the whole guarantee. +/// Every arm is unconditional — see the [`AnyParser`] docs for why no +/// `cfg` is needed. Add a line here when `mk_langs!` gains a language. +/// +/// [`AnyParser`]: crate::langs::AnyParser +macro_rules! with_any_parser { + ($any:expr, |$p:ident| $body:expr) => { + match $any { + $crate::langs::AnyParser::Javascript($p) => $body, + $crate::langs::AnyParser::Mozjs($p) => $body, + $crate::langs::AnyParser::Java($p) => $body, + $crate::langs::AnyParser::Go($p) => $body, + $crate::langs::AnyParser::Kotlin($p) => $body, + $crate::langs::AnyParser::Lua($p) => $body, + $crate::langs::AnyParser::Rust($p) => $body, + $crate::langs::AnyParser::Tcl($p) => $body, + $crate::langs::AnyParser::Irules($p) => $body, + $crate::langs::AnyParser::C($p) => $body, + $crate::langs::AnyParser::Cpp($p) => $body, + $crate::langs::AnyParser::Mozcpp($p) => $body, + $crate::langs::AnyParser::Objc($p) => $body, + $crate::langs::AnyParser::Csharp($p) => $body, + $crate::langs::AnyParser::Elixir($p) => $body, + $crate::langs::AnyParser::Python($p) => $body, + $crate::langs::AnyParser::Tsx($p) => $body, + $crate::langs::AnyParser::Typescript($p) => $body, + $crate::langs::AnyParser::Bash($p) => $body, + $crate::langs::AnyParser::Ccomment($p) => $body, + $crate::langs::AnyParser::Preproc($p) => $body, + $crate::langs::AnyParser::Perl($p) => $body, + $crate::langs::AnyParser::Php($p) => $body, + $crate::langs::AnyParser::Ruby($p) => $body, + $crate::langs::AnyParser::Groovy($p) => $body, } - }; } @@ -786,4 +656,5 @@ pub(crate) use kind_sets::{ }; pub(crate) use { get_language, mk_action, mk_code, mk_emacs_mode, mk_extensions, mk_lang, mk_langs, + with_any_parser, }; diff --git a/src/metric_suite.rs b/src/metric_suite.rs new file mode 100644 index 000000000..d040f5497 --- /dev/null +++ b/src/metric_suite.rs @@ -0,0 +1,86 @@ +//! The metric half of the per-language parser contract. +//! +//! [`ParserTrait`] answers "what is this node?" through its `Checker` / +//! `Getter` classifiers and knows nothing about metrics. [`MetricSuite`] +//! layers the thirteen per-metric `compute` implementations on top, keyed +//! on the same `*Code` tag, so the metric walks can bound on one trait +//! while the parse layer stays free of them (#1376). The blanket impl +//! below is the only implementor: every `Parser` whose tag implements +//! every metric trait is a `MetricSuite`, which is true of all of them. + +use crate::abc::Abc; +use crate::alterator::Alterator; +use crate::checker::Checker; +use crate::cognitive::Cognitive; +use crate::cyclomatic::Cyclomatic; +use crate::getter::Getter; +use crate::halstead::Halstead; +use crate::loc::Loc; +use crate::mi::Mi; +use crate::nargs::NArgs; +use crate::nexits::Exit; +use crate::nom::Nom; +use crate::npa::Npa; +use crate::npm::Npm; +use crate::parser::Parser; +use crate::tokens::Tokens; +use crate::traits::{LanguageInfo, ParserTrait}; +use crate::wmc::Wmc; + +/// Per-language metric implementations reachable from a parser. +/// +/// Each associated type is the `*Code` tag itself; the walk calls +/// `T::Cognitive::compute(...)` and so on. Bound a walk on this trait +/// when it computes a metric, and on [`ParserTrait`] alone when it only +/// classifies nodes. +pub(crate) trait MetricSuite: ParserTrait { + type Cognitive: Cognitive; + type Cyclomatic: Cyclomatic; + type Halstead: Halstead; + type Loc: Loc; + type Nom: Nom; + type Mi: Mi; + type NArgs: NArgs; + type Exit: Exit; + type Wmc: Wmc; + type Abc: Abc; + type Npm: Npm; + type Npa: Npa; + type Tokens: Tokens; +} + +impl MetricSuite for Parser +where + T: 'static + + LanguageInfo + + Alterator + + Checker + + Getter + + Abc + + Cognitive + + Cyclomatic + + Exit + + Halstead + + Loc + + Mi + + NArgs + + Nom + + Npa + + Npm + + Tokens + + Wmc, +{ + type Cognitive = T; + type Cyclomatic = T; + type Halstead = T; + type Loc = T; + type Nom = T; + type Mi = T; + type NArgs = T; + type Exit = T; + type Wmc = T; + type Abc = T; + type Npm = T; + type Npa = T; + type Tokens = T; +} diff --git a/src/metrics/cognitive.rs b/src/metrics/cognitive.rs index 20db1aff6..7a2c6e674 100644 --- a/src/metrics/cognitive.rs +++ b/src/metrics/cognitive.rs @@ -9899,7 +9899,7 @@ end", /// a definition nested in *conditionals*, the `stops` entry on one /// nested in another *function*. Both are covered below, plus the two /// shapes the fix must leave alone. - fn check_js_function_boundary(filename: &str) { + fn check_js_function_boundary(filename: &str) { fn score(space: &FuncSpace, name: &str) -> u64 { function_space(space, name).metrics.cognitive.cognitive() } diff --git a/src/metrics/container_scope_tests.rs b/src/metrics/container_scope_tests.rs index 7de085d9a..e8fb3cfb8 100644 --- a/src/metrics/container_scope_tests.rs +++ b/src/metrics/container_scope_tests.rs @@ -589,7 +589,7 @@ fn the_file_root_keeps_its_rollup() { /// Asserted here on the serialized kind rather than by walking nodes and /// checking `is_func_space_with_code(n) ⇒ get_space_kind_with_code(n) != /// Unknown` directly. `Checker` / `Getter` methods are static and -/// monomorphised per parser type; `AstInner` hands out a `root_node` but +/// monomorphised per parser type; `AnyParser` hands out a `root_node` but /// no LANG-generic way to invoke them against it, so the node-walk form /// would need a new `run_*` dispatch arm on the macro — production /// surface grown to host a test. The promoted-but-`Unknown` space *is* diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 2dfd4d3d3..ca1979ab5 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -484,7 +484,7 @@ mod tests { // reported location names the language row instead of a shared line // no assertion message distinguishes. #[track_caller] - fn assert_ops_operands( + fn assert_ops_operands( source: &str, file: &str, expected_n2: usize, @@ -517,7 +517,7 @@ mod tests { /// plain `fn` that cannot capture a loop variable, so they reach /// for the closure-taking helper it wraps. Three copies of that /// dance is two too many. - fn assert_halstead_counts( + fn assert_halstead_counts( source: &str, file: &str, expected: [u64; 4], @@ -3826,7 +3826,7 @@ end", // Balanced openers must count once and render folded (no bare // `(`/`[`, no n1 inflation) — the property #768 feared was broken. - fn assert_folded_openers(source: &str, file: &str) { + fn assert_folded_openers(source: &str, file: &str) { let path = PathBuf::from(file); let parser = T::new(source.as_bytes().to_vec(), &path, None); let ops = crate::ops::ops_inner(&parser, None).expect("ops walk succeeds"); @@ -5571,7 +5571,7 @@ f() { /// Every row is measured in *both* dialects, so a fix applied to /// one getter and not its clone fails here. `braced_word_shed`'s /// drift assertion likewise runs against both grammars. - fn check_braced_word_cases( + fn check_braced_word_cases( cases: &[BracedWordCase], file: &str, kinds: &BracedWordKinds, @@ -5947,7 +5947,7 @@ f() { /// Runs one #1318 table against one dialect. Both dialects run /// every shared row, so a fix that reached one getter and not its /// clone fails here. - fn check_braced_word_value_cases( + fn check_braced_word_value_cases( cases: &[BracedWordValueCase], file: &str, ) { @@ -8044,7 +8044,7 @@ f() { /// of the same `HalsteadMaps` — which is worth knowing before /// reading it as independent corroboration of the count. #[track_caller] - fn assert_char_literal_operands(file: &str, label: &str) { + fn assert_char_literal_operands(file: &str, label: &str) { // `char` x4 and `int` are text-keyed primitive operators, so // n1 = 4 (`;`, `=`, `char`, `int`) and N1 = 5 + 5 + 4 + 1. // Operands: `a`..`e`, plus `'x'` (twice), `'y'`, `'\n'`, `'ab'`. @@ -8256,7 +8256,7 @@ f() { /// `field_expression` instead of the `this` leaf would hold `n2` at /// 6 while the vocabulary silently became `this->x`. #[track_caller] - fn assert_this_receiver_parity(file: &str, label: &str) { + fn assert_this_receiver_parity(file: &str, label: &str) { // Operators, keyed by kind_id except the text-keyed primitives: // `{` x3 (class body, `m1`, `m2`), `int` x3, `;` x4 (the field, // the two returns, the struct terminator), `(` x2, `return` x2, diff --git a/src/metrics/nom.rs b/src/metrics/nom.rs index 9726b05f6..99faf8b79 100644 --- a/src/metrics/nom.rs +++ b/src/metrics/nom.rs @@ -956,7 +956,7 @@ mod tests { ); } - fn check_returned_object_arrow_nom(file_name: &str) { + fn check_returned_object_arrow_nom(file_name: &str) { check_metrics::( "function f() { return { foo: x => x }; }", file_name, diff --git a/src/ops.rs b/src/ops.rs index d928345b7..44899ceda 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -16,7 +16,7 @@ use crate::spaces::{SpaceKind, line_span, push_children}; use crate::halstead::{Halstead, HalsteadMaps}; -use crate::traits::ParserTrait; +use crate::MetricSuite; /// All operands and operators of a space. #[derive(Debug, Clone)] @@ -119,7 +119,7 @@ struct State<'a> { /// frame and `ops_inner` would return [`MetricsError::EmptyRoot`] for an /// input where `metrics()` succeeds (issue #789). A `Unit` root needs no /// wrapper, so nothing is pushed in that case. -fn push_synthetic_unit_root( +fn push_synthetic_unit_root( state_stack: &mut Vec, node: &Node, code: &[u8], @@ -161,7 +161,7 @@ crate::observation::counter!(space_kind_lookups); /// property and Elixir's reads the `Call` target text and scans the /// ancestor chain for an enclosing `quote` block, so it is not free on /// every node either. -fn classify_space_kind<'a, T: ParserTrait>( +fn classify_space_kind<'a, T: MetricSuite>( node: &Node<'a>, code: &[u8], ancestors: Ancestors<'a, '_>, @@ -230,7 +230,7 @@ fn sorted_vocabulary(mut keys: Vec<&[u8]>) -> Vec { rendered } -fn compute_operators_and_operands(state: &mut State) { +fn compute_operators_and_operands(state: &mut State) { let maps = &state.halstead_maps; // Primitive-type operators live in a second map (keyed by text rather @@ -255,7 +255,7 @@ fn compute_operators_and_operands(state: &mut State) { /// call would rebuild (and, since #1091, re-sort) the whole file's /// vocabulary once per level-drop in the walk, and every result but the /// last would be overwritten. -fn finalize(state_stack: &mut Vec, diff_level: usize) { +fn finalize(state_stack: &mut Vec, diff_level: usize) { for _ in 0..diff_level { if state_stack.len() < 2 { break; @@ -298,7 +298,7 @@ struct Walk { /// is whatever the caller passes in `name`; `name_was_lossy` is left at /// its `false` default because an explicit `String` name is never lossy. /// Mirrors [`crate::spaces::metrics_inner`]. -pub(crate) fn ops_inner( +pub(crate) fn ops_inner( parser: &T, name: Option, ) -> Result { diff --git a/src/parser.rs b/src/parser.rs index 5248feef1..fd6564c45 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -10,21 +10,8 @@ use std::marker::PhantomData; use std::path::Path; use std::sync::Arc; -use crate::abc::Abc; use crate::checker::Checker; -use crate::cognitive::Cognitive; -use crate::cyclomatic::Cyclomatic; -use crate::halstead::Halstead; -use crate::loc::Loc; -use crate::mi::Mi; -use crate::nargs::NArgs; -use crate::nexits::Exit; use crate::node::Ancestors; -use crate::nom::Nom; -use crate::npa::Npa; -use crate::npm::Npm; -use crate::tokens::Tokens; -use crate::wmc::Wmc; use crate::alterator::Alterator; use crate::getter::Getter; @@ -42,25 +29,7 @@ use crate::traits::*; /// of the language code tags (`RustCode`, `PythonCode`, etc.) declared /// by the internal `mk_code!` macro. #[derive(Debug)] -pub(crate) struct Parser< - T: LanguageInfo - + Alterator - + Checker - + Getter - + Abc - + Cognitive - + Cyclomatic - + Exit - + Halstead - + Loc - + Mi - + NArgs - + Nom - + Npa - + Npm - + Tokens - + Wmc, -> { +pub(crate) struct Parser { code: Vec, tree: Tree, phantom: PhantomData, @@ -113,42 +82,9 @@ fn get_fake_code( } } -impl< - T: 'static - + LanguageInfo - + Alterator - + Checker - + Getter - + Abc - + Cognitive - + Cyclomatic - + Exit - + Halstead - + Loc - + Mi - + NArgs - + Nom - + Npa - + Npm - + Tokens - + Wmc, -> ParserTrait for Parser -{ +impl ParserTrait for Parser { type Checker = T; type Getter = T; - type Cognitive = T; - type Cyclomatic = T; - type Halstead = T; - type Loc = T; - type Nom = T; - type Mi = T; - type NArgs = T; - type Exit = T; - type Wmc = T; - type Abc = T; - type Npm = T; - type Npa = T; - type Tokens = T; fn new(code: Vec, path: &Path, pr: Option>) -> Self { let fake_code = get_fake_code::(&code, path, pr); @@ -248,27 +184,7 @@ impl< } } -impl< - T: 'static - + LanguageInfo - + Alterator - + Checker - + Getter - + Abc - + Cognitive - + Cyclomatic - + Exit - + Halstead - + Loc - + Mi - + NArgs - + Nom - + Npa - + Npm - + Tokens - + Wmc, -> Parser -{ +impl Parser { /// Builds a [`Parser`] from a pre-parsed [`tree_sitter::Tree`] /// and the matching source bytes. /// diff --git a/src/spaces.rs b/src/spaces.rs index 1fb8059d1..084aa8eb8 100644 --- a/src/spaces.rs +++ b/src/spaces.rs @@ -482,7 +482,7 @@ pub struct Source<'a> { /// let _ = ast.metrics(MetricsOptions::default()).expect("walker succeeds"); /// ``` pub struct Ast { - inner: crate::langs::AstInner, + inner: crate::langs::AnyParser, name: Option, } diff --git a/src/spaces/ast.rs b/src/spaces/ast.rs index 6cf016766..e0a1e0a0b 100644 --- a/src/spaces/ast.rs +++ b/src/spaces/ast.rs @@ -7,6 +7,10 @@ use super::*; +use crate::langs::AnyParser; +use crate::macros::with_any_parser; +use crate::ops::ops_inner; + impl fmt::Debug for Ast { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // The held parser owns a `tree_sitter::Tree` and a `Vec`; @@ -39,8 +43,7 @@ impl Ast { preproc_path, preproc, } = source; - let inner = - crate::langs::ast_parse_dispatch(lang, code.into_owned(), preproc_path, preproc)?; + let inner = AnyParser::parse(lang, code.into_owned(), preproc_path, preproc)?; Ok(Self { inner, name }) } @@ -67,7 +70,7 @@ impl Ast { code: Vec, name: Option, ) -> Result { - let inner = crate::langs::ast_from_tree_dispatch(lang, tree, code)?; + let inner = AnyParser::from_tree(lang, tree, code)?; Ok(Self { inner, name }) } @@ -130,7 +133,11 @@ impl Ast { /// [`SpaceKind::Unit`] [`FuncSpace`] before walking, so this method /// does not return `Err` in practice today. pub fn metrics(&self, options: MetricsOptions) -> Result { - self.inner.run_metrics(self.name.clone(), options) + with_any_parser!(&self.inner, |p| metrics_inner( + p, + self.name.clone(), + options + )) } /// Return every operator and operand of each space in the held parse. @@ -164,7 +171,7 @@ impl Ast { /// assert!(!ops.name_was_lossy); /// ``` pub fn ops(&self) -> Result { - self.inner.run_ops(self.name.clone()) + with_any_parser!(&self.inner, |p| ops_inner(p, self.name.clone())) } /// Source language of the parsed tree. @@ -180,7 +187,7 @@ impl Ast { #[must_use] #[inline] pub fn source(&self) -> &[u8] { - self.inner.code_bytes() + with_any_parser!(&self.inner, |p| p.code()) } /// Display name carried through to [`FuncSpace::name`] by every @@ -201,7 +208,7 @@ impl Ast { #[must_use] #[inline] pub fn as_tree_sitter(&self) -> &tree_sitter::Tree { - self.inner.ts_tree() + with_any_parser!(&self.inner, |p| p.ts_tree()) } /// Strip non-doc comments from the held parse, returning the source @@ -221,7 +228,7 @@ impl Ast { /// ``` #[must_use] pub fn strip_comments(&self) -> Option> { - self.inner.run_strip_comments() + with_any_parser!(&self.inner, |p| crate::comment_rm::rm_comments(p)) } /// Detect the span of every function in the held parse. Safe to call @@ -239,7 +246,7 @@ impl Ast { /// ``` #[must_use] pub fn functions(&self) -> Vec { - self.inner.run_functions() + with_any_parser!(&self.inner, |p| crate::function::function(p)) } /// Build the [`AstResponse`](crate::AstResponse) node tree for the held @@ -264,7 +271,7 @@ impl Ast { /// ``` #[must_use] pub fn dump(&self, cfg: crate::AstCfg) -> crate::AstResponse { - self.inner.run_dump(cfg) + with_any_parser!(&self.inner, |p| crate::ast::dump_inner(p, cfg)) } /// Count `(matching, total)` nodes in the held parse, where a node @@ -286,7 +293,7 @@ impl Ast { /// ``` #[must_use] pub fn count(&self, filters: &[String]) -> (usize, usize) { - self.inner.run_count(filters) + with_any_parser!(&self.inner, |p| crate::count::count(p, filters)) } /// Find every node in the held parse whose kind is named in @@ -299,7 +306,7 @@ impl Ast { /// Currently infallible; the [`Result`] wrapper is reserved for a /// future strict-parsing mode (matching the other `Ast` walkers). pub fn find(&self, filters: &[String]) -> Result>, MetricsError> { - self.inner.run_find(filters) + with_any_parser!(&self.inner, |p| crate::find::find(p, filters)) } /// Collect every in-source suppression marker (`// bca: suppress …`) @@ -307,7 +314,7 @@ impl Ast { /// tree is reused. #[must_use] pub fn suppressions(&self) -> Vec { - self.inner.run_suppressions() + with_any_parser!(&self.inner, |p| crate::suppression::suppression_markers(p)) } /// Borrow the root [`Node`] of the held parse for callers that drive @@ -316,6 +323,6 @@ impl Ast { #[must_use] #[inline] pub fn root_node(&self) -> Node<'_> { - self.inner.root_node() + with_any_parser!(&self.inner, |p| p.root()) } } diff --git a/src/spaces/compute.rs b/src/spaces/compute.rs index 2449679f0..b002076cd 100644 --- a/src/spaces/compute.rs +++ b/src/spaces/compute.rs @@ -7,6 +7,7 @@ //! `crate::spaces::analyze` (and `pub(crate) metrics_inner`) is preserved. use super::*; +use crate::MetricSuite; use crate::diag::warn; // Walks that ended with a cognitive nesting slot still live. Freeing @@ -24,7 +25,7 @@ crate::observation::counter!(nesting_slots_retained); /// absorbed all of its children is wasted work, not a partial sum. Only /// [`finalize_state`] calls it, once per space (#1106). #[inline] -fn compute_halstead_and_mi(state: &mut State, selected: MetricSet) { +fn compute_halstead_and_mi(state: &mut State, selected: MetricSet) { if selected.contains(Metric::Halstead) { state .halstead_maps @@ -55,7 +56,7 @@ fn compute_halstead_and_mi(state: &mut State, selected: MetricSe /// until this runs, and an `Unknown` parent silently drops every /// method's cyclomatic from its class WMC. #[inline] -fn compute_wmc(state: &mut State, selected: MetricSet) { +fn compute_wmc(state: &mut State, selected: MetricSet) { if selected.contains(Metric::Wmc) { T::Wmc::compute( state.space.kind, @@ -82,7 +83,7 @@ fn compute_wmc(state: &mut State, selected: MetricSet) { /// all-zero block on every file root, since a unit is a member scope /// like any other. #[inline] -fn note_member_scope(state: &mut State, selected: MetricSet) { +fn note_member_scope(state: &mut State, selected: MetricSet) { let kind = state.space.kind; if selected.contains(Metric::Npm) && ::HAS_MEMBERS { state.space.metrics.npm.set_space_kind(kind); @@ -224,7 +225,7 @@ fn anchor_unit_sloc_span(state: &mut State, selected: MetricSet) { /// `mi::Stats` both have no-op `merge`s, so nothing reads a parent's /// intermediate Halstead/MI, and this call overwrites them from the final /// maps anyway (#1106). -fn finalize_state(state: &mut State, selected: MetricSet) { +fn finalize_state(state: &mut State, selected: MetricSet) { anchor_unit_sloc_span(state, selected); compute_minmax(state, selected); compute_sum(state, selected); @@ -234,7 +235,7 @@ fn finalize_state(state: &mut State, selected: MetricSet) { compute_averages(state, selected); } -fn finalize(state_stack: &mut Vec, diff_level: usize, selected: MetricSet) { +fn finalize(state_stack: &mut Vec, diff_level: usize, selected: MetricSet) { if state_stack.is_empty() { return; } @@ -324,7 +325,7 @@ struct NodeFacts { // cost saving for `with_only(&[Metric::Loc])`. Extracted from // `metrics_inner` so the walker stays under clippy's 100-line ceiling. #[inline] -fn compute_per_node<'a, T: ParserTrait>( +fn compute_per_node<'a, T: MetricSuite>( state: &mut State<'a>, node: &Node<'a>, code: &'a [u8], @@ -402,7 +403,7 @@ fn compute_per_node<'a, T: ParserTrait>( /// had already stopped agreeing with. [`anchor_unit_sloc_span`] now /// derives every unit's span from the one [`crate::spaces::line_span`] /// recorded, this frame included. -fn push_synthetic_unit_root( +fn push_synthetic_unit_root( state_stack: &mut Vec, node: &Node, code: &[u8], @@ -434,7 +435,7 @@ fn push_synthetic_unit_root( /// per-`Call` source-text keyword scan, so it is far from a cheap enum /// compare (issue #522; the `Loc` unit flag that used to force it on /// every node went away with #1067). -fn open_func_space<'a, T: ParserTrait>( +fn open_func_space<'a, T: MetricSuite>( state_stack: &mut Vec>, node: &Node<'a>, code: &'a [u8], @@ -639,7 +640,7 @@ pub(crate) fn push_children<'a, 's, Tag: Copy>( &stack[first..] } -pub(crate) fn metrics_inner( +pub(crate) fn metrics_inner( parser: &T, name: Option, options: MetricsOptions, diff --git a/src/spaces_tests.rs b/src/spaces_tests.rs index 74a507263..e879eacd1 100644 --- a/src/spaces_tests.rs +++ b/src/spaces_tests.rs @@ -2,7 +2,7 @@ use crate::MetricsOptions; use crate::node::Ancestors; use crate::spaces::metrics_inner; use crate::test_support::check_func_space; -use crate::{CppParser, ParserTrait, SpaceKind}; +use crate::{CppParser, MetricSuite, ParserTrait, SpaceKind}; /// `SpaceKind` is `#[non_exhaustive]` (#551); the attribute is a /// compile-time forward-compat contract and must not change the @@ -189,7 +189,7 @@ fn cpp_error_root_yields_unit_top_level_space() { /// wrapper path. Issue #220 tracks finding additional per-grammar /// fixtures that surface ERROR roots so each language can have /// both a contract test and a wrapper-exercising test. -fn assert_top_level_space_is_unit_contract(source: &str, filename: &str) { +fn assert_top_level_space_is_unit_contract(source: &str, filename: &str) { let path = std::path::PathBuf::from(filename); let parser = P::new(source.as_bytes().to_vec(), &path, None); let space = metrics_inner( @@ -226,7 +226,7 @@ fn assert_top_level_space_is_unit_contract(source: &str, filenam /// the contract-only path. Use this for languages where a fixture /// is known to make the grammar return ERROR (currently: Lua, C++ /// via mozcpp). -fn assert_partial_input_yields_synthetic_unit_wrapper( +fn assert_partial_input_yields_synthetic_unit_wrapper( source: &str, filename: &str, ) { diff --git a/src/test_support.rs b/src/test_support.rs index 574867784..c16215f04 100644 --- a/src/test_support.rs +++ b/src/test_support.rs @@ -14,7 +14,8 @@ use crate::node::{Node, Tree}; use crate::spaces::metrics_inner; use crate::traits::LanguageInfo; use crate::{ - CodeMetrics, FuncSpace, LANG, Metric, MetricsOptions, ParserTrait, Source, SpaceKind, analyze, + CodeMetrics, FuncSpace, LANG, Metric, MetricSuite, MetricsOptions, ParserTrait, Source, + SpaceKind, analyze, }; /// Parses `source` as `filename` under `options` and hands the resulting @@ -24,7 +25,7 @@ use crate::{ /// normalise it on the way in: CRLF/CR collapse to LF, and the trailing /// newline is regularised to exactly one. Use [`metrics_verbatim`] when a /// test's input must reach the parser untouched. -fn check_func_space_with( +fn check_func_space_with( source: &str, filename: &str, options: MetricsOptions, @@ -49,7 +50,7 @@ fn check_func_space_with( /// is what made the unit suite pay for ~15 walks per assertion (#1127). /// This full-set variant remains for tests whose subject *is* the whole /// surface. -pub(crate) fn check_func_space( +pub(crate) fn check_func_space( source: &str, filename: &str, check: F, @@ -64,7 +65,7 @@ pub(crate) fn check_func_space( /// `get_space_kind` run regardless of the selection — so structural /// assertions (`assert_child_space_kind`, `child_space`, nesting) hold /// identically under either variant. -pub(crate) fn check_func_space_only( +pub(crate) fn check_func_space_only( source: &str, filename: &str, metrics: &[Metric], @@ -92,7 +93,7 @@ pub(crate) fn check_func_space_only( /// `metric_selection_parity` in `src/spaces_tests.rs` pins that across /// each metric and a multi-language fixture set, so a migrated test /// asserting the same numbers is asserting the same thing. -pub(crate) fn check_metrics_only( +pub(crate) fn check_metrics_only( source: &str, filename: &str, metrics: &[Metric], @@ -122,7 +123,7 @@ pub(crate) fn check_metrics_only( /// ``` macro_rules! check_metrics_only_shim { ($name:ident, $($metric:ident),+ $(,)?) => { - fn $name( + fn $name( source: &str, filename: &str, check: fn($crate::CodeMetrics), @@ -142,7 +143,7 @@ macro_rules! check_metrics_only_shim { /// delegating to [`check_func_space_only`]. macro_rules! check_func_space_only_shim { ($name:ident, $($metric:ident),+ $(,)?) => { - fn $name( + fn $name( source: &str, filename: &str, check: F, diff --git a/src/traits.rs b/src/traits.rs index bf7c8b20d..1b803b4a6 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -9,27 +9,14 @@ use std::path::Path; use std::sync::Arc; -use crate::abc::Abc; use crate::alterator::Alterator; use crate::checker::Checker; -use crate::cognitive::Cognitive; -use crate::cyclomatic::Cyclomatic; use crate::getter::Getter; -use crate::halstead::Halstead; use crate::langs::*; -use crate::loc::Loc; -use crate::mi::Mi; -use crate::nargs::NArgs; -use crate::nexits::Exit; use crate::node::Ancestors; use crate::node::Node; -use crate::nom::Nom; -use crate::npa::Npa; -use crate::npm::Npm; use crate::parser::Filter; use crate::preproc::PreprocResults; -use crate::tokens::Tokens; -use crate::wmc::Wmc; /// Static identification of a language code tag. /// @@ -45,27 +32,17 @@ pub(crate) trait LanguageInfo { } // Internal language-dispatch trait reached only by the macro-generated -// `Parser` impls in `src/parser.rs` and the `AstInner` dispatch in -// `src/macros/mod.rs`. The 15 associated types are not part of any -// documented extension contract — metric extraction is driven through -// the public [`crate::Ast`] / [`crate::analyze`] seam, not by -// implementing this trait. See STABILITY.md. +// `Parser` impls in `src/parser.rs` and the `AnyParser` dispatch in +// `src/macros/mod.rs`. It carries the *parse* half of a language: the +// tree plus its `Checker` / `Getter` classifiers. The per-metric +// associated types live on the separate [`crate::MetricSuite`] supertrait +// (#1376), so this half has no dependency on the metric modules. Not +// part of any documented extension contract — metric extraction is +// driven through the public [`crate::Ast`] / [`crate::analyze`] seam, +// not by implementing this trait. See STABILITY.md. pub(crate) trait ParserTrait { type Checker: Alterator + Checker; type Getter: Getter; - type Cognitive: Cognitive; - type Cyclomatic: Cyclomatic; - type Halstead: Halstead; - type Loc: Loc; - type Nom: Nom; - type Mi: Mi; - type NArgs: NArgs; - type Exit: Exit; - type Wmc: Wmc; - type Abc: Abc; - type Npm: Npm; - type Npa: Npa; - type Tokens: Tokens; fn new(code: Vec, path: &Path, pr: Option>) -> Self; fn root(&self) -> Node<'_>; From a5be6134cde831fe4534fa9da3c998954f9ab842 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Wed, 9 Sep 2026 16:25:51 -0700 Subject: [PATCH 2/9] refactor(lib): move classification helpers beside their traits The Elixir keyword-Call, Tcl command-name and Python alias helpers were defined in metrics::cognitive / metrics::npa yet imported by the Checker and Getter impls, so the parse layer depended on the metric layer. They now live in a lang_helpers module the metrics import instead. SpaceKind and HalsteadType are Getter return types, so they move out of spaces / metrics::halstead into their own modules; both stay re-exported at their public paths. The two termcolor helpers leave tools.rs for output::color, and FromPathError leaves error.rs so that file holds only the parse-layer MetricsError. Preparatory step for #1376: no public path changes, no metric moves. --- src/checker.rs | 2 +- src/checker/elixir.rs | 4 +- src/checker/python.rs | 4 +- src/error.rs | 149 ++----------------------------- src/from_path_error.rs | 148 ++++++++++++++++++++++++++++++ src/function.rs | 2 +- src/getter.rs | 4 +- src/getter/elixir.rs | 4 +- src/halstead_type.rs | 17 ++++ src/lang_helpers.rs | 18 ++++ src/lang_helpers/elixir.rs | 81 +++++++++++++++++ src/lang_helpers/python.rs | 37 ++++++++ src/lang_helpers/tcl.rs | 34 +++++++ src/lib.rs | 7 +- src/metrics/abc/elixir.rs | 2 +- src/metrics/abc/tcl.rs | 2 +- src/metrics/cognitive.rs | 128 +------------------------- src/metrics/cyclomatic/elixir.rs | 2 +- src/metrics/cyclomatic/tcl.rs | 2 +- src/metrics/halstead.rs | 14 +-- src/metrics/loc.rs | 2 +- src/metrics/nexits.rs | 2 +- src/metrics/npa.rs | 3 + src/metrics/npa/elixir.rs | 2 +- src/metrics/npa/shared.rs | 15 ---- src/metrics/npm.rs | 3 +- src/metrics/npm/elixir.rs | 2 +- src/output/color.rs | 17 +++- src/output/dump.rs | 2 +- src/output/dump_metrics.rs | 2 +- src/output/dump_ops.rs | 2 +- src/{spaces => }/space_kind.rs | 46 ++++++++-- src/spaces.rs | 37 +------- src/spaces/options.rs | 2 +- src/tools.rs | 16 ---- 35 files changed, 439 insertions(+), 375 deletions(-) create mode 100644 src/from_path_error.rs create mode 100644 src/halstead_type.rs create mode 100644 src/lang_helpers.rs create mode 100644 src/lang_helpers/elixir.rs create mode 100644 src/lang_helpers/python.rs create mode 100644 src/lang_helpers/tcl.rs rename src/{spaces => }/space_kind.rs (70%) diff --git a/src/checker.rs b/src/checker.rs index cc40d885a..57ca04eee 100644 --- a/src/checker.rs +++ b/src/checker.rs @@ -1810,7 +1810,7 @@ mod tests { // half of the set is covered by the drift guard above. #[test] fn python_is_lambda_matches_live_lambda_and_agrees_with_is_closure() { - use crate::metrics::cognitive::python_is_lambda; + use crate::lang_helpers::python::python_is_lambda; let parser = parse_python("def f():\n return lambda x: x and x\n"); let lambda = find_first_kind(&parser, Python::Lambda as u16) diff --git a/src/checker/elixir.rs b/src/checker/elixir.rs index daeb5cee1..e27b7d469 100644 --- a/src/checker/elixir.rs +++ b/src/checker/elixir.rs @@ -36,7 +36,7 @@ impl Checker for ElixirCode { code: &[u8], ancestors: Ancestors<'a, '_>, ) -> bool { - use crate::metrics::cognitive::{ + use crate::lang_helpers::elixir::{ elixir_call_keyword, elixir_is_class_macro, elixir_is_inside_quote_block, elixir_is_method_macro, }; @@ -57,7 +57,7 @@ impl Checker for ElixirCode { } fn is_func_with_code<'a>(node: &Node<'a>, code: &[u8], ancestors: Ancestors<'a, '_>) -> bool { - use crate::metrics::cognitive::{ + use crate::lang_helpers::elixir::{ elixir_call_keyword, elixir_is_inside_quote_block, elixir_is_method_macro, }; let Some(kw) = elixir_call_keyword(node, code) else { diff --git a/src/checker/python.rs b/src/checker/python.rs index 834d13c86..81758c83b 100644 --- a/src/checker/python.rs +++ b/src/checker/python.rs @@ -41,7 +41,7 @@ impl Checker for PythonCode { // nom/nargs or desync from cognitive (issues #419/#422; lesson 2 // in lessons_learned.md). The drift-guard test below asserts // `Lambda2` stays unseen until then. - crate::metrics::cognitive::python_is_lambda(node) + crate::lang_helpers::python::python_is_lambda(node) } fn is_call(node: &Node) -> bool { @@ -90,7 +90,7 @@ impl Checker for PythonCode { fn is_else_if<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { node.kind_id() == Python::IfStatement && ancestors.iter(node).next().is_some_and(|(parent, above)| { - crate::metrics::npa::python_is_block(&parent) + crate::lang_helpers::python::python_is_block(&parent) && parent.children().filter(Node::is_named).count() == 1 && above .parent(&parent) diff --git a/src/error.rs b/src/error.rs index 8ba343064..7049a9400 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,5 +1,10 @@ //! Error type returned from the library's top-level entry points. //! +//! [`FromPathError`](crate::FromPathError), the richer error of the +//! file-backed `Ast::from_path`, lives in the root crate's +//! `from_path_error` module; this one is the parse-layer error every +//! dispatch entry point returns. +//! //! Prior to this module, every entry point returned `Option<…>` and //! collapsed parse failure, empty input, and disabled-language builds //! into a single `None`. [`MetricsError`] distinguishes those cases so @@ -109,147 +114,3 @@ impl std::fmt::Display for MetricsError { } impl std::error::Error for MetricsError {} - -/// Error returned by [`Ast::from_path`][crate::Ast::from_path]. -/// -/// `from_path` reads, language-detects, and parses a file in one call, so it -/// can fail in more ways than the in-memory [`Ast::parse`][crate::Ast::parse] -/// (which only reports [`MetricsError`]). Unlike [`analyze`][crate::analyze], -/// `from_path` does not silently skip files: every reason it cannot produce a -/// tree surfaces as a distinct variant so the caller — who asked for *this* -/// file's tree — learns why. -/// -/// The enum is `#[non_exhaustive]`; match with a trailing `_` arm to stay -/// forward-compatible. -#[non_exhaustive] -#[derive(Debug)] -pub enum FromPathError { - /// The file could not be read (a genuine I/O fault: missing file, - /// permission denied, hardware error). Carries the underlying - /// [`std::io::Error`]. - Io(std::io::Error), - /// The path is not valid UTF-8. The path doubles as the resulting - /// [`FuncSpace`][crate::FuncSpace] name (an identifier used as a map key - /// and in JSON output), so a lossy conversion is rejected rather than - /// silently corrupting correlation — mirroring `analyze`'s strict - /// default. - NonUtf8Path, - /// The file is empty, too small, binary, or encoded in an unsupported - /// encoding (UTF-16, invalid UTF-8) — the same files - /// [`analyze`][crate::analyze] skips. `from_path` reuses the library's - /// text reader for byte-exact metric parity with `analyze`, so these - /// inputs cannot yield a tree. - Unreadable, - /// No language is registered for the path (unknown extension and no - /// recognizable shebang / mode line). - UnknownLanguage, - /// The detected language's per-language Cargo feature is not enabled in - /// this build (carries the [`MetricsError::LanguageDisabled`] raised by - /// [`Ast::parse`][crate::Ast::parse]). - Parse(MetricsError), -} - -impl std::fmt::Display for FromPathError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Io(e) => write!(f, "could not read file: {e}"), - Self::NonUtf8Path => f.write_str("path is not valid UTF-8"), - Self::Unreadable => { - f.write_str("file is empty, binary, or not valid UTF-8 source text") - } - Self::UnknownLanguage => f.write_str("no language is registered for this path"), - Self::Parse(e) => write!(f, "{e}"), - } - } -} - -impl std::error::Error for FromPathError { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match self { - Self::Io(e) => Some(e), - Self::Parse(e) => Some(e), - _ => None, - } - } -} - -impl From for FromPathError { - fn from(e: MetricsError) -> Self { - Self::Parse(e) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::error::Error as _; - use std::io::{Error as IoError, ErrorKind}; - - // `FromPathError`'s `Display` / `source` impls and its `From` - // conversion are part of the stable error surface but are only - // reached on `from_path` failure paths that no other test drives. - // Pin the message shape (substring, not exact wording) and the - // `source` chaining contract, which is what `?`-propagating callers - // and `anyhow`-style reporters rely on. - - #[test] - fn from_path_error_display_covers_every_variant() { - let io = FromPathError::Io(IoError::new(ErrorKind::PermissionDenied, "denied")); - assert!(io.to_string().contains("could not read file")); - assert!(io.to_string().contains("denied")); - - assert!( - FromPathError::NonUtf8Path - .to_string() - .contains("not valid UTF-8") - ); - assert!( - FromPathError::Unreadable - .to_string() - .contains("empty, binary") - ); - assert!( - FromPathError::UnknownLanguage - .to_string() - .contains("no language is registered") - ); - - // `Parse` delegates to the wrapped `MetricsError`'s `Display`. - let parse = FromPathError::Parse(MetricsError::LanguageDisabled(LANG::Rust)); - assert_eq!( - parse.to_string(), - MetricsError::LanguageDisabled(LANG::Rust).to_string() - ); - } - - #[test] - fn from_path_error_source_chains_only_for_wrapping_variants() { - // `Io` and `Parse` wrap an underlying error and must expose it; - // the leaf variants must report no source. - let io = FromPathError::Io(IoError::new(ErrorKind::NotFound, "missing")); - assert!(io.source().is_some(), "Io must chain to the io::Error"); - - let parse = FromPathError::Parse(MetricsError::EmptyRoot); - assert!(parse.source().is_some(), "Parse must chain to MetricsError"); - - for leaf in [ - FromPathError::NonUtf8Path, - FromPathError::Unreadable, - FromPathError::UnknownLanguage, - ] { - assert!(leaf.source().is_none(), "{leaf:?} must report no source"); - } - } - - #[test] - fn metrics_error_converts_into_parse_variant() { - let converted: FromPathError = MetricsError::LanguageDisabled(LANG::Cpp).into(); - assert!( - matches!( - converted, - FromPathError::Parse(MetricsError::LanguageDisabled(LANG::Cpp)) - ), - "From must wrap into Parse" - ); - } -} diff --git a/src/from_path_error.rs b/src/from_path_error.rs new file mode 100644 index 000000000..68cbe213c --- /dev/null +++ b/src/from_path_error.rs @@ -0,0 +1,148 @@ +//! Error type of the file-backed [`Ast::from_path`][crate::Ast::from_path]. + +use crate::MetricsError; + +/// Error returned by [`Ast::from_path`][crate::Ast::from_path]. +/// +/// `from_path` reads, language-detects, and parses a file in one call, so it +/// can fail in more ways than the in-memory [`Ast::parse`][crate::Ast::parse] +/// (which only reports [`MetricsError`]). Unlike [`analyze`][crate::analyze], +/// `from_path` does not silently skip files: every reason it cannot produce a +/// tree surfaces as a distinct variant so the caller — who asked for *this* +/// file's tree — learns why. +/// +/// The enum is `#[non_exhaustive]`; match with a trailing `_` arm to stay +/// forward-compatible. +#[non_exhaustive] +#[derive(Debug)] +pub enum FromPathError { + /// The file could not be read (a genuine I/O fault: missing file, + /// permission denied, hardware error). Carries the underlying + /// [`std::io::Error`]. + Io(std::io::Error), + /// The path is not valid UTF-8. The path doubles as the resulting + /// [`FuncSpace`][crate::FuncSpace] name (an identifier used as a map key + /// and in JSON output), so a lossy conversion is rejected rather than + /// silently corrupting correlation — mirroring `analyze`'s strict + /// default. + NonUtf8Path, + /// The file is empty, too small, binary, or encoded in an unsupported + /// encoding (UTF-16, invalid UTF-8) — the same files + /// [`analyze`][crate::analyze] skips. `from_path` reuses the library's + /// text reader for byte-exact metric parity with `analyze`, so these + /// inputs cannot yield a tree. + Unreadable, + /// No language is registered for the path (unknown extension and no + /// recognizable shebang / mode line). + UnknownLanguage, + /// The detected language's per-language Cargo feature is not enabled in + /// this build (carries the [`MetricsError::LanguageDisabled`] raised by + /// [`Ast::parse`][crate::Ast::parse]). + Parse(MetricsError), +} + +impl std::fmt::Display for FromPathError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Io(e) => write!(f, "could not read file: {e}"), + Self::NonUtf8Path => f.write_str("path is not valid UTF-8"), + Self::Unreadable => { + f.write_str("file is empty, binary, or not valid UTF-8 source text") + } + Self::UnknownLanguage => f.write_str("no language is registered for this path"), + Self::Parse(e) => write!(f, "{e}"), + } + } +} + +impl std::error::Error for FromPathError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Io(e) => Some(e), + Self::Parse(e) => Some(e), + _ => None, + } + } +} + +impl From for FromPathError { + fn from(e: MetricsError) -> Self { + Self::Parse(e) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::LANG; + use std::error::Error as _; + use std::io::{Error as IoError, ErrorKind}; + + // `FromPathError`'s `Display` / `source` impls and its `From` + // conversion are part of the stable error surface but are only + // reached on `from_path` failure paths that no other test drives. + // Pin the message shape (substring, not exact wording) and the + // `source` chaining contract, which is what `?`-propagating callers + // and `anyhow`-style reporters rely on. + + #[test] + fn from_path_error_display_covers_every_variant() { + let io = FromPathError::Io(IoError::new(ErrorKind::PermissionDenied, "denied")); + assert!(io.to_string().contains("could not read file")); + assert!(io.to_string().contains("denied")); + + assert!( + FromPathError::NonUtf8Path + .to_string() + .contains("not valid UTF-8") + ); + assert!( + FromPathError::Unreadable + .to_string() + .contains("empty, binary") + ); + assert!( + FromPathError::UnknownLanguage + .to_string() + .contains("no language is registered") + ); + + // `Parse` delegates to the wrapped `MetricsError`'s `Display`. + let parse = FromPathError::Parse(MetricsError::LanguageDisabled(LANG::Rust)); + assert_eq!( + parse.to_string(), + MetricsError::LanguageDisabled(LANG::Rust).to_string() + ); + } + + #[test] + fn from_path_error_source_chains_only_for_wrapping_variants() { + // `Io` and `Parse` wrap an underlying error and must expose it; + // the leaf variants must report no source. + let io = FromPathError::Io(IoError::new(ErrorKind::NotFound, "missing")); + assert!(io.source().is_some(), "Io must chain to the io::Error"); + + let parse = FromPathError::Parse(MetricsError::EmptyRoot); + assert!(parse.source().is_some(), "Parse must chain to MetricsError"); + + for leaf in [ + FromPathError::NonUtf8Path, + FromPathError::Unreadable, + FromPathError::UnknownLanguage, + ] { + assert!(leaf.source().is_none(), "{leaf:?} must report no source"); + } + } + + #[test] + fn metrics_error_converts_into_parse_variant() { + let converted: FromPathError = MetricsError::LanguageDisabled(LANG::Cpp).into(); + assert!( + matches!( + converted, + FromPathError::Parse(MetricsError::LanguageDisabled(LANG::Cpp)) + ), + "From must wrap into Parse" + ); + } +} diff --git a/src/function.rs b/src/function.rs index 70202c580..e359991d1 100644 --- a/src/function.rs +++ b/src/function.rs @@ -21,7 +21,7 @@ use crate::traits::{ParserTrait, Search}; use crate::checker::Checker; use crate::getter::Getter; -use crate::tools::{color, intense_color}; +use crate::output::color::{color, intense_color}; /// Function span data. #[derive(Debug)] diff --git a/src/getter.rs b/src/getter.rs index 5d3abdd27..d873527fd 100644 --- a/src/getter.rs +++ b/src/getter.rs @@ -6,9 +6,9 @@ // function so the per-language impl blocks stay readable. #![allow(clippy::wildcard_imports, clippy::enum_glob_use)] -use crate::metrics::halstead::HalsteadType; +use crate::halstead_type::HalsteadType; -use crate::spaces::SpaceKind; +use crate::space_kind::SpaceKind; use crate::traits::Search; use crate::*; diff --git a/src/getter/elixir.rs b/src/getter/elixir.rs index 75af3d5cb..d25c88f91 100644 --- a/src/getter/elixir.rs +++ b/src/getter/elixir.rs @@ -57,7 +57,7 @@ impl Getter for ElixirCode { code: &[u8], ancestors: Ancestors<'a, '_>, ) -> SpaceKind { - use crate::metrics::cognitive::{ + use crate::lang_helpers::elixir::{ elixir_call_keyword, elixir_is_class_macro, elixir_is_inside_quote_block, elixir_is_method_macro, }; @@ -103,7 +103,7 @@ impl Getter for ElixirCode { ) -> Option<&'a str> { use Elixir as E; - use crate::metrics::cognitive::{ + use crate::lang_helpers::elixir::{ elixir_call_keyword, elixir_is_class_macro, elixir_is_inside_quote_block, elixir_is_method_macro, }; diff --git a/src/halstead_type.rs b/src/halstead_type.rs new file mode 100644 index 000000000..be3b50613 --- /dev/null +++ b/src/halstead_type.rs @@ -0,0 +1,17 @@ +//! The operator / operand / unknown classification a `Getter` assigns +//! to each node for the Halstead metric. +//! +//! Defined beside the classifiers rather than in `metrics::halstead` +//! because `Getter::get_op_type` returns it: the parse layer names the +//! type, the metric consumes it (#1376). `metrics::halstead` re-exports +//! it under its historical public path. + +/// Specifies the type of nodes accepted by the `Halstead` metric. +pub enum HalsteadType { + /// The node is an `Halstead` operator + Operator, + /// The node is an `Halstead` operand + Operand, + /// The node is unknown to the `Halstead` metric + Unknown, +} diff --git a/src/lang_helpers.rs b/src/lang_helpers.rs new file mode 100644 index 000000000..317f92004 --- /dev/null +++ b/src/lang_helpers.rs @@ -0,0 +1,18 @@ +//! Byte-level identity helpers shared by the classifiers and the metrics. +//! +//! Some questions a walk asks are not answerable from a node's kind: an +//! Elixir `def` and a `quote` are both `Call` nodes told apart only by +//! their target text, a Tcl `switch` is a plain `command` whose leading +//! word names the builtin, and Python's aliased `block` / `lambda` kinds +//! must be normalised at one site (grammar-dispatch §9 and §10). These +//! helpers read the bytes once so `Checker`, `Getter`, and every metric +//! agree on the answer. +//! +//! They live beside the classifiers rather than in a metric module +//! because the classifiers consult them: a `Checker` impl that imported +//! a helper *from* `metrics::cognitive` would make the parse layer depend +//! on the metric layer, the inversion #1376 exists to remove. + +pub(crate) mod elixir; +pub(crate) mod python; +pub(crate) mod tcl; diff --git a/src/lang_helpers/elixir.rs b/src/lang_helpers/elixir.rs new file mode 100644 index 000000000..2fcfe5318 --- /dev/null +++ b/src/lang_helpers/elixir.rs @@ -0,0 +1,81 @@ +//! Elixir: keyword `Call` identity (`def`, `defmodule`, `quote`, …). + +use crate::Elixir; +use crate::node::{Ancestors, Node}; + +/// Reads the text of the `target` field of an Elixir `Call` node. +/// +/// Most of Elixir's control-flow constructs (`if`, `unless`, `for`, +/// `while`, `case`, `cond`, `with`, `try`) and method-defining macros +/// (`def`, `defp`, `defmacro`, …) parse as `Call` nodes whose `target` +/// is an `Identifier` whose source text spells the keyword. The +/// `Cyclomatic` and `Exit` impls already follow this pattern; this +/// helper centralises the byte-text lookup so `Cognitive` and `Abc` +/// can share it. +/// +/// Returns `None` for Calls whose target is not a simple identifier +/// (e.g. `Module.func(…)` parses as `RemoteCallWithParentheses` with +/// the dotted name as target) or when the bytes are not valid UTF-8. +pub(crate) fn elixir_call_keyword<'a>(node: &'a Node<'a>, code: &'a [u8]) -> Option<&'a str> { + if node.kind_id() != Elixir::Call as u16 { + return None; + } + let target = node.child_by_field_name("target")?; + if target.kind_id() != Elixir::Identifier as u16 { + return None; + } + target.utf8_text(code) +} + +/// Method-defining macros (`def`, `defp`, `defmacro`, `defmacrop`). The set +/// is duplicated across checker, getter, and several metric impls +/// because each consults it from a different trait surface; centralising +/// the literal here keeps future additions (e.g. `defguard`) consistent. +#[inline] +pub(crate) fn elixir_is_method_macro(kw: &str) -> bool { + matches!(kw, "def" | "defp" | "defmacro" | "defmacrop") +} + +/// Class-defining macro (`defmodule`). Paired with [`elixir_is_method_macro`] +/// where a caller needs both ("any space-opening declaration"). +#[inline] +pub(crate) fn elixir_is_class_macro(kw: &str) -> bool { + kw == "defmodule" +} + +/// Returns true when `node` is lexically nested inside the `do_block` of a +/// `quote do … end` Call (Elixir's metaprogramming template). A `def` / +/// `defp` / `defmacro` / `defmacrop` inside `quote` does not define a +/// method of any enclosing module — the syntax tree is a code template +/// emitted later, when the surrounding macro is invoked. Treating those +/// quoted Calls as methods inflates `Wmc` and disagrees with `Npm`'s +/// direct-children classification (#310). +/// +/// Walks the ancestor chain looking for a `quote` Call ancestor. Stops at +/// the first match (true) or at the root (false). Each step is a single +/// `child_by_field_name("target")` + identifier byte compare, so the cost +/// is O(steps) when `ancestors` is known — with `Ancestors::unknown` each +/// step additionally pays `Node::parent`'s O(depth) (#1084). +pub(crate) fn elixir_is_inside_quote_block<'a>( + node: &Node<'a>, + code: &[u8], + ancestors: Ancestors<'a, '_>, +) -> bool { + ancestors + .iter(node) + .any(|(n, _)| elixir_call_keyword(&n, code) == Some("quote")) +} + +/// Iterates the direct-child `Call` nodes inside the `do_block` of an +/// Elixir Call (typically a `defmodule`). Used by `Npm` / `Npa` to scan +/// a module body for method-defining macros / `defstruct` without +/// descending into nested modules. Yields no items when the Call has +/// no `do_block`. +pub(crate) fn elixir_do_block_call_children<'a>( + node: &'a Node<'a>, +) -> impl Iterator> + 'a { + node.children() + .filter(|child| child.kind_id() == Elixir::DoBlock as u16) + .flat_map(|do_block| do_block.children()) + .filter(|stmt| stmt.kind_id() == Elixir::Call as u16) +} diff --git a/src/lang_helpers/python.rs b/src/lang_helpers/python.rs new file mode 100644 index 000000000..1a901a65b --- /dev/null +++ b/src/lang_helpers/python.rs @@ -0,0 +1,37 @@ +//! Python: the aliased `block` and `lambda` kind sets. + +use crate::Python; +use crate::node::Node; + +/// Whether `node` is a Python `lambda` expression, under either of the +/// grammar's two aliased kind_ids: `Lambda` (196, the concrete +/// production emitted today) and `Lambda2` (197, the currently-unseen +/// hidden alias). `Lambda3` (73) is the `lambda` *keyword* token, not a +/// closure node, and is intentionally excluded. +/// +/// This is the single normalization chokepoint for the lambda-alias set +/// — mirroring [`python_is_block`] for the block aliases (#419). It +/// is reused by the cognitive lambda-scope walks below and by +/// `PythonCode::is_closure`, so a future grammar bump +/// that promotes `Lambda2` to a concrete node is handled in exactly one +/// place rather than drifting across sites (#422). The +/// `python_hidden_block_and_lambda_aliases_stay_unseen` drift guard in +/// `checker.rs` trips on such a bump. +pub(crate) fn python_is_lambda(node: &Node) -> bool { + matches!(node.kind_id().into(), Python::Lambda | Python::Lambda2) +} + +/// Single normalization point for Python's aliased `block` kind_ids. +/// +/// tree-sitter-python lists two `kind_id`s that both stringify to +/// `"block"`: `Block` (135, the hidden `_block` supertype) and +/// `Block2` (160, the concrete production). Empirically only `Block2` +/// is ever emitted for real block bodies (function, class, if/for, +/// while/try/with), so `Block` is dead today — but a future grammar +/// bump could promote the supertype to a concrete node. Routing every +/// "is this a block?" check through here means such a bump is handled +/// at one site instead of silently undercounting at several (issue +/// #419; lesson 2 / 34 / 56 in docs/development/lessons_learned.md). +pub(crate) fn python_is_block(node: &Node) -> bool { + matches!(node.kind_id().into(), Python::Block | Python::Block2) +} diff --git a/src/lang_helpers/tcl.rs b/src/lang_helpers/tcl.rs new file mode 100644 index 000000000..baf7a881a --- /dev/null +++ b/src/lang_helpers/tcl.rs @@ -0,0 +1,34 @@ +//! Tcl: the leading word of a `command` node. + +use crate::Tcl; +use crate::node::Node; + +/// Reads the leading word of a Tcl `command` node when it is a plain +/// `simple_word` (`switch`, `for`, `puts`, …). Returns `None` for any other +/// node kind, for commands whose leading word is computed (`$cmd`, `[cmd]` +/// parse it as `variable_substitution` / `command_substitution`, never +/// statically resolvable to a builtin), and for non-UTF-8 bytes. Shared by +/// the out-of-band control-flow detectors below (grammar-dispatch §10: +/// identity questions read the bytes). +/// +/// A *literal* name in a quoted or braced spelling is also unresolved, and +/// that is a deliberate limitation: `"for" {set i 0} {$i < 3} {incr i} {…}` +/// and `{for} …` are legal Tcl that still invoke the builtin, but the +/// grammar parses their name as `quoted_word` / `braced_word` rather than +/// `simple_word`, so they score as plain commands. The `simple_word` gate +/// is what keeps the computed forms out; matching the quoted spellings +/// would mean unquoting the bytes for a style no real Tcl uses. +/// +/// Callers dispatch on the returned name so each `command` node resolves it +/// exactly once per metric walk — the helpers below take the resolved +/// identity as a precondition rather than re-deriving it. +pub(crate) fn tcl_command_name<'a>(node: &'a Node<'a>, code: &'a [u8]) -> Option<&'a str> { + if node.kind_id() != Tcl::Command as u16 { + return None; + } + let name = node.child_by_field_name("name")?; + if name.kind_id() != Tcl::SimpleWord as u16 { + return None; + } + name.utf8_text(code) +} diff --git a/src/lib.rs b/src/lib.rs index b46489074..2e8c5af9d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -131,6 +131,9 @@ mod c_macro; mod cfg_predicate; mod checker; mod getter; +mod halstead_type; +mod lang_helpers; +mod space_kind; // Fast hashing for the walk's integer-keyed maps. Shared by `spaces` // (node ids) and `metrics::halstead` (grammar `kind_id`s); `metrics::loc` // was the third until #1109 moved its line sets to a bitset. The module @@ -251,7 +254,9 @@ mod diag; // --- Errors --- mod error; -pub use crate::error::{FromPathError, MetricsError}; +pub use crate::error::MetricsError; +mod from_path_error; +pub use crate::from_path_error::FromPathError; // --- Metric selection --- mod metric_set; diff --git a/src/metrics/abc/elixir.rs b/src/metrics/abc/elixir.rs index 81dc5ca4d..2942fc359 100644 --- a/src/metrics/abc/elixir.rs +++ b/src/metrics/abc/elixir.rs @@ -195,7 +195,7 @@ impl Abc for ElixirCode { // helper to look up the keyword, but apply different // policies on top. E::Call => { - let keyword = super::cognitive::elixir_call_keyword(node, code); + let keyword = crate::lang_helpers::elixir::elixir_call_keyword(node, code); let is_definition_or_directive = matches!( keyword, Some( diff --git a/src/metrics/abc/tcl.rs b/src/metrics/abc/tcl.rs index 3232330c5..89c780fc5 100644 --- a/src/metrics/abc/tcl.rs +++ b/src/metrics/abc/tcl.rs @@ -311,6 +311,6 @@ impl Abc for TclCode { // `[pick] x`) therefore stays a branch: it is not statically a builtin, // which is what the assignment classification claims. fn tcl_command_is_assignment(node: &Node, code: &[u8]) -> bool { - crate::metrics::cognitive::tcl_command_name(node, code) + crate::lang_helpers::tcl::tcl_command_name(node, code) .is_some_and(|name| TCL_ASSIGNMENT_COMMANDS.contains(&name)) } diff --git a/src/metrics/cognitive.rs b/src/metrics/cognitive.rs index 7a2c6e674..359f37e93 100644 --- a/src/metrics/cognitive.rs +++ b/src/metrics/cognitive.rs @@ -27,6 +27,9 @@ use crate::spaces::{Nesting, NestingMap}; use std::fmt; use crate::checker::Checker; +use crate::lang_helpers::elixir::{elixir_call_keyword, elixir_is_method_macro}; +use crate::lang_helpers::python::python_is_lambda; +use crate::lang_helpers::tcl::tcl_command_name; use crate::macros::implement_metric_trait; use crate::*; @@ -348,24 +351,6 @@ fn increase_nesting(stats: &mut Stats, nesting: &mut Nesting) { stats.boolean_seq.reset(); } -/// Whether `node` is a Python `lambda` expression, under either of the -/// grammar's two aliased kind_ids: `Lambda` (196, the concrete -/// production emitted today) and `Lambda2` (197, the currently-unseen -/// hidden alias). `Lambda3` (73) is the `lambda` *keyword* token, not a -/// closure node, and is intentionally excluded. -/// -/// This is the single normalization chokepoint for the lambda-alias set -/// — mirroring `npa::python_is_block` for the block aliases (#419). It -/// is reused by the cognitive lambda-scope walks below and by -/// [`PythonCode::is_closure`](crate::checker), so a future grammar bump -/// that promotes `Lambda2` to a concrete node is handled in exactly one -/// place rather than drifting across sites (#422). The -/// `python_hidden_block_and_lambda_aliases_stay_unseen` drift guard in -/// `checker.rs` trips on such a bump. -pub(crate) fn python_is_lambda(node: &Node) -> bool { - matches!(node.kind_id().into(), Python::Lambda | Python::Lambda2) -} - macro_rules! js_cognitive { ($lang:ident) => { fn compute<'a>( @@ -557,60 +542,6 @@ mod tcl; mod tsx; mod typescript; -// Reads the text of the `target` field of an Elixir `Call` node. -// -// Most of Elixir's control-flow constructs (`if`, `unless`, `for`, -// `while`, `case`, `cond`, `with`, `try`) and method-defining macros -// (`def`, `defp`, `defmacro`, …) parse as `Call` nodes whose `target` -// is an `Identifier` whose source text spells the keyword. The -// `Cyclomatic` and `Exit` impls already follow this pattern; this -// helper centralises the byte-text lookup so `Cognitive` and `Abc` -// can share it. -// -// Returns `None` for Calls whose target is not a simple identifier -// (e.g. `Module.func(…)` parses as `RemoteCallWithParentheses` with -// the dotted name as target) or when the bytes are not valid UTF-8. -pub(crate) fn elixir_call_keyword<'a>(node: &'a Node<'a>, code: &'a [u8]) -> Option<&'a str> { - if node.kind_id() != Elixir::Call as u16 { - return None; - } - let target = node.child_by_field_name("target")?; - if target.kind_id() != Elixir::Identifier as u16 { - return None; - } - target.utf8_text(code) -} - -// Reads the leading word of a Tcl `command` node when it is a plain -// `simple_word` (`switch`, `for`, `puts`, …). Returns `None` for any other -// node kind, for commands whose leading word is computed (`$cmd`, `[cmd]` -// parse it as `variable_substitution` / `command_substitution`, never -// statically resolvable to a builtin), and for non-UTF-8 bytes. Shared by -// the out-of-band control-flow detectors below (grammar-dispatch §10: -// identity questions read the bytes). -// -// A *literal* name in a quoted or braced spelling is also unresolved, and -// that is a deliberate limitation: `"for" {set i 0} {$i < 3} {incr i} {…}` -// and `{for} …` are legal Tcl that still invoke the builtin, but the -// grammar parses their name as `quoted_word` / `braced_word` rather than -// `simple_word`, so they score as plain commands. The `simple_word` gate -// is what keeps the computed forms out; matching the quoted spellings -// would mean unquoting the bytes for a style no real Tcl uses. -// -// Callers dispatch on the returned name so each `command` node resolves it -// exactly once per metric walk — the helpers below take the resolved -// identity as a precondition rather than re-deriving it. -pub(crate) fn tcl_command_name<'a>(node: &'a Node<'a>, code: &'a [u8]) -> Option<&'a str> { - if node.kind_id() != Tcl::Command as u16 { - return None; - } - let name = node.child_by_field_name("name")?; - if name.kind_id() != Tcl::SimpleWord as u16 { - return None; - } - name.utf8_text(code) -} - // Tcl's `switch` is a generic `command` (no dedicated kind_id, unlike // `if`/`while`/`foreach`/`catch`), so the kind-dispatch in the Cognitive // and Cyclomatic impls never sees it (issue #467, lesson 19). Both metrics @@ -733,59 +664,6 @@ pub(crate) fn irules_switch_decision_arms(node: &Node, code: &[u8]) -> Option bool { - matches!(kw, "def" | "defp" | "defmacro" | "defmacrop") -} - -// Class-defining macro (`defmodule`). Paired with [`elixir_is_method_macro`] -// where a caller needs both ("any space-opening declaration"). -#[inline] -pub(crate) fn elixir_is_class_macro(kw: &str) -> bool { - kw == "defmodule" -} - -// Returns true when `node` is lexically nested inside the `do_block` of a -// `quote do … end` Call (Elixir's metaprogramming template). A `def` / -// `defp` / `defmacro` / `defmacrop` inside `quote` does not define a -// method of any enclosing module — the syntax tree is a code template -// emitted later, when the surrounding macro is invoked. Treating those -// quoted Calls as methods inflates `Wmc` and disagrees with `Npm`'s -// direct-children classification (#310). -// -// Walks the ancestor chain looking for a `quote` Call ancestor. Stops at -// the first match (true) or at the root (false). Each step is a single -// `child_by_field_name("target")` + identifier byte compare, so the cost -// is O(steps) when `ancestors` is known — with `Ancestors::unknown` each -// step additionally pays `Node::parent`'s O(depth) (#1084). -pub(crate) fn elixir_is_inside_quote_block<'a>( - node: &Node<'a>, - code: &[u8], - ancestors: Ancestors<'a, '_>, -) -> bool { - ancestors - .iter(node) - .any(|(n, _)| elixir_call_keyword(&n, code) == Some("quote")) -} - -// Iterates the direct-child `Call` nodes inside the `do_block` of an -// Elixir Call (typically a `defmodule`). Used by `Npm` / `Npa` to scan -// a module body for method-defining macros / `defstruct` without -// descending into nested modules. Yields no items when the Call has -// no `do_block`. -pub(crate) fn elixir_do_block_call_children<'a>( - node: &'a Node<'a>, -) -> impl Iterator> + 'a { - node.children() - .filter(|child| child.kind_id() == Elixir::DoBlock as u16) - .flat_map(|do_block| do_block.children()) - .filter(|stmt| stmt.kind_id() == Elixir::Call as u16) -} - implement_metric_trait!(Cognitive, PreprocCode, CcommentCode); #[cfg(test)] diff --git a/src/metrics/cyclomatic/elixir.rs b/src/metrics/cyclomatic/elixir.rs index 14d4751a6..c4103909b 100644 --- a/src/metrics/cyclomatic/elixir.rs +++ b/src/metrics/cyclomatic/elixir.rs @@ -125,7 +125,7 @@ fn elixir_is_default_clause<'a>( .next() .is_some_and(|(parent, _)| parent.kind_id() == E::DoBlock as u16) && chain.next().is_some_and(|(grandparent, _)| { - crate::metrics::cognitive::elixir_call_keyword(&grandparent, code) + crate::lang_helpers::elixir::elixir_call_keyword(&grandparent, code) == Some("cond") }) } diff --git a/src/metrics/cyclomatic/tcl.rs b/src/metrics/cyclomatic/tcl.rs index 6f70467b5..d57000ab6 100644 --- a/src/metrics/cyclomatic/tcl.rs +++ b/src/metrics/cyclomatic/tcl.rs @@ -30,7 +30,7 @@ impl Cyclomatic for TclCode { // Tcl `switch` and `for` are generic `command`s with no dedicated // kind (issues #467, #1264), so the kind dispatch above never // sees them. The leading word is resolved once and dispatched on. - Tcl::Command => match crate::metrics::cognitive::tcl_command_name(node, code) { + Tcl::Command => match crate::lang_helpers::tcl::tcl_command_name(node, code) { // Mirroring the C-family convention (see // `impl_cyclomatic_c_family`): each non-`default` arm is a // decision point in standard CCN, while modified CCN diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index ca1979ab5..5f50ed4d9 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -28,6 +28,10 @@ use std::collections::HashMap; use std::fmt; +// Re-exported here so the public `metrics::halstead::HalsteadType` path +// survives the enum's move beside `Getter`, whose `get_op_type` returns it. +pub use crate::halstead_type::HalsteadType; + use crate::checker::Checker; use crate::getter::Getter; use crate::int_hash::IntKeyHashMap; @@ -45,16 +49,6 @@ pub struct Stats { operands: u64, } -/// Specifies the type of nodes accepted by the `Halstead` metric. -pub enum HalsteadType { - /// The node is an `Halstead` operator - Operator, - /// The node is an `Halstead` operand - Operand, - /// The node is unknown to the `Halstead` metric - Unknown, -} - /// Per-space operator / operand occurrence maps used to compute the /// Halstead `Stats` struct. One map per distinct operator (`kind_id`) /// and one per distinct operand (`text`); merged across nested spaces. diff --git a/src/metrics/loc.rs b/src/metrics/loc.rs index 3bf51776b..f195e7f22 100644 --- a/src/metrics/loc.rs +++ b/src/metrics/loc.rs @@ -37,7 +37,7 @@ #![warn(clippy::arithmetic_side_effects)] use crate::checker::Checker; -use crate::metrics::npa::python_is_block; +use crate::lang_helpers::python::python_is_block; use std::fmt; use crate::macros::implement_metric_trait; diff --git a/src/metrics/nexits.rs b/src/metrics/nexits.rs index 598313999..75fdb906c 100644 --- a/src/metrics/nexits.rs +++ b/src/metrics/nexits.rs @@ -369,7 +369,7 @@ impl Exit for TclCode { // `error` in argument position (`puts error`) is a `word_list` // child and is not counted. if matches!( - crate::metrics::cognitive::tcl_command_name(node, code), + crate::lang_helpers::tcl::tcl_command_name(node, code), Some("return" | "error" | "throw" | "exit") ) { stats.exit += 1; diff --git a/src/metrics/npa.rs b/src/metrics/npa.rs index ffaac9846..b869eeacc 100644 --- a/src/metrics/npa.rs +++ b/src/metrics/npa.rs @@ -372,6 +372,9 @@ macro_rules! impl_npa_java_like { mod shared; pub(crate) use shared::*; +// Reached by the per-language submodules through `use super::*`. +use crate::lang_helpers::python::python_is_block; + // TypeScript / TSX share the same OOP node shape: `class_declaration` // and `abstract_class_declaration` both contain a `class_body`; // `interface_declaration` contains an `interface_body`. The diff --git a/src/metrics/npa/elixir.rs b/src/metrics/npa/elixir.rs index 50e27305f..8888004fe 100644 --- a/src/metrics/npa/elixir.rs +++ b/src/metrics/npa/elixir.rs @@ -29,7 +29,7 @@ impl Npa for ElixirCode { _ancestors: Ancestors<'a, '_>, stats: &mut Stats, ) { - use crate::metrics::cognitive::{elixir_call_keyword, elixir_do_block_call_children}; + use crate::lang_helpers::elixir::{elixir_call_keyword, elixir_do_block_call_children}; // The space-opening node for a `defmodule` Call is the node // itself, and the walker pushes that space before running any diff --git a/src/metrics/npa/shared.rs b/src/metrics/npa/shared.rs index 4a4e8279a..e4c634ed0 100644 --- a/src/metrics/npa/shared.rs +++ b/src/metrics/npa/shared.rs @@ -679,21 +679,6 @@ pub(crate) fn rust_item_is_public(node: &Node) -> bool { .any(|vis| rust_visibility_modifier_is_public(&vis)) } -// Single normalization point for Python's aliased `block` kind_ids. -// -// tree-sitter-python lists two `kind_id`s that both stringify to -// `"block"`: `Block` (135, the hidden `_block` supertype) and -// `Block2` (160, the concrete production). Empirically only `Block2` -// is ever emitted for real block bodies (function, class, if/for, -// while/try/with), so `Block` is dead today — but a future grammar -// bump could promote the supertype to a concrete node. Routing every -// "is this a block?" check through here means such a bump is handled -// at one site instead of silently undercounting at several (issue -// #419; lesson 2 / 34 / 56 in docs/development/lessons_learned.md). -pub(crate) fn python_is_block(node: &Node) -> bool { - matches!(node.kind_id().into(), Python::Block | Python::Block2) -} - // Kotlin's grammar models classes and interfaces under a single // `class_declaration` node; the `class` / `interface` keyword child // disambiguates. A `ClassBody` belongs to an interface iff its parent diff --git a/src/metrics/npm.rs b/src/metrics/npm.rs index 4ce9ba60c..26ea73238 100644 --- a/src/metrics/npm.rs +++ b/src/metrics/npm.rs @@ -20,9 +20,10 @@ use std::fmt; use crate::checker::{Checker, csharp_accessor_count}; +use crate::lang_helpers::python::python_is_block; use crate::langs::*; use crate::macros::implement_metric_trait; -use crate::metrics::npa::{accessibility_ratio, python_is_block, ts_member_is_public}; +use crate::metrics::npa::{accessibility_ratio, ts_member_is_public}; use crate::node::Node; use crate::*; diff --git a/src/metrics/npm/elixir.rs b/src/metrics/npm/elixir.rs index 1463aa0e2..a8d758c1b 100644 --- a/src/metrics/npm/elixir.rs +++ b/src/metrics/npm/elixir.rs @@ -23,7 +23,7 @@ impl Npm for ElixirCode { _ancestors: Ancestors<'a, '_>, stats: &mut Stats, ) { - use crate::metrics::cognitive::{elixir_call_keyword, elixir_do_block_call_children}; + use crate::lang_helpers::elixir::{elixir_call_keyword, elixir_do_block_call_children}; // The space-opening node for a `defmodule` Call is the node // itself, and the walker pushes that space before running any diff --git a/src/output/color.rs b/src/output/color.rs index 083ea4b73..413dfccb8 100644 --- a/src/output/color.rs +++ b/src/output/color.rs @@ -16,7 +16,7 @@ use std::io::{StdoutLock, Write}; -use termcolor::{Buffer, BufferWriter, ColorChoice, ColorSpec, WriteColor}; +use termcolor::{Buffer, BufferWriter, Color, ColorChoice, ColorSpec, WriteColor}; /// Whether the terminal dump serializers ([`crate::dump_root`], /// [`crate::dump_ops`], [`crate::dump_node`], @@ -243,6 +243,21 @@ where rendered } +// Accept `&mut dyn WriteColor` rather than `&mut StandardStreamLock` so +// tests (e.g. `function::dump_spans`) can substitute `termcolor::NoColor` +// over a `Vec` to capture the rendered bytes. Production callers +// continue to pass `&mut StandardStreamLock`, which unsized-coerces to +// the trait object at the call site. +#[inline] +pub(crate) fn color(stdout: &mut dyn WriteColor, color: Color) -> std::io::Result<()> { + stdout.set_color(ColorSpec::new().set_fg(Some(color))) +} + +#[inline] +pub(crate) fn intense_color(stdout: &mut dyn WriteColor, color: Color) -> std::io::Result<()> { + stdout.set_color(ColorSpec::new().set_fg(Some(color)).set_intense(true)) +} + #[cfg(test)] mod tests { use std::cell::RefCell; diff --git a/src/output/dump.rs b/src/output/dump.rs index 869b283de..75831a425 100644 --- a/src/output/dump.rs +++ b/src/output/dump.rs @@ -11,7 +11,7 @@ use termcolor::{Color, WriteColor}; use crate::node::Node; use crate::output::ColorMode; use crate::output::color::print_to_stdout; -use crate::tools::{color, intense_color}; +use crate::output::color::{color, intense_color}; /// Dumps the `AST` of a code. /// diff --git a/src/output/dump_metrics.rs b/src/output/dump_metrics.rs index 40cacb6d3..bda28e0bf 100644 --- a/src/output/dump_metrics.rs +++ b/src/output/dump_metrics.rs @@ -32,7 +32,7 @@ use crate::output::{ColorMode, branch_glyphs}; use crate::spaces::{CodeMetrics, FuncSpace}; use crate::wire; -use crate::tools::{color, intense_color}; +use crate::output::color::{color, intense_color}; /// Decimal places used when rendering a non-integer float in the text /// dump. JSON output keeps full precision; the terminal view trades the diff --git a/src/output/dump_ops.rs b/src/output/dump_ops.rs index 398129142..113eac659 100644 --- a/src/output/dump_ops.rs +++ b/src/output/dump_ops.rs @@ -4,7 +4,7 @@ use crate::ops::Ops; use crate::output::color::print_to_stdout; use crate::output::{ColorMode, branch_glyphs}; -use crate::tools::{color, intense_color}; +use crate::output::color::{color, intense_color}; /// Dumps all operands and operators of a code. /// diff --git a/src/spaces/space_kind.rs b/src/space_kind.rs similarity index 70% rename from src/spaces/space_kind.rs rename to src/space_kind.rs index bc922eb62..f2ad84c0b 100644 --- a/src/spaces/space_kind.rs +++ b/src/space_kind.rs @@ -1,11 +1,43 @@ -//! Inherent and `Display` impl blocks for [`super::SpaceKind`]. +//! The kinds of space the walk opens: functions and the containers +//! (class, struct, trait, impl, namespace, interface) plus the file unit. //! -//! Split out of `spaces.rs` to keep that module focused on the public -//! API type definitions. The blocks are moved verbatim; method and -//! trait resolution is by type, so `crate::spaces::SpaceKind`'s methods -//! and `Display` impl resolve unchanged. +//! Defined beside the classifiers because `Getter::get_space_kind` +//! returns it; `spaces` re-exports it under its historical public path +//! (#1376). -use super::*; +use std::fmt; + +use serde::{Deserialize, Serialize}; + +/// The list of supported space kinds. +// New space kinds land as languages are added (a future module-, mixin-, +// or enum-style space), so this is marked `#[non_exhaustive]` to keep +// such additions additive rather than a 2.0 break. CLI/web consumers +// matching on it already carry a `_ =>` arm. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +#[non_exhaustive] +pub enum SpaceKind { + /// An unknown space + #[default] + Unknown, + /// A function space + Function, + /// A class space + Class, + /// A struct space + Struct, + /// A `Rust` trait space + Trait, + /// A `Rust` implementation space + Impl, + /// A general space + Unit, + /// A `C/C++` namespace + Namespace, + /// An interface + Interface, +} impl SpaceKind { /// Parse a [`SpaceKind`] from its lowercase serialized form — the @@ -17,7 +49,7 @@ impl SpaceKind { /// This is the single source of truth for the string-to-kind mapping a /// consumer needs when it reads a serialized `kind` (the Python /// `to_sarif` binding uses it to apply per-metric threshold scope via - /// [`crate::metric_catalog::MetricScope::admits`]). A round-trip test + /// `metric_catalog::MetricScope::admits`). A round-trip test /// pins it against the serde representation so the two cannot drift. #[must_use] pub fn from_serialized(serialized: &str) -> Self { diff --git a/src/spaces.rs b/src/spaces.rs index 084aa8eb8..b1e83c94b 100644 --- a/src/spaces.rs +++ b/src/spaces.rs @@ -19,7 +19,6 @@ use std::borrow::Cow; -use serde::{Deserialize, Serialize}; use std::fmt; use std::path::Path; use std::sync::Arc; @@ -101,11 +100,13 @@ mod ast; mod code_metrics; mod options; mod source; -mod space_kind; // `analyze` is `pub` — re-exported from `lib.rs`, so it must stay // reachable at `crate::spaces::analyze`. pub use compute::analyze; +// `SpaceKind` is defined beside `Getter` (whose `get_space_kind` returns +// it) and re-exported here so `crate::spaces::SpaceKind` keeps resolving. +pub use crate::space_kind::SpaceKind; // `metrics_inner` and `push_children` are `pub(crate)` — `metrics_inner` // is re-exported from `lib.rs` and `push_children` is consumed by // `crate::ops`, so both must stay reachable at their `crate::spaces::` @@ -118,36 +119,6 @@ pub(crate) use compute::{metrics_inner, push_children}; #[cfg(test)] use compute::apply_suppression; -/// The list of supported space kinds. -// New space kinds land as languages are added (a future module-, mixin-, -// or enum-style space), so this is marked `#[non_exhaustive]` to keep -// such additions additive rather than a 2.0 break. CLI/web consumers -// matching on it already carry a `_ =>` arm. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -#[non_exhaustive] -pub enum SpaceKind { - /// An unknown space - #[default] - Unknown, - /// A function space - Function, - /// A class space - Class, - /// A struct space - Struct, - /// A `Rust` trait space - Trait, - /// A `Rust` implementation space - Impl, - /// A general space - Unit, - /// A `C/C++` namespace - Namespace, - /// An interface - Interface, -} - /// All metrics data. /// /// The set of metrics actually computed is governed by @@ -210,7 +181,7 @@ pub struct CodeMetrics { /// [`MetricsOptions::with_only`] the bitfield is restricted to the /// caller's selection plus auto-added dependencies. /// - /// The [`Serialize`] impl consults this set to elide fields the + /// The [`Serialize`](serde::Serialize) impl consults this set to elide fields the /// caller did not select. The field itself is not serialized. pub selected: MetricSet, } diff --git a/src/spaces/options.rs b/src/spaces/options.rs index 2c3665333..18a5ce0e2 100644 --- a/src/spaces/options.rs +++ b/src/spaces/options.rs @@ -51,7 +51,7 @@ impl MetricsOptions { /// Restrict computation to the given metrics. Metrics outside /// this set are skipped during the walk; their `Stats` fields on /// [`CodeMetrics`] remain at their `Default` value and are - /// elided from the [`Serialize`] output. Pass an empty slice to + /// elided from the [`Serialize`](serde::Serialize) output. Pass an empty slice to /// disable every metric (the walker still runs and produces the /// space tree, but no metric values are populated). /// diff --git a/src/tools.rs b/src/tools.rs index 79f74bf63..aa940e719 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -26,7 +26,6 @@ use std::path::{Component, Path, PathBuf}; use std::sync::OnceLock; use regex::bytes::Regex; -use termcolor::{Color, ColorSpec, WriteColor}; use crate::langs::*; @@ -878,21 +877,6 @@ fn min_distance_candidates(possibilities: &[PathBuf], current_path: &Path) -> Ve path_min.into_iter().cloned().collect() } -// Accept `&mut dyn WriteColor` rather than `&mut StandardStreamLock` so -// tests (e.g. `function::dump_spans`) can substitute `termcolor::NoColor` -// over a `Vec` to capture the rendered bytes. Production callers -// continue to pass `&mut StandardStreamLock`, which unsized-coerces to -// the trait object at the call site. -#[inline] -pub(crate) fn color(stdout: &mut dyn WriteColor, color: Color) -> std::io::Result<()> { - stdout.set_color(ColorSpec::new().set_fg(Some(color))) -} - -#[inline] -pub(crate) fn intense_color(stdout: &mut dyn WriteColor, color: Color) -> std::io::Result<()> { - stdout.set_color(ColorSpec::new().set_fg(Some(color)).set_intense(true)) -} - #[cfg(test)] #[path = "tools_tests.rs"] mod tests; From 10898e77a1a37aaa9d02e221e568bf86205794bc Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Thu, 10 Sep 2026 09:09:37 -0700 Subject: [PATCH 3/9] refactor(ast): move the parse layer into big-code-analysis-ast The generated kind enums, LANG and language detection, Node and the tree-sitter wrappers, the Checker / Getter / Alterator classifiers and their per-language impls, the C-family preprocessor pass, comment stripping, the AST dump, and node counting / finding now live in a published sub-crate the root pins at an exact version. The root keeps the metric walk, output formats, suppression and VCS metrics, and re-exports everything it re-exported before at the same paths, so no caller of big-code-analysis changes. The dependency runs one way only: the sub-crate builds and tests with this crate absent from its graph. Every per-language Cargo feature forwards to it under the same name, so a --features rust build still links exactly one grammar, and its own feature-matrix legs resolve what they name rather than unifying every language back on. Two types crossed with Getter because they are its return types. SpaceKind keeps its name and its public paths. HalsteadType is renamed TokenRole: it answers whether a node acts as an operator or an operand, which the grammar decides and any structural consumer can ask, so naming it after the single metric that reads it made the parse layer look like it carried metric vocabulary. The old name survives as a deprecated alias at metrics::halstead::HalsteadType until 3.0, pinned by a guard that fails to compile if it is dropped or re-pointed. Node's accessors that the walk uses become public and documented, and MetricsError moves with LANG::tree_sitter_language, which returns it. Gates, workflows, baselines and the enums codegen path follow the move. No metric value changes. Fixes #1376 --- .bca-baseline.toml | 576 ++++++++-------- .github/workflows/ci.yml | 10 +- .github/workflows/fuzz.yml | 1 + .github/workflows/mutation-test.yml | 5 +- .github/workflows/pages.yml | 2 + .github/workflows/release.yml | 21 +- .pre-commit-config.yaml | 13 +- .rustfmt-bail-baseline.txt | 44 +- .taplo.toml | 1 + CHANGELOG.md | 37 + Cargo.lock | 38 +- Cargo.toml | 181 +++-- Makefile | 29 +- STABILITY.md | 97 ++- big-code-analysis-ast/Cargo.toml | 168 +++++ big-code-analysis-ast/README.md | 32 + .../src}/alterator.rs | 6 +- {src => big-code-analysis-ast/src}/ast.rs | 13 +- big-code-analysis-ast/src/c_declarator.rs | 282 ++++++++ .../src}/c_langs_macros/c_macros.rs | 0 .../src}/c_langs_macros/c_specials.rs | 0 .../src}/c_langs_macros/mod.rs | 5 +- {src => big-code-analysis-ast/src}/c_macro.rs | 6 +- .../src}/cfg_predicate.rs | 3 +- {src => big-code-analysis-ast/src}/checker.rs | 61 +- .../src}/checker/bash.rs | 0 .../src}/checker/c.rs | 0 .../src}/checker/ccomment.rs | 0 .../src}/checker/cpp.rs | 0 .../src}/checker/csharp.rs | 0 .../src}/checker/elixir.rs | 0 .../src}/checker/go.rs | 0 .../src}/checker/groovy.rs | 0 .../src}/checker/irules.rs | 0 .../src}/checker/java.rs | 0 .../src}/checker/javascript.rs | 0 .../src}/checker/kotlin.rs | 0 .../src}/checker/lua.rs | 0 .../src}/checker/mozcpp.rs | 0 .../src}/checker/mozjs.rs | 0 .../src}/checker/objc.rs | 0 .../src}/checker/perl.rs | 0 .../src}/checker/php.rs | 0 .../src}/checker/preproc.rs | 0 .../src}/checker/python.rs | 0 .../src}/checker/ruby.rs | 3 +- .../src}/checker/rust.rs | 0 .../src}/checker/tcl.rs | 0 .../src}/checker/tsx.rs | 0 .../src}/checker/typescript.rs | 0 .../src}/comment_rm.rs | 6 +- {src => big-code-analysis-ast/src}/count.rs | 10 +- {src => big-code-analysis-ast/src}/error.rs | 33 +- {src => big-code-analysis-ast/src}/find.rs | 6 +- {src => big-code-analysis-ast/src}/getter.rs | 101 ++- .../src}/getter/bash.rs | 16 +- .../src}/getter/c.rs | 8 +- .../src}/getter/ccomment.rs | 0 .../src}/getter/cpp.rs | 12 +- .../src}/getter/csharp.rs | 12 +- .../src}/getter/elixir.rs | 10 +- .../src}/getter/go.rs | 8 +- .../src}/getter/groovy.rs | 10 +- .../src}/getter/irules.rs | 16 +- .../src}/getter/java.rs | 8 +- .../src}/getter/javascript.rs | 0 .../src}/getter/kotlin.rs | 16 +- .../src}/getter/lua.rs | 8 +- .../src}/getter/mozcpp.rs | 12 +- .../src}/getter/mozjs.rs | 0 .../src}/getter/objc.rs | 10 +- .../src}/getter/perl.rs | 12 +- .../src}/getter/php.rs | 26 +- .../src}/getter/preproc.rs | 0 .../src}/getter/python.rs | 18 +- .../src}/getter/ruby.rs | 14 +- .../src}/getter/rust.rs | 20 +- .../src}/getter/tcl.rs | 16 +- .../src}/getter/tsx.rs | 0 .../src}/getter/typescript.rs | 0 .../src}/lang_helpers.rs | 6 +- .../src}/lang_helpers/elixir.rs | 17 +- .../src}/lang_helpers/python.rs | 12 +- .../src}/lang_helpers/tcl.rs | 4 +- {src => big-code-analysis-ast/src}/langs.rs | 99 +-- .../src}/language_enum_roundtrip.rs | 0 .../src}/languages/language_bash.rs | 3 + .../src}/languages/language_c.rs | 3 + .../src}/languages/language_ccomment.rs | 3 + .../src}/languages/language_cpp.rs | 3 + .../src}/languages/language_csharp.rs | 3 + .../src}/languages/language_elixir.rs | 3 + .../src}/languages/language_go.rs | 3 + .../src}/languages/language_groovy.rs | 3 + .../src}/languages/language_irules.rs | 3 + .../src}/languages/language_java.rs | 3 + .../src}/languages/language_javascript.rs | 3 + .../src}/languages/language_kotlin.rs | 3 + .../src}/languages/language_lua.rs | 3 + .../src}/languages/language_mozcpp.rs | 3 + .../src}/languages/language_mozjs.rs | 3 + .../src}/languages/language_objc.rs | 3 + .../src}/languages/language_perl.rs | 3 + .../src}/languages/language_php.rs | 3 + .../src}/languages/language_preproc.rs | 3 + .../src}/languages/language_python.rs | 3 + .../src}/languages/language_ruby.rs | 3 + .../src}/languages/language_rust.rs | 3 + .../src}/languages/language_tcl.rs | 3 + .../src}/languages/language_tsx.rs | 3 + .../src}/languages/language_typescript.rs | 3 + .../src}/languages/mod.rs | 0 big-code-analysis-ast/src/lib.rs | 147 ++++ .../src}/macros/kind_sets.rs | 63 +- big-code-analysis-ast/src/macros/mod.rs | 581 ++++++++++++++++ {src => big-code-analysis-ast/src}/node.rs | 370 +++++----- .../src}/node/parser_cache.rs | 26 +- big-code-analysis-ast/src/observation.rs | 90 +++ {src => big-code-analysis-ast/src}/parser.rs | 12 +- {src => big-code-analysis-ast/src}/preproc.rs | 16 +- .../src}/preproc_tests.rs | 45 +- .../src}/recursion.rs | 11 +- .../src}/space_kind.rs | 2 +- big-code-analysis-ast/src/test_support.rs | 63 ++ big-code-analysis-ast/src/token_role.rs | 26 + {src => big-code-analysis-ast/src}/tools.rs | 20 +- .../src}/tools_tests.rs | 0 {src => big-code-analysis-ast/src}/traits.rs | 49 +- .../src/html_report/styling.rs | 5 +- docs/development/lessons_learned.md | 5 +- enums/templates/rust.rs | 3 + fuzz/Cargo.lock | 22 +- fuzz/fuzz_targets/preproc_macro.rs | 3 +- recreate-grammars.sh | 4 +- src/c_declarator.rs | 641 ------------------ src/c_family_space_names_tests.rs | 365 ++++++++++ src/halstead_type.rs | 17 - src/lib.rs | 145 ++-- src/lib_docs_tests.rs | 49 ++ src/macros/mod.rs | 534 +-------------- src/metrics/halstead.rs | 87 ++- src/metrics/loc/bash.rs | 6 +- src/metrics/npa.rs | 1 - src/metrics/npm.rs | 1 - src/observation.rs | 74 -- src/observation_tests.rs | 170 +++++ src/output/dump.rs | 3 +- src/spaces.rs | 11 +- src/spaces/ast.rs | 14 +- src/spaces_tests.rs | 52 +- src/test_support.rs | 58 +- tests/api/ast_seam_test.rs | 47 ++ tests/grammars/alterator_string_flattening.rs | 2 +- tests/parity/halstead_set_target_parity.rs | 3 +- utils/check-diagnostic-prefix-test.py | 5 +- utils/check-enums-codegen-drift-test.py | 56 +- utils/check-enums-codegen-drift.sh | 18 +- utils/check-grammar-crate-test.py | 10 +- utils/check-grammar-crate.py | 4 +- utils/check-publish-metadata-test.py | 40 +- utils/check-publish-metadata.py | 2 +- utils/check-rustfmt-bail-test.py | 26 +- utils/check-rustfmt-bail.py | 17 +- utils/check-versions-test.py | 20 +- utils/check-versions.py | 15 +- 165 files changed, 3701 insertions(+), 2610 deletions(-) create mode 100644 big-code-analysis-ast/Cargo.toml create mode 100644 big-code-analysis-ast/README.md rename {src => big-code-analysis-ast/src}/alterator.rs (99%) rename {src => big-code-analysis-ast/src}/ast.rs (98%) create mode 100644 big-code-analysis-ast/src/c_declarator.rs rename {src => big-code-analysis-ast/src}/c_langs_macros/c_macros.rs (100%) rename {src => big-code-analysis-ast/src}/c_langs_macros/c_specials.rs (100%) rename {src => big-code-analysis-ast/src}/c_langs_macros/mod.rs (96%) rename {src => big-code-analysis-ast/src}/c_macro.rs (99%) rename {src => big-code-analysis-ast/src}/cfg_predicate.rs (99%) rename {src => big-code-analysis-ast/src}/checker.rs (98%) rename {src => big-code-analysis-ast/src}/checker/bash.rs (100%) rename {src => big-code-analysis-ast/src}/checker/c.rs (100%) rename {src => big-code-analysis-ast/src}/checker/ccomment.rs (100%) rename {src => big-code-analysis-ast/src}/checker/cpp.rs (100%) rename {src => big-code-analysis-ast/src}/checker/csharp.rs (100%) rename {src => big-code-analysis-ast/src}/checker/elixir.rs (100%) rename {src => big-code-analysis-ast/src}/checker/go.rs (100%) rename {src => big-code-analysis-ast/src}/checker/groovy.rs (100%) rename {src => big-code-analysis-ast/src}/checker/irules.rs (100%) rename {src => big-code-analysis-ast/src}/checker/java.rs (100%) rename {src => big-code-analysis-ast/src}/checker/javascript.rs (100%) rename {src => big-code-analysis-ast/src}/checker/kotlin.rs (100%) rename {src => big-code-analysis-ast/src}/checker/lua.rs (100%) rename {src => big-code-analysis-ast/src}/checker/mozcpp.rs (100%) rename {src => big-code-analysis-ast/src}/checker/mozjs.rs (100%) rename {src => big-code-analysis-ast/src}/checker/objc.rs (100%) rename {src => big-code-analysis-ast/src}/checker/perl.rs (100%) rename {src => big-code-analysis-ast/src}/checker/php.rs (100%) rename {src => big-code-analysis-ast/src}/checker/preproc.rs (100%) rename {src => big-code-analysis-ast/src}/checker/python.rs (100%) rename {src => big-code-analysis-ast/src}/checker/ruby.rs (97%) rename {src => big-code-analysis-ast/src}/checker/rust.rs (100%) rename {src => big-code-analysis-ast/src}/checker/tcl.rs (100%) rename {src => big-code-analysis-ast/src}/checker/tsx.rs (100%) rename {src => big-code-analysis-ast/src}/checker/typescript.rs (100%) rename {src => big-code-analysis-ast/src}/comment_rm.rs (98%) rename {src => big-code-analysis-ast/src}/count.rs (97%) rename {src => big-code-analysis-ast/src}/error.rs (81%) rename {src => big-code-analysis-ast/src}/find.rs (93%) rename {src => big-code-analysis-ast/src}/getter.rs (95%) rename {src => big-code-analysis-ast/src}/getter/bash.rs (96%) rename {src => big-code-analysis-ast/src}/getter/c.rs (97%) rename {src => big-code-analysis-ast/src}/getter/ccomment.rs (100%) rename {src => big-code-analysis-ast/src}/getter/cpp.rs (97%) rename {src => big-code-analysis-ast/src}/getter/csharp.rs (96%) rename {src => big-code-analysis-ast/src}/getter/elixir.rs (98%) rename {src => big-code-analysis-ast/src}/getter/go.rs (93%) rename {src => big-code-analysis-ast/src}/getter/groovy.rs (98%) rename {src => big-code-analysis-ast/src}/getter/irules.rs (96%) rename {src => big-code-analysis-ast/src}/getter/java.rs (97%) rename {src => big-code-analysis-ast/src}/getter/javascript.rs (100%) rename {src => big-code-analysis-ast/src}/getter/kotlin.rs (97%) rename {src => big-code-analysis-ast/src}/getter/lua.rs (93%) rename {src => big-code-analysis-ast/src}/getter/mozcpp.rs (96%) rename {src => big-code-analysis-ast/src}/getter/mozjs.rs (100%) rename {src => big-code-analysis-ast/src}/getter/objc.rs (97%) rename {src => big-code-analysis-ast/src}/getter/perl.rs (98%) rename {src => big-code-analysis-ast/src}/getter/php.rs (96%) rename {src => big-code-analysis-ast/src}/getter/preproc.rs (100%) rename {src => big-code-analysis-ast/src}/getter/python.rs (92%) rename {src => big-code-analysis-ast/src}/getter/ruby.rs (98%) rename {src => big-code-analysis-ast/src}/getter/rust.rs (90%) rename {src => big-code-analysis-ast/src}/getter/tcl.rs (97%) rename {src => big-code-analysis-ast/src}/getter/tsx.rs (100%) rename {src => big-code-analysis-ast/src}/getter/typescript.rs (100%) rename {src => big-code-analysis-ast/src}/lang_helpers.rs (92%) rename {src => big-code-analysis-ast/src}/lang_helpers/elixir.rs (90%) rename {src => big-code-analysis-ast/src}/lang_helpers/python.rs (85%) rename {src => big-code-analysis-ast/src}/lang_helpers/tcl.rs (93%) rename {src => big-code-analysis-ast/src}/langs.rs (85%) rename {src => big-code-analysis-ast/src}/language_enum_roundtrip.rs (100%) rename {src => big-code-analysis-ast/src}/languages/language_bash.rs (99%) rename {src => big-code-analysis-ast/src}/languages/language_c.rs (99%) rename {src => big-code-analysis-ast/src}/languages/language_ccomment.rs (93%) rename {src => big-code-analysis-ast/src}/languages/language_cpp.rs (99%) rename {src => big-code-analysis-ast/src}/languages/language_csharp.rs (99%) rename {src => big-code-analysis-ast/src}/languages/language_elixir.rs (99%) rename {src => big-code-analysis-ast/src}/languages/language_go.rs (99%) rename {src => big-code-analysis-ast/src}/languages/language_groovy.rs (99%) rename {src => big-code-analysis-ast/src}/languages/language_irules.rs (98%) rename {src => big-code-analysis-ast/src}/languages/language_java.rs (99%) rename {src => big-code-analysis-ast/src}/languages/language_javascript.rs (99%) rename {src => big-code-analysis-ast/src}/languages/language_kotlin.rs (99%) rename {src => big-code-analysis-ast/src}/languages/language_lua.rs (98%) rename {src => big-code-analysis-ast/src}/languages/language_mozcpp.rs (99%) rename {src => big-code-analysis-ast/src}/languages/language_mozjs.rs (99%) rename {src => big-code-analysis-ast/src}/languages/language_objc.rs (99%) rename {src => big-code-analysis-ast/src}/languages/language_perl.rs (99%) rename {src => big-code-analysis-ast/src}/languages/language_php.rs (99%) rename {src => big-code-analysis-ast/src}/languages/language_preproc.rs (95%) rename {src => big-code-analysis-ast/src}/languages/language_python.rs (99%) rename {src => big-code-analysis-ast/src}/languages/language_ruby.rs (99%) rename {src => big-code-analysis-ast/src}/languages/language_rust.rs (99%) rename {src => big-code-analysis-ast/src}/languages/language_tcl.rs (98%) rename {src => big-code-analysis-ast/src}/languages/language_tsx.rs (99%) rename {src => big-code-analysis-ast/src}/languages/language_typescript.rs (99%) rename {src => big-code-analysis-ast/src}/languages/mod.rs (100%) create mode 100644 big-code-analysis-ast/src/lib.rs rename {src => big-code-analysis-ast/src}/macros/kind_sets.rs (96%) create mode 100644 big-code-analysis-ast/src/macros/mod.rs rename {src => big-code-analysis-ast/src}/node.rs (86%) rename {src => big-code-analysis-ast/src}/node/parser_cache.rs (88%) create mode 100644 big-code-analysis-ast/src/observation.rs rename {src => big-code-analysis-ast/src}/parser.rs (96%) rename {src => big-code-analysis-ast/src}/preproc.rs (98%) rename {src => big-code-analysis-ast/src}/preproc_tests.rs (97%) rename {src => big-code-analysis-ast/src}/recursion.rs (95%) rename {src => big-code-analysis-ast/src}/space_kind.rs (98%) create mode 100644 big-code-analysis-ast/src/test_support.rs create mode 100644 big-code-analysis-ast/src/token_role.rs rename {src => big-code-analysis-ast/src}/tools.rs (98%) rename {src => big-code-analysis-ast/src}/tools_tests.rs (100%) rename {src => big-code-analysis-ast/src}/traits.rs (51%) delete mode 100644 src/c_declarator.rs create mode 100644 src/c_family_space_names_tests.rs delete mode 100644 src/halstead_type.rs create mode 100644 src/lib_docs_tests.rs delete mode 100644 src/observation.rs create mode 100644 src/observation_tests.rs diff --git a/.bca-baseline.toml b/.bca-baseline.toml index 19b29f1e9..6f0accfdf 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -11,6 +11,294 @@ version = 6 tier = "soft" headroom = 0.95 +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "" +metric = "loc.ploc" +value = 545.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "Alterator::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "Alterator::get_ast_node" +metric = "nargs" +value = 6.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "Alterator::get_default" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "BashCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "CCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "CppCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "CsharpCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "ElixirCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "GoCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "GroovyCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "IrulesCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "JavaCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "JavascriptCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "KotlinCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "LuaCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "MozcppCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "MozjsCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "ObjcCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "PerlCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "PhpCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "PythonCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "RubyCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "RustCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "TclCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "TsxCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/alterator.rs" +qualified = "TypescriptCode::alterate" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/ast.rs" +qualified = "AstNode::with_field_name" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/ast.rs" +qualified = "Span::new" +metric = "nargs" +value = 6.0 + +[[entry]] +path = "big-code-analysis-ast/src/ast.rs" +qualified = "build" +metric = "halstead.effort" +value = 72581.41181251501 + +[[entry]] +path = "big-code-analysis-ast/src/c_macro.rs" +qualified = "replace" +metric = "halstead.effort" +value = 82074.58162485641 + +[[entry]] +path = "big-code-analysis-ast/src/c_macro.rs" +qualified = "step_normal" +metric = "halstead.effort" +value = 66515.67234293568 + +[[entry]] +path = "big-code-analysis-ast/src/c_macro.rs" +qualified = "step_normal" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/c_macro.rs" +qualified = "step_normal" +metric = "nexits" +value = 6.0 + +[[entry]] +path = "big-code-analysis-ast/src/c_macro.rs" +qualified = "step_raw_string" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/getter/bash.rs" +qualified = "BashCode::get_op_type" +metric = "halstead.effort" +value = 57421.05047463073 + +[[entry]] +path = "big-code-analysis-ast/src/getter/elixir.rs" +qualified = "ElixirCode::get_op_type" +metric = "halstead.effort" +value = 68425.65670070829 + +[[entry]] +path = "big-code-analysis-ast/src/getter/irules.rs" +qualified = "IrulesCode::get_op_type" +metric = "halstead.effort" +value = 51602.8394982239 + +[[entry]] +path = "big-code-analysis-ast/src/getter/perl.rs" +qualified = "PerlCode::get_op_type" +metric = "halstead.effort" +value = 106425.53731125958 + +[[entry]] +path = "big-code-analysis-ast/src/getter/php.rs" +qualified = "PhpCode::get_op_type" +metric = "halstead.effort" +value = 92711.94569253008 + +[[entry]] +path = "big-code-analysis-ast/src/getter/python.rs" +qualified = "PythonCode::get_op_type" +metric = "halstead.effort" +value = 48346.66496041094 + +[[entry]] +path = "big-code-analysis-ast/src/getter/ruby.rs" +qualified = "RubyCode::get_op_type" +metric = "halstead.effort" +value = 120805.48245794396 + +[[entry]] +path = "big-code-analysis-ast/src/node.rs" +qualified = "Node<'a>" +metric = "nom" +value = 33.0 + +[[entry]] +path = "big-code-analysis-ast/src/parser.rs" +qualified = "Parser::filters" +metric = "halstead.effort" +value = 58860.61554860244 + +[[entry]] +path = "big-code-analysis-ast/src/preproc.rs" +qualified = "accumulate_reachable_includes" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/preproc.rs" +qualified = "classify_preproc_node" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/preproc.rs" +qualified = "record_indirect_includes" +metric = "nargs" +value = 5.0 + +[[entry]] +path = "big-code-analysis-ast/src/tools.rs" +qualified = "read_gated" +metric = "nexits" +value = 7.0 + [[entry]] path = "big-code-analysis-cli/src/baseline.rs" qualified = "Baseline::from_str" @@ -593,258 +881,6 @@ qualified = "options_from" metric = "nexits" value = 8.0 -[[entry]] -path = "src/alterator.rs" -qualified = "" -metric = "loc.ploc" -value = 545.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "Alterator::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "Alterator::get_ast_node" -metric = "nargs" -value = 6.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "Alterator::get_default" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "BashCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "CCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "CppCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "CsharpCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "ElixirCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "GoCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "GroovyCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "IrulesCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "JavaCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "JavascriptCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "KotlinCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "LuaCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "MozcppCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "MozjsCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "ObjcCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "PerlCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "PhpCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "PythonCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "RubyCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "RustCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "TclCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "TsxCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/alterator.rs" -qualified = "TypescriptCode::alterate" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/ast.rs" -qualified = "AstNode::with_field_name" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/ast.rs" -qualified = "Span::new" -metric = "nargs" -value = 6.0 - -[[entry]] -path = "src/ast.rs" -qualified = "build" -metric = "halstead.effort" -value = 72581.41181251501 - -[[entry]] -path = "src/c_macro.rs" -qualified = "replace" -metric = "halstead.effort" -value = 82074.58162485641 - -[[entry]] -path = "src/c_macro.rs" -qualified = "step_normal" -metric = "halstead.effort" -value = 66515.67234293568 - -[[entry]] -path = "src/c_macro.rs" -qualified = "step_normal" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/c_macro.rs" -qualified = "step_normal" -metric = "nexits" -value = 6.0 - -[[entry]] -path = "src/c_macro.rs" -qualified = "step_raw_string" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/getter/bash.rs" -qualified = "BashCode::get_op_type" -metric = "halstead.effort" -value = 57421.05047463073 - -[[entry]] -path = "src/getter/elixir.rs" -qualified = "ElixirCode::get_op_type" -metric = "halstead.effort" -value = 68425.65670070829 - -[[entry]] -path = "src/getter/irules.rs" -qualified = "IrulesCode::get_op_type" -metric = "halstead.effort" -value = 51602.8394982239 - -[[entry]] -path = "src/getter/perl.rs" -qualified = "PerlCode::get_op_type" -metric = "halstead.effort" -value = 106425.53731125958 - -[[entry]] -path = "src/getter/php.rs" -qualified = "PhpCode::get_op_type" -metric = "halstead.effort" -value = 92711.94569253008 - -[[entry]] -path = "src/getter/python.rs" -qualified = "PythonCode::get_op_type" -metric = "halstead.effort" -value = 48346.66496041094 - -[[entry]] -path = "src/getter/ruby.rs" -qualified = "RubyCode::get_op_type" -metric = "halstead.effort" -value = 120865.54836941602 - [[entry]] path = "src/metrics/abc/csharp.rs" qualified = "csharp_inspect_container" @@ -1037,12 +1073,6 @@ qualified = "python_self_attr_name_bytes" metric = "nexits" value = 6.0 -[[entry]] -path = "src/node.rs" -qualified = "Node<'a>" -metric = "nom" -value = 33.0 - [[entry]] path = "src/ops.rs" qualified = "ops_inner" @@ -1133,30 +1163,6 @@ qualified = "metric_values" metric = "halstead.effort" value = 80351.11907066956 -[[entry]] -path = "src/parser.rs" -qualified = "Parser::filters" -metric = "halstead.effort" -value = 58860.61554860244 - -[[entry]] -path = "src/preproc.rs" -qualified = "accumulate_reachable_includes" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/preproc.rs" -qualified = "classify_preproc_node" -metric = "nargs" -value = 5.0 - -[[entry]] -path = "src/preproc.rs" -qualified = "record_indirect_includes" -metric = "nargs" -value = 5.0 - [[entry]] path = "src/spaces.rs" qualified = "FuncSpace::new" @@ -1211,12 +1217,6 @@ qualified = "parse_native" metric = "nexits" value = 6.0 -[[entry]] -path = "src/tools.rs" -qualified = "read_gated" -metric = "nexits" -value = 7.0 - [[entry]] path = "src/vcs/bus_factor.rs" qualified = "compute" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21bfb3fea..c23cd2afe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -297,7 +297,7 @@ jobs: # warnings` is repeated here deliberately. Keep the two in # step if the workflow-level value ever gains a flag. RUSTFLAGS: "-D warnings --cfg chain_audit" - run: cargo test -p big-code-analysis --lib --all-features --locked + run: cargo test -p big-code-analysis -p big-code-analysis-ast --lib --all-features --locked coverage: name: coverage @@ -403,6 +403,14 @@ jobs: # `Tsx` variant). - name: minimal-langs (lib) flags: --no-default-features --features rust,typescript -p big-code-analysis + # The parse layer builds on its own too (#1376): the three lib + # legs again, against the crate that actually links the grammars. + - name: default (ast) + flags: -p big-code-analysis-ast + - name: no-default-features (ast) + flags: --no-default-features -p big-code-analysis-ast + - name: minimal-langs (ast) + flags: --no-default-features --features rust,typescript -p big-code-analysis-ast - name: default (cli) flags: -p big-code-analysis-cli - name: no-default-features (cli) diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index b1d1dd168..cb5203dea 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -21,6 +21,7 @@ on: pull_request: paths: - 'src/**' + - 'big-code-analysis-ast/**' - 'fuzz/**' - 'Cargo.toml' - 'Cargo.lock' diff --git a/.github/workflows/mutation-test.yml b/.github/workflows/mutation-test.yml index c835b7ce5..84f15049b 100644 --- a/.github/workflows/mutation-test.yml +++ b/.github/workflows/mutation-test.yml @@ -53,13 +53,14 @@ jobs: mkdir -p target/mutants cargo mutants \ --package big-code-analysis \ + --package big-code-analysis-ast \ --no-shuffle \ --in-place \ -j 2 \ --minimum-test-timeout 120 \ -f src/metrics/ \ - -f src/checker.rs \ - -f src/getter.rs \ + -f big-code-analysis-ast/src/checker.rs \ + -f big-code-analysis-ast/src/getter.rs \ --output target/mutants \ | tee target/mutants/mutants.log diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index ffe9470b5..7dd5a7f99 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -12,6 +12,7 @@ on: paths: - 'big-code-analysis-book/**' - 'src/**' + - 'big-code-analysis-ast/**' - 'big-code-analysis-cli/**' - 'big-code-analysis-web/**' - 'big-code-analysis-py/**' @@ -35,6 +36,7 @@ on: paths: - 'big-code-analysis-book/**' - 'src/**' + - 'big-code-analysis-ast/**' - 'big-code-analysis-cli/**' - 'big-code-analysis-web/**' - 'big-code-analysis-py/**' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index de609e915..b0b748f9f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -138,7 +138,7 @@ jobs: cargo publish --dry-run --locked --manifest-path "$d/Cargo.toml" done - # The three top-level crates get no dry-run, because none is + # The four top-level crates get no dry-run, because none is # possible here. Each pins an internal dependency at # `=` — the parent on the five leaves, the CLI and web # crates on the parent — and the Lockstep version policy @@ -1454,6 +1454,25 @@ jobs: fi done + # The parse layer (#1376) is a `=X.Y.Z` path dependency of the + # root crate, so it publishes between the leaves and the root for + # the same reason the leaves publish first. + - name: Publish big-code-analysis-ast + env: + VERSION: ${{ needs.preflight.outputs.version }} + CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} + run: | + set -euo pipefail + INDEX="https://index.crates.io/bi/g-/big-code-analysis-ast" + # Bash-glob substring match: see leaf-publish step for rationale. + BODY=$(curl -sfL "$INDEX" 2>/dev/null || true) + NEEDLE="\"vers\":\"${VERSION}\"" + if [[ "${BODY}" == *"${NEEDLE}"* ]]; then + echo "big-code-analysis-ast ${VERSION} already on crates.io — skipping" + else + cargo publish -p big-code-analysis-ast --locked + fi + - name: Publish big-code-analysis env: VERSION: ${{ needs.preflight.outputs.version }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bf3287928..568b1f318 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -206,7 +206,7 @@ repos: - id: check-publish-metadata name: check-publish-metadata language: system - files: '^(Cargo\.(toml|lock)|(big-code-analysis-(cli|web|py|bench)|xtask|tree-sitter-(ccomment|mozcpp|mozjs|preproc|tcl))/Cargo\.toml|utils/check-publish-metadata\.py)$' + files: '^(Cargo\.(toml|lock)|(big-code-analysis-(ast|cli|web|py|bench)|xtask|tree-sitter-(ccomment|mozcpp|mozjs|preproc|tcl))/Cargo\.toml|utils/check-publish-metadata\.py)$' entry: python3 utils/check-publish-metadata.py pass_filenames: false @@ -251,18 +251,19 @@ repos: pass_filenames: false # Sync-test for check-grammar-crate.py's EXTENSIONS table against - # src/langs.rs (#869). Re-runs when the script, its test, or the - # source-of-truth language table changes. + # big-code-analysis-ast/src/langs.rs (#869). Re-runs when the + # script, its test, or the source-of-truth language table changes. - id: check-grammar-crate-test name: check-grammar-crate-test language: system - files: '^(utils/check-grammar-crate(-test)?\.py|src/langs\.rs)$' + files: '^(utils/check-grammar-crate(-test)?\.py|big-code-analysis-ast/src/langs\.rs)$' entry: python3 -m unittest -q utils/check-grammar-crate-test.py pass_filenames: false # Block enums codegen drift (#405). Runs the codegen into a # tempdir and diffs against the checked-in - # `src/c_langs_macros/*.rs` and `src/languages/language_*.rs` + # `big-code-analysis-ast/src/c_langs_macros/*.rs` and + # `big-code-analysis-ast/src/languages/language_*.rs` # files. Triggered by edits to inputs that actually affect # codegen output: the source crate's src/, templates, data # files, and Cargo.toml — plus the checked-in artifacts @@ -273,7 +274,7 @@ repos: - id: enums-codegen-drift name: enums-codegen-drift language: system - files: '^(enums/(src|templates|data)/.+|enums/Cargo\.toml|src/c_langs_macros/[^/]+\.rs|src/languages/language_[^/]+\.rs|utils/check-enums-codegen-drift\.sh)$' + files: '^(enums/(src|templates|data)/.+|enums/Cargo\.toml|big-code-analysis-ast/src/c_langs_macros/[^/]+\.rs|big-code-analysis-ast/src/languages/language_[^/]+\.rs|utils/check-enums-codegen-drift\.sh)$' entry: bash utils/check-enums-codegen-drift.sh pass_filenames: false diff --git a/.rustfmt-bail-baseline.txt b/.rustfmt-bail-baseline.txt index 30a12c092..c05424a45 100644 --- a/.rustfmt-bail-baseline.txt +++ b/.rustfmt-bail-baseline.txt @@ -30,7 +30,8 @@ # comment to hoist in these: # # enums/src/macros.rs `Lang::$camel => stringify!($camel)` -# src/macros/mod.rs `mk_lang!` / `mk_action!` / `mk_langs!` +# big-code-analysis-ast/src/macros/mod.rs +# `mk_lang!` / `mk_action!` / `mk_langs!` # bodies; every arm is `LANG::$camel => …` # src/metrics/cyclomatic.rs `impl_cyclomatic_c_family!`, whose arm # `… | $ternary $(| $short_circuit)+ =>` @@ -44,7 +45,8 @@ # kind #1136 decided to keep, so the count grew by one with nothing new # to hoist. Hand-check the arm's formatting instead. # -# src/getter/ruby.rs 3 -> 4, the `BQUOTE` subshell guard +# big-code-analysis-ast/src/getter/ruby.rs +# 3 -> 4, the `BQUOTE` subshell guard # (#1360) in `get_op_type` # src/vcs/error.rs `classify_error_variants!` (#1245): its # matcher, plus the 11 `$pat => $sample` @@ -113,26 +115,26 @@ # not checking it, and it does drift: before #1136 `src/getter/java.rs` # carried a 252-character match arm. +big-code-analysis-ast/src/getter.rs 3 +big-code-analysis-ast/src/getter/bash.rs 3 +big-code-analysis-ast/src/getter/c.rs 3 +big-code-analysis-ast/src/getter/cpp.rs 5 +big-code-analysis-ast/src/getter/csharp.rs 5 +big-code-analysis-ast/src/getter/elixir.rs 3 +big-code-analysis-ast/src/getter/go.rs 1 +big-code-analysis-ast/src/getter/groovy.rs 5 +big-code-analysis-ast/src/getter/irules.rs 6 +big-code-analysis-ast/src/getter/java.rs 2 +big-code-analysis-ast/src/getter/kotlin.rs 4 +big-code-analysis-ast/src/getter/lua.rs 3 +big-code-analysis-ast/src/getter/mozcpp.rs 5 +big-code-analysis-ast/src/getter/perl.rs 3 +big-code-analysis-ast/src/getter/php.rs 11 +big-code-analysis-ast/src/getter/python.rs 7 +big-code-analysis-ast/src/getter/ruby.rs 4 +big-code-analysis-ast/src/getter/tcl.rs 6 +big-code-analysis-ast/src/macros/mod.rs 14 enums/src/macros.rs 1 -src/getter.rs 3 -src/getter/bash.rs 3 -src/getter/c.rs 3 -src/getter/cpp.rs 5 -src/getter/csharp.rs 5 -src/getter/elixir.rs 3 -src/getter/go.rs 1 -src/getter/groovy.rs 5 -src/getter/irules.rs 6 -src/getter/java.rs 2 -src/getter/kotlin.rs 4 -src/getter/lua.rs 3 -src/getter/mozcpp.rs 5 -src/getter/perl.rs 3 -src/getter/php.rs 11 -src/getter/python.rs 7 -src/getter/ruby.rs 4 -src/getter/tcl.rs 6 -src/macros/mod.rs 38 src/metrics/abc/elixir.rs 5 src/metrics/cognitive/perl.rs 8 src/metrics/cyclomatic.rs 7 diff --git a/.taplo.toml b/.taplo.toml index 4e8888d29..fcc1a92b3 100644 --- a/.taplo.toml +++ b/.taplo.toml @@ -7,6 +7,7 @@ # its own conventions. include = [ "Cargo.toml", + "big-code-analysis-ast/**/*.toml", "big-code-analysis-cli/**/*.toml", "big-code-analysis-py/**/*.toml", "big-code-analysis-web/**/*.toml", diff --git a/CHANGELOG.md b/CHANGELOG.md index 27a9d35ae..376e67a26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,34 @@ for historical reference. ### Added +- The parse and classification layer now lives in its own published + crate, `big-code-analysis-ast` (#1376). It depends on nothing in this + crate, in either direction, so it can be built and tested alone: the generated per-grammar + kind enums, `LANG`, `Node`, `Checker` / `Getter` / `Alterator`, the + C-family preprocessor pass, comment stripping, the AST dump and node + counting / finding. The root crate depends on it at an exact `=X.Y.Z` + pin, forwards every per-language Cargo feature to it under the same + name, and re-exports everything it re-exported before at the same + paths, so no caller of `big-code-analysis` changes. The sub-crate is + internal plumbing with no stability promise of its own (see + `STABILITY.md` and its README); it exists so a second structural + consumer can share the classifiers without the metric walk. +- `Node` gains public, documented accessors for the parts of a + tree-sitter node the metric walk reads: `kind`, `kind_id`, `id`, + `is_named`, `utf8_text`, the byte / row / column position accessors, + `child`, `child_count`, `child_by_field_name`, `children`, + `children_with`, `cursor`, `parent`, and `previous_sibling`. Their + signatures are shape-stable; their values follow the tree-sitter pin + (see the escape-hatches section of `STABILITY.md`). +- `SpaceKind` and the operator/operand classification are now defined in + `big-code-analysis-ast` (they are `Getter` return types) and + re-exported at their existing paths; `MetricsError` likewise. One + consequence is additive on the published surface: `SpaceKind` gains a + public `is_member_scope()` — the walk consults it across the new crate + boundary, so it can no longer be `pub(crate)`. It answers "does this + kind roll up `npm` / `npa` members", i.e. anything but `Function` and + `Unknown`. + - Per-space *own* value for `nargs` in the serialized wire shape: `nargs.value` (#1236). `nargs.total` remains the subtree sum; the new field is the per-space scalar `bca check --threshold nargs=N` has @@ -69,6 +97,15 @@ for historical reference. ### Changed +- `metrics::halstead::HalsteadType` is renamed **`TokenRole`** and moves + to `big_code_analysis_ast::token_role`. It answers whether a node acts + as an operator or an operand, which the grammar decides and any + structural consumer can use; naming it after the one metric that reads + it made the parse layer look like it carried metric vocabulary. + `HalsteadType` remains as a deprecated type alias at its old path, so + no code breaks; the alias goes away at `3.0`. The variants are + unchanged. + - The `tree-sitter` runtime is `=0.26.13`, up one upstream patch release, pinned in lockstep across the root manifest, `enums`, the five vendored `bca-tree-sitter-*` crates, and the `fuzz` lockfile. diff --git a/Cargo.lock b/Cargo.lock index 863957fe3..5dfed5005 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -417,27 +417,16 @@ dependencies = [ name = "big-code-analysis" version = "2.2.1" dependencies = [ - "aho-corasick", - "bca-tree-sitter-ccomment", - "bca-tree-sitter-mozcpp", - "bca-tree-sitter-mozjs", - "bca-tree-sitter-preproc", - "bca-tree-sitter-tcl", + "big-code-analysis-ast", "bstr", "ciborium", "crossbeam", "csv", - "dekobon-tree-sitter-groovy", "gix", "globset", "hmac", "insta", "jsonschema", - "num-derive", - "num-format", - "num-traits", - "petgraph", - "pretty_assertions", "quick-xml", "regex", "serde", @@ -448,6 +437,30 @@ dependencies = [ "termcolor", "toml", "tree-sitter", + "walkdir", +] + +[[package]] +name = "big-code-analysis-ast" +version = "2.2.1" +dependencies = [ + "aho-corasick", + "bca-tree-sitter-ccomment", + "bca-tree-sitter-mozcpp", + "bca-tree-sitter-mozjs", + "bca-tree-sitter-preproc", + "bca-tree-sitter-tcl", + "dekobon-tree-sitter-groovy", + "num-derive", + "num-format", + "num-traits", + "petgraph", + "pretty_assertions", + "regex", + "serde", + "serde_json", + "tempfile", + "tree-sitter", "tree-sitter-bash", "tree-sitter-c", "tree-sitter-c-sharp", @@ -466,7 +479,6 @@ dependencies = [ "tree-sitter-ruby", "tree-sitter-rust", "tree-sitter-typescript", - "walkdir", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index ef6fd21f5..3c1d0d463 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] resolver = "3" members = [ + "big-code-analysis-ast", "big-code-analysis-bench", "big-code-analysis-cli", "big-code-analysis-py", @@ -19,7 +20,12 @@ members = [ # `cargo build --workspace`, but a bare `cargo build` at the repo root # should not pull in artefacts that have no shipping counterpart in a # Rust-only build. -default-members = [".", "big-code-analysis-cli", "big-code-analysis-web"] +default-members = [ + ".", + "big-code-analysis-ast", + "big-code-analysis-cli", + "big-code-analysis-web", +] exclude = [ "enums", # The cargo-fuzz crate (#1154). Excluded so `cargo test --workspace` @@ -126,7 +132,7 @@ module_name_repetitions = "allow" # `clippy::unwrap_used` is deliberately NOT declared here. It is set as # `#![cfg_attr(not(test), warn(clippy::unwrap_used))]` at each production -# crate root instead (the eight lib/bin roots), because a Cargo lint +# crate root instead (the nine lib/bin roots), because a Cargo lint # applies to every target of its package and the `unwrap` ban is a # production rule: this workspace's test targets hold 1_023 legitimate # `unwrap()` calls, against 0 in production. `cfg(test)` is set for @@ -179,7 +185,6 @@ all-features = true rustdoc-args = ["--cfg", "docsrs"] [dependencies] -aho-corasick = "^1.0" # `bstr` carries the raw, possibly-non-UTF-8 bytes git stores for paths # and author identities through the VCS pipeline; UTF-8 conversion is # deferred to the output boundary with explicit error handling (per the @@ -210,10 +215,6 @@ gix = { version = "^0.87", default-features = false, features = [ # listed explicitly or `gix-hash::Kind` compiles with no variants. "sha1", ], optional = true } -num-derive = "^0.5" -num-format = "^0.4" -num-traits = "^0.2" -petgraph = "^0.8" regex = "^1.7" serde.workspace = true # `float_roundtrip` makes serde_json's float *parser* bit-exact, so a value @@ -241,54 +242,29 @@ hmac = "=0.13.0" termcolor = "^1.2" tree-sitter.workspace = true -# Grammar crates are gated behind per-language Cargo features (see -# `[features]` below). The default feature set `all-languages` enables -# every grammar so a bare `cargo build` matches the historical -# behaviour; consumers that only need a subset of languages can opt -# into a narrower set with `--no-default-features --features rust,…`. -tree-sitter-bash = { workspace = true, optional = true } -tree-sitter-c-sharp = { workspace = true, optional = true } -tree-sitter-elixir = { workspace = true, optional = true } -tree-sitter-go = { workspace = true, optional = true } -dekobon-tree-sitter-groovy = { workspace = true, optional = true } -tree-sitter-irules = { workspace = true, optional = true } -tree-sitter-java = { workspace = true, optional = true } -tree-sitter-javascript = { workspace = true, optional = true } -tree-sitter-kotlin-ng = { workspace = true, optional = true } -tree-sitter-lua = { workspace = true, optional = true } -tree-sitter-objc = { workspace = true, optional = true } -tree-sitter-perl = { workspace = true, optional = true } -tree-sitter-php = { workspace = true, optional = true } -tree-sitter-python = { workspace = true, optional = true } -tree-sitter-ruby = { workspace = true, optional = true } -tree-sitter-rust = { workspace = true, optional = true } -tree-sitter-typescript = { workspace = true, optional = true } -tree-sitter-ccomment = { workspace = true, optional = true } -tree-sitter-mozcpp = { workspace = true, optional = true } -tree-sitter-cpp = { workspace = true, optional = true } -tree-sitter-c = { workspace = true, optional = true } -tree-sitter-mozjs = { workspace = true, optional = true } -tree-sitter-preproc = { workspace = true, optional = true } -tree-sitter-tcl = { workspace = true, optional = true } +# The parse and classification layer (#1376): the generated kind enums, +# `LANG`, `Node`, the `Checker` / `Getter` / `Alterator` classifiers and +# the C preprocessor pass. Version-locked like the vendored grammar leaves; +# the per-language features below forward to it, so a `--features rust` +# build of this crate still links exactly one grammar. +big-code-analysis-ast = { path = "big-code-analysis-ast", version = "=2.2.1", default-features = false } # Per-language Cargo features. `default = ["all-languages"]` keeps the # library's historical "every grammar compiled in" behaviour for # callers that take the crate as a black box (the CLI and web crates # pin to `features = ["all-languages"]` for exactly this reason). # -# Each language feature pulls in only the grammar crate(s) the -# matching `src/languages/language_*.rs` module needs at runtime — the -# token enums themselves carry no grammar-crate references and stay -# unconditionally compiled, so the `LANG` enum, the per-language -# `*Code` / `*Parser` tags, and the `Tree::new` / `analyze` dispatch -# surface remain identical across feature sets. Disabling a feature -# strips the grammar crate from the dep graph; calling into a -# disabled variant produces `Err(MetricsError::LanguageDisabled(LANG))` -# from every entry point that returns a `Result`. -# -# `cpp`, `c`, and `mozcpp` share the C-family helper crates -# (`tree-sitter-ccomment`, `tree-sitter-preproc`); listing the deps on -# each leaves their union enabled when any of those features is on. +# Since #1376 the grammar crates are dependencies of +# `big-code-analysis-ast`, and every language feature here forwards to +# the same-named feature there — one name per language on both sides, +# so `--no-default-features --features rust` still links exactly one +# grammar. The token enums, the `LANG` enum, the per-language `*Code` / +# `*Parser` tags and the dispatch surface are compiled unconditionally +# in that crate; a disabled feature only strips the grammar from the dep +# graph, and calling into its variant produces +# `Err(MetricsError::LanguageDisabled(LANG))` from every entry point +# that returns a `Result`. Root-side `#[cfg(feature = …)]` gates (all in +# tests) keep working because the names are the same. [features] default = ["all-languages"] # Change-history (VCS) metrics (issue #328). `vcs` is the umbrella @@ -303,6 +279,14 @@ default = ["all-languages"] vcs = ["vcs-git"] vcs-git = ["dep:gix", "dep:bstr"] all-languages = [ + # Forwarded so the sub-crate sees its own umbrella feature and not just + # the 22 language features below. Its + # `#![cfg_attr(not(feature = "all-languages"), allow(dead_code))]` + # keys on exactly this name, so without the line a `-p + # big-code-analysis --all-features` build relaxes dead-code in the + # parse layer while every one of its languages is live — the opposite + # of what that attribute's comment promises. + "big-code-analysis-ast/all-languages", "bash", "c", "cpp", @@ -326,67 +310,60 @@ all-languages = [ "tcl", "typescript", ] -bash = ["dep:tree-sitter-bash"] -# Since #720 the `Cpp` LANG variant uses the upstream community -# `tree-sitter-cpp` grammar (see `get_language!(tree_sitter_cpp)` in -# `src/macros.rs`); the Mozilla fork moved to the opt-in `mozcpp` -# feature / `LANG::Mozcpp` below. The `Ccomment` and `Preproc` LANG -# variants are internal C-family helpers (comment stripping + -# preprocessor directives) shared by all three C-family features, so -# `cpp`, `c` (added in #721), and `mozcpp` each pull them. NOTE: the -# `cpp` dep set changed in #720 (`tree-sitter-mozcpp` → `tree-sitter-cpp`) -# — a SemVer break for `--no-default-features` consumers, recorded in -# CHANGELOG under the 2.0 milestone. -# Internal: the C-family comment-stripping (`Ccomment`) and -# preprocessor (`Preproc`) helper variants. Enabled automatically by -# every C-family language feature below (`cpp`, `c`, `mozcpp`) so those -# helper LANG variants are compiled in whenever any C-family grammar is. -# Not meant to be selected on its own — it pulls no user-facing language. -# (Before #721 these crates were listed directly on each C-family -# feature, but the `Ccomment` / `Preproc` LANG variants were gated on -# `cpp` alone, so a `--features c` / `--features mozcpp` build without -# `cpp` had the crates yet disabled the variants — breaking `strip-comments` -# / `preproc` for C-only and Mozcpp-only builds.) -c-family-helpers = ["dep:tree-sitter-ccomment", "dep:tree-sitter-preproc"] -cpp = ["dep:tree-sitter-cpp", "c-family-helpers"] -# Dedicated C language (`LANG::C`, upstream `tree-sitter-c`), added in -# #721. Owns `.c`; shares the C-family `ccomment` / `preproc` helpers. -c = ["dep:tree-sitter-c", "c-family-helpers"] -# Opt-in Mozilla/Gecko C++ dialect (vendored `tree-sitter-mozcpp` fork, -# `LANG::Mozcpp`). Owns zero file extensions — selected only explicitly -# (`--language mozcpp`, manifest, API), mirroring `mozjs` since #507. -# Enabling `mozcpp` alone therefore analyzes no files by extension. -mozcpp = ["dep:tree-sitter-mozcpp", "c-family-helpers"] -csharp = ["dep:tree-sitter-c-sharp"] -elixir = ["dep:tree-sitter-elixir"] -go = ["dep:tree-sitter-go"] -groovy = ["dep:dekobon-tree-sitter-groovy"] -irules = ["dep:tree-sitter-irules"] -java = ["dep:tree-sitter-java"] -javascript = ["dep:tree-sitter-javascript"] -kotlin = ["dep:tree-sitter-kotlin-ng"] -lua = ["dep:tree-sitter-lua"] -mozjs = ["dep:tree-sitter-mozjs"] -# Objective-C (`LANG::Objc`, upstream `tree-sitter-objc`). Owns `.m`; -# `.mm` Objective-C++ stays on `LANG::Cpp` (the ObjC grammar parses C -# but not the C++ half of `.mm` files) — see #724. -objc = ["dep:tree-sitter-objc"] -perl = ["dep:tree-sitter-perl"] -php = ["dep:tree-sitter-php"] -python = ["dep:tree-sitter-python"] -ruby = ["dep:tree-sitter-ruby"] -rust = ["dep:tree-sitter-rust"] -tcl = ["dep:tree-sitter-tcl"] -typescript = ["dep:tree-sitter-typescript"] +bash = ["big-code-analysis-ast/bash"] +# The C-family trio and their shared `Ccomment` / `Preproc` helper +# variants: the rationale (and the #720 / #721 / #724 history) lives on +# the same-named features in `big-code-analysis-ast/Cargo.toml`. +# `c-family-helpers` is internal there and forwarded here only so the +# root's feature set stays a superset of the sub-crate's. +c-family-helpers = ["big-code-analysis-ast/c-family-helpers"] +cpp = ["big-code-analysis-ast/cpp"] +c = ["big-code-analysis-ast/c"] +mozcpp = ["big-code-analysis-ast/mozcpp"] +csharp = ["big-code-analysis-ast/csharp"] +elixir = ["big-code-analysis-ast/elixir"] +go = ["big-code-analysis-ast/go"] +groovy = ["big-code-analysis-ast/groovy"] +irules = ["big-code-analysis-ast/irules"] +java = ["big-code-analysis-ast/java"] +javascript = ["big-code-analysis-ast/javascript"] +kotlin = ["big-code-analysis-ast/kotlin"] +lua = ["big-code-analysis-ast/lua"] +mozjs = ["big-code-analysis-ast/mozjs"] +objc = ["big-code-analysis-ast/objc"] +perl = ["big-code-analysis-ast/perl"] +php = ["big-code-analysis-ast/php"] +python = ["big-code-analysis-ast/python"] +ruby = ["big-code-analysis-ast/ruby"] +rust = ["big-code-analysis-ast/rust"] +tcl = ["big-code-analysis-ast/tcl"] +typescript = ["big-code-analysis-ast/typescript"] [lints] workspace = true [dev-dependencies] +# The parse-and-inspect helpers the metric tests share with the parse +# layer's own tests. Feature unification keeps `test-support` out of the +# non-test build. +# +# `default-features = false` is load-bearing, not tidiness: cargo unions +# the feature sets of a package's normal and dev dependencies whenever a +# dev target is in the build, so omitting it re-enables `all-languages` +# on the parse layer for every `cargo test` / `cargo check --all-targets` +# — including the `no-default-features (lib)` and `minimal-langs (lib)` +# CI legs, which then link all 25 grammars and stop testing the subset +# they exist to test. It also failed the two `LanguageDisabled` +# assertions in `tests/api/ast_seam_test.rs`, because the root's +# `javascript` was off while the parse layer's was on. The language +# features come from the normal dependency above; this entry adds only +# `test-support`. +big-code-analysis-ast = { path = "big-code-analysis-ast", version = "=2.2.1", default-features = false, features = [ + "test-support", +] } insta = { version = "1.29.0", features = ["yaml", "json", "redactions"] } jsonschema = "^0.49" tempfile.workspace = true -pretty_assertions = "^1.3" quick-xml = "^0.41" # Test-only: the integration corpus harness and the concurrent_files # unit tests walk fixture trees; production code no longer walks (#495). diff --git a/Makefile b/Makefile index 1c6d4b058..26d7c8de5 100644 --- a/Makefile +++ b/Makefile @@ -135,7 +135,7 @@ help: @echo " grammar-marker-sync-test Self-tests for the grammar-marker-sync gate" @echo " check-versions Enforce lockstep version invariant across owned crates" @echo " check-versions-test Self-tests for the check-versions gate" - @echo " check-grammar-crate-test Sync-test EXTENSIONS table vs src/langs.rs" + @echo " check-grammar-crate-test Sync-test EXTENSIONS table vs langs.rs" @echo " check-excluded-manifests Assert excluded crates root a workspace, declare lints, and =-pin grammars" @echo " check-excluded-manifests-test Self-tests for the check-excluded-manifests gate" @echo " check-ruff-lockstep Assert the ruff-pre-commit rev, uv.lock, and the requirements export agree" @@ -322,17 +322,18 @@ test-doc: # triples the lib suite's wall time. Runs in the `chain-audit` CI lane; # run it locally around any change to a walk's truncate/push bookkeeping. # -# Library-scoped, matching that lane: all five walks that thread a chain -# live in the root crate, so the CLI / web / integration tiers would -# re-pay the quadratic cost without reaching an assertion the lib tests -# do not already reach. +# Library-scoped, matching that lane: the five walks that thread a chain +# live in the root crate (`spaces::compute`, `ops`, `suppression`) and +# in `big-code-analysis-ast` (`comment_rm`, `Search::act_on_node`), so +# the CLI / web / integration tiers would re-pay the quadratic cost +# without reaching an assertion the lib tests do not already reach. # # RUSTFLAGS rather than a Cargo feature on purpose: `make test` passes # `--all-features`, so a feature would switch the audit back on in the # very inner loop this target exists to keep fast. chain-audit: RUSTFLAGS="$${RUSTFLAGS:-} --cfg chain_audit" \ - cargo test -p big-code-analysis --lib --all-features + cargo test -p big-code-analysis -p big-code-analysis-ast --lib --all-features # `cargo insta test` shells out to `cargo test`, not $(TEST_CMD): insta's # nextest integration needs `--test-runner nextest`, and under it insta @@ -465,8 +466,9 @@ grammar-marker-sync: # Enums-codegen drift gate. Closes #405: running any # `recreate-grammars.sh` invocation silently regenerated -# `src/c_langs_macros/{c_macros,c_specials}.rs` to a pre- -# optimization form. This gate runs the codegen into a tempdir +# `big-code-analysis-ast/src/c_langs_macros/{c_macros,c_specials}.rs` +# to a pre-optimization form. This gate runs the codegen into a +# tempdir # and diffs against the checked-in files; drift fails. enums-codegen-drift: @echo "Checking enums codegen drift..." @@ -586,7 +588,7 @@ check-ruff-lockstep-test: @(cd $(BASE_DIR) && python3 -m unittest -q utils/check-ruff-lockstep-test.py) # Publish-metadata gate (#1224). `cargo publish --dry-run` is the -# natural pre-tag check, but it cannot run for the three top-level +# natural pre-tag check, but it cannot run for the four top-level # crates: each pins an internal dependency at `=`, and the # Lockstep policy makes that the version being released — which is by # definition not yet on crates.io. The workaround it replaces skipped @@ -652,8 +654,9 @@ check-safety-doc-pin-test: @(cd $(BASE_DIR) && python3 -m unittest -q utils/check-safety-doc-pin-test.py) # Sync gate for check-grammar-crate.py's EXTENSIONS table. Re-derives -# the grammar -> extension mapping from src/langs.rs `mk_langs!` and -# fails if the hand-maintained table has drifted (#869). Static — no +# the grammar -> extension mapping from the parse layer's langs.rs +# `mk_langs!` and fails if the hand-maintained table has drifted +# (#869). Static — no # network, no cargo — so it rides the cheap test DAG alongside the # other helper-script self-tests. check-grammar-crate-test: @@ -1440,7 +1443,7 @@ doc-check-docsrs: @if rustup toolchain list 2>/dev/null | grep -q '^nightly'; then \ echo "Building docs.rs-style rustdoc (nightly, --cfg docsrs, -D warnings)..."; \ RUSTDOCFLAGS="--cfg docsrs -D warnings" \ - cargo +nightly doc --no-deps -p big-code-analysis --all-features; \ + cargo +nightly doc --no-deps -p big-code-analysis -p big-code-analysis-ast --all-features; \ else \ echo "nightly toolchain not found; skipping docs.rs-style doc check"; \ fi @@ -2026,7 +2029,7 @@ verify-changelog: # # Only the five vendored grammar leaves can be dry-run, so they are all # that last sentence covers. They carry no internal pins, so `cargo -# publish --dry-run` resolves for them at any time. The three top-level +# publish --dry-run` resolves for them at any time. The four top-level # crates cannot: each pins an internal # dependency at `=` and the Lockstep policy makes that the # version being released, which is by definition not yet on diff --git a/STABILITY.md b/STABILITY.md index e01c2d03d..6019ef931 100644 --- a/STABILITY.md +++ b/STABILITY.md @@ -45,7 +45,9 @@ section. - **Language identification** - `LANG` enum (generated by the `mk_langs!` macro invoked from - `src/langs.rs`; the macro itself lives in `src/macros/mod.rs`): + `big-code-analysis-ast/src/langs.rs`; the macro itself lives in + `big-code-analysis-ast/src/macros/mod.rs`, and the root crate + re-exports the enum unchanged): variants are additive. Adding a new variant in a minor bump is allowed; renaming or removing one is a `3.0` break. Derives `Hash` and implements `Display` (the `name` string) and @@ -60,8 +62,10 @@ section. and the Python bindings). The human-pretty `c/c++` / `c#` display forms were dropped at `2.0`, a break in the serialized `language` value. - - `get_language_for_file`, `guess_language` in `src/tools.rs`. -- **File readers** (`src/tools.rs`) + - `get_language_for_file`, `guess_language` in + `big-code-analysis-ast/src/tools.rs`, re-exported from the root. +- **File readers** (`big-code-analysis-ast/src/tools.rs`, re-exported + from the root) - `read_file`, `read_file_with_eol`, `normalize_eol`. Their signatures are fixed for `2.x`; in particular `read_file_with_eol` keeps its `io::Result>>` @@ -160,7 +164,8 @@ section. `ParseMetricError` / `ParseLangError` convention; it is deliberately exhaustive so callers can match both failure modes without a wildcard. - - `MetricsError` in `src/error.rs`: carries `#[non_exhaustive]`, + - `MetricsError` in `big-code-analysis-ast/src/error.rs` + (re-exported from the root): carries `#[non_exhaustive]`, so adding variants is additive. Current variants are `LanguageDisabled(LANG)` (the only one produced today) and the reserved-for-future `EmptyRoot`. The previously-reserved @@ -170,8 +175,9 @@ section. `Display` impls are stable; the exact wording of `Display` output is not. - **Result shapes** - - `FuncSpace`, `CodeMetrics`, `SpaceKind`, `Metrics` in - `src/spaces.rs`. These are the JSON / YAML / TOML / CBOR + - `FuncSpace`, `CodeMetrics`, `Metrics` in `src/spaces.rs`, and + `SpaceKind` in `big-code-analysis-ast/src/space_kind.rs` + (re-exported at `crate::SpaceKind`). These are the JSON / YAML / TOML / CBOR serialization roots and have downstream consumers. `FuncSpace` and `CodeMetrics` derive `PartialEq` (#552) so callers can compare analyses structurally; `Eq` is omitted because the @@ -193,7 +199,8 @@ section. detail — the contract is that these trees tear down in constant stack, not the presence of any particular `Drop` body. - `FunctionSpan` in `src/function.rs`. - - `PreprocResults` and `PreprocFile` in `src/preproc.rs` — the + - `PreprocResults` and `PreprocFile` in + `big-code-analysis-ast/src/preproc.rs` (re-exported from the root) — the serialization roots of a `bca preproc` document, on `--output` and on stdout alike (one `to_string` feeds both destinations). Their fields keep their `HashMap` / `HashSet` types, so *iteration* order @@ -423,24 +430,32 @@ re-finalizes under any key without a re-walk). The following are explicitly **not** part of the shape contract: -- Anything marked `#[doc(hidden)]` (see `src/traits.rs` for current - examples). These exist for macro plumbing and may move at any - time, including in patch bumps. -- The per-language `*Code` / `*Parser` types - (`RustCode`, `PythonCode`, …). They are public because the - `mk_langs!` macro emits them, but they are intended to be reached - through `LANG` rather than referenced by name. -- `Parser` and the per-language `Checker` / `Getter` / `Alterator` - trait impls: these are internal plumbing. As of `2.0` they are - `pub(crate)`, not a public extension surface: `ParserTrait`, - `Parser`, `Filter`, `Cursor`, and the per-metric compute traits - (`Cognitive`, `Cyclomatic`, `Halstead`, `Loc`, `Mi`, `Nom`, - `NArgs`, `Exit`, `Abc`, `Npa`, `Npm`, `Tokens`, `Wmc`) were all - demoted from their former `#[doc(hidden)]`-but-`pub` state to - `pub(crate)`. `LanguageInfo` was likewise demoted to `pub(crate)`, - and the `Callback` trait / `AstCallback` dispatch were removed at - `2.0`. None of these are reachable from the public API or appear in - the curated rustdoc; treat them as internal plumbing. +- Anything marked `#[doc(hidden)]`. These exist for macro plumbing + and may move at any time, including in patch bumps. +- **The `big-code-analysis-ast` crate.** Since #1376 the parse and + classification layer — the generated kind enums, the per-language + `*Code` / `*Parser` types (`RustCode`, `PythonCode`, …), `Parser`, + `ParserTrait`, `LanguageInfo`, `Search`, `Filter`, `Cursor`, + `Ancestors`, the `Checker` / `Getter` / `Alterator` traits and their + per-language impls, `AnyParser` and the `with_any_parser!` dispatch + macro — lives in a separate published crate that this one pins at an + exact `=X.Y.Z` version. Those items are `pub` *there* because this + crate needs them, and they are documented there, but **none of them + is part of this contract**: this crate does not re-export them, and + the sub-crate promises nothing about its own names, signatures or + module paths from one release to the next (see its README). The + items this crate *does* re-export from it — `LANG`, `ParseLangError`, + `Node`, `MetricsError`, `SpaceKind`, `metrics::halstead::TokenRole`, + the AST dump types, `Count` / `CountCollector`, the preprocessor types + and functions, and the file readers — are covered exactly as the + sections above describe, through this crate's paths. Depend on + `big-code-analysis`; depend on `big-code-analysis-ast` directly only + if you accept re-pinning on every release. +- The per-metric compute traits (`Cognitive`, `Cyclomatic`, `Halstead`, + `Loc`, `Mi`, `Nom`, `NArgs`, `Exit`, `Abc`, `Npa`, `Npm`, `Tokens`, + `Wmc`) and the `MetricSuite` supertrait that keys them on a parser: + `pub(crate)` in this crate since `2.0`, not an extension surface. The + `Callback` trait / `AstCallback` dispatch were removed at `2.0`. [changelog]: ./CHANGELOG.md @@ -591,7 +606,8 @@ upstream grammar contract; depend on them only when you need to reach the raw tree-sitter surface. - **`Node` exposes its `tree_sitter::Node` through an accessor.** - `Node<'a>` in `src/node.rs` wraps the tree-sitter node; the inner + `Node<'a>` (defined in `big-code-analysis-ast/src/node.rs`, re-exported + here) wraps the tree-sitter node; the inner field is private and the node is reached through `Node::as_tree_sitter(&self) -> tree_sitter::Node<'a>` (the node is `Copy`, so it is returned by value). Anything you do through @@ -599,7 +615,28 @@ reach the raw tree-sitter surface. move whenever we bump that pin (typically under a minor release). This mirrors the higher-level `Ast::as_tree_sitter` seam below; reach for `Node::as_tree_sitter` only when you already hold a - `Node` from this surface. **(breaking, 2.0)** The pre-2.0 shape + `Node` from this surface. + + Since the #1376 split `Node` also exposes the accessors the metric + walk itself uses — `kind`, `kind_id`, `id`, `is_named`, `utf8_text`, + `start_byte` / `end_byte`, `start_position` / `end_position`, + `start_row` / `end_row` / `end_line`, `child`, `child_count`, + `child_by_field_name`, `children`, `children_with`, `cursor`, + `parent` and `previous_sibling`. Their *signatures* are shape-stable + within `2.x` (additive from here). Their + *values* follow the tree-sitter pin exactly as `as_tree_sitter` does: + `kind()` strings and `kind_id()` numbers are the pinned grammar's and + move when it is bumped. `parent` and `previous_sibling` resolve from + the tree root and cost `O(depth)` per call; a walk over a whole tree + should keep its own ancestor chain instead. + + `Node`'s remaining methods — the `Search` walk helpers and the + ancestor-chain-taking `has_sibling` / `count_specific_ancestors` / + `parent_grandparent_match` — are deliberately **not** covered. They + name `Search`, `Ancestors` and `Checker`, which this crate does not + re-export, so they are not callable from `big-code-analysis` alone; + reaching them means depending on `big-code-analysis-ast` directly, + which carries no promise at all. **(breaking, 2.0)** The pre-2.0 shape was `pub struct Node<'a>(pub OtherNode<'a>)`, a public tuple field welding the value-not-stable `tree_sitter::Node` into the stable struct's layout. #556 demoted the field to private and @@ -646,9 +683,9 @@ reach the raw tree-sitter surface. `descendants_by_kind()` traversal helpers and `as_tree_sitter()` escape hatch are shape-stable; the raw node *kinds* and `kind_id`s it surfaces follow the `tree-sitter` pin in the same value-not-stable sense as the - re-export. The language-dispatched `AstInner` enum and - the matching `ast_*_dispatch` helpers stay `pub(crate)`; only `Ast` - is exposed. At `2.0` the path-positional `get_function_spaces*` / + re-export. The language-dispatched `AnyParser` enum it wraps is a + `big-code-analysis-ast` item and not re-exported; only `Ast` is + exposed here. At `2.0` the path-positional `get_function_spaces*` / `metrics_from_tree` / `get_ops` shims, the parser-generic `metrics` / `metrics_with_options` / `operands_and_operators` functions, and the `action` / `Callback` dispatch were all removed in favor of these diff --git a/big-code-analysis-ast/Cargo.toml b/big-code-analysis-ast/Cargo.toml new file mode 100644 index 000000000..15ec678fc --- /dev/null +++ b/big-code-analysis-ast/Cargo.toml @@ -0,0 +1,168 @@ +[package] +name = "big-code-analysis-ast" +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true +documentation = "https://docs.rs/big-code-analysis-ast/" +readme = "README.md" +keywords = ["metrics", "tree-sitter"] +description = "Parse and classification layer behind big-code-analysis (internal, version-locked)" + +[package.metadata.docs.rs] +all-features = true +# Mirrors the root manifest, so `make doc-check-docsrs` (which builds +# both crates with `--cfg docsrs`) reproduces what docs.rs renders. +rustdoc-args = ["--cfg", "docsrs"] + +[dependencies] +aho-corasick = "^1.0" +num-derive = "^0.5" +num-format = "^0.4" +num-traits = "^0.2" +petgraph = "^0.8" +regex = "^1.7" +serde.workspace = true + +tree-sitter.workspace = true +# Grammar crates are gated behind per-language Cargo features (see +# `[features]` below). The default feature set `all-languages` enables +# every grammar so a bare `cargo build` matches the historical +# behaviour; consumers that only need a subset of languages can opt +# into a narrower set with `--no-default-features --features rust,…`. +tree-sitter-bash = { workspace = true, optional = true } +tree-sitter-c-sharp = { workspace = true, optional = true } +tree-sitter-elixir = { workspace = true, optional = true } +tree-sitter-go = { workspace = true, optional = true } +dekobon-tree-sitter-groovy = { workspace = true, optional = true } +tree-sitter-irules = { workspace = true, optional = true } +tree-sitter-java = { workspace = true, optional = true } +tree-sitter-javascript = { workspace = true, optional = true } +tree-sitter-kotlin-ng = { workspace = true, optional = true } +tree-sitter-lua = { workspace = true, optional = true } +tree-sitter-objc = { workspace = true, optional = true } +tree-sitter-perl = { workspace = true, optional = true } +tree-sitter-php = { workspace = true, optional = true } +tree-sitter-python = { workspace = true, optional = true } +tree-sitter-ruby = { workspace = true, optional = true } +tree-sitter-rust = { workspace = true, optional = true } +tree-sitter-typescript = { workspace = true, optional = true } +tree-sitter-ccomment = { workspace = true, optional = true } +tree-sitter-mozcpp = { workspace = true, optional = true } +tree-sitter-cpp = { workspace = true, optional = true } +tree-sitter-c = { workspace = true, optional = true } +tree-sitter-mozjs = { workspace = true, optional = true } +tree-sitter-preproc = { workspace = true, optional = true } +tree-sitter-tcl = { workspace = true, optional = true } + +# Per-language Cargo features. `default = ["all-languages"]` keeps the +# library's historical "every grammar compiled in" behaviour for +# callers that take the crate as a black box. `big-code-analysis` +# forwards each of these under the same name, so a `--features rust` +# build of either crate links exactly one grammar. +# +# Each language feature pulls in only the grammar crate(s) the +# matching `src/languages/language_*.rs` module needs at runtime — the +# token enums themselves carry no grammar-crate references and stay +# unconditionally compiled, so the `LANG` enum, the per-language +# `*Code` / `*Parser` tags, and the `Tree::new` / `analyze` dispatch +# surface remain identical across feature sets. Disabling a feature +# strips the grammar crate from the dep graph; calling into a +# disabled variant produces `Err(MetricsError::LanguageDisabled(LANG))` +# from every entry point that returns a `Result`. +# +# `cpp`, `c`, and `mozcpp` share the C-family helper crates +# (`tree-sitter-ccomment`, `tree-sitter-preproc`); listing the deps on +# each leaves their union enabled when any of those features is on. +[features] +default = ["all-languages"] +# Exposes the parse-and-inspect test helpers in `test_support` to a +# dependent crate's test target (the root crate's metric tests use them). +# Never enable it in a shipping build. +test-support = [] +all-languages = [ + "bash", + "c", + "cpp", + "csharp", + "elixir", + "go", + "groovy", + "irules", + "java", + "javascript", + "kotlin", + "lua", + "mozcpp", + "mozjs", + "objc", + "perl", + "php", + "python", + "ruby", + "rust", + "tcl", + "typescript", +] +bash = ["dep:tree-sitter-bash"] +# Since #720 the `Cpp` LANG variant uses the upstream community +# `tree-sitter-cpp` grammar (see `get_language!(tree_sitter_cpp)` in +# `src/macros.rs`); the Mozilla fork moved to the opt-in `mozcpp` +# feature / `LANG::Mozcpp` below. The `Ccomment` and `Preproc` LANG +# variants are internal C-family helpers (comment stripping + +# preprocessor directives) shared by all three C-family features, so +# `cpp`, `c` (added in #721), and `mozcpp` each pull them. NOTE: the +# `cpp` dep set changed in #720 (`tree-sitter-mozcpp` → `tree-sitter-cpp`) +# — a SemVer break for `--no-default-features` consumers, recorded in +# CHANGELOG under the 2.0 milestone. +# Internal: the C-family comment-stripping (`Ccomment`) and +# preprocessor (`Preproc`) helper variants. Enabled automatically by +# every C-family language feature below (`cpp`, `c`, `mozcpp`) so those +# helper LANG variants are compiled in whenever any C-family grammar is. +# Not meant to be selected on its own — it pulls no user-facing language. +# (Before #721 these crates were listed directly on each C-family +# feature, but the `Ccomment` / `Preproc` LANG variants were gated on +# `cpp` alone, so a `--features c` / `--features mozcpp` build without +# `cpp` had the crates yet disabled the variants — breaking `strip-comments` +# / `preproc` for C-only and Mozcpp-only builds.) +c-family-helpers = ["dep:tree-sitter-ccomment", "dep:tree-sitter-preproc"] +cpp = ["dep:tree-sitter-cpp", "c-family-helpers"] +# Dedicated C language (`LANG::C`, upstream `tree-sitter-c`), added in +# #721. Owns `.c`; shares the C-family `ccomment` / `preproc` helpers. +c = ["dep:tree-sitter-c", "c-family-helpers"] +# Opt-in Mozilla/Gecko C++ dialect (vendored `tree-sitter-mozcpp` fork, +# `LANG::Mozcpp`). Owns zero file extensions — selected only explicitly +# (`--language mozcpp`, manifest, API), mirroring `mozjs` since #507. +# Enabling `mozcpp` alone therefore analyzes no files by extension. +mozcpp = ["dep:tree-sitter-mozcpp", "c-family-helpers"] +csharp = ["dep:tree-sitter-c-sharp"] +elixir = ["dep:tree-sitter-elixir"] +go = ["dep:tree-sitter-go"] +groovy = ["dep:dekobon-tree-sitter-groovy"] +irules = ["dep:tree-sitter-irules"] +java = ["dep:tree-sitter-java"] +javascript = ["dep:tree-sitter-javascript"] +kotlin = ["dep:tree-sitter-kotlin-ng"] +lua = ["dep:tree-sitter-lua"] +mozjs = ["dep:tree-sitter-mozjs"] +# Objective-C (`LANG::Objc`, upstream `tree-sitter-objc`). Owns `.m`; +# `.mm` Objective-C++ stays on `LANG::Cpp` (the ObjC grammar parses C +# but not the C++ half of `.mm` files) — see #724. +objc = ["dep:tree-sitter-objc"] +perl = ["dep:tree-sitter-perl"] +php = ["dep:tree-sitter-php"] +python = ["dep:tree-sitter-python"] +ruby = ["dep:tree-sitter-ruby"] +rust = ["dep:tree-sitter-rust"] +tcl = ["dep:tree-sitter-tcl"] +typescript = ["dep:tree-sitter-typescript"] + +[lints] +workspace = true + +[dev-dependencies] +pretty_assertions = "^1.3" +serde_json.workspace = true +tempfile.workspace = true diff --git a/big-code-analysis-ast/README.md b/big-code-analysis-ast/README.md new file mode 100644 index 000000000..b02b76218 --- /dev/null +++ b/big-code-analysis-ast/README.md @@ -0,0 +1,32 @@ +# big-code-analysis-ast + +The parse and classification layer behind +[`big-code-analysis`](https://crates.io/crates/big-code-analysis): the +tree-sitter wrappers (`Node`, `Ancestors`), the generated per-grammar +kind enums, the `LANG` enum and language detection, the `Checker` / +`Getter` / `Alterator` classifiers, the C-family preprocessor pass, +comment stripping, and the AST dump. It computes no metric. + +## This crate is internal plumbing + +It is published only so that `big-code-analysis` can be. That crate +pins it at an exact `=X.Y.Z` version, the two are released together, +and nothing here carries a stability promise of its own: names, +signatures and module paths may change in any release, including a +patch. Depend on `big-code-analysis` and reach what it re-exports +(`LANG`, `Node`, `MetricsError`, `SpaceKind`, the AST dump types, the +preprocessor types, the file readers), which its +[`STABILITY.md`](https://github.com/dekobon/big-code-analysis/blob/main/STABILITY.md) +covers. Depend on this crate directly only when you accept re-pinning +on every release. + +The split exists (#1376) so a second structural consumer — a linter, a +call-graph builder, a language server — can share one classification +layer without the metric machinery. Every per-language Cargo feature +of `big-code-analysis` (`rust`, `python`, …, `all-languages`) is a +feature of this crate under the same name, and the grammar crates are +dependencies here. + +## License + +MPL-2.0, like the rest of the repository. diff --git a/src/alterator.rs b/big-code-analysis-ast/src/alterator.rs similarity index 99% rename from src/alterator.rs rename to big-code-analysis-ast/src/alterator.rs index 7df21a17a..8903d179d 100644 --- a/src/alterator.rs +++ b/big-code-analysis-ast/src/alterator.rs @@ -6,12 +6,16 @@ // function so the per-language impl blocks stay readable. #![allow(clippy::wildcard_imports, clippy::enum_glob_use)] +//! Per-language AST simplification for the dump: the [`Alterator`] hook a +//! language implements to reshape a node before it is rendered as an +//! [`AstNode`]. + use crate::*; /// A trait to create a richer `AST` node for a programming language, mainly /// thought to be sent on the network. Crate-internal extension over /// [`Checker`], used only by the per-language `Parser` impls. -pub(crate) trait Alterator +pub trait Alterator where Self: Checker, { diff --git a/src/ast.rs b/big-code-analysis-ast/src/ast.rs similarity index 98% rename from src/ast.rs rename to big-code-analysis-ast/src/ast.rs index 0ce1b5e4b..0e1621ce0 100644 --- a/src/ast.rs +++ b/big-code-analysis-ast/src/ast.rs @@ -6,6 +6,9 @@ // function so the per-language impl blocks stay readable. #![allow(clippy::enum_glob_use, clippy::if_not_else, clippy::wildcard_imports)] +//! The AST dump: [`AstNode`], [`Span`], [`AstCfg`], [`AstResponse`], and +//! the walk that builds them. + use serde::{Deserialize, Serialize}; use crate::*; @@ -17,7 +20,7 @@ use crate::*; /// `{start_line, start_col, end_line, end_col, start_byte, end_byte}`. The /// line/column pairs are 1-based; the byte offsets are 0-based half-open /// (`[start_byte, end_byte)`) indices into the parsed source bytes -/// ([`Ast::source`](crate::Ast::source)). The `*_line` vocabulary aligns the +/// (`big_code_analysis::Ast::source`). The `*_line` vocabulary aligns the /// `/ast` span field names with the `/function` and `/metrics` endpoints /// (`start_line` / `end_line`), so a client correlating spans across /// endpoints no longer special-cases `*_row` vs `*_line` per endpoint @@ -322,10 +325,10 @@ pub struct AstCfg { pub span: bool, } -/// Build the AST dump for `parser` under `cfg`. Backs [`crate::Ast::dump`]; -/// the AST-extraction analogue of [`crate::spaces::metrics_inner`] / -/// [`crate::ops::ops_inner`]. -pub(crate) fn dump_inner(parser: &T, cfg: AstCfg) -> AstResponse { +/// Build the AST dump for `parser` under `cfg`. Backs `big_code_analysis::Ast::dump`; +/// the AST-extraction analogue of `big_code_analysis::spaces::metrics_inner` / +/// `big_code_analysis::ops::ops_inner`. +pub fn dump_inner(parser: &T, cfg: AstCfg) -> AstResponse { AstResponse { id: cfg.id, language: cfg.language, diff --git a/big-code-analysis-ast/src/c_declarator.rs b/big-code-analysis-ast/src/c_declarator.rs new file mode 100644 index 000000000..e81ebc369 --- /dev/null +++ b/big-code-analysis-ast/src/c_declarator.rs @@ -0,0 +1,282 @@ +//! The C-family declarator-chain walk, shared by the two surfaces that +//! need it. +//! +//! C declarator syntax nests outward from the declared name, so neither +//! a function's parameter list nor its name is reliably a child of the +//! function node itself. Both are found from the one node +//! [`innermost_declarator`] returns — the innermost link on the chain +//! that is the function's own declarator rather than its return type's: +//! +//! - `big_code_analysis::metrics::nargs` reads its `parameters` field (#1200). +//! - The `Getter::get_func_space_name` impls for C, C++, mozcpp and +//! Objective-C read its name side through [`declarator_name`] (#1208). +//! +//! Keeping one walk keeps the two answers about the same function from +//! disagreeing, which is how #1208 arose: the arity came from this +//! chain and the name came from a leftmost pre-order search that stopped +//! one level too early. The invariant is one *function*, one walk — +//! not one node: an unexpanded function-like macro puts the arity and +//! the name on two links of the chain, which each function's own doc +//! below explains (#1213). + +use crate::checker::Checker; +use crate::node::Node; + +/// The innermost declarator along a C-family function's declarator +/// chain: the node whose `parameters` field holds the function's *own* +/// formal arguments. [`declarator_name`] takes the name from the same +/// node's `declarator` field. +/// +/// C declarator syntax nests outward from the declared name, so a +/// function node's `declarator` field is only the function's parameter +/// list when the return type is plain. Anything the return type +/// contributes — a `*`, a `&`, a parenthesised group — wraps the +/// `function_declarator` that owns the real list, and the outermost +/// `parameters` a chain carries can belong to the *return type* rather +/// than to the function (`int (*f(int a))(int b)` returns a pointer to a +/// one-argument function and itself takes one argument). Taking the +/// innermost is what makes both of those come out right (#1200), and +/// the same node carries the name `f` that a leftmost search misses +/// (#1208). +/// +/// "Innermost" has one exception, and it is the one shape where the +/// grammar's reading and the preprocessor's disagree. An unexpanded +/// function-like macro — `RUN_STATS_METHOD(allocate)(JNIEnv *env, +/// jclass clazz)`, which is what every JNI shim looks like — parses as +/// a `function_declarator` sitting in another one's `declarator` field, +/// so the innermost list is the macro's `(allocate)` and the function's +/// own arguments are discarded. Neither language permits that chain: a +/// function may not return a function type (C11 6.7.6.3p1, C++ +/// `[dcl.fct]`), so a legitimate function returning a function pointer +/// always interposes a `parenthesized_declarator`, and the direct +/// nesting can only be a macro (or an `ERROR`, below). The walk stops +/// at the outer link there and reports the function's arity (#1213). +/// +/// The walk is by field name, per `.claude/rules/grammar-dispatch.md` +/// §3, which also sidesteps §1: `PointerDeclarator2`, +/// `FunctionDeclarator2`/`3` and `ReferenceDeclarator2`/`3`/`4` are +/// numeric-suffix aliases that a `kind_id` match would have to +/// enumerate and would silently regress on the next grammar bump. +/// +/// Three of the rules on the chain expose no field at all, which is why +/// the field alone is not enough. Every entry below is from the pinned +/// grammars' `node-types.json` (`tree-sitter-cpp` 0.23.4, +/// `tree-sitter-c` 0.24.2, `tree-sitter-objc` 3.0.2, vendored +/// `tree-sitter-mozcpp`), and the fieldless list is the complete set of +/// `*_declarator` rules with no fields that a *function definition's* +/// name side can reach — the rest (`variadic_declarator`, +/// `structured_binding_declarator`, Objective-C's `keyword_declarator` +/// and `struct_declarator`, and the `abstract_*` family) sit in +/// parameter, binding or type position, never here. +/// +/// | rule | `declarator` field | +/// | --- | --- | +/// | `pointer_declarator` | required | +/// | `function_declarator` | required | +/// | `abstract_function_declarator` | **optional** | +/// | `reference_declarator` | **absent — no fields** | +/// | `parenthesized_declarator` | **absent — no fields** | +/// | `attributed_declarator` | **absent — no fields** | +/// +/// In all three fieldless rules the inner declarator is the last +/// *named* child once attributes are set aside, so that is the +/// fallback: +/// +/// - `reference_declarator` is `seq(choice('&', '&&'), _declarator)`. +/// - `parenthesized_declarator` is `seq('(', +/// optional(ms_call_modifier), _declarator, ')')` — last rather than +/// sole, so `int (__cdecl *f(int a))(int b)` does not defeat it. +/// - `attributed_declarator` is `seq(_declarator, +/// repeat1(attribute_declaration))`, the one rule that puts the +/// declarator **first**. Excluding `attribute_declaration` — its only +/// non-declarator child type in all four grammars — restores "last" +/// as the right answer, and without that exclusion +/// `int f(int a, int b) [[deprecated]]` reports 0. +/// +/// `template_argument_list` is excluded for the same reason as +/// `attribute_declaration`, and it is the one exclusion the fallback +/// needs beyond the three rules above. The fallback also runs on the +/// *name* forms, which have no `declarator` field either, and two of +/// them — `template_function` and `template_method` — put their argument +/// list last: `void f(int a)`. A type argument +/// spelling a function type carries a `parameters` field of its own, so +/// descending into it made that function read as taking two arguments +/// and made its name resolve to nothing at all, the abstract declarator +/// the chain landed on spelling no identifier. Excluding the argument +/// list leaves the name itself as the last named child, which terminates +/// the chain where it should. +/// +/// Comments are excluded for the same reason, tree-sitter admitting one +/// anywhere. +/// +/// The fallback stops at a node that already carries `parameters`, +/// which is the C++ lambda: `abstract_function_declarator`'s +/// `declarator` field is optional, so `[](int a, int (*cb)(int x))` +/// would otherwise descend into the `parameter_list` and return `cb`'s +/// `(int x)` — one argument instead of two. +/// +/// Every step strictly descends a finite tree, so the walk terminates +/// without a depth cap. +/// +/// # ERROR-recovery trees are outside this contract +/// +/// Every rule above is the grammar's, and none of them holds once +/// tree-sitter starts recovering. An unexpanded macro in declarator +/// position — `T *f() TF_ATTRIBUTE_NOINLINE { … }` — puts the real +/// `function_declarator` inside an `ERROR` node and leaves the macro's +/// `field_identifier` as the `pointer_declarator`'s last named child, +/// so the fallback follows the macro and the walk answers `None`. +/// +/// Give that macro an argument — `T *f() TF_LOCKS_EXCLUDED(mu_) { … }`, +/// which is the spelling the TensorFlow / Abseil annotations actually +/// take — and it is a `function_declarator` carrying `parameters`, so +/// the walk answers with the *macro's* name rather than with nothing. +/// That is the one shape this change made worse: the leftmost pre-order +/// search it replaced descended into the `ERROR` and got `f` right. +/// `a_parenthesised_macro_takes_the_name_of_the_function_it_annotates` +/// pins it. +/// +/// Recovery also manufactures the direct `function_declarator` nesting +/// the macro rule keys on, from source containing no macro-obscured +/// declarator at all. A *statement* macro followed by an `if` — +/// TensorFlow's `TF_ASSIGN_OR_RETURN(bool ok, Try(x)); if (ok) { … }` — +/// recovers into a `function_declarator` whose `declarator` field is the +/// macro call and whose `parameters` field is the `if` **condition**. So +/// the rule changes the answer for 19 of the 46 corpus spaces it +/// touches, from the macro's argument count to the condition's, neither +/// of which is an arity. There is no fixture for it: whether the +/// grammar recovers this way depends on where the line breaks fall, +/// tree-sitter costing a recovery by the extent it skips, so any pinned +/// spelling would be a claim about whitespace (#1213). +/// +/// Whatever any strategy returns there is arbitrary, and the walk does +/// not try to be clever about it. Measured over `DeepSpeech` and +/// `pdf.js` (14,269 files), moving the four getters onto this walk +/// named 46 previously-nameless function spaces, un-named 2 and renamed +/// 4 — 354 nameless spaces down to 310, a net 44. All six of the latter +/// sit inside recovery subtrees: one of the un-named had been reporting +/// an `if` statement's callee as a function name, and one of the renamed +/// is the `TF_LOCKS_EXCLUDED` case above (#1208). +#[must_use] +pub fn innermost_declarator<'tree, T: Checker>(node: &Node<'tree>) -> Option> { + // The chain starts at the `declarator` field rather than at `node` + // so the last-named-child fallback can never fire on the function + // node itself and walk into its body. The walk runs outside-in, so + // the innermost qualifying link is the last one it yields. + std::iter::successors( + node.child_by_field_name("declarator"), + |current| match current.child_by_field_name("declarator") { + // An unexpanded function-like macro standing in for the + // declarator, which is the shape JNI shims take. Neither C + // nor C++ lets a function return a function type (C11 + // 6.7.6.3p1, C++ `[dcl.fct]`), so a `function_declarator` + // directly inside another one's `declarator` field is not a + // declarator chain at all: the outer list is the function's + // own and the inner one holds the macro's arguments. Both + // links have to be tested — a pointer return puts a + // `function_declarator` in a `pointer_declarator`'s + // `declarator` field, and stopping *there* would end the + // chain on a node carrying no `parameters` and report 0 + // (#1213). + Some(inner) + if current.kind() == FUNCTION_DECLARATOR && inner.kind() == FUNCTION_DECLARATOR => + { + None + } + Some(declarator) => Some(declarator), + None if current.child_by_field_name("parameters").is_some() => None, + None => current + .children() + .filter(|child| { + child.is_named() + && !T::is_comment(child) + && !matches!(child.kind(), ATTRIBUTE | TEMPLATE_ARGUMENTS) + }) + .last(), + }, + ) + // A conversion operator's `declarator` field is the type it converts + // *to*, not its name side: `operator int (*)(int x)` takes no + // arguments, and everything from here inward describes that + // function-pointer type. Cutting the chain restores the 0 the + // pre-#1200 code reported by never finding `parameters` at all. + .take_while(|link| link.kind() != CONVERSION_OPERATOR) + .filter(|link| link.child_by_field_name("parameters").is_some()) + .last() +} + +/// The node spelling a C-family function's name. +/// +/// It is the `declarator` field of [`innermost_declarator`], and it is a +/// separate function only so the four `get_func_space_name` impls state +/// that pairing once instead of four times. Each caller still gates the +/// result on its own grammar's identifier kinds: what counts as a name +/// is where C, C++ and Objective-C differ (`destructor_name`, +/// `qualified_identifier`, `operator_name`, `template_function`), and a +/// kind this module accepted on their behalf would be a claim about +/// four grammars made in a module that reads none of them. +/// +/// The macro shape [`innermost_declarator`] stops at is the one place +/// the name and the arity come off different nodes. There that +/// `declarator` field is the macro *invocation* — itself a +/// `function_declarator`, which no getter's identifier gate accepts — so +/// the walk descends through it to the identifier the macro spells. +/// `RUN_STATS_METHOD` is the only name in the source: the real +/// `Java_…_allocate` exists only after `##` pasting, and it is the token +/// a reader greps for. This is why the module doc states the invariant +/// per *function* rather than per node (#1213). +#[must_use] +pub fn declarator_name<'tree, T: Checker>(node: &Node<'tree>) -> Option> { + // A run of them rather than one: `A(b)(c)(int x)` is two nested + // invocations and the name is still `A`. Each step descends a finite + // tree, so this terminates for the same reason the walk above does. + // + // Written as a chain rather than a `while` with `?` inside it + // deliberately. The loop form needs an early return for "this + // `function_declarator` has no `declarator` field", which the + // grammars declare required and only an `ERROR` could violate — two + // arms no test can reach, and coverage counts them. + // + // The two forms are not identical on that unreachable input, which is + // worth stating rather than leaving for the next reader to rediscover + // (#1220). On an `ERROR` tree where a `function_declarator` lacks its + // `declarator` field, the loop returned `None` and this chain yields + // that `function_declarator` itself — the whole declarator span in + // place of a name. + // + // Nothing observes the difference, and the reason is external to this + // module: every caller gates the result on its own grammar's + // identifier kinds (`TypeIdentifier | Identifier | FieldIdentifier`, + // plus the C++ name forms in `getter/cpp.rs` and `getter/mozcpp.rs`), + // and `function_declarator` is in none of those lists. The `matches!` + // falls through and `get_func_space_name` returns `None` — the same + // answer the loop gave. That dependency is the thing to preserve: a + // getter that widened its gate to accept `function_declarator` would + // start naming functions after their whole declarator span on + // malformed input, and this comment is the only place that says so. + std::iter::successors( + innermost_declarator::(node)?.child_by_field_name("declarator"), + |link| { + if link.kind() == FUNCTION_DECLARATOR { + link.child_by_field_name("declarator") + } else { + None + } + }, + ) + .last() +} + +/// Compared by `kind()` string rather than `kind_id`, per +/// `.claude/rules/grammar-dispatch.md` §1: every rule below carries +/// numeric-suffix aliases across the four C-family grammars, and a +/// `kind_id` match would have to enumerate every one of them and would +/// regress silently on the next grammar bump. C and Objective-C simply +/// never emit `operator_cast`. +const CONVERSION_OPERATOR: &str = "operator_cast"; +const ATTRIBUTE: &str = "attribute_declaration"; +const TEMPLATE_ARGUMENTS: &str = "template_argument_list"; +/// Carries the most aliases of the four — `FunctionDeclarator2` through +/// `FunctionDeclarator5` in `tree-sitter-c` alone — so it is the one the +/// `kind()`-string rule above most needs to cover. +const FUNCTION_DECLARATOR: &str = "function_declarator"; diff --git a/src/c_langs_macros/c_macros.rs b/big-code-analysis-ast/src/c_langs_macros/c_macros.rs similarity index 100% rename from src/c_langs_macros/c_macros.rs rename to big-code-analysis-ast/src/c_langs_macros/c_macros.rs diff --git a/src/c_langs_macros/c_specials.rs b/big-code-analysis-ast/src/c_langs_macros/c_specials.rs similarity index 100% rename from src/c_langs_macros/c_specials.rs rename to big-code-analysis-ast/src/c_langs_macros/c_specials.rs diff --git a/src/c_langs_macros/mod.rs b/big-code-analysis-ast/src/c_langs_macros/mod.rs similarity index 96% rename from src/c_langs_macros/mod.rs rename to big-code-analysis-ast/src/c_langs_macros/mod.rs index 3290fecb3..850e9aecd 100644 --- a/src/c_langs_macros/mod.rs +++ b/big-code-analysis-ast/src/c_langs_macros/mod.rs @@ -1,3 +1,6 @@ +//! Generated tables of predefined and special C-family macro names +//! (`enums -l c_macros`). + mod c_macros; pub(crate) use c_macros::*; @@ -103,7 +106,7 @@ mod tests { let root = parser.root(); if debug || root.has_error() { eprintln!("Sample (MOZCPP) {n}: {sample}"); - dump_node(&v_sample, &root, -1, None, None).unwrap(); + eprintln!("{}", root.as_tree_sitter().to_sexp()); } assert!(!root.has_error()); } diff --git a/src/c_macro.rs b/big-code-analysis-ast/src/c_macro.rs similarity index 99% rename from src/c_macro.rs rename to big-code-analysis-ast/src/c_macro.rs index 0f8f64dcc..bfd124b29 100644 --- a/src/c_macro.rs +++ b/big-code-analysis-ast/src/c_macro.rs @@ -14,20 +14,22 @@ // that is the #126 shape and the one a reader cannot check locally. #![warn(clippy::indexing_slicing)] +//! C-family `#define` masking applied to the source before parsing. + use std::borrow::Borrow; use std::collections::HashSet; use std::hash::Hash; use crate::c_langs_macros::is_predefined_macros; -/// The macro-name oracle [`replace`] masks against. +/// The macro-name oracle `replace` masks against. /// /// Generic over the set's element type so the crate can pass a /// `HashSet<&str>` borrowed straight out of the preprocessor results /// (`preproc::visible_macros`) while the published /// `preproc::get_macros` shape (`HashSet`) keeps working /// unchanged. Only `contains` is ever asked of it. -pub(crate) trait MacroName: Borrow + Eq + Hash {} +pub trait MacroName: Borrow + Eq + Hash {} impl + Eq + Hash> MacroName for T {} diff --git a/src/cfg_predicate.rs b/big-code-analysis-ast/src/cfg_predicate.rs similarity index 99% rename from src/cfg_predicate.rs rename to big-code-analysis-ast/src/cfg_predicate.rs index 6efb8bc46..c37e6638b 100644 --- a/src/cfg_predicate.rs +++ b/big-code-analysis-ast/src/cfg_predicate.rs @@ -30,7 +30,8 @@ use std::{iter, ops::Range}; /// /// The slow path collapses interior whitespace and retries, tolerating /// unusual spacing like `# [ cfg ( test ) ]`. -pub(crate) fn attribute_marks_test(body: &str) -> bool { +#[must_use] +pub fn attribute_marks_test(body: &str) -> bool { let matches_test = |s: &str| { matches!(s, "test" | "rstest" | "wasm_bindgen_test" | "test_case") || s.ends_with("::test") diff --git a/src/checker.rs b/big-code-analysis-ast/src/checker.rs similarity index 98% rename from src/checker.rs rename to big-code-analysis-ast/src/checker.rs index 57ca04eee..6797bd0b1 100644 --- a/src/checker.rs +++ b/big-code-analysis-ast/src/checker.rs @@ -6,6 +6,8 @@ // function so the per-language impl blocks stay readable. #![allow(clippy::wildcard_imports, clippy::enum_glob_use)] +//! The [`Checker`] classification predicates and their per-language impls. + use std::sync::OnceLock; use aho_corasick::AhoCorasick; @@ -313,9 +315,15 @@ fn get_aho_corasick_match(code: &[u8]) -> bool { /// so adding a language no longer requires copy-pasting `-> false` stubs. /// The flip side is that the compiler cannot flag a forgotten override, so /// each language's per-metric tests are the safety net for completeness. +/// +/// `#[doc(hidden)]` for the reason the crate root gives: this is +/// plumbing, and the surface the README points a reader at is `LANG` / +/// `Node` / `SpaceKind`, not a 200-method classifier trait. #[doc(hidden)] -pub(crate) trait Checker { +pub trait Checker { + /// Whether `node` is a comment. #[inline] + #[must_use] fn is_comment(_: &Node) -> bool { false } @@ -326,10 +334,14 @@ pub(crate) trait Checker { /// a macro token; reaching it with [`Node::parent`] costs /// `O(depth)` per comment (#1096). #[inline] + #[must_use] fn is_useful_comment<'a>(_: &Node<'a>, _: &[u8], _: Ancestors<'a, '_>) -> bool { false } + /// Whether `node` opens a space: a function, closure, or one of the + /// containers (class, struct, trait, impl, namespace, interface). #[inline] + #[must_use] fn is_func_space(_: &Node) -> bool { false } @@ -344,23 +356,30 @@ pub(crate) trait Checker { /// Every other grammar answers `is_func` from the node's own kind /// and ignores the parameter. #[inline] + #[must_use] fn is_func<'a>(_: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> bool { false } /// Whether `node` is an anonymous function — the complement of - /// [`is_func`] on the grammars that express both. Takes `ancestors` + /// [`is_func`](Self::is_func) on the grammars that express both. Takes `ancestors` /// for the same reason, plus one of its own: Ruby's `{ … }` block is /// a closure only when its parent is not the `Lambda` that already /// counted it. #[inline] + #[must_use] fn is_closure<'a>(_: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> bool { false } + /// Whether `node` is a call expression. #[inline] + #[must_use] fn is_call(_: &Node) -> bool { false } + /// Whether `node`, met among a parameter list's children, is + /// punctuation or a keyword rather than a parameter (`nargs` skips it). #[inline] + #[must_use] fn is_non_arg(_: &Node) -> bool { false } @@ -369,7 +388,7 @@ pub(crate) trait Checker { /// /// C's `int f(void)` declares zero parameters, but the grammar /// emits a real `parameter_declaration` for the `void`, so the - /// negative filters in [`crate::nargs`] counted it as one — `f` and + /// negative filters in `big_code_analysis::nargs` counted it as one — `f` and /// `int f(int)` both reported 1 despite one of them taking no /// argument at all. /// @@ -379,6 +398,7 @@ pub(crate) trait Checker { /// `void f(int)` really does take one. Only the bytes tell them /// apart — `.claude/rules/grammar-dispatch.md` §10. #[inline] + #[must_use] fn is_empty_param_marker(_param: &Node, _code: &[u8]) -> bool { false } @@ -399,10 +419,14 @@ pub(crate) trait Checker { /// so a MISSING or zero-width ERROR node under `parameters` on a /// broken parse cannot silently conjure an argument. #[inline] + #[must_use] fn is_bare_param(_: &Node) -> bool { false } + /// Whether `node` is a string literal, under every aliased kind the + /// grammar emits for one. #[inline] + #[must_use] fn is_string(_: &Node) -> bool { false } @@ -415,14 +439,20 @@ pub(crate) trait Checker { /// either from the node alone costs `O(depth)` (#1084). Languages /// with a dedicated `elsif` clause node ignore it. #[inline] + #[must_use] fn is_else_if(_: &Node, _: Ancestors<'_, '_>) -> bool { false } + /// Whether `node` is a primitive type name (`int`, `double`, …), + /// which Halstead keys by text so each distinct primitive counts once. #[inline] + #[must_use] fn is_primitive(_node: &Node) -> bool { false } + /// Whether `node` is, or contains, a tree-sitter `ERROR` node. + #[must_use] fn is_error(node: &Node) -> bool { node.has_error() } @@ -441,6 +471,7 @@ pub(crate) trait Checker { /// before an item, and resolving siblings from the node alone /// costs `O(depth)` per step (#1100). #[inline] + #[must_use] fn should_skip_subtree<'a>( _node: &Node<'a>, _code: &[u8], @@ -449,7 +480,7 @@ pub(crate) trait Checker { false } - /// Source-aware variant of [`is_func_space`]. The default forwards + /// Source-aware variant of [`is_func_space`](Self::is_func_space). The default forwards /// to the byte-less predicate so languages whose function-space /// classification is encoded in distinct grammar productions (Java, /// Rust, Python, …) need no override. Languages whose function @@ -463,6 +494,7 @@ pub(crate) trait Checker { /// (Elixir's `quote` template) can answer without paying /// `Node::parent`'s `O(depth)` per step (#1084). #[inline] + #[must_use] fn is_func_space_with_code<'a>( node: &Node<'a>, _code: &[u8], @@ -471,9 +503,10 @@ pub(crate) trait Checker { Self::is_func_space(node) } - /// Source-aware variant of [`is_func`]. Same rationale as - /// [`is_func_space_with_code`] (#275). + /// Source-aware variant of [`is_func`](Self::is_func). Same rationale as + /// [`is_func_space_with_code`](Self::is_func_space_with_code) (#275). #[inline] + #[must_use] fn is_func_with_code<'a>(node: &Node<'a>, _code: &[u8], ancestors: Ancestors<'a, '_>) -> bool { Self::is_func(node, ancestors) } @@ -489,6 +522,7 @@ pub(crate) trait Checker { /// `elixir_call_keyword` call answers both halves at once /// (#310 follow-on perf). #[inline] + #[must_use] fn promotes_to_func_space_with_code<'a>( node: &Node<'a>, code: &[u8], @@ -531,6 +565,7 @@ pub(crate) trait Checker { /// without an override (`.claude/rules/grammar-dispatch.md` §7). /// One spelling has nothing to disagree with. #[inline] + #[must_use] fn is_non_member_function<'a>( _node: &Node<'a>, _code: &[u8], @@ -560,7 +595,7 @@ mod perl; mod php; mod preproc; mod python; -pub(crate) mod ruby; +pub mod ruby; mod rust; mod tcl; mod tsx; @@ -573,7 +608,8 @@ mod typescript; /// on exactly which `object_creation_expression` nodes open a Class space /// (#463); a lambda is a distinct `lambda_expression` node and never /// reaches this path. -pub(crate) fn java_anonymous_class_body<'a>(node: &Node<'a>) -> Option> { +#[must_use] +pub fn java_anonymous_class_body<'a>(node: &Node<'a>) -> Option> { node.first_child(|id| id == Java::ClassBody as u16) } @@ -585,7 +621,8 @@ pub(crate) fn java_anonymous_class_body<'a>(node: &Node<'a>) -> Option> /// Shared by `csharp_member_has_accessors` (here) and the npm reference /// `csharp_count_member`, which keeps its own `.max(1)` fallback so an /// accessor-less expression-bodied member still counts as one method (#464). -pub(crate) fn csharp_accessor_count(node: &Node) -> usize { +#[must_use] +pub fn csharp_accessor_count(node: &Node) -> usize { node.children() .filter(|c| c.kind_id() == Csharp::AccessorList as u16) .flat_map(|list| list.children()) @@ -605,7 +642,8 @@ pub(crate) fn csharp_accessor_count(node: &Node) -> usize { /// children. This mirrors the npm reference (`csharp_count_member`): the /// member counts as its accessor count, falling back to 1 for the /// accessor-less expression-bodied form (#464 indexer, #472 property). -pub(crate) fn csharp_member_has_accessors(node: &Node) -> bool { +#[must_use] +pub fn csharp_member_has_accessors(node: &Node) -> bool { csharp_accessor_count(node) > 0 } @@ -620,7 +658,8 @@ pub(crate) fn csharp_member_has_accessors(node: &Node) -> bool { /// comparison is what separates it from an unnamed parameter of any /// other type: `f(int)` and `f(void)` are the same shape and different /// arities. -pub(crate) fn c_family_void_parameter(param: &Node, code: &[u8]) -> bool { +#[must_use] +pub fn c_family_void_parameter(param: &Node, code: &[u8]) -> bool { param.child_by_field_name("declarator").is_none() && param .child_by_field_name("type") diff --git a/src/checker/bash.rs b/big-code-analysis-ast/src/checker/bash.rs similarity index 100% rename from src/checker/bash.rs rename to big-code-analysis-ast/src/checker/bash.rs diff --git a/src/checker/c.rs b/big-code-analysis-ast/src/checker/c.rs similarity index 100% rename from src/checker/c.rs rename to big-code-analysis-ast/src/checker/c.rs diff --git a/src/checker/ccomment.rs b/big-code-analysis-ast/src/checker/ccomment.rs similarity index 100% rename from src/checker/ccomment.rs rename to big-code-analysis-ast/src/checker/ccomment.rs diff --git a/src/checker/cpp.rs b/big-code-analysis-ast/src/checker/cpp.rs similarity index 100% rename from src/checker/cpp.rs rename to big-code-analysis-ast/src/checker/cpp.rs diff --git a/src/checker/csharp.rs b/big-code-analysis-ast/src/checker/csharp.rs similarity index 100% rename from src/checker/csharp.rs rename to big-code-analysis-ast/src/checker/csharp.rs diff --git a/src/checker/elixir.rs b/big-code-analysis-ast/src/checker/elixir.rs similarity index 100% rename from src/checker/elixir.rs rename to big-code-analysis-ast/src/checker/elixir.rs diff --git a/src/checker/go.rs b/big-code-analysis-ast/src/checker/go.rs similarity index 100% rename from src/checker/go.rs rename to big-code-analysis-ast/src/checker/go.rs diff --git a/src/checker/groovy.rs b/big-code-analysis-ast/src/checker/groovy.rs similarity index 100% rename from src/checker/groovy.rs rename to big-code-analysis-ast/src/checker/groovy.rs diff --git a/src/checker/irules.rs b/big-code-analysis-ast/src/checker/irules.rs similarity index 100% rename from src/checker/irules.rs rename to big-code-analysis-ast/src/checker/irules.rs diff --git a/src/checker/java.rs b/big-code-analysis-ast/src/checker/java.rs similarity index 100% rename from src/checker/java.rs rename to big-code-analysis-ast/src/checker/java.rs diff --git a/src/checker/javascript.rs b/big-code-analysis-ast/src/checker/javascript.rs similarity index 100% rename from src/checker/javascript.rs rename to big-code-analysis-ast/src/checker/javascript.rs diff --git a/src/checker/kotlin.rs b/big-code-analysis-ast/src/checker/kotlin.rs similarity index 100% rename from src/checker/kotlin.rs rename to big-code-analysis-ast/src/checker/kotlin.rs diff --git a/src/checker/lua.rs b/big-code-analysis-ast/src/checker/lua.rs similarity index 100% rename from src/checker/lua.rs rename to big-code-analysis-ast/src/checker/lua.rs diff --git a/src/checker/mozcpp.rs b/big-code-analysis-ast/src/checker/mozcpp.rs similarity index 100% rename from src/checker/mozcpp.rs rename to big-code-analysis-ast/src/checker/mozcpp.rs diff --git a/src/checker/mozjs.rs b/big-code-analysis-ast/src/checker/mozjs.rs similarity index 100% rename from src/checker/mozjs.rs rename to big-code-analysis-ast/src/checker/mozjs.rs diff --git a/src/checker/objc.rs b/big-code-analysis-ast/src/checker/objc.rs similarity index 100% rename from src/checker/objc.rs rename to big-code-analysis-ast/src/checker/objc.rs diff --git a/src/checker/perl.rs b/big-code-analysis-ast/src/checker/perl.rs similarity index 100% rename from src/checker/perl.rs rename to big-code-analysis-ast/src/checker/perl.rs diff --git a/src/checker/php.rs b/big-code-analysis-ast/src/checker/php.rs similarity index 100% rename from src/checker/php.rs rename to big-code-analysis-ast/src/checker/php.rs diff --git a/src/checker/preproc.rs b/big-code-analysis-ast/src/checker/preproc.rs similarity index 100% rename from src/checker/preproc.rs rename to big-code-analysis-ast/src/checker/preproc.rs diff --git a/src/checker/python.rs b/big-code-analysis-ast/src/checker/python.rs similarity index 100% rename from src/checker/python.rs rename to big-code-analysis-ast/src/checker/python.rs diff --git a/src/checker/ruby.rs b/big-code-analysis-ast/src/checker/ruby.rs similarity index 97% rename from src/checker/ruby.rs rename to big-code-analysis-ast/src/checker/ruby.rs index b33f82b40..d8940a57e 100644 --- a/src/checker/ruby.rs +++ b/big-code-analysis-ast/src/checker/ruby.rs @@ -22,7 +22,8 @@ use super::*; /// The parent comes off the caller's chain: all consumers run per node /// from a walk, and `Node::parent` costs `O(depth)` because /// `tree_sitter` resolves it by descending from the root (#1088). -pub(crate) fn is_stabby_lambda_body<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { +#[must_use] +pub fn is_stabby_lambda_body<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> bool { ancestors .parent(node) .is_some_and(|parent| parent.kind_id() == Ruby::Lambda) diff --git a/src/checker/rust.rs b/big-code-analysis-ast/src/checker/rust.rs similarity index 100% rename from src/checker/rust.rs rename to big-code-analysis-ast/src/checker/rust.rs diff --git a/src/checker/tcl.rs b/big-code-analysis-ast/src/checker/tcl.rs similarity index 100% rename from src/checker/tcl.rs rename to big-code-analysis-ast/src/checker/tcl.rs diff --git a/src/checker/tsx.rs b/big-code-analysis-ast/src/checker/tsx.rs similarity index 100% rename from src/checker/tsx.rs rename to big-code-analysis-ast/src/checker/tsx.rs diff --git a/src/checker/typescript.rs b/big-code-analysis-ast/src/checker/typescript.rs similarity index 100% rename from src/checker/typescript.rs rename to big-code-analysis-ast/src/checker/typescript.rs diff --git a/src/comment_rm.rs b/big-code-analysis-ast/src/comment_rm.rs similarity index 98% rename from src/comment_rm.rs rename to big-code-analysis-ast/src/comment_rm.rs index f77b51634..6d14a1f3f 100644 --- a/src/comment_rm.rs +++ b/big-code-analysis-ast/src/comment_rm.rs @@ -6,6 +6,8 @@ // function so the per-language impl blocks stay readable. #![allow(clippy::enum_glob_use, clippy::if_not_else, clippy::wildcard_imports)] +//! Comment stripping over a parsed tree. + use crate::checker::Checker; use crate::node::{Ancestors, Node}; use crate::traits::ParserTrait; @@ -83,8 +85,8 @@ impl LineEnding { } /// Removes comments from a code. Crate-internal walk core reached -/// through the [`crate::Ast::strip_comments`] seam. -pub(crate) fn rm_comments(parser: &T) -> Option> { +/// through the `big_code_analysis::Ast::strip_comments` seam. +pub fn rm_comments(parser: &T) -> Option> { let node = parser.root(); let mut stack = Vec::new(); let mut cursor = node.cursor(); diff --git a/src/count.rs b/big-code-analysis-ast/src/count.rs similarity index 97% rename from src/count.rs rename to big-code-analysis-ast/src/count.rs index 0a3a5fea3..914057542 100644 --- a/src/count.rs +++ b/big-code-analysis-ast/src/count.rs @@ -17,6 +17,8 @@ // function so the per-language impl blocks stay readable. #![allow(clippy::wildcard_imports, clippy::enum_glob_use)] +//! Node counting by kind or category, and the shared [`CountCollector`]. + use num_format::{Locale, ToFormattedString}; use std::fmt; use std::sync::{Arc, Mutex}; @@ -25,8 +27,8 @@ use crate::traits::ParserTrait; /// Counts the types of nodes specified in the input slice and the /// number of nodes in a code. Crate-internal walk core reached through -/// the [`crate::Ast::count`] seam. -pub(crate) fn count(parser: &T, filters: &[String]) -> (usize, usize) { +/// the `big_code_analysis::Ast::count` seam. +pub fn count(parser: &T, filters: &[String]) -> (usize, usize) { let filters = parser.filters(filters); let node = parser.root(); let mut cursor = node.cursor(); @@ -50,14 +52,14 @@ pub(crate) fn count(parser: &T, filters: &[String]) -> (usize, u } /// Opaque, shareable collector that accumulates a [`Count`] across the -/// worker threads of a [`crate::ConcurrentRunner`] walk. +/// worker threads of a `big_code_analysis::ConcurrentRunner` walk. /// /// Wraps the shared `Arc>` behind a newtype so callers do /// not handle the synchronization machinery directly. [`Clone`] is a /// cheap reference-count bump, so each worker /// can hold its own handle to the same tally while the config still /// satisfies the `'static + Send + Sync` bound of -/// [`crate::ConcurrentRunner`]. Recover the final tally with +/// `big_code_analysis::ConcurrentRunner`. Recover the final tally with /// [`CountCollector::into_count`] once every worker has joined. #[derive(Debug, Clone)] pub struct CountCollector(Arc>); diff --git a/src/error.rs b/big-code-analysis-ast/src/error.rs similarity index 81% rename from src/error.rs rename to big-code-analysis-ast/src/error.rs index 7049a9400..473288841 100644 --- a/src/error.rs +++ b/big-code-analysis-ast/src/error.rs @@ -1,6 +1,6 @@ //! Error type returned from the library's top-level entry points. //! -//! [`FromPathError`](crate::FromPathError), the richer error of the +//! `big_code_analysis::FromPathError`, the richer error of the //! file-backed `Ast::from_path`, lives in the root crate's //! `from_path_error` module; this one is the parse-layer error every //! dispatch entry point returns. @@ -38,22 +38,24 @@ use crate::LANG; /// produced today: every dispatch entry point emits it when the /// caller selects a /// [`LANG`] whose per-language Cargo feature is not enabled in the -/// current build (see #252). The example exercises the happy path -/// and demonstrates the exhaustive-with-`_` match shape that callers -/// should adopt to stay forward-compatible with future variants. +/// current build (see #252). The example demonstrates the +/// exhaustive-with-`_` match shape that callers should adopt to stay +/// forward-compatible with future variants. /// -/// ``` -/// use big_code_analysis::{analyze, MetricsError, MetricsOptions, Source, LANG}; +/// [`AnyParser::parse`](crate::langs::AnyParser::parse) stands in for +/// `big_code_analysis::analyze` here — it is the same dispatch, one +/// layer down, and it is reachable from this crate alone. The match +/// arms are the ones an `analyze` caller writes. /// -/// let source = Source::new(LANG::Cpp, b"int a = 42;"); -/// let result = analyze(source, MetricsOptions::default()); +/// ``` +/// use big_code_analysis_ast::{AnyParser, MetricsError, LANG}; /// -/// // Today this call succeeds; the match below documents the shape -/// // callers must adopt so adding a future variant is non-breaking. -/// assert!(result.is_ok()); +/// let result = AnyParser::parse(LANG::Cpp, b"int a = 42;".to_vec(), None, None); /// /// match result { -/// Ok(_space) => {} +/// Ok(_parser) => { +/// // The `cpp` feature is enabled in this build. +/// } /// Err(MetricsError::EmptyRoot) => { /// // Reserved: walker produced no top-level FuncSpace. /// } @@ -69,7 +71,7 @@ use crate::LANG; #[non_exhaustive] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MetricsError { - /// The walker produced no top-level [`FuncSpace`][crate::FuncSpace]. + /// The walker produced no top-level `big_code_analysis::FuncSpace`. /// /// Reserved — not produced today. `metrics_with_options` always /// pushes a synthetic top-level [`SpaceKind::Unit`][crate::SpaceKind] @@ -81,13 +83,12 @@ pub enum MetricsError { /// drain to empty (e.g. an option that suppresses the synthetic /// root for sources with no parseable structure). /// - /// [`FuncSpace`]: crate::FuncSpace EmptyRoot, /// The requested [`LANG`] is not enabled in this build. /// /// Produced by every dispatch entry point - /// ([`crate::analyze`], [`crate::Ast::parse`], - /// [`crate::Ast::from_tree_sitter`], and + /// (`big_code_analysis::analyze`, `big_code_analysis::Ast::parse`, + /// `big_code_analysis::Ast::from_tree_sitter`, and /// [`crate::LANG::tree_sitter_language`]) /// when the caller selects a [`LANG`] variant whose per-language /// Cargo feature is not enabled in the current build — see the diff --git a/src/find.rs b/big-code-analysis-ast/src/find.rs similarity index 93% rename from src/find.rs rename to big-code-analysis-ast/src/find.rs index 83ee9b7be..4c4b7b359 100644 --- a/src/find.rs +++ b/big-code-analysis-ast/src/find.rs @@ -6,13 +6,15 @@ // function so the per-language impl blocks stay readable. #![allow(clippy::wildcard_imports, clippy::enum_glob_use)] +//! Node finding by kind or category. + use crate::node::Node; use crate::error::MetricsError; use crate::traits::ParserTrait; /// Finds the types of nodes specified in the input slice. Crate-internal -/// walk core reached through the [`crate::Ast::find`] seam. +/// walk core reached through the `big_code_analysis::Ast::find` seam. /// /// "No matches" is represented by `Ok(Vec::new())` rather than an /// error — it is a normal outcome, not a failure mode. The @@ -30,7 +32,7 @@ use crate::traits::ParserTrait; // `unnecessary_wraps` would have us drop it and break that uniform // `?`-able shape across the walk cores. #[allow(clippy::unnecessary_wraps)] -pub(crate) fn find<'a, T: ParserTrait>( +pub fn find<'a, T: ParserTrait>( parser: &'a T, filters: &[String], ) -> Result>, MetricsError> { diff --git a/src/getter.rs b/big-code-analysis-ast/src/getter.rs similarity index 95% rename from src/getter.rs rename to big-code-analysis-ast/src/getter.rs index d873527fd..f2443e713 100644 --- a/src/getter.rs +++ b/big-code-analysis-ast/src/getter.rs @@ -6,7 +6,10 @@ // function so the per-language impl blocks stay readable. #![allow(clippy::wildcard_imports, clippy::enum_glob_use)] -use crate::halstead_type::HalsteadType; +//! The [`Getter`] accessors — names, space kinds, Halstead classes — and +//! their per-language impls. + +use crate::token_role::TokenRole; use crate::space_kind::SpaceKind; use crate::traits::Search; @@ -19,12 +22,12 @@ use crate::*; /// precondition; the two guards here cover two unrelated failure modes: /// /// * `std::str::from_utf8` rejects non-UTF-8 bytes. This one is -/// reachable from ordinary use — [`crate::Ast::parse`] accepts +/// reachable from ordinary use — `big_code_analysis::Ast::parse` accepts /// arbitrary bytes, so a node span in a partially-binary source need /// not be valid UTF-8. /// * `code.get` bounds-checks the range. This one is reachable only by /// violating the same-parse precondition documented on `Getter`, i.e. -/// [`crate::Ast::from_tree_sitter`] adopting a tree built from longer +/// `big_code_analysis::Ast::from_tree_sitter` adopting a tree built from longer /// source than the `code` passed alongside it. /// /// Both degrade to `None`. The walker stores a space's name as @@ -131,7 +134,7 @@ macro_rules! impl_js_family_get_op_type { operand_extras: [$($operand_extra:ident),* $(,)?] $(, predefined_void: $predefined_type:ident)? $(,)? ) => { - fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use $lang::*; // TS/TSX only: a `void` return / parameter type is parsed as a @@ -154,7 +157,7 @@ macro_rules! impl_js_family_get_op_type { .child(0) .is_some_and(|child| child.kind_id() == Void as u16) { - return HalsteadType::Unknown; + return TokenRole::Unknown; } )? @@ -189,7 +192,7 @@ macro_rules! impl_js_family_get_op_type { SLASH if ancestors.parent_has_kind(node, Regex as u16) => { - HalsteadType::Unknown + TokenRole::Unknown } Export | Import | Import2 | Extends | DOT | From | LPAREN | COMMA | As | STAR | GTGT | GTGTGT | COLON | Return | Delete | Throw | Break | Continue | If @@ -208,7 +211,7 @@ macro_rules! impl_js_family_get_op_type { // groups across languages, skewing n1/n2 for accessor-heavy // code (#695). | Set | Get - $(| $op_extra)* => HalsteadType::Operator, + $(| $op_extra)* => TokenRole::Operator, // `Regex` is the literal's own node and contributes one // operand, the way Ruby's `Regex` and Elixir's `Sigil` // do. It was in neither arm before #1314, so `/abc/g` @@ -236,7 +239,7 @@ macro_rules! impl_js_family_get_op_type { // still yields three. Identifier | PropertyIdentifier | PrivatePropertyIdentifier | MetaProperty | String | Number | True | False | Null | This | Super | Undefined | Regex - $(| $operand_extra)* => HalsteadType::Operand, + $(| $operand_extra)* => TokenRole::Operand, // A `` `...` `` is a string literal; without interpolation it // mirrors `"..."` and contributes one operand. When it has a // `TemplateSubstitution` child the inner expression is already @@ -246,7 +249,7 @@ macro_rules! impl_js_family_get_op_type { TemplateString => { Self::string_operand_type(node, &[TemplateSubstitution as u16]) } - _ => HalsteadType::Unknown, + _ => TokenRole::Unknown, } } }; @@ -259,7 +262,8 @@ macro_rules! impl_js_family_get_op_type { /// restate the rule — `` / `` / `` / `` in /// Kotlin, Java and Groovy all do (#1184). Calling `Self::…` there would /// recurse. -pub(crate) fn default_func_space_name<'a, 'tree>( +#[must_use] +pub fn default_func_space_name<'a, 'tree>( node: &Node<'tree>, code: &'a [u8], _ancestors: Ancestors<'tree, '_>, @@ -279,8 +283,8 @@ pub(crate) fn default_func_space_name<'a, 'tree>( /// /// Every method taking a `code: &[u8]` next to a `&Node` slices `code` /// by that node's byte range. `code` must be the exact buffer `node` was -/// parsed from — [`crate::Ast::source`] for a node obtained from the -/// same [`crate::Ast`]. Pairing a node with any other buffer reads the +/// parsed from — `big_code_analysis::Ast::source` for a node obtained from the +/// same `big_code_analysis::Ast`. Pairing a node with any other buffer reads the /// wrong bytes at best and panics on an out-of-bounds index at worst; /// the same precondition is documented on [`crate::dump_node`] (#795). /// `node_text` — reached from the default `get_func_space_name` and @@ -294,36 +298,36 @@ pub(crate) fn default_func_space_name<'a, 'tree>( /// parameters because the ids are same-typed and positional: a /// transposed pair compiles and silently inverts the rule for that /// dialect. -pub(crate) struct BracedWordKinds { +pub struct BracedWordKinds { /// `braced_word_simple`, the literal *value* form. - pub(crate) value: u16, + pub value: u16, /// `braced_word`, the *script* form. - pub(crate) script: u16, + pub script: u16, /// `comment`, the one named child of a script that is not a /// command. - pub(crate) comment: u16, + pub comment: u16, /// `command`, the generic command node. Its `name` field carries /// the leading word [`Getter::is_value_braced_word`] recognises the /// construct by, and it is the only parent a *generic* argument /// list hangs from. The hidden `_command` supertype (`Command2` in /// both enums) is deliberately absent: the parser never emits it /// (grammar-dispatch §2). - pub(crate) command: u16, + pub command: u16, /// `word_list`, a command's `arguments` field — and, in both /// grammars, also the argument list of the modelled `namespace` /// construct, which is why the rule reads the grandparent rather /// than stopping here. - pub(crate) word_list: u16, + pub word_list: u16, /// `simple_word`, the only spelling of a command name this rule /// resolves. A computed name (`$cmd {…}`, `[pick] {…}`) parses as /// `variable_substitution` / `command_substitution` and is not /// statically resolvable at all. - pub(crate) simple_word: u16, + pub simple_word: u16, /// `argument`, one entry of a `proc` parameter list. A modelled /// slot that holds a *value* rather than a script: a defaulted /// parameter (`proc p {a {b {x y}}}`) spells its default as a /// `braced_word`, and a default is never evaluated as code. - pub(crate) argument: u16, + pub argument: u16, /// `{`, the brace opener — the *only* node /// [`Getter::braced_word_op_type`] may revise. A braced word's /// operator children are not the opener alone: `_terminator` is a @@ -333,7 +337,7 @@ pub(crate) struct BracedWordKinds { /// therefore swallowed those separators too, taking /// `lappend x {puts a ; puts b}` to `n1` 0 / `N1` 0 and /// `halstead.effort` — a gated threshold metric — to `0.0`. - pub(crate) open_brace: u16, + pub open_brace: u16, } /// The core Tcl-family commands that evaluate a braced argument as a @@ -405,7 +409,17 @@ const SCRIPT_TAKING_COMMANDS: [&str; 8] = [ /// ([`Getter::is_switch_arm`]). Spelling it twice would let one drift. const SWITCH_COMMAND: &str = "switch"; -pub(crate) trait Getter { +/// Per-language accessors that *name* and *classify* what a node is: +/// the function or space name, the [`SpaceKind`] a node opens, and the +/// [`TokenRole`] of a leaf. +/// +/// Every method has a default that answers "nothing" (`None`, +/// [`SpaceKind::Unknown`], [`TokenRole::Unknown`]); a language +/// overrides only the ones its grammar expresses. +pub trait Getter { + /// Names the function `node` declares. Defaults to + /// [`get_func_space_name`](Self::get_func_space_name). + #[must_use] fn get_func_name<'a, 'tree>( node: &Node<'tree>, code: &'a [u8], @@ -420,6 +434,7 @@ pub(crate) trait Getter { /// needs it: its `def` / `defmodule` heads are ordinary `Call` /// nodes, and one inside a `quote` template names no space at all, /// which is a question about what encloses the call (#1088). + #[must_use] fn get_func_space_name<'a, 'tree>( node: &Node<'tree>, code: &'a [u8], @@ -428,11 +443,14 @@ pub(crate) trait Getter { default_func_space_name(node, code, ancestors) } + /// The kind of space `node` opens, or [`SpaceKind::Unknown`] when it + /// opens none. + #[must_use] fn get_space_kind(_node: &Node) -> SpaceKind { SpaceKind::Unknown } - /// Source-aware variant of [`get_space_kind`]. The default + /// Source-aware variant of [`get_space_kind`](Self::get_space_kind). The default /// forwards to the byte-less classifier; languages whose space /// kinds are encoded in macro identifier text (Elixir's /// `defmodule` / `def` / `defp` / `defmacro` / `defmacrop` Calls) @@ -443,6 +461,7 @@ pub(crate) trait Getter { /// needs it to see whether the `Call` sits inside a `quote` /// template without paying `Node::parent`'s `O(depth)` (#1084). #[inline] + #[must_use] fn get_space_kind_with_code<'a>( node: &Node<'a>, _code: &[u8], @@ -461,8 +480,9 @@ pub(crate) trait Getter { /// namespace identifier in both C++ grammars, Bash's `$name`, and /// iRules' `$var`. Reaching those parents with [`Node::parent`] /// instead costs `O(depth)` per node (#1096). - fn get_op_type<'a>(_node: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> HalsteadType { - HalsteadType::Unknown + #[must_use] + fn get_op_type<'a>(_node: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> TokenRole { + TokenRole::Unknown } /// Source-aware variant of [`get_op_type`]. The default forwards @@ -476,11 +496,12 @@ pub(crate) trait Getter { /// /// [`get_op_type`]: Self::get_op_type #[inline] + #[must_use] fn get_op_type_with_code<'a>( node: &Node<'a>, _code: &[u8], ancestors: Ancestors<'a, '_>, - ) -> HalsteadType { + ) -> TokenRole { Self::get_op_type(node, ancestors) } @@ -493,6 +514,7 @@ pub(crate) trait Getter { /// distinct `"a "` operand and break parity with the long `${a}` /// form (#454). #[inline] + #[must_use] fn get_operand_id<'a>( node: &Node<'a>, code: &'a [u8], @@ -504,7 +526,7 @@ pub(crate) trait Getter { /// Classifies a string-literal `node` as a single Halstead /// operand, *unless* it wraps an interpolation child drawn from /// `interp_kinds` — in which case the wrapper yields - /// [`HalsteadType::Unknown`] because the inner expressions are + /// [`TokenRole::Unknown`] because the inner expressions are /// walked and counted separately. Counting the wrapper too would /// double-count their contribution to `N2`. /// @@ -513,11 +535,12 @@ pub(crate) trait Getter { /// (#183 / #184 / #191 / #192 / #199 / #277, …). Each language /// supplies only its own grammar's interpolation child-kind ids; /// the per-call rationale lives at each call site. - fn string_operand_type(node: &Node, interp_kinds: &[u16]) -> HalsteadType { + #[must_use] + fn string_operand_type(node: &Node, interp_kinds: &[u16]) -> TokenRole { if node.wraps_any(interp_kinds) { - HalsteadType::Unknown + TokenRole::Unknown } else { - HalsteadType::Operand + TokenRole::Operand } } @@ -562,6 +585,7 @@ pub(crate) trait Getter { /// /// [`braced_word_op_type`]: Self::braced_word_op_type /// [`get_op_type_with_code`]: Self::get_op_type_with_code + #[must_use] fn is_subsumed_braced_word<'a>( node: &Node<'a>, ancestors: Ancestors<'a, '_>, @@ -602,6 +626,7 @@ pub(crate) trait Getter { /// caller decides that case before asking, because a name is /// always a literal and must not pick up the `switch`-arm rescue /// below. + #[must_use] fn generic_argument_command<'tree, 'chain>( word: &Node<'tree>, ancestors: Ancestors<'tree, 'chain>, @@ -624,7 +649,7 @@ pub(crate) trait Getter { /// them; recognition is out-of-band, by the enclosing command's /// leading word (grammar-dispatch §9). A word that fills a modelled /// construct's slot is a script; a word passed to a command in - /// [`SCRIPT_TAKING_COMMANDS`] is a script; **everything else is a + /// `SCRIPT_TAKING_COMMANDS` is a script; **everything else is a /// value**. /// /// That default is the load-bearing choice, and it is deliberately @@ -670,6 +695,7 @@ pub(crate) trait Getter { /// per such block, plus the vocabulary entry in the case above. /// /// [`braced_word_op_type`]: Self::braced_word_op_type + #[must_use] fn is_value_braced_word<'a>( word: &Node<'a>, code: &[u8], @@ -740,6 +766,7 @@ pub(crate) trait Getter { /// argument of such a command. /// /// [`is_switch_arm`]: Self::is_switch_arm + #[must_use] fn is_switch_arm_body(word: &Node<'_>, command: &Node<'_>) -> bool { command .child_by_field_name("arguments") @@ -761,6 +788,7 @@ pub(crate) trait Getter { /// (`"eval" {…}`) is legal Tcl that this deliberately leaves /// unresolved — the same limitation `tcl_command_name` records for /// the Cognitive and Cyclomatic walkers. + #[must_use] fn command_leading_word<'c>( command: &Node<'_>, code: &'c [u8], @@ -796,6 +824,7 @@ pub(crate) trait Getter { /// called `a`, which is exactly what the value default is for /// everywhere else. iRules models `switch` with `switch_arm` /// children, so its arm bodies never reach here. + #[must_use] fn is_switch_arm<'a>( command: &Node<'a>, code: &[u8], @@ -857,12 +886,13 @@ pub(crate) trait Getter { /// [`is_switch_arm_body`]: Self::is_switch_arm_body /// /// [`get_op_type`]: Self::get_op_type + #[must_use] fn braced_word_op_type<'a>( node: &Node<'a>, code: &[u8], ancestors: Ancestors<'a, '_>, kinds: &BracedWordKinds, - ) -> HalsteadType { + ) -> TokenRole { let base = Self::get_op_type(node, ancestors); // Only the opener can change answer, and the test has to say so // rather than infer it from the parent kind. A braced word's @@ -872,7 +902,7 @@ pub(crate) trait Getter { // and both dialects list `SEMI` as an operator. The closer is // unclassified and a `\n` terminator is not an operator, so // those two are the whole set. - if node.kind_id() != kinds.open_brace || !matches!(base, HalsteadType::Operator) { + if node.kind_id() != kinds.open_brace || !matches!(base, TokenRole::Operator) { return base; } let quotes_a_literal = ancestors.iter(node).next().is_some_and(|(parent, above)| { @@ -880,12 +910,15 @@ pub(crate) trait Getter { && Self::is_value_braced_word(&parent, code, above, kinds) }); if quotes_a_literal { - HalsteadType::Unknown + TokenRole::Unknown } else { base } } + /// The grammar's name for the operator kind `id`, used to render a + /// Halstead operator; empty for languages that do not report them. + #[must_use] fn get_operator_id_as_str(_id: u16) -> &'static str { "" } diff --git a/src/getter/bash.rs b/big-code-analysis-ast/src/getter/bash.rs similarity index 96% rename from src/getter/bash.rs rename to big-code-analysis-ast/src/getter/bash.rs index a5211850f..dc21d20d3 100644 --- a/src/getter/bash.rs +++ b/big-code-analysis-ast/src/getter/bash.rs @@ -31,7 +31,7 @@ impl Getter for BashCode { } } - fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { match node.kind_id().into() { // Control flow and declaration keywords Bash::If | Bash::Then | Bash::Fi | Bash::Elif | Bash::Else @@ -71,7 +71,7 @@ impl Getter for BashCode { | Bash::LTLTLT // Ternary operator | Bash::QMARK | Bash::QMARK2 - => HalsteadType::Operator, + => TokenRole::Operator, // Quoted strings count as one operand when they are inert. // When they contain any `$var`/`${...}`/`$(...)`/`$((...))` @@ -86,9 +86,9 @@ impl Getter for BashCode { // note on `command_name` below, which it shares a shape with. Bash::String | Bash::RawString | Bash::AnsiCString => { if bash_string_has_expansion(node) { - HalsteadType::Unknown + TokenRole::Unknown } else { - HalsteadType::Operand + TokenRole::Operand } } @@ -156,7 +156,7 @@ impl Getter for BashCode { Bash::Word | Bash::Word2 | Bash::Word3 | Bash::Word4 | Bash::Number | Bash::Number2 | Bash::NumberToken1 | Bash::NumberToken2 | Bash::SimpleExpansion - => HalsteadType::Operand, + => TokenRole::Operand, // `variable_name` / `special_variable_name` are operands when // they stand alone — the `name` in `name=value`, or the inner @@ -172,12 +172,12 @@ impl Getter for BashCode { Bash::VariableName | Bash::VariableName2 | Bash::VariableName3 | Bash::SpecialVariableName | Bash::SpecialVariableName2 => { if ancestors.parent_has_kind(node, Bash::SimpleExpansion as u16) { - HalsteadType::Unknown + TokenRole::Unknown } else { - HalsteadType::Operand + TokenRole::Operand } } - _ => HalsteadType::Unknown, + _ => TokenRole::Unknown, } } diff --git a/src/getter/c.rs b/big-code-analysis-ast/src/getter/c.rs similarity index 97% rename from src/getter/c.rs rename to big-code-analysis-ast/src/getter/c.rs index d0bbbbaa4..5e472d74a 100644 --- a/src/getter/c.rs +++ b/big-code-analysis-ast/src/getter/c.rs @@ -50,7 +50,7 @@ impl Getter for CCode { } } - fn get_op_type<'a>(node: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> TokenRole { use C::*; // C's operator alphabet is the C++ set minus the C++-only forms @@ -77,7 +77,7 @@ impl Getter for CCode { // modifier has a distinct kind_id, so keying by kind_id (the default // `operators` store) keeps them distinct in n1 while `long long`'s // two `long` tokens correctly fold to one n1 entry but two N1 hits. - | Signed | Unsigned | Long | Short => HalsteadType::Operator, + | Signed | Unsigned | Long | Short => TokenRole::Operator, // `CharLiteral` joins the operand list here for the whole // C family — `cpp.rs`, `mozcpp.rs` and `objc.rs` carry the // same arm and point back at this note (#1316). Before it a @@ -108,8 +108,8 @@ impl Getter for CCode { // `c_family_char_literal_is_not_a_string` in `checker.rs` // pins it both ways. Identifier | TypeIdentifier | FieldIdentifier | StringLiteral | CharLiteral - | NumberLiteral | True | False | Null | DOTDOTDOT => HalsteadType::Operand, - _ => HalsteadType::Unknown, + | NumberLiteral | True | False | Null | DOTDOTDOT => TokenRole::Operand, + _ => TokenRole::Unknown, } } diff --git a/src/getter/ccomment.rs b/big-code-analysis-ast/src/getter/ccomment.rs similarity index 100% rename from src/getter/ccomment.rs rename to big-code-analysis-ast/src/getter/ccomment.rs diff --git a/src/getter/cpp.rs b/big-code-analysis-ast/src/getter/cpp.rs similarity index 97% rename from src/getter/cpp.rs rename to big-code-analysis-ast/src/getter/cpp.rs index 404b58b44..1fe5faca1 100644 --- a/src/getter/cpp.rs +++ b/big-code-analysis-ast/src/getter/cpp.rs @@ -71,7 +71,7 @@ impl Getter for CppCode { } } - fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Cpp::*; // `LPAREN2` here (and the `LBRACK2`/`LBRACK3` aliases in the @@ -107,7 +107,7 @@ impl Getter for CppCode { LPAREN if ancestors.parent_has_kind(node, RawStringLiteral as u16) => { - HalsteadType::Unknown + TokenRole::Unknown } DOT | DOTSTAR | LPAREN | LPAREN2 | COMMA | STAR | GTGT | COLON | SEMI | Return | Break | Continue | If | Else | Switch | Case | Default | For | While | Goto | Do @@ -127,7 +127,7 @@ impl Getter for CppCode { // modifier has a distinct kind_id, so keying by kind_id (the default // `operators` store) keeps them distinct in n1 while `long long`'s // two `long` tokens correctly fold to one n1 entry but two N1 hits. - | Signed | Unsigned | Long | Short => HalsteadType::Operator, + | Signed | Unsigned | Long | Short => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only // classified node in a character literal, so it bills one @@ -190,16 +190,16 @@ impl Getter for CppCode { // construct is unaffected either way. Identifier | TypeIdentifier | FieldIdentifier | RawStringLiteral | StringLiteral | CharLiteral | NumberLiteral | True | False | Null | This | DOTDOTDOT => { - HalsteadType::Operand + TokenRole::Operand } // A namespace identifier is an operand only where it // *names* a namespace; the same kind also spells the // qualifier in `ns::thing`, which the final arm leaves // `Unknown` (#1096). NamespaceIdentifier if ancestors.parent_has_kind(node, NamespaceDefinition as u16) => { - HalsteadType::Operand + TokenRole::Operand } - _ => HalsteadType::Unknown, + _ => TokenRole::Unknown, } } diff --git a/src/getter/csharp.rs b/big-code-analysis-ast/src/getter/csharp.rs similarity index 96% rename from src/getter/csharp.rs rename to big-code-analysis-ast/src/getter/csharp.rs index e87191d52..b13520227 100644 --- a/src/getter/csharp.rs +++ b/big-code-analysis-ast/src/getter/csharp.rs @@ -45,7 +45,7 @@ impl Getter for CsharpCode { } } - fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Csharp::*; match node.kind_id().into() { @@ -78,7 +78,7 @@ impl Getter for CsharpCode { | AMPEQ | PIPEEQ | CARETEQ | LTLTEQ | GTGTEQ | GTGTGTEQ | QMARKQMARKEQ // Predefined / primitive types | PredefinedType - => HalsteadType::Operator, + => TokenRole::Operator, // `boolean_literal: choice('true', 'false')` wraps the // keyword leaf, so a literal reaches the walker twice; // listing both kinds inflated `N2` by one per literal while @@ -92,8 +92,8 @@ impl Getter for CsharpCode { // an overloaded operator's *name* is better counted as an // operator, as `operator +` already is, is #1296. True | False => match ancestors.parent(node).map(|p| p.kind_id().into()) { - Some(BooleanLiteral) => HalsteadType::Unknown, - _ => HalsteadType::Operand, + Some(BooleanLiteral) => TokenRole::Unknown, + _ => TokenRole::Operand, }, // Operands: identifiers and literals. `NullLiteral` is a // childless leaf, so it needs no such guard. @@ -111,7 +111,7 @@ impl Getter for CsharpCode { Identifier | IntegerLiteral | RealLiteral | BooleanLiteral | NullLiteral | CharacterLiteral | StringLiteral | VerbatimStringLiteral | RawStringLiteral - => HalsteadType::Operand, + => TokenRole::Operand, // `$"..."` counts as one operand when inert. When it carries // any `Interpolation` child the inner expressions are // already walked and classified as operands; counting the @@ -121,7 +121,7 @@ impl Getter for CsharpCode { InterpolatedStringExpression => { Self::string_operand_type(node, &[Interpolation as u16]) } - _ => HalsteadType::Unknown, + _ => TokenRole::Unknown, } } diff --git a/src/getter/elixir.rs b/big-code-analysis-ast/src/getter/elixir.rs similarity index 98% rename from src/getter/elixir.rs rename to big-code-analysis-ast/src/getter/elixir.rs index d25c88f91..19b1220fc 100644 --- a/src/getter/elixir.rs +++ b/big-code-analysis-ast/src/getter/elixir.rs @@ -139,7 +139,7 @@ impl Getter for ElixirCode { Some("") } - fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Elixir as E; match node.kind_id().into() { @@ -162,7 +162,7 @@ impl Getter for ElixirCode { E::SLASH | E::LPAREN | E::LBRACE | E::LBRACK | E::LT | E::GT | E::PIPE if ancestors.parent_has_kind(node, E::Sigil as u16) => { - HalsteadType::Unknown + TokenRole::Unknown } // Reserved-word keywords that have dedicated token kinds in // the grammar — block delimiters, exception clauses, the @@ -211,7 +211,7 @@ impl Getter for ElixirCode { | E::EQGT | E::BSLASHBSLASH | E::EQTILDE | E::SLASHSLASH // Custom / less common Elixir operators | E::LTTILDE | E::TILDEGT | E::LTTILDEGT | E::LTLTTILDE | E::TILDEGTGT - => HalsteadType::Operator, + => TokenRole::Operator, // String literals contribute exactly one operand each when // they are inert. When they carry an `interpolation` child, @@ -255,9 +255,9 @@ impl Getter for ElixirCode { | E::Integer | E::Float | E::Char | E::Atom | E::Atom2 | E::QuotedAtom | E::Boolean | E::Nil - => HalsteadType::Operand, + => TokenRole::Operand, - _ => HalsteadType::Unknown, + _ => TokenRole::Unknown, } } diff --git a/src/getter/go.rs b/big-code-analysis-ast/src/getter/go.rs similarity index 93% rename from src/getter/go.rs rename to big-code-analysis-ast/src/getter/go.rs index ce827dd76..1d0ae976c 100644 --- a/src/getter/go.rs +++ b/big-code-analysis-ast/src/getter/go.rs @@ -16,7 +16,7 @@ impl Getter for GoCode { } } - fn get_op_type<'a>(node: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> TokenRole { use Go as G; match node.kind_id().into() { @@ -35,15 +35,15 @@ impl Getter for GoCode { | G::LTLT | G::GTGT | G::AMPCARET | G::LTDASH | G::PLUSPLUS | G::DASHDASH | G::PLUSEQ | G::DASHEQ | G::STAREQ | G::SLASHEQ | G::PERCENTEQ | G::AMPEQ | G::PIPEEQ | G::CARETEQ | G::LTLTEQ | G::GTGTEQ | G::AMPCARETEQ - => HalsteadType::Operator, + => TokenRole::Operator, // Operands: identifiers and literals G::Identifier | G::Identifier2 | G::Identifier3 | G::BlankIdentifier | G::FieldIdentifier | G::PackageIdentifier | G::TypeIdentifier | G::LabelName | G::IntLiteral | G::FloatLiteral | G::ImaginaryLiteral | G::RuneLiteral | G::InterpretedStringLiteral | G::RawStringLiteral | G::Nil | G::True | G::False | G::Iota - => HalsteadType::Operand, - _ => HalsteadType::Unknown, + => TokenRole::Operand, + _ => TokenRole::Unknown, } } diff --git a/src/getter/groovy.rs b/big-code-analysis-ast/src/getter/groovy.rs similarity index 98% rename from src/getter/groovy.rs rename to big-code-analysis-ast/src/getter/groovy.rs index 264e08d8c..05f603f70 100644 --- a/src/getter/groovy.rs +++ b/big-code-analysis-ast/src/getter/groovy.rs @@ -80,7 +80,7 @@ impl Getter for GroovyCode { } } - fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Groovy::*; // Mirrors `JavaCode`'s minimal classification — modifiers // (`Public`, `Static`, …), declaration keywords (`Class`, @@ -127,7 +127,7 @@ impl Getter for GroovyCode { SLASH if ancestors.parent_has_kind(node, StringLiteral as u16) => { - HalsteadType::Unknown + TokenRole::Unknown } // Control-flow + keyword operators (mirrors Java's set, // minus tokens that no longer exist in the dekobon grammar @@ -152,7 +152,7 @@ impl Getter for GroovyCode { // implication `==>`, and spread-map `*:`. | DOTDOT | DOTDOTLT | LTDOTDOT | LTDOTDOTLT | QMARKCOLON | QMARKEQ | QMARKDOT | QMARKQMARKDOT | STARDOT | DOTAMP | DOTAT | QMARKLBRACK | EQEQEQ | BANGEQEQ - | LTEQGT | EQTILDE | EQEQTILDE | EQEQGT | STARCOLON => HalsteadType::Operator, + | LTEQGT | EQTILDE | EQEQTILDE | EQEQGT | STARCOLON => TokenRole::Operator, // `QualifiedName` (a `package` / `import` path) and // `QualifiedType` (`java.util.Map` in type position) were @@ -176,7 +176,7 @@ impl Getter for GroovyCode { // `identifier`, and a single-segment type is a bare // `type_identifier` the grammar does not wrap at all. Identifier | TypeIdentifier | NullLiteral | True | False | NumberLiteral => { - HalsteadType::Operand + TokenRole::Operand } // A Groovy GString interpolates inner expressions whose @@ -197,7 +197,7 @@ impl Getter for GroovyCode { ], ), - _ => HalsteadType::Unknown, + _ => TokenRole::Unknown, } } diff --git a/src/getter/irules.rs b/big-code-analysis-ast/src/getter/irules.rs similarity index 96% rename from src/getter/irules.rs rename to big-code-analysis-ast/src/getter/irules.rs index 663f35b4e..b341a1805 100644 --- a/src/getter/irules.rs +++ b/big-code-analysis-ast/src/getter/irules.rs @@ -32,7 +32,7 @@ impl Getter for IrulesCode { } } - fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { match node.kind_id().into() { // The braced-word rule (#1314, #1354 / #1317) — the twin of // the Tcl arm, which carries the derivation and every @@ -43,7 +43,7 @@ impl Getter for IrulesCode { // the value form is `BracedWordSimple` (133), which admits // the same six child kinds Tcl's does. _ if Self::is_subsumed_braced_word(node, ancestors, &BRACED_WORD_KINDS) => { - HalsteadType::Unknown + TokenRole::Unknown } // Anonymous keyword tokens (the `*2` aliases are the keyword // literals; the unsuffixed high-id variants are the statement @@ -128,7 +128,7 @@ impl Getter for IrulesCode { | Irules::PIPEPIPE | Irules::Or // Ternary conditional operator. - | Irules::QMARK => HalsteadType::Operator, + | Irules::QMARK => TokenRole::Operator, // `Id` (named, id 49) is a standalone identifier operand — e.g. // a `set` target (`set s …` → `s`). But it is ALSO the inner @@ -145,9 +145,9 @@ impl Getter for IrulesCode { // non-surfacing token.) Irules::Id => { if ancestors.parent_has_kind(node, Irules::VariableSubstitution as u16) { - HalsteadType::Unknown + TokenRole::Unknown } else { - HalsteadType::Operand + TokenRole::Operand } } @@ -174,7 +174,7 @@ impl Getter for IrulesCode { | Irules::EventName | Irules::BracedWord | Irules::BracedWordSimple - | Irules::VariableSubstitution => HalsteadType::Operand, + | Irules::VariableSubstitution => TokenRole::Operand, // Double-quoted strings count as a single operand when inert // (`"hello world"`). When they carry a `$var` or `[cmd]` @@ -190,7 +190,7 @@ impl Getter for IrulesCode { ], ), - _ => HalsteadType::Unknown, + _ => TokenRole::Unknown, } } @@ -211,7 +211,7 @@ impl Getter for IrulesCode { node: &Node<'a>, code: &[u8], ancestors: Ancestors<'a, '_>, - ) -> HalsteadType { + ) -> TokenRole { Self::braced_word_op_type(node, code, ancestors, &BRACED_WORD_KINDS) } diff --git a/src/getter/java.rs b/big-code-analysis-ast/src/getter/java.rs similarity index 97% rename from src/getter/java.rs rename to big-code-analysis-ast/src/getter/java.rs index 672b7fdcc..efdf1966f 100644 --- a/src/getter/java.rs +++ b/big-code-analysis-ast/src/getter/java.rs @@ -78,7 +78,7 @@ impl Getter for JavaCode { } } - fn get_op_type<'a>(node: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> TokenRole { use Java::*; // Some guides that informed grammar choice for Halstead // keywords, operators, literals: https://docs.oracle.com/javase/specs/jls/se18/html/jls-3.html#jls-3.12 @@ -101,17 +101,17 @@ impl Getter for JavaCode { // primitive types | Byte | Short | Int | Long | Char | Float | Double | BooleanType => { - HalsteadType::Operator + TokenRole::Operator }, // Operands: variables, constants, literals Identifier | NullLiteral | ClassLiteral | True | False | StringLiteral | CharacterLiteral | HexIntegerLiteral | OctalIntegerLiteral | BinaryIntegerLiteral | DecimalIntegerLiteral | HexFloatingPointLiteral | DecimalFloatingPointLiteral => { - HalsteadType::Operand + TokenRole::Operand }, _ => { - HalsteadType::Unknown + TokenRole::Unknown }, } } diff --git a/src/getter/javascript.rs b/big-code-analysis-ast/src/getter/javascript.rs similarity index 100% rename from src/getter/javascript.rs rename to big-code-analysis-ast/src/getter/javascript.rs diff --git a/src/getter/kotlin.rs b/big-code-analysis-ast/src/getter/kotlin.rs similarity index 97% rename from src/getter/kotlin.rs rename to big-code-analysis-ast/src/getter/kotlin.rs index b4ba8623b..abeefc156 100644 --- a/src/getter/kotlin.rs +++ b/big-code-analysis-ast/src/getter/kotlin.rs @@ -210,7 +210,7 @@ impl Getter for KotlinCode { } } - fn get_op_type<'a>(node: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> TokenRole { use Kotlin::*; match node.kind_id().into() { @@ -231,10 +231,10 @@ impl Getter for KotlinCode { // Operator: logical and misc | AMPAMP | PIPEPIPE | BANG | BANGBANG | QMARK | QMARKCOLON | QMARKDOT - | DOTDOT | DOTDOTLT | DASHGT | COLON => HalsteadType::Operator, + | DOTDOT | DOTDOTLT | DASHGT | COLON => TokenRole::Operator, // Operands: identifiers and literals Identifier | NumberLiteral | FloatLiteral | CharacterLiteral | Label => { - HalsteadType::Operand + TokenRole::Operand } // Regression #191: a Kotlin string template wraps an // `Interpolation` child (the long `"${expr}"` form) whose @@ -249,7 +249,7 @@ impl Getter for KotlinCode { StringLiteral | MultilineStringLiteral => { Self::string_operand_type(node, &[Interpolation as u16]) } - _ => HalsteadType::Unknown, + _ => TokenRole::Unknown, } } @@ -257,7 +257,7 @@ impl Getter for KotlinCode { node: &Node<'a>, code: &[u8], ancestors: Ancestors<'a, '_>, - ) -> HalsteadType { + ) -> TokenRole { use Kotlin::*; match node.kind_id().into() { @@ -275,7 +275,7 @@ impl Getter for KotlinCode { StringContent | StringContent2 | StringContent3 if kotlin_is_short_interp_name(node, ancestors.previous_sibling(node), code) => { - HalsteadType::Operand + TokenRole::Operand } // The wrapping literal is Unknown when it interpolates in // either form (long `${expr}` or short `$name`); the inner @@ -283,9 +283,9 @@ impl Getter for KotlinCode { // plain literal has no interpolation and stays one operand. StringLiteral | MultilineStringLiteral => { if kotlin_string_has_interp(node, code) { - HalsteadType::Unknown + TokenRole::Unknown } else { - HalsteadType::Operand + TokenRole::Operand } } _ => Self::get_op_type(node, ancestors), diff --git a/src/getter/lua.rs b/big-code-analysis-ast/src/getter/lua.rs similarity index 93% rename from src/getter/lua.rs rename to big-code-analysis-ast/src/getter/lua.rs index 89e95626f..9c5782167 100644 --- a/src/getter/lua.rs +++ b/big-code-analysis-ast/src/getter/lua.rs @@ -15,7 +15,7 @@ impl Getter for LuaCode { } } - fn get_op_type<'a>(node: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> TokenRole { match node.kind_id().into() { // Control-flow and declaration keywords Lua::If @@ -79,13 +79,13 @@ impl Getter for LuaCode { | Lua::EQ // `break` is a named leaf node (no anonymous keyword child), so it must be // matched directly here — unlike `return`/`goto` which are anonymous tokens. - | Lua::BreakStatement => HalsteadType::Operator, + | Lua::BreakStatement => TokenRole::Operator, // Operands: identifiers and literals Lua::Identifier | Lua::Number | Lua::String | Lua::True | Lua::False | Lua::Nil - | Lua::VarargExpression => HalsteadType::Operand, + | Lua::VarargExpression => TokenRole::Operand, - _ => HalsteadType::Unknown, + _ => TokenRole::Unknown, } } diff --git a/src/getter/mozcpp.rs b/big-code-analysis-ast/src/getter/mozcpp.rs similarity index 96% rename from src/getter/mozcpp.rs rename to big-code-analysis-ast/src/getter/mozcpp.rs index 70b3b4a8a..11e8a8491 100644 --- a/src/getter/mozcpp.rs +++ b/big-code-analysis-ast/src/getter/mozcpp.rs @@ -71,7 +71,7 @@ impl Getter for MozcppCode { } } - fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Mozcpp::*; // `LPAREN2` is a defensive arm (collapsed to `LPAREN` before @@ -86,7 +86,7 @@ impl Getter for MozcppCode { LPAREN if ancestors.parent_has_kind(node, RawStringLiteral as u16) => { - HalsteadType::Unknown + TokenRole::Unknown } DOT | DOTSTAR | LPAREN | LPAREN2 | COMMA | STAR | GTGT | COLON | SEMI | Return | Break | Continue | If | Else | Switch | Case | Default | For | While | Goto | Do @@ -106,7 +106,7 @@ impl Getter for MozcppCode { // modifier has a distinct kind_id, so keying by kind_id (the default // `operators` store) keeps them distinct in n1 while `long long`'s // two `long` tokens correctly fold to one n1 entry but two N1 hits. - | Signed | Unsigned | Long | Short => HalsteadType::Operator, + | Signed | Unsigned | Long | Short => TokenRole::Operator, // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only // classified node in a character literal, so it bills one @@ -126,16 +126,16 @@ impl Getter for MozcppCode { // `LPAREN` arm above relies on. Identifier | TypeIdentifier | FieldIdentifier | RawStringLiteral | StringLiteral | CharLiteral | NumberLiteral | True | False | Null | This | DOTDOTDOT => { - HalsteadType::Operand + TokenRole::Operand } // A namespace identifier is an operand only where it // *names* a namespace; the same kind also spells the // qualifier in `ns::thing`, which the final arm leaves // `Unknown` (#1096). NamespaceIdentifier if ancestors.parent_has_kind(node, NamespaceDefinition as u16) => { - HalsteadType::Operand + TokenRole::Operand } - _ => HalsteadType::Unknown, + _ => TokenRole::Unknown, } } diff --git a/src/getter/mozjs.rs b/big-code-analysis-ast/src/getter/mozjs.rs similarity index 100% rename from src/getter/mozjs.rs rename to big-code-analysis-ast/src/getter/mozjs.rs diff --git a/src/getter/objc.rs b/big-code-analysis-ast/src/getter/objc.rs similarity index 97% rename from src/getter/objc.rs rename to big-code-analysis-ast/src/getter/objc.rs index 3828ff958..902ea4485 100644 --- a/src/getter/objc.rs +++ b/big-code-analysis-ast/src/getter/objc.rs @@ -67,7 +67,7 @@ impl Getter for ObjcCode { } } - fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Objc::*; // ObjC is C plus message sends, blocks, and the `@`-directives. @@ -90,7 +90,7 @@ impl Getter for ObjcCode { // byte twice and planted a phantom `@` in n1 for a file // whose only `@` was in NSString literals (grammar-dispatch // §5, the compound-leaf guard). - AT if ancestors.parent_has_kind(node, StringLiteral as u16) => HalsteadType::Unknown, + AT if ancestors.parent_has_kind(node, StringLiteral as u16) => TokenRole::Unknown, // The C operator set, then the ObjC-specific structural // keywords / markers from `In` onwards. DOT | LPAREN | LPAREN2 | COMMA | STAR | GTGT | COLON | SEMI | Return | Break @@ -101,7 +101,7 @@ impl Getter for ObjcCode { | CARETEQ | PIPEEQ | LBRACK | LBRACE | QMARK | PrimitiveType | TypeSpecifier | Sizeof | Signed | Unsigned | Long | Short | In | AT | ATtry | ATcatch | ATfinally | ATthrow | ATsynchronized | ATautoreleasepool | ATselector | ATencode => { - HalsteadType::Operator + TokenRole::Operator } // `CharLiteral` — the full derivation lives on the same arm // in `src/getter/c.rs` (#1316): the wrapper is the only @@ -109,8 +109,8 @@ impl Getter for ObjcCode { // operand per literal, keyed by text, and `Checker::is_string` // deliberately stays without a `CharLiteral` arm. Identifier | TypeIdentifier | FieldIdentifier | StringLiteral | CharLiteral - | NumberLiteral | True | False | Null | DOTDOTDOT => HalsteadType::Operand, - _ => HalsteadType::Unknown, + | NumberLiteral | True | False | Null | DOTDOTDOT => TokenRole::Operand, + _ => TokenRole::Unknown, } } diff --git a/src/getter/perl.rs b/big-code-analysis-ast/src/getter/perl.rs similarity index 98% rename from src/getter/perl.rs rename to big-code-analysis-ast/src/getter/perl.rs index 2dabdf0e5..e2d706eab 100644 --- a/src/getter/perl.rs +++ b/big-code-analysis-ast/src/getter/perl.rs @@ -14,7 +14,7 @@ impl Getter for PerlCode { } } - fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Perl as P; match node.kind_id().into() { @@ -35,7 +35,7 @@ impl Getter for PerlCode { P::SLASH if ancestors.parent_has_kind(node, P::PatternMatcher as u16) => { - HalsteadType::Unknown + TokenRole::Unknown } // Control-flow and declaration keywords. `Perl::Sub` is the // `sub` keyword (token id 16); `Perl::SUB` is the `__SUB__` @@ -78,7 +78,7 @@ impl Getter for PerlCode { // split is the semantic one above, not this. `get_operator_id_as_str` below renders them // `s///` and `tr///` rather than as raw kind names. | P::SubstitutionPatternS | P::TransliterationTrOrY - => HalsteadType::Operator, + => TokenRole::Operator, // `package_name`, `package_variable` and `typeglob` each // spell a whole name — `Data::Dumper`, `$Foo::count`, // `*STDOUT` — and are operands in their own right below, so @@ -157,7 +157,7 @@ impl Getter for PerlCode { Some(P::PackageName | P::PackageVariable | P::Typeglob) ) => { - HalsteadType::Unknown + TokenRole::Unknown } // Operands: identifiers and literals. Non-interpolating // string literals (`'…'`, `q{…}`) are leaf operands; the @@ -180,7 +180,7 @@ impl Getter for PerlCode { | P::True | P::False | P::SpecialLiteral | P::StringSingleQuoted | P::StringQQuoted | P::FILE | P::LINE | P::SUB | P::PACKAGE - => HalsteadType::Operand, + => TokenRole::Operand, // Perl's interpolating string-like literals count as one // operand when inert. When they carry an `Interpolation` // child the inner scalar / array / hash variables are @@ -226,7 +226,7 @@ impl Getter for PerlCode { | P::PatternMatcher | P::PatternMatcherM | P::RegexPatternQr | P::WordListQw => { Self::string_operand_type(node, &[P::Interpolation as u16, P::ListItem as u16]) } - _ => HalsteadType::Unknown, + _ => TokenRole::Unknown, } } diff --git a/src/getter/php.rs b/big-code-analysis-ast/src/getter/php.rs similarity index 96% rename from src/getter/php.rs rename to big-code-analysis-ast/src/getter/php.rs index fdf5514e9..7ce683119 100644 --- a/src/getter/php.rs +++ b/big-code-analysis-ast/src/getter/php.rs @@ -28,7 +28,7 @@ impl Getter for PhpCode { } } - fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Php::*; match node.kind_id().into() { // String-interpolation opener. `LBRACE` is *both* the @@ -79,7 +79,7 @@ impl Getter for PhpCode { ) ) => { - HalsteadType::Unknown + TokenRole::Unknown } // Operator: control-flow keywords If | Else | Elseif | Endif @@ -137,7 +137,7 @@ impl Getter for PhpCode { // Operator: string concat | DOT - => HalsteadType::Operator, + => TokenRole::Operator, // `name` is a genuine operand almost everywhere — function, // class, const, member (`->prop`) and namespace-component @@ -158,8 +158,8 @@ impl Getter for PhpCode { // It rides along defensively so a grammar bump that promotes // it inherits the guard rather than the bug. Name | Name2 => match ancestors.parent(node).map(|p| p.kind_id().into()) { - Some(VariableName | DynamicVariableName) => HalsteadType::Unknown, - _ => HalsteadType::Operand, + Some(VariableName | DynamicVariableName) => TokenRole::Unknown, + _ => TokenRole::Operand, }, // Variable-variable syntax nests these wrappers: `$$x` is a @@ -173,8 +173,8 @@ impl Getter for PhpCode { // `$x` hangs off a `binary_expression`) is unaffected. VariableName | DynamicVariableName => { match ancestors.parent(node).map(|p| p.kind_id().into()) { - Some(DynamicVariableName) => HalsteadType::Unknown, - _ => HalsteadType::Operand, + Some(DynamicVariableName) => TokenRole::Unknown, + _ => TokenRole::Operand, } } @@ -249,8 +249,8 @@ impl Getter for PhpCode { // only the node it is attributed to moved. Int | Bool | Array | Object | String2 | Float2 | Null2 => { match ancestors.parent(node).map(|p| p.kind_id().into()) { - Some(PrimitiveType) => HalsteadType::Unknown, - _ => HalsteadType::Operand, + Some(PrimitiveType) => TokenRole::Unknown, + _ => TokenRole::Operand, } } @@ -268,7 +268,7 @@ impl Getter for PhpCode { | Boolean | Null | BottomType | PrimitiveType | CastType - => HalsteadType::Operand, + => TokenRole::Operand, // `EncapsedString` (double-quoted), `Heredoc`, and // `ShellCommandExpression` (backticks) count as one @@ -323,13 +323,13 @@ impl Getter for PhpCode { || (kind == HeredocBody as u16 && c.wraps_any(PHP_INTERP_KINDS)) }); if has_interp { - HalsteadType::Unknown + TokenRole::Unknown } else { - HalsteadType::Operand + TokenRole::Operand } } - _ => HalsteadType::Unknown, + _ => TokenRole::Unknown, } } diff --git a/src/getter/preproc.rs b/big-code-analysis-ast/src/getter/preproc.rs similarity index 100% rename from src/getter/preproc.rs rename to big-code-analysis-ast/src/getter/preproc.rs diff --git a/src/getter/python.rs b/big-code-analysis-ast/src/getter/python.rs similarity index 92% rename from src/getter/python.rs rename to big-code-analysis-ast/src/getter/python.rs index 6434825fc..f038eaad8 100644 --- a/src/getter/python.rs +++ b/big-code-analysis-ast/src/getter/python.rs @@ -13,7 +13,7 @@ impl Getter for PythonCode { } } - fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Python::*; match node.kind_id().into() { @@ -25,8 +25,8 @@ impl Getter for PythonCode { // operator below, so the leaf must yield Unknown — otherwise // `a not in b` would count `not` + `in` as two operators (#413). Not | In | Is => match ancestors.parent(node).map(|p| p.kind_id().into()) { - Some(Notin | Isnot) => HalsteadType::Unknown, - _ => HalsteadType::Operator, + Some(Notin | Isnot) => TokenRole::Unknown, + _ => TokenRole::Operator, }, Import | DOT | From | COMMA | As | STAR | GTGT | Assert | COLONEQ | Return | Def | Del | Raise | Pass | Break | Continue | If | Elif | Else | Async | For @@ -54,18 +54,18 @@ impl Getter for PythonCode { // Lambda/Lambda2 expression nodes that wrap it, to avoid the same // node+keyword double count fixed for await (#413). | Lambda3 => { - HalsteadType::Operator + TokenRole::Operator } - Identifier | Integer | Float | True | False | None => HalsteadType::Operand, + Identifier | Integer | Float | True | False | None => TokenRole::Operand, String => { // Docstring / module-level string statement: an `ExpressionStatement` // whose only child is the string. Skip those. let mut climb = ancestors.iter(node); let Some((parent, _)) = climb.next() else { - return HalsteadType::Unknown; + return TokenRole::Unknown; }; if parent.kind_id() == ExpressionStatement && parent.child_count() == 1 { - return HalsteadType::Unknown; + return TokenRole::Unknown; } // Implicit-concatenation docstring (`"""doc""" "more"`): the // adjacent literals are wrapped in a `concatenated_string`, @@ -80,7 +80,7 @@ impl Getter for PythonCode { && grandparent.child_count() == 1 }) { - return HalsteadType::Unknown; + return TokenRole::Unknown; } // Regression #191: an f-string wraps `Interpolation` children // whose inner expressions are walked and counted separately. @@ -88,7 +88,7 @@ impl Getter for PythonCode { // pattern as #180 for Bash/Elixir and #184 for PHP). Self::string_operand_type(node, &[Interpolation as u16]) } - _ => HalsteadType::Unknown, + _ => TokenRole::Unknown, } } diff --git a/src/getter/ruby.rs b/big-code-analysis-ast/src/getter/ruby.rs similarity index 98% rename from src/getter/ruby.rs rename to big-code-analysis-ast/src/getter/ruby.rs index caacf0b13..a0f6d5df1 100644 --- a/src/getter/ruby.rs +++ b/big-code-analysis-ast/src/getter/ruby.rs @@ -18,7 +18,7 @@ impl Getter for RubyCode { } } - fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Ruby as R; match node.kind_id().into() { @@ -47,7 +47,7 @@ impl Getter for RubyCode { R::SLASH | R::SLASH2 if ancestors.parent_has_kind(node, R::Regex as u16) => { - HalsteadType::Unknown + TokenRole::Unknown } // Subshell delimiter punctuation — the second delimiter // family of the fabrication the arm above removes for @@ -89,7 +89,7 @@ impl Getter for RubyCode { // `ruby_subshell_start_alias_never_reaches_kind_id` // instead. R::BQUOTE if ancestors.parent_has_kind(node, R::Subshell as u16) => { - HalsteadType::Unknown + TokenRole::Unknown } // Control-flow keyword tokens. tree-sitter-ruby gives each // keyword its own anonymous numbered variant (e.g. `If2` is @@ -162,7 +162,7 @@ impl Getter for RubyCode { | R::EQGT | R::QMARK | R::DOTDOT | R::DOTDOTDOT // Subshell backtick used as method-name marker (def `...) | R::BQUOTE - => HalsteadType::Operator, + => TokenRole::Operator, // String-like literals contribute one operand each when // inert. The wrapper is suppressed exactly when it holds a @@ -292,7 +292,7 @@ impl Getter for RubyCode { .parent(node) .is_some_and(|p| matches!(p.kind_id().into(), R::Complex | R::Rational)) => { - HalsteadType::Unknown + TokenRole::Unknown } // Operands: identifiers and literals. @@ -307,9 +307,9 @@ impl Getter for RubyCode { | R::True | R::False | R::Nil | R::Zelf | R::Super | R::Line | R::File | R::Encoding - => HalsteadType::Operand, + => TokenRole::Operand, - _ => HalsteadType::Unknown, + _ => TokenRole::Unknown, } } diff --git a/src/getter/rust.rs b/big-code-analysis-ast/src/getter/rust.rs similarity index 90% rename from src/getter/rust.rs rename to big-code-analysis-ast/src/getter/rust.rs index 676864a17..bce73cd04 100644 --- a/src/getter/rust.rs +++ b/big-code-analysis-ast/src/getter/rust.rs @@ -33,7 +33,7 @@ impl Getter for RustCode { } } - fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Rust::*; match node.kind_id().into() { @@ -44,22 +44,22 @@ impl Getter for RustCode { // Similarly, exclude `/` when it corresponds to the third slash in `///` (`OuterDocCommentMarker`) PIPEPIPE | SLASH => match ancestors.parent(node) { Some(parent) if matches!(parent.kind_id().into(), BinaryExpression) => { - HalsteadType::Operator + TokenRole::Operator } - _ => HalsteadType::Unknown, + _ => TokenRole::Unknown, }, // Ensure `!` is counted as an operator unless it belongs to an `InnerDocCommentMarker` `//!` BANG => match ancestors.parent(node) { Some(parent) if !matches!(parent.kind_id().into(), InnerDocCommentMarker) => { - HalsteadType::Operator + TokenRole::Operator } - _ => HalsteadType::Unknown, + _ => TokenRole::Unknown, }, // COLONCOLON (`::`) is the path-segment separator. C++, Java, // C#, and Kotlin all classify it as an operator; omitting it // here (issue #394) silently dropped every path expression // (`std::collections::HashMap`, `Vec::new`, `T::method`) into - // HalsteadType::Unknown, deflating n1/N1 for path-heavy code. + // TokenRole::Unknown, deflating n1/N1 for path-heavy code. // // The 14 declaration/visibility keywords (Const, Static, Enum, // Struct, Trait, Impl, Use, Mod, Pub, Type, Union, Where, @@ -78,18 +78,18 @@ impl Getter for RustCode { | PrimitiveType11 | PrimitiveType12 | PrimitiveType13 | PrimitiveType14 | PrimitiveType15 | PrimitiveType16 | PrimitiveType17 | Fn | SEMI | COLONCOLON | Const | Static | Enum | Struct | Trait | Impl | Use | Mod | Pub | Type | Union - | Where | Extern | Dyn => HalsteadType::Operator, + | Where | Extern | Dyn => TokenRole::Operator, // FieldIdentifier (e.g. `p.x`) and TypeIdentifier (e.g. `Vec`, // `HashMap`) are operand-class names — C++ and Go classify them // the same way (see arms ~588 and ~862 below). Omitting them - // here silently dropped both into HalsteadType::Unknown, + // here silently dropped both into TokenRole::Unknown, // deflating n2/N2 and the derived vocabulary/volume/effort // estimates (issue #390). Identifier | TypeIdentifier | FieldIdentifier | StringLiteral | RawStringLiteral | IntegerLiteral | FloatLiteral | BooleanLiteral | Zelf | CharLiteral | UNDERSCORE => { - HalsteadType::Operand + TokenRole::Operand } - _ => HalsteadType::Unknown, + _ => TokenRole::Unknown, } } diff --git a/src/getter/tcl.rs b/big-code-analysis-ast/src/getter/tcl.rs similarity index 97% rename from src/getter/tcl.rs rename to big-code-analysis-ast/src/getter/tcl.rs index f44ab7ba8..921df714c 100644 --- a/src/getter/tcl.rs +++ b/big-code-analysis-ast/src/getter/tcl.rs @@ -32,7 +32,7 @@ impl Getter for TclCode { } } - fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> HalsteadType { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { match node.kind_id().into() { // The braced-word rule (#1354 / #1317), whose two halves // `Getter::is_subsumed_braced_word` states once for both @@ -141,7 +141,7 @@ impl Getter for TclCode { // every branching metric on opposite sides of the same // bytes. _ if Self::is_subsumed_braced_word(node, ancestors, &BRACED_WORD_KINDS) => { - HalsteadType::Unknown + TokenRole::Unknown } // Anonymous keyword tokens (control-flow and declaration keywords). Tcl::Proc @@ -204,7 +204,7 @@ impl Getter for TclCode { | Tcl::AMPAMP | Tcl::PIPEPIPE // Ternary conditional operator. - | Tcl::QMARK => HalsteadType::Operator, + | Tcl::QMARK => TokenRole::Operator, // The anonymous `id` token (`Id2`) is the kind the parser // actually emits, in *both* of its positions: as a standalone @@ -220,9 +220,9 @@ impl Getter for TclCode { // classifies it identically. Tcl::Id | Tcl::Id2 => { if ancestors.parent_has_kind(node, Tcl::VariableSubstitution as u16) { - HalsteadType::Unknown + TokenRole::Unknown } else { - HalsteadType::Operand + TokenRole::Operand } } @@ -251,7 +251,7 @@ impl Getter for TclCode { | Tcl::Number | Tcl::BracedWord | Tcl::BracedWordSimple - | Tcl::VariableSubstitution => HalsteadType::Operand, + | Tcl::VariableSubstitution => TokenRole::Operand, // Double-quoted strings count as a single operand when inert // (`"hello world"`). When they carry a `$var` or `[cmd]` @@ -268,7 +268,7 @@ impl Getter for TclCode { ], ), - _ => HalsteadType::Unknown, + _ => TokenRole::Unknown, } } @@ -285,7 +285,7 @@ impl Getter for TclCode { node: &Node<'a>, code: &[u8], ancestors: Ancestors<'a, '_>, - ) -> HalsteadType { + ) -> TokenRole { Self::braced_word_op_type(node, code, ancestors, &BRACED_WORD_KINDS) } diff --git a/src/getter/tsx.rs b/big-code-analysis-ast/src/getter/tsx.rs similarity index 100% rename from src/getter/tsx.rs rename to big-code-analysis-ast/src/getter/tsx.rs diff --git a/src/getter/typescript.rs b/big-code-analysis-ast/src/getter/typescript.rs similarity index 100% rename from src/getter/typescript.rs rename to big-code-analysis-ast/src/getter/typescript.rs diff --git a/src/lang_helpers.rs b/big-code-analysis-ast/src/lang_helpers.rs similarity index 92% rename from src/lang_helpers.rs rename to big-code-analysis-ast/src/lang_helpers.rs index 317f92004..2c42f98c0 100644 --- a/src/lang_helpers.rs +++ b/big-code-analysis-ast/src/lang_helpers.rs @@ -13,6 +13,6 @@ //! a helper *from* `metrics::cognitive` would make the parse layer depend //! on the metric layer, the inversion #1376 exists to remove. -pub(crate) mod elixir; -pub(crate) mod python; -pub(crate) mod tcl; +pub mod elixir; +pub mod python; +pub mod tcl; diff --git a/src/lang_helpers/elixir.rs b/big-code-analysis-ast/src/lang_helpers/elixir.rs similarity index 90% rename from src/lang_helpers/elixir.rs rename to big-code-analysis-ast/src/lang_helpers/elixir.rs index 2fcfe5318..2c040e3db 100644 --- a/src/lang_helpers/elixir.rs +++ b/big-code-analysis-ast/src/lang_helpers/elixir.rs @@ -16,7 +16,9 @@ use crate::node::{Ancestors, Node}; /// Returns `None` for Calls whose target is not a simple identifier /// (e.g. `Module.func(…)` parses as `RemoteCallWithParentheses` with /// the dotted name as target) or when the bytes are not valid UTF-8. -pub(crate) fn elixir_call_keyword<'a>(node: &'a Node<'a>, code: &'a [u8]) -> Option<&'a str> { +#[inline] +#[must_use] +pub fn elixir_call_keyword<'a>(node: &'a Node<'a>, code: &'a [u8]) -> Option<&'a str> { if node.kind_id() != Elixir::Call as u16 { return None; } @@ -32,14 +34,16 @@ pub(crate) fn elixir_call_keyword<'a>(node: &'a Node<'a>, code: &'a [u8]) -> Opt /// because each consults it from a different trait surface; centralising /// the literal here keeps future additions (e.g. `defguard`) consistent. #[inline] -pub(crate) fn elixir_is_method_macro(kw: &str) -> bool { +#[must_use] +pub fn elixir_is_method_macro(kw: &str) -> bool { matches!(kw, "def" | "defp" | "defmacro" | "defmacrop") } /// Class-defining macro (`defmodule`). Paired with [`elixir_is_method_macro`] /// where a caller needs both ("any space-opening declaration"). #[inline] -pub(crate) fn elixir_is_class_macro(kw: &str) -> bool { +#[must_use] +pub fn elixir_is_class_macro(kw: &str) -> bool { kw == "defmodule" } @@ -56,7 +60,9 @@ pub(crate) fn elixir_is_class_macro(kw: &str) -> bool { /// `child_by_field_name("target")` + identifier byte compare, so the cost /// is O(steps) when `ancestors` is known — with `Ancestors::unknown` each /// step additionally pays `Node::parent`'s O(depth) (#1084). -pub(crate) fn elixir_is_inside_quote_block<'a>( +#[inline] +#[must_use] +pub fn elixir_is_inside_quote_block<'a>( node: &Node<'a>, code: &[u8], ancestors: Ancestors<'a, '_>, @@ -71,7 +77,8 @@ pub(crate) fn elixir_is_inside_quote_block<'a>( /// a module body for method-defining macros / `defstruct` without /// descending into nested modules. Yields no items when the Call has /// no `do_block`. -pub(crate) fn elixir_do_block_call_children<'a>( +#[inline] +pub fn elixir_do_block_call_children<'a>( node: &'a Node<'a>, ) -> impl Iterator> + 'a { node.children() diff --git a/src/lang_helpers/python.rs b/big-code-analysis-ast/src/lang_helpers/python.rs similarity index 85% rename from src/lang_helpers/python.rs rename to big-code-analysis-ast/src/lang_helpers/python.rs index 1a901a65b..14f3fca1c 100644 --- a/src/lang_helpers/python.rs +++ b/big-code-analysis-ast/src/lang_helpers/python.rs @@ -11,13 +11,15 @@ use crate::node::Node; /// /// This is the single normalization chokepoint for the lambda-alias set /// — mirroring [`python_is_block`] for the block aliases (#419). It -/// is reused by the cognitive lambda-scope walks below and by -/// `PythonCode::is_closure`, so a future grammar bump +/// is reused by `PythonCode::is_closure` here and by the cognitive +/// lambda-scope walks in `big-code-analysis`, so a future grammar bump /// that promotes `Lambda2` to a concrete node is handled in exactly one /// place rather than drifting across sites (#422). The /// `python_hidden_block_and_lambda_aliases_stay_unseen` drift guard in /// `checker.rs` trips on such a bump. -pub(crate) fn python_is_lambda(node: &Node) -> bool { +#[inline] +#[must_use] +pub fn python_is_lambda(node: &Node) -> bool { matches!(node.kind_id().into(), Python::Lambda | Python::Lambda2) } @@ -32,6 +34,8 @@ pub(crate) fn python_is_lambda(node: &Node) -> bool { /// "is this a block?" check through here means such a bump is handled /// at one site instead of silently undercounting at several (issue /// #419; lesson 2 / 34 / 56 in docs/development/lessons_learned.md). -pub(crate) fn python_is_block(node: &Node) -> bool { +#[inline] +#[must_use] +pub fn python_is_block(node: &Node) -> bool { matches!(node.kind_id().into(), Python::Block | Python::Block2) } diff --git a/src/lang_helpers/tcl.rs b/big-code-analysis-ast/src/lang_helpers/tcl.rs similarity index 93% rename from src/lang_helpers/tcl.rs rename to big-code-analysis-ast/src/lang_helpers/tcl.rs index baf7a881a..b6125a17e 100644 --- a/src/lang_helpers/tcl.rs +++ b/big-code-analysis-ast/src/lang_helpers/tcl.rs @@ -22,7 +22,9 @@ use crate::node::Node; /// Callers dispatch on the returned name so each `command` node resolves it /// exactly once per metric walk — the helpers below take the resolved /// identity as a precondition rather than re-deriving it. -pub(crate) fn tcl_command_name<'a>(node: &'a Node<'a>, code: &'a [u8]) -> Option<&'a str> { +#[inline] +#[must_use] +pub fn tcl_command_name<'a>(node: &'a Node<'a>, code: &'a [u8]) -> Option<&'a str> { if node.kind_id() != Tcl::Command as u16 { return None; } diff --git a/src/langs.rs b/big-code-analysis-ast/src/langs.rs similarity index 85% rename from src/langs.rs rename to big-code-analysis-ast/src/langs.rs index 3f2bfe4da..2b2846594 100644 --- a/src/langs.rs +++ b/big-code-analysis-ast/src/langs.rs @@ -6,6 +6,9 @@ // function so the per-language impl blocks stay readable. #![allow(clippy::wildcard_imports, clippy::enum_glob_use)] +//! The [`LANG`] enum, extension and emacs-mode lookup, the per-language +//! `*Code` tags and `*Parser` aliases, and [`AnyParser`]. + use std::path::Path; use std::sync::Arc; use tree_sitter::Language; @@ -388,9 +391,9 @@ mod tests { /// manifest pins both exist regardless of the enabled language set. #[test] fn grammar_version_matches_cargo_toml_pin() { - // CARGO_MANIFEST_DIR is the root crate's dir, which is the workspace - // root, so this is the manifest that carries `[workspace.dependencies]`. - let manifest = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/Cargo.toml")); + // CARGO_MANIFEST_DIR is this crate's dir, one level below the + // workspace root whose manifest carries `[workspace.dependencies]`. + let manifest = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/../Cargo.toml")); // LANG variant -> its `[workspace.dependencies]` key. Tsx and // Typescript share the one `tree-sitter-typescript` crate; the @@ -519,50 +522,6 @@ mod tests { } } - // Regression guard for issue #262: the `MetricsError::EmptyRoot` - // variant is documented as "Reserved — not produced today". - // `metrics_with_options` pushes a synthetic top-level Unit - // `FuncSpace` before walking, so every parse — including empty, - // whitespace-only, and comment-only input — currently returns - // `Ok(FuncSpace { kind: Unit, .. })`. If the walker is ever - // changed to legitimately drain its state stack (e.g. by - // dropping the synthetic root), this test will start failing - // and the variant docs must be revisited. - #[test] - fn empty_and_comment_only_input_never_returns_empty_root() { - use crate::{MetricsOptions, Source, SpaceKind, analyze}; - - // Pair every enabled language with sources that would, by - // the old (false) variant doc, surface `EmptyRoot`. The - // comment syntaxes cover line and block forms across the - // supported language families. - let inputs: &[&[u8]] = &[b"", b" \n\t\n", b"// just a comment\n", b"/* block */\n"]; - - for lang in LANG::into_enum_iter() { - if !lang.is_enabled() { - continue; - } - for src in inputs { - let space = analyze(Source::new(lang, src), MetricsOptions::default()) - .unwrap_or_else(|err| { - panic!( - "{} on input {:?} unexpectedly returned {err:?}; \ - EmptyRoot is documented as not produced today", - lang.name(), - String::from_utf8_lossy(src), - ) - }); - assert_eq!( - space.kind, - SpaceKind::Unit, - "{} on input {:?} produced a non-Unit top-level FuncSpace", - lang.name(), - String::from_utf8_lossy(src), - ); - } - } - } - // `Display` must agree with `name` for every variant — the // impl delegates to it, so this pins that contract against future // refactors that might diverge the two. @@ -787,50 +746,4 @@ mod tests { "expected LanguageDisabled display to mention `rust`, got {rendered:?}", ); } - - // Drift guard for the crate-level `## Supported Languages` rustdoc - // list in `src/lib.rs` (#769): every LANG variant's canonical slug - // — the single source of truth from `name()` — must appear in that - // list as a backtick-delimited token. Without this guard, adding a - // language (or renaming a slug) silently desyncs the docs.rs - // landing page, which is exactly how Objective-C went missing for a - // full release after #724 shipped it. - // - // The slug set is derived from `LANG::into_enum_iter()`, which is - // compiled unconditionally (the enum surface is feature-independent; - // only the grammar crates are gated), so this test is robust under - // `--no-default-features` and any per-language feature subset — no - // `all-languages` gate needed. Distinct slugs are deduplicated, so - // shared-slug families do not require one bullet per variant. - #[test] - fn supported_languages_rustdoc_lists_every_slug() { - // Bound the search to the `## Supported Languages` section so an - // incidental backtick match elsewhere in the module docs (e.g. - // a slug named in the metrics section) cannot mask a real - // omission from the list itself. - const SECTION_HEADER: &str = "## Supported Languages"; - const NEXT_HEADER: &str = "## Supported Metrics"; - - let lib_rs = std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/src/lib.rs")) - .expect("src/lib.rs is readable from CARGO_MANIFEST_DIR"); - - let section_start = lib_rs - .find(SECTION_HEADER) - .expect("rustdoc must contain a `## Supported Languages` section"); - let section_end = lib_rs[section_start..] - .find(NEXT_HEADER) - .map(|offset| section_start + offset) - .expect("`## Supported Languages` must be followed by `## Supported Metrics`"); - let section = &lib_rs[section_start..section_end]; - - for lang in LANG::into_enum_iter() { - let slug_token = format!("`{}`", lang.name()); - assert!( - section.contains(&slug_token), - "LANG::{lang:?} slug {slug_token} is missing from the \ - `## Supported Languages` rustdoc list in src/lib.rs — \ - add an entry there (see #769)", - ); - } - } } diff --git a/src/language_enum_roundtrip.rs b/big-code-analysis-ast/src/language_enum_roundtrip.rs similarity index 100% rename from src/language_enum_roundtrip.rs rename to big-code-analysis-ast/src/language_enum_roundtrip.rs diff --git a/src/languages/language_bash.rs b/big-code-analysis-ast/src/languages/language_bash.rs similarity index 99% rename from src/languages/language_bash.rs rename to big-code-analysis-ast/src/languages/language_bash.rs index 5b25fbaed..5aad4cc58 100644 --- a/src/languages/language_bash.rs +++ b/big-code-analysis-ast/src/languages/language_bash.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_c.rs b/big-code-analysis-ast/src/languages/language_c.rs similarity index 99% rename from src/languages/language_c.rs rename to big-code-analysis-ast/src/languages/language_c.rs index 1d668ac8b..d1a9d8ff2 100644 --- a/src/languages/language_c.rs +++ b/big-code-analysis-ast/src/languages/language_c.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_ccomment.rs b/big-code-analysis-ast/src/languages/language_ccomment.rs similarity index 93% rename from src/languages/language_ccomment.rs rename to big-code-analysis-ast/src/languages/language_ccomment.rs index 516807295..44e9ee267 100644 --- a/src/languages/language_ccomment.rs +++ b/big-code-analysis-ast/src/languages/language_ccomment.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_cpp.rs b/big-code-analysis-ast/src/languages/language_cpp.rs similarity index 99% rename from src/languages/language_cpp.rs rename to big-code-analysis-ast/src/languages/language_cpp.rs index a2b16362a..d57ea5aa1 100644 --- a/src/languages/language_cpp.rs +++ b/big-code-analysis-ast/src/languages/language_cpp.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_csharp.rs b/big-code-analysis-ast/src/languages/language_csharp.rs similarity index 99% rename from src/languages/language_csharp.rs rename to big-code-analysis-ast/src/languages/language_csharp.rs index 2ef36f36b..dcc104fa2 100644 --- a/src/languages/language_csharp.rs +++ b/big-code-analysis-ast/src/languages/language_csharp.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_elixir.rs b/big-code-analysis-ast/src/languages/language_elixir.rs similarity index 99% rename from src/languages/language_elixir.rs rename to big-code-analysis-ast/src/languages/language_elixir.rs index 0dd35a34b..c7ce60649 100644 --- a/src/languages/language_elixir.rs +++ b/big-code-analysis-ast/src/languages/language_elixir.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_go.rs b/big-code-analysis-ast/src/languages/language_go.rs similarity index 99% rename from src/languages/language_go.rs rename to big-code-analysis-ast/src/languages/language_go.rs index 4a737faef..28902284d 100644 --- a/src/languages/language_go.rs +++ b/big-code-analysis-ast/src/languages/language_go.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_groovy.rs b/big-code-analysis-ast/src/languages/language_groovy.rs similarity index 99% rename from src/languages/language_groovy.rs rename to big-code-analysis-ast/src/languages/language_groovy.rs index adeb2a134..9ec3a8e57 100644 --- a/src/languages/language_groovy.rs +++ b/big-code-analysis-ast/src/languages/language_groovy.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_irules.rs b/big-code-analysis-ast/src/languages/language_irules.rs similarity index 98% rename from src/languages/language_irules.rs rename to big-code-analysis-ast/src/languages/language_irules.rs index a725a13f4..2ee65b017 100644 --- a/src/languages/language_irules.rs +++ b/big-code-analysis-ast/src/languages/language_irules.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_java.rs b/big-code-analysis-ast/src/languages/language_java.rs similarity index 99% rename from src/languages/language_java.rs rename to big-code-analysis-ast/src/languages/language_java.rs index e895c20e4..7942cf908 100644 --- a/src/languages/language_java.rs +++ b/big-code-analysis-ast/src/languages/language_java.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_javascript.rs b/big-code-analysis-ast/src/languages/language_javascript.rs similarity index 99% rename from src/languages/language_javascript.rs rename to big-code-analysis-ast/src/languages/language_javascript.rs index 2e45a09bf..1dfa9445b 100644 --- a/src/languages/language_javascript.rs +++ b/big-code-analysis-ast/src/languages/language_javascript.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_kotlin.rs b/big-code-analysis-ast/src/languages/language_kotlin.rs similarity index 99% rename from src/languages/language_kotlin.rs rename to big-code-analysis-ast/src/languages/language_kotlin.rs index 972252edf..bdab6f27c 100644 --- a/src/languages/language_kotlin.rs +++ b/big-code-analysis-ast/src/languages/language_kotlin.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_lua.rs b/big-code-analysis-ast/src/languages/language_lua.rs similarity index 98% rename from src/languages/language_lua.rs rename to big-code-analysis-ast/src/languages/language_lua.rs index d2ac68532..5aa8b59f4 100644 --- a/src/languages/language_lua.rs +++ b/big-code-analysis-ast/src/languages/language_lua.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_mozcpp.rs b/big-code-analysis-ast/src/languages/language_mozcpp.rs similarity index 99% rename from src/languages/language_mozcpp.rs rename to big-code-analysis-ast/src/languages/language_mozcpp.rs index 78c80aa28..4efa86e45 100644 --- a/src/languages/language_mozcpp.rs +++ b/big-code-analysis-ast/src/languages/language_mozcpp.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_mozjs.rs b/big-code-analysis-ast/src/languages/language_mozjs.rs similarity index 99% rename from src/languages/language_mozjs.rs rename to big-code-analysis-ast/src/languages/language_mozjs.rs index a36400893..b4e3f0ccc 100644 --- a/src/languages/language_mozjs.rs +++ b/big-code-analysis-ast/src/languages/language_mozjs.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_objc.rs b/big-code-analysis-ast/src/languages/language_objc.rs similarity index 99% rename from src/languages/language_objc.rs rename to big-code-analysis-ast/src/languages/language_objc.rs index 1b6dad80d..567a07b83 100644 --- a/src/languages/language_objc.rs +++ b/big-code-analysis-ast/src/languages/language_objc.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_perl.rs b/big-code-analysis-ast/src/languages/language_perl.rs similarity index 99% rename from src/languages/language_perl.rs rename to big-code-analysis-ast/src/languages/language_perl.rs index 890c282a2..93d2ad7a4 100644 --- a/src/languages/language_perl.rs +++ b/big-code-analysis-ast/src/languages/language_perl.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_php.rs b/big-code-analysis-ast/src/languages/language_php.rs similarity index 99% rename from src/languages/language_php.rs rename to big-code-analysis-ast/src/languages/language_php.rs index e86f61a7c..e32c3326a 100644 --- a/src/languages/language_php.rs +++ b/big-code-analysis-ast/src/languages/language_php.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_preproc.rs b/big-code-analysis-ast/src/languages/language_preproc.rs similarity index 95% rename from src/languages/language_preproc.rs rename to big-code-analysis-ast/src/languages/language_preproc.rs index b9d508cf3..dacd0d5fa 100644 --- a/src/languages/language_preproc.rs +++ b/big-code-analysis-ast/src/languages/language_preproc.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_python.rs b/big-code-analysis-ast/src/languages/language_python.rs similarity index 99% rename from src/languages/language_python.rs rename to big-code-analysis-ast/src/languages/language_python.rs index b9b9f734c..0425311a0 100644 --- a/src/languages/language_python.rs +++ b/big-code-analysis-ast/src/languages/language_python.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_ruby.rs b/big-code-analysis-ast/src/languages/language_ruby.rs similarity index 99% rename from src/languages/language_ruby.rs rename to big-code-analysis-ast/src/languages/language_ruby.rs index bb2f3b07e..0325efb1d 100644 --- a/src/languages/language_ruby.rs +++ b/big-code-analysis-ast/src/languages/language_ruby.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_rust.rs b/big-code-analysis-ast/src/languages/language_rust.rs similarity index 99% rename from src/languages/language_rust.rs rename to big-code-analysis-ast/src/languages/language_rust.rs index 4ae830e80..22123c6a1 100644 --- a/src/languages/language_rust.rs +++ b/big-code-analysis-ast/src/languages/language_rust.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_tcl.rs b/big-code-analysis-ast/src/languages/language_tcl.rs similarity index 98% rename from src/languages/language_tcl.rs rename to big-code-analysis-ast/src/languages/language_tcl.rs index 317df661a..1592d3904 100644 --- a/src/languages/language_tcl.rs +++ b/big-code-analysis-ast/src/languages/language_tcl.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_tsx.rs b/big-code-analysis-ast/src/languages/language_tsx.rs similarity index 99% rename from src/languages/language_tsx.rs rename to big-code-analysis-ast/src/languages/language_tsx.rs index e33a7f40c..fce1fd36f 100644 --- a/src/languages/language_tsx.rs +++ b/big-code-analysis-ast/src/languages/language_tsx.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/language_typescript.rs b/big-code-analysis-ast/src/languages/language_typescript.rs similarity index 99% rename from src/languages/language_typescript.rs rename to big-code-analysis-ast/src/languages/language_typescript.rs index 0726cb565..8195e1f9e 100644 --- a/src/languages/language_typescript.rs +++ b/big-code-analysis-ast/src/languages/language_typescript.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/src/languages/mod.rs b/big-code-analysis-ast/src/languages/mod.rs similarity index 100% rename from src/languages/mod.rs rename to big-code-analysis-ast/src/languages/mod.rs diff --git a/big-code-analysis-ast/src/lib.rs b/big-code-analysis-ast/src/lib.rs new file mode 100644 index 000000000..8b0c47300 --- /dev/null +++ b/big-code-analysis-ast/src/lib.rs @@ -0,0 +1,147 @@ +// Per-language modules deliberately consume the macro-generated +// tree-sitter token enums via `use crate::*` and `use Foo::*` inside +// match expressions — explicit imports would list dozens of variants per +// arm and obscure the per-language token sets that are the point of +// these files. Allowed at the module level rather than per function so +// the per-language impl blocks stay readable. +#![allow(clippy::doc_markdown, clippy::enum_glob_use, clippy::wildcard_imports)] +// Per-language Cargo features let a downstream build only a subset of +// grammars. In such a build the code for the disabled languages — their +// macro-generated `*Code` / `*Parser` tags plus the getter / checker +// helpers only those languages reach — is compiled but never +// constructed, so `-D dead-code` fires on items that are all live in the +// default `all-languages` build. Relax dead-code to a warning only when +// the full language set is NOT enabled; the default build and +// `--all-features` (and thus the primary CI gate and `make pre-commit`) +// still hard-deny it, so genuine dead code is caught there. +#![cfg_attr(not(feature = "all-languages"), allow(dead_code))] + +//! The parse and classification layer behind +//! [`big-code-analysis`](https://crates.io/crates/big-code-analysis). +//! +//! This crate turns source bytes into a [`tree_sitter`] tree and +//! answers structural questions about it: which [`LANG`] a file is in, +//! what a [`Node`] is (`is_func`, `is_call`, `is_string`, … through +//! [`Checker`]), what it is called and which Halstead class or +//! [`SpaceKind`] it opens ([`Getter`]), and how it should be rendered +//! as an [`AstNode`] ([`Alterator`]). It also owns the C-family +//! preprocessor pass ([`preproc`]), comment stripping, node counting and +//! finding, and language detection ([`guess_language`]). It computes no +//! metric. +//! +//! # Stability +//! +//! **This crate is internal plumbing.** It exists so the metric walk in +//! `big-code-analysis` and any future structural consumer (a linter, a +//! call-graph builder, a language server) can share one classification +//! layer without the metric machinery. `big-code-analysis` pins it at +//! an exact `=X.Y.Z` version, the two are released together, and no +//! item here carries a stability promise of its own: names, signatures +//! and module paths may change in any release. Depend on +//! `big-code-analysis` and reach what it re-exports; depend on this +//! crate directly only when you accept re-pinning on every release. +//! +//! The one contract that does hold is the one `big-code-analysis` +//! documents in its `STABILITY.md` for the items it re-exports from +//! here (`LANG`, `Node`, `MetricsError`, `SpaceKind`, the AST dump +//! types, the preprocessor types, the file readers). Those are stable +//! *through that crate*. +//! +//! # Layout +//! +//! - [`languages`]: one generated enum per grammar, one variant per +//! tree-sitter kind id. +//! - [`langs`]: the [`LANG`] enum, extension / emacs-mode lookup, the +//! per-language `*Code` tags and `*Parser` aliases, and [`AnyParser`], +//! the runtime-dispatched parser every consumer matches on with +//! [`with_any_parser!`]. +//! - [`node`], [`traits`], [`parser`]: the tree-sitter wrappers and the +//! [`ParserTrait`] / [`Search`] / [`LanguageInfo`] contracts. +//! - [`checker`], [`getter`], [`alterator`], [`lang_helpers`]: the +//! per-language classifiers and the byte-level identity helpers they +//! share with the metrics. +//! - [`preproc`], `c_macro`, [`c_declarator`]: the C-family pipeline. +//! - `comment_rm`, [`ast`], [`count`], `find`, [`tools`]: the +//! per-file operations reached through `big_code_analysis::Ast`. +//! +//! The unlinked names above are crate-private modules, reached only +//! through the `AnyParser` methods `mk_action!` generates. + +#![allow(clippy::upper_case_acronyms)] +// Production-only `unwrap()` ban. See `[workspace.lints.clippy]` in the +// root `Cargo.toml` for why this is a per-root attribute and not a +// Cargo lint (#1227). +#![cfg_attr(not(test), warn(clippy::unwrap_used))] + +// The `pub(crate)` entries below are named by nothing outside this +// crate, so they stay narrow per AGENTS.md ("widen visibility only when +// an item is re-exported from `lib.rs`"). `comment_rm` and `find` look +// like exceptions and are not: they are reached through `$crate::` in +// `mk_action!`, which expands here. +pub mod alterator; +pub mod ast; +pub mod c_declarator; +pub(crate) mod c_langs_macros; +pub(crate) mod c_macro; +pub(crate) mod cfg_predicate; +pub mod checker; +pub(crate) mod comment_rm; +pub mod count; +pub mod error; +pub(crate) mod find; +pub mod getter; +pub mod lang_helpers; +pub mod langs; +pub mod languages; +pub mod macros; +pub mod node; +pub mod observation; +pub mod parser; +pub mod preproc; +pub mod recursion; +pub mod space_kind; +pub mod token_role; +pub mod tools; +pub mod traits; + +#[cfg(test)] +mod language_enum_roundtrip; + +/// Parse-and-inspect helpers for tests, in this crate and in +/// `big-code-analysis`'s metric tests (behind the `test-support` +/// feature). Never part of a shipping build. +#[cfg(any(test, feature = "test-support"))] +#[doc(hidden)] +pub mod test_support; + +// Flat re-exports. The per-language modules reach every token enum, tag +// and classifier through `use crate::*`; `big-code-analysis` does the +// same through one `pub use big_code_analysis_ast::*`, so the +// names below are the crate's working vocabulary rather than a curated +// public surface. +pub use crate::alterator::Alterator; +pub use crate::ast::{AstCfg, AstNode, AstPayload, AstResponse, MAX_AST_SERIALIZE_DEPTH, Span}; +pub use crate::checker::*; +pub use crate::count::{Count, CountCollector}; +pub use crate::error::MetricsError; +pub use crate::getter::Getter; +pub use crate::langs::*; +pub use crate::languages::*; +pub use crate::macros::ParseLangError; +pub use crate::node::{Ancestors, Node}; +pub use crate::parser::Parser; +pub use crate::preproc::{ + PreprocDiagnostic, PreprocFile, PreprocResults, fix_includes, get_macros, preprocess, +}; +pub use crate::space_kind::SpaceKind; +pub use crate::token_role::TokenRole; +pub use crate::tools::{ + SkipReason, get_language_for_file, guess_language, is_generated, normalize_eol, read_file, + read_file_with_eol, read_file_with_eol_classified, write_file, +}; +pub use crate::traits::{LanguageInfo, ParserTrait, Search}; + +/// Re-export of the underlying `tree-sitter` crate, so a consumer can +/// build a [`tree_sitter::Tree`] against the exact grammar version this +/// crate is pinned to. +pub use ::tree_sitter; diff --git a/src/macros/kind_sets.rs b/big-code-analysis-ast/src/macros/kind_sets.rs similarity index 96% rename from src/macros/kind_sets.rs rename to big-code-analysis-ast/src/macros/kind_sets.rs index 3edcdb227..95e343a08 100644 --- a/src/macros/kind_sets.rs +++ b/big-code-analysis-ast/src/macros/kind_sets.rs @@ -10,7 +10,7 @@ // membership -- and every per-call grammar rationale comment travels // with its macro (see `.claude/rules/macro-comments.md`). // -// Re-exported below via `pub(crate) use` and again from +// Re-exported below via `pub use` and again from // `macros/mod.rs`, so every existing `crate::macros::` import in // `checker.rs`, `metrics/npa.rs`, and `metrics/abc.rs` keeps resolving // unchanged. @@ -21,6 +21,8 @@ // here keeps every match site in lockstep, so a future grammar bump that // adds another numbered variant is a one-line edit instead of a scatter // of 4-5 sites. +#[macro_export] +#[doc(hidden)] macro_rules! csharp_invocation_expr_kinds { () => { $crate::Csharp::InvocationExpression @@ -29,6 +31,8 @@ macro_rules! csharp_invocation_expr_kinds { }; } +#[macro_export] +#[doc(hidden)] macro_rules! csharp_paren_expr_kinds { () => { $crate::Csharp::ParenthesizedExpression @@ -37,6 +41,8 @@ macro_rules! csharp_paren_expr_kinds { }; } +#[macro_export] +#[doc(hidden)] macro_rules! csharp_prefix_unary_expr_kinds { () => { $crate::Csharp::PrefixUnaryExpression | $crate::Csharp::PrefixUnaryExpression2 @@ -60,6 +66,8 @@ macro_rules! csharp_prefix_unary_expr_kinds { // Before #372 only the first three (invocation / identifier / // boolean) were recognised, so all five kinds above silently scored // zero conditions in `if` / `while` / `do` / ternary contexts. +#[macro_export] +#[doc(hidden)] macro_rules! csharp_bool_terminal_kinds { () => { $crate::Csharp::InvocationExpression @@ -75,12 +83,16 @@ macro_rules! csharp_bool_terminal_kinds { }; } +#[macro_export] +#[doc(hidden)] macro_rules! csharp_var_decl_kinds { () => { $crate::Csharp::VariableDeclaration | $crate::Csharp::VariableDeclaration2 }; } +#[macro_export] +#[doc(hidden)] macro_rules! csharp_var_declarator_kinds { () => { $crate::Csharp::VariableDeclarator | $crate::Csharp::VariableDeclarator2 @@ -102,6 +114,8 @@ macro_rules! csharp_var_declarator_kinds { // `java_walk_ternary`, and the two branches of `java_walk_for_statement` // (the latter ORs in `SEMI | RPAREN` at the call site to also recognise // the empty-condition `for (;;)` form). +#[macro_export] +#[doc(hidden)] macro_rules! java_bool_terminal_kinds { () => { $crate::Java::MethodInvocation @@ -127,6 +141,8 @@ macro_rules! java_bool_terminal_kinds { // `CastExpression`, `ParenthesizedTypeCast`, `InstanceofExpression`); // the dekobon Groovy grammar has no `await` or `array_access` // analogues, so those collapse out of the C# set. +#[macro_export] +#[doc(hidden)] macro_rules! groovy_bool_terminal_kinds { () => { $crate::Groovy::MethodInvocation @@ -148,6 +164,8 @@ macro_rules! groovy_bool_terminal_kinds { // `_count_unary_conditions`) consumes the same set in both // helpers, so hoisting to a macro removes the literal duplication. +#[macro_export] +#[doc(hidden)] macro_rules! rust_bool_terminal_kinds { // `ScopedIdentifier` (`crate::FLAG`, `ns::flag`) and // `AwaitExpression` (`ready().await`) are both idiomatic shapes @@ -166,6 +184,8 @@ macro_rules! rust_bool_terminal_kinds { }; } +#[macro_export] +#[doc(hidden)] macro_rules! go_bool_terminal_kinds { // Aliased Identifier kind_ids (lesson #2): tree-sitter-go emits // `identifier` under three numeric ids (1, 60, 61) depending on @@ -185,6 +205,8 @@ macro_rules! go_bool_terminal_kinds { }; } +#[macro_export] +#[doc(hidden)] macro_rules! cpp_bool_terminal_kinds { // Matches on node-kind NAMES, not one grammar's enum discriminants, // so it is correct for every C-family grammar: the shared ABC helpers @@ -215,6 +237,8 @@ macro_rules! cpp_bool_terminal_kinds { }; } +#[macro_export] +#[doc(hidden)] macro_rules! php_bool_terminal_kinds { // Aliased kind_ids (lesson 2): // - `name` has two ids (1, 211) @@ -249,6 +273,8 @@ macro_rules! php_bool_terminal_kinds { }; } +#[macro_export] +#[doc(hidden)] macro_rules! python_bool_terminal_kinds { // `Await` (`await ready()`) evaluates to a boolean in idiomatic // async Python — mirrors the C# fix in #372 (lesson 19) which @@ -273,6 +299,8 @@ macro_rules! python_bool_terminal_kinds { }; } +#[macro_export] +#[doc(hidden)] macro_rules! perl_bool_terminal_kinds { () => { $crate::Perl::Identifier @@ -295,6 +323,8 @@ macro_rules! perl_bool_terminal_kinds { }; } +#[macro_export] +#[doc(hidden)] macro_rules! lua_bool_terminal_kinds { () => { $crate::Lua::Identifier @@ -311,6 +341,8 @@ macro_rules! lua_bool_terminal_kinds { }; } +#[macro_export] +#[doc(hidden)] macro_rules! tcl_bool_terminal_kinds { () => { $crate::Tcl::SimpleWord @@ -326,6 +358,8 @@ macro_rules! tcl_bool_terminal_kinds { // iRules counterpart of `tcl_bool_terminal_kinds!` (the grammar is a Tcl // dialect, so the terminal-operand set is the same shape). +#[macro_export] +#[doc(hidden)] macro_rules! irules_bool_terminal_kinds { () => { $crate::Irules::SimpleWord @@ -349,6 +383,8 @@ macro_rules! irules_bool_terminal_kinds { // generic, which silently dropped `MemberExpression2` (the kind // runtime emits for `obj.foo`) for all four languages. +#[macro_export] +#[doc(hidden)] macro_rules! javascript_bool_terminal_kinds { // `AwaitExpression` (`await ready()`) is in the terminal set // mirroring the C# reference (lesson 19). `Number` is a @@ -372,6 +408,8 @@ macro_rules! javascript_bool_terminal_kinds { }; } +#[macro_export] +#[doc(hidden)] macro_rules! mozjs_bool_terminal_kinds { // `AwaitExpression` (`await ready()`) is in the terminal set // mirroring the C# reference (lesson 19). `Number` is a @@ -394,6 +432,8 @@ macro_rules! mozjs_bool_terminal_kinds { }; } +#[macro_export] +#[doc(hidden)] macro_rules! typescript_bool_terminal_kinds { // `AwaitExpression` (`await ready()`) is in the terminal set // mirroring the C# reference (lesson 19). `Number` (the numeric @@ -421,6 +461,8 @@ macro_rules! typescript_bool_terminal_kinds { }; } +#[macro_export] +#[doc(hidden)] macro_rules! tsx_bool_terminal_kinds { // `AwaitExpression` (`await ready()`) is in the terminal set // mirroring the C# reference (lesson 19). `Number` (the numeric @@ -459,6 +501,8 @@ macro_rules! tsx_bool_terminal_kinds { // (`arr[0]`), and `this_expression`. Comparison operands (`x > 0`) are // themselves `binary_expression` nodes, so they are absent from this set // and contribute nothing — matching the paper's "only unary conditions". +#[macro_export] +#[doc(hidden)] macro_rules! kotlin_bool_terminal_kinds { () => { $crate::Kotlin::Identifier @@ -478,6 +522,8 @@ macro_rules! kotlin_bool_terminal_kinds { // (`@ivar`, `@@cvar`, `$gvar`), `constant`, `element_reference` // (`items[0]`), and `integer`. Comparison operands (`x > 0`) are nested // `binary` nodes, so they are absent here and contribute nothing. +#[macro_export] +#[doc(hidden)] macro_rules! ruby_bool_terminal_kinds { () => { $crate::Ruby::Identifier @@ -507,6 +553,8 @@ macro_rules! ruby_bool_terminal_kinds { // parse as `boolean`, verified by AST dump), `nil`, `atom`, `integer`, // and `access_call` (`xs[i]`). Comparison operands are nested // `binary_operator` nodes and so contribute nothing. +#[macro_export] +#[doc(hidden)] macro_rules! elixir_bool_terminal_kinds { () => { $crate::Elixir::Identifier @@ -535,6 +583,8 @@ macro_rules! elixir_bool_terminal_kinds { // `AwaitExpression` and other shapes; this body is the historical // floor, not the current set). #[allow(unused_macros)] +#[macro_export] +#[doc(hidden)] macro_rules! js_family_bool_terminal_kinds { ($Lang:ident) => { $crate::$Lang::Identifier @@ -546,14 +596,3 @@ macro_rules! js_family_bool_terminal_kinds { | $crate::$Lang::SubscriptExpression }; } - -pub(crate) use { - cpp_bool_terminal_kinds, csharp_bool_terminal_kinds, csharp_invocation_expr_kinds, - csharp_paren_expr_kinds, csharp_prefix_unary_expr_kinds, csharp_var_decl_kinds, - csharp_var_declarator_kinds, elixir_bool_terminal_kinds, go_bool_terminal_kinds, - groovy_bool_terminal_kinds, irules_bool_terminal_kinds, java_bool_terminal_kinds, - javascript_bool_terminal_kinds, kotlin_bool_terminal_kinds, lua_bool_terminal_kinds, - mozjs_bool_terminal_kinds, perl_bool_terminal_kinds, php_bool_terminal_kinds, - python_bool_terminal_kinds, ruby_bool_terminal_kinds, rust_bool_terminal_kinds, - tcl_bool_terminal_kinds, tsx_bool_terminal_kinds, typescript_bool_terminal_kinds, -}; diff --git a/big-code-analysis-ast/src/macros/mod.rs b/big-code-analysis-ast/src/macros/mod.rs new file mode 100644 index 000000000..34fdadffe --- /dev/null +++ b/big-code-analysis-ast/src/macros/mod.rs @@ -0,0 +1,581 @@ +//! The `mk_langs!` family that generates [`langs`](crate::langs), the +//! [`with_any_parser!`](crate::with_any_parser) dispatch macro, and the +//! kind-set aliases. + +// `get_language!` is invoked only from feature-gated arms in `mk_lang!` +// (one arm per `LANG::*` variant whose per-language Cargo feature is +// enabled). A build with `--no-default-features` and no language +// feature has no remaining call sites; suppress the lint for that +// pathological-but-valid configuration. +#[allow(unused_macros)] +macro_rules! get_language { + (tree_sitter_typescript) => { + tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into() + }; + (tree_sitter_tsx) => { + tree_sitter_typescript::LANGUAGE_TSX.into() + }; + (tree_sitter_php) => { + tree_sitter_php::LANGUAGE_PHP.into() + }; + ($name:ident) => { + $name::LANGUAGE.into() + }; +} + +macro_rules! mk_lang { + ( $( ($feature:literal, $camel:ident, $name:ident, $display: expr, $description:expr, $version:literal) ),* ) => { + /// The list of supported languages. + /// + /// Every variant is always defined regardless of the Cargo + /// feature set: per-language features only gate the grammar + /// crate references, never the enum surface itself. Disabled + /// variants surface at runtime as + /// [`crate::MetricsError::LanguageDisabled`] from every entry + /// point that returns a `Result`. + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] + pub enum LANG { + $( + #[doc = $description] + $camel, + )* + } + impl LANG { + /// Return an iterator over the supported languages. + /// + /// # Examples + /// + /// ``` + /// use big_code_analysis_ast::LANG; + /// + /// for lang in LANG::into_enum_iter() { + /// println!("{:?}", lang); + /// } + /// ``` + pub fn into_enum_iter() -> impl Iterator { + use LANG::*; + [$( $camel, )*].into_iter() + } + + /// Returns the name of a language as a `&str`. + /// + /// # Examples + /// + /// ``` + /// use big_code_analysis_ast::LANG; + /// + /// println!("{}", LANG::Rust.name()); + /// ``` + pub fn name(&self) -> &'static str { + match self { + $( + LANG::$camel => $display, + )* + } + } + + /// Returns the pinned tree-sitter grammar crate version that + /// backs this variant (e.g. `"0.25.1"` for [`LANG::Bash`]). + /// + /// The value mirrors the `=X.Y.Z` pin in the workspace + /// `Cargo.toml` and is independent of the per-language Cargo + /// feature: it is returned even for a variant whose feature is + /// disabled in the current build (a build-time constant, no + /// grammar crate reference). A drift test in `src/langs.rs` + /// asserts every value here matches the manifest pin. + /// + /// # Grammars vs. forks + /// + /// For languages backed by an upstream crates.io grammar + /// (`bash`, `rust`, `python`, `typescript`, …) this is the + /// exact upstream grammar version, so a consumer migrating + /// matchers off py-tree-sitter can line node-kind vocabularies + /// up against the same pin. For the vendored big-code-analysis + /// forks (`mozcpp`, `mozjs`, `tcl`, `ccomment`, `preproc`, + /// `kotlin`) the value is the **fork crate's** version + /// (published as `bca-tree-sitter-*` / `tree-sitter-kotlin-ng`), + /// not an upstream tree-sitter grammar semver — there is no + /// upstream release to compare against. + /// + /// This is part of the value-not-stable surface: the returned + /// version changes whenever the grammar pin is bumped. + #[must_use] + pub fn grammar_version(&self) -> &'static str { + match self { + $( + LANG::$camel => $version, + )* + } + } + + /// Reports whether this variant's grammar crate is + /// compiled into the current build. + /// + /// Returns `false` for variants whose per-language Cargo + /// feature is disabled; calling + /// [`Self::tree_sitter_language`], `big_code_analysis::analyze`, + /// or any other dispatcher with such a variant will + /// return [`crate::MetricsError::LanguageDisabled`]. + #[must_use] + pub fn is_enabled(&self) -> bool { + match self { + $( + #[cfg(feature = $feature)] + LANG::$camel => true, + #[cfg(not(feature = $feature))] + LANG::$camel => false, + )* + } + } + + // Returns a tree-sitter language paired with this variant, + // or `Err(LanguageDisabled)` when the matching Cargo + // feature is off. This is the internal entry point used + // by `Tree::new` to construct a parser; the public + // counterpart is `tree_sitter_language`. + pub(crate) fn get_ts_language(&self) -> Result { + match self { + $( + #[cfg(feature = $feature)] + LANG::$camel => Ok(get_language!($name)), + #[cfg(not(feature = $feature))] + LANG::$camel => Err(crate::MetricsError::LanguageDisabled(*self)), + )* + } + } + + /// Returns the [`tree_sitter::Language`] grammar used by + /// this variant. + /// + /// Useful when feeding a caller-built + /// [`tree_sitter::Parser`] into the + /// `big_code_analysis::Ast::from_tree_sitter` entry point — the + /// language returned here is the one the metric walker + /// expects for `kind_id` matching, so the trees agree + /// structurally. + /// + /// This method is part of the value-not-stable surface: + /// the underlying `tree-sitter-*` grammar pin may bump + /// in any minor release, which can change `Language` + /// equality on the caller side. + /// + /// # Errors + /// + /// Returns [`crate::MetricsError::LanguageDisabled`] when + /// the variant's per-language Cargo feature is not + /// enabled in the current build (see the `[features]` + /// table in the root `Cargo.toml`). + /// + /// # Examples + /// + /// ``` + /// use big_code_analysis_ast::LANG; + /// + /// let _lang = LANG::Rust.tree_sitter_language().expect("rust feature enabled"); + /// ``` + pub fn tree_sitter_language(&self) -> Result<::tree_sitter::Language, crate::MetricsError> { + self.get_ts_language() + } + } + + /// Renders the language's canonical lowercase slug, identical to + /// [`LANG::name`]. + /// + /// Every variant has a distinct slug, so `Display` is injective + /// and a `Display` → [`FromStr`](std::str::FromStr) round-trip + /// returns the original variant (see the round-trip test in + /// `src/langs.rs`). The slug is the single canonical identifier + /// used across every surface (CLI JSON, web `/metrics`, the + /// Python bindings): it contains no punctuation and is always a + /// valid `FromStr` lookup token. + impl ::std::fmt::Display for LANG { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + f.write_str(self.name()) + } + } + + /// Parses a [`LANG`] from its [`Display`](std::fmt::Display) + /// spelling (the canonical lowercase [`LANG::name`] slug, e.g. + /// `"rust"`, `"cpp"`, `"csharp"`, `"tsx"`). + /// + /// Matching is case-sensitive and exact, mirroring + /// `big_code_analysis::Metric`'s `FromStr`: only the canonical + /// lowercase slug is accepted. File extensions and emacs modes + /// are deliberately *not* accepted here — use + /// [`get_from_ext`](crate::get_from_ext) / + /// [`get_from_emacs_mode`](crate::get_from_emacs_mode) for those. + /// + /// Every variant has a distinct slug, so this is the exact + /// inverse of [`Display`](std::fmt::Display): the round-trip + /// `LANG::from_str(&lang.to_string())` returns the original + /// variant for every `LANG`. + impl ::std::str::FromStr for LANG { + type Err = $crate::macros::ParseLangError; + + fn from_str(s: &str) -> Result { + LANG::into_enum_iter() + .find(|lang| lang.name() == s) + .ok_or_else(|| $crate::macros::ParseLangError::new(s)) + } + } + }; +} + +/// Error returned by [`LANG`](crate::LANG)'s +/// [`FromStr`](std::str::FromStr) impl when the input is not a +/// recognised language name. +/// +/// Holds the offending input verbatim so wrapper layers can format +/// their own user-facing message; mirrors +/// `big_code_analysis::ParseMetricError`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParseLangError(String); + +impl ParseLangError { + // Constructor kept `pub(crate)` so the macro-generated `FromStr` + // impl in `crate::langs` can build the error without exposing the + // private field across module boundaries. + pub(crate) fn new(input: &str) -> Self { + Self(input.to_owned()) + } + + /// The rejected input that failed to parse as a language name. + /// + /// Lets callers recover the offending string programmatically + /// rather than scraping it out of the [`Display`](std::fmt::Display) + /// output. Mirrors + /// `big_code_analysis::ParseMetricError::input`. + #[must_use] + pub fn input(&self) -> &str { + &self.0 + } +} + +impl ::std::fmt::Display for ParseLangError { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + write!(f, "unknown language: {}", self.0) + } +} + +impl ::std::error::Error for ParseLangError {} + +macro_rules! mk_action { + ( $( ($feature:literal, $camel:ident, $parser:ident) ),* ) => { + /// A parsed tree plus its source bytes for a language chosen at + /// runtime — one variant per [`LANG`], each holding that + /// language's `Parser`. The public seam is + /// `big_code_analysis::Ast`; this enum is the language-dispatched carrier + /// it wraps, and the value a caller matches on to reach a + /// concrete parser (`with_any_parser!`). + /// + /// Every variant exists regardless of the Cargo feature set: a + /// `Parser` *type* needs no grammar crate, only + /// [`Self::parse`] / [`Self::from_tree`] do, and those are the + /// arms that are feature-gated. A disabled language is therefore + /// never *constructed* — the constructors return + /// `Err(LanguageDisabled)` for it — but it can always be *named*, + /// which is what lets `with_any_parser!` be written once without + /// any `cfg` of its own (#1376). + pub enum AnyParser { + $( + #[doc = concat!("The `", stringify!($camel), "` parser.")] + $camel($parser), + )* + } + + impl AnyParser { + /// Parse `source` as `lang`. + /// + /// `Parser::new` keys the C-family macro-expansion lookup off + /// the caller-supplied path; callers analysing in-memory + /// snippets pass `None` and get the empty `Path` (`""`), + /// which the lookup ignores. That path never leaks into a + /// display name — `Ast` carries the name separately. + /// `source` is taken by value so an owned buffer moves + /// straight into the parser instead of being copied. + /// + /// # Errors + /// + /// `MetricsError::LanguageDisabled` when `lang`'s Cargo + /// feature is not enabled in this build. + pub fn parse( + lang: LANG, + source: Vec, + preproc_path: Option<&Path>, + preproc: Option>, + ) -> Result { + let preproc_path = preproc_path.unwrap_or(Path::new("")); + match lang { + $( + #[cfg(feature = $feature)] + LANG::$camel => Ok(AnyParser::$camel($parser::new(source, preproc_path, preproc))), + #[cfg(not(feature = $feature))] + LANG::$camel => { + let _ = (source, preproc_path, preproc); + Err(MetricsError::LanguageDisabled(lang)) + }, + )* + } + } + + /// Adopt a caller-built [`tree_sitter::Tree`] produced from + /// `source` with `lang`'s grammar. + /// + /// # Errors + /// + /// `MetricsError::LanguageDisabled` when `lang`'s Cargo + /// feature is not enabled in this build. + pub fn from_tree( + lang: LANG, + tree: ::tree_sitter::Tree, + source: Vec, + ) -> Result { + match lang { + $( + #[cfg(feature = $feature)] + LANG::$camel => Ok(AnyParser::$camel($parser::from_tree(tree, source))), + #[cfg(not(feature = $feature))] + LANG::$camel => { + let _ = (tree, source); + Err(MetricsError::LanguageDisabled(lang)) + }, + )* + } + } + + /// The language this parser was built for. + #[must_use] + pub fn language(&self) -> LANG { + match self { + $( AnyParser::$camel(_) => LANG::$camel, )* + } + } + + /// The bytes the tree was parsed from — after `#define` + /// expansion for the C family, so not necessarily the bytes + /// handed to [`Self::parse`]. + #[must_use] + pub fn code(&self) -> &[u8] { + $crate::with_any_parser!(self, |p| p.code()) + } + + /// The held [`tree_sitter::Tree`]. + #[must_use] + pub fn ts_tree(&self) -> &::tree_sitter::Tree { + $crate::with_any_parser!(self, |p| p.ts_tree()) + } + + /// The root [`Node`] of the held tree. + #[must_use] + pub fn root_node(&self) -> Node<'_> { + $crate::with_any_parser!(self, |p| p.root()) + } + + /// The source with non-doc comments removed, or `None` when + /// there was nothing to strip. + #[must_use] + pub fn strip_comments(&self) -> Option> { + $crate::with_any_parser!(self, |p| $crate::comment_rm::rm_comments(p)) + } + + /// The AST dump under `cfg`. + #[must_use] + pub fn dump(&self, cfg: $crate::ast::AstCfg) -> $crate::ast::AstResponse { + $crate::with_any_parser!(self, |p| $crate::ast::dump_inner(p, cfg)) + } + + /// `(matching, total)` node counts for `filters` — the same + /// vocabulary [`ParserTrait::filters`] accepts. + #[must_use] + pub fn count(&self, filters: &[String]) -> (usize, usize) { + $crate::with_any_parser!(self, |p| $crate::count::count(p, filters)) + } + + /// Every node matching `filters`, in source order. The nodes + /// borrow the held tree. + /// + /// # Errors + /// + /// Currently infallible; the `Result` is reserved for a + /// future strict-parsing mode. + pub fn find(&self, filters: &[String]) -> Result>, MetricsError> { + $crate::with_any_parser!(self, |p| $crate::find::find(p, filters)) + } + } + }; +} + +/// Dispatches over every [`AnyParser`] variant, binding the concrete +/// `Parser` to `$p` and evaluating `$body` once per arm. +/// +/// Written out by hand rather than generated inside `mk_action!` so the +/// arm list stays a plain match: a variant missing here is a +/// non-exhaustive-match compile error, which is the whole guarantee. +/// Every arm is unconditional — see the [`AnyParser`] docs for why no +/// `cfg` is needed. Add a line here when `mk_langs!` gains a language. +/// +/// [`AnyParser`]: crate::langs::AnyParser +#[macro_export] +macro_rules! with_any_parser { + ($any:expr, |$p:ident| $body:expr) => { + match $any { + $crate::langs::AnyParser::Javascript($p) => $body, + $crate::langs::AnyParser::Mozjs($p) => $body, + $crate::langs::AnyParser::Java($p) => $body, + $crate::langs::AnyParser::Go($p) => $body, + $crate::langs::AnyParser::Kotlin($p) => $body, + $crate::langs::AnyParser::Lua($p) => $body, + $crate::langs::AnyParser::Rust($p) => $body, + $crate::langs::AnyParser::Tcl($p) => $body, + $crate::langs::AnyParser::Irules($p) => $body, + $crate::langs::AnyParser::C($p) => $body, + $crate::langs::AnyParser::Cpp($p) => $body, + $crate::langs::AnyParser::Mozcpp($p) => $body, + $crate::langs::AnyParser::Objc($p) => $body, + $crate::langs::AnyParser::Csharp($p) => $body, + $crate::langs::AnyParser::Elixir($p) => $body, + $crate::langs::AnyParser::Python($p) => $body, + $crate::langs::AnyParser::Tsx($p) => $body, + $crate::langs::AnyParser::Typescript($p) => $body, + $crate::langs::AnyParser::Bash($p) => $body, + $crate::langs::AnyParser::Ccomment($p) => $body, + $crate::langs::AnyParser::Preproc($p) => $body, + $crate::langs::AnyParser::Perl($p) => $body, + $crate::langs::AnyParser::Php($p) => $body, + $crate::langs::AnyParser::Ruby($p) => $body, + $crate::langs::AnyParser::Groovy($p) => $body, + } + }; +} + +macro_rules! mk_extensions { + ( $( ($camel:ident, [ $( $ext:ident ),* ]) ),* ) => { + /// Detects the language associated to the input file extension. + /// + /// # Examples + /// + /// ``` + /// use big_code_analysis_ast::get_from_ext; + /// + /// let ext = "rs"; + /// + /// get_from_ext(ext).unwrap(); + /// ``` + pub fn get_from_ext(ext: &str) -> Option{ + match ext { + $( + $( + stringify!($ext) => Some(LANG::$camel), + )* + )* + _ => None, + } + } + + impl LANG { + /// Returns the file extensions recognised for this language. + /// + /// The returned list is the same one consulted by + /// [`get_from_ext`] and [`crate::get_language_for_file`]. + /// Helper variants without user-facing files (`Ccomment`, + /// `Preproc`) return an empty slice. + /// + /// # Examples + /// + /// ``` + /// use big_code_analysis_ast::LANG; + /// + /// assert!(LANG::Rust.extensions().contains(&"rs")); + /// ``` + #[must_use] + pub fn extensions(&self) -> &'static [&'static str] { + match self { + $( + LANG::$camel => &[ $( stringify!($ext), )* ], + )* + } + } + } + }; +} + +macro_rules! mk_emacs_mode { + ( $( ($camel:ident, [ $( $emacs_mode:expr ),* ]) ),* ) => { + /// Detects the language associated to the input `Emacs` mode. + /// + /// An `Emacs` mode is used to detect a language according to + /// particular text-information contained in a file. + /// + /// # Examples + /// + /// ``` + /// use big_code_analysis_ast::get_from_emacs_mode; + /// + /// let emacs_mode = "rust"; + /// + /// get_from_emacs_mode(emacs_mode).unwrap(); + /// ``` + pub fn get_from_emacs_mode(mode: &str) -> Option{ + match mode { + $( + $( + $emacs_mode => Some(LANG::$camel), + )* + )* + _ => None, + } + } + }; +} + +macro_rules! mk_code { + ( $( ($camel:ident, $code:ident, $parser:ident, $name:ident, $docname:expr) ),* ) => { + $( + #[doc = concat!("Per-language code type tag for ", $docname, "; carries no data.")] + pub struct $code { _guard: (), } + + impl LanguageInfo for $code { + type BaseLang = $camel; + + fn lang() -> LANG { + LANG::$camel + } + } + + #[doc = "The `"] + #[doc = $docname] + #[doc = "` language parser."] + pub type $parser = Parser<$code>; + )* + }; +} + +macro_rules! mk_langs { + ( $( ($feature:literal, $camel:ident, $description: expr, $display: expr, $code:ident, $parser:ident, $name:ident, [ $( $ext:ident ),* ], [ $( $emacs_mode:expr ),* ], $version:literal) ),* ) => { + mk_lang!($( ($feature, $camel, $name, $display, $description, $version) ),*); + mk_action!($( ($feature, $camel, $parser) ),*); + mk_extensions!($( ($camel, [ $( $ext ),* ]) ),*); + mk_emacs_mode!($( ($camel, [ $( $emacs_mode ),* ]) ),*); + mk_code!($( ($camel, $code, $parser, $name, stringify!($camel)) ),*); + }; +} + +mod kind_sets; + +// The kind-set macros are `#[macro_export]`ed (they live at the crate +// root for `big-code-analysis`'s metric modules); re-exported here so +// the `crate::macros::` spelling the classifiers use keeps working. +pub use crate::{ + cpp_bool_terminal_kinds, csharp_bool_terminal_kinds, csharp_invocation_expr_kinds, + csharp_paren_expr_kinds, csharp_prefix_unary_expr_kinds, csharp_var_decl_kinds, + csharp_var_declarator_kinds, elixir_bool_terminal_kinds, go_bool_terminal_kinds, + groovy_bool_terminal_kinds, irules_bool_terminal_kinds, java_bool_terminal_kinds, + javascript_bool_terminal_kinds, kotlin_bool_terminal_kinds, lua_bool_terminal_kinds, + mozjs_bool_terminal_kinds, perl_bool_terminal_kinds, php_bool_terminal_kinds, + python_bool_terminal_kinds, ruby_bool_terminal_kinds, rust_bool_terminal_kinds, + tcl_bool_terminal_kinds, tsx_bool_terminal_kinds, typescript_bool_terminal_kinds, + with_any_parser, +}; +pub(crate) use { + get_language, mk_action, mk_code, mk_emacs_mode, mk_extensions, mk_lang, mk_langs, +}; diff --git a/src/node.rs b/big-code-analysis-ast/src/node.rs similarity index 86% rename from src/node.rs rename to big-code-analysis-ast/src/node.rs index 0233c7c37..9ba749d21 100644 --- a/src/node.rs +++ b/big-code-analysis-ast/src/node.rs @@ -10,6 +10,8 @@ clippy::cast_sign_loss )] +//! Tree-sitter wrappers: `Tree`, [`Node`], [`Ancestors`], and the cursors. + mod parser_cache; use tree_sitter::Node as OtherNode; @@ -31,7 +33,10 @@ use parser_cache::parse_on_scratch_parser; // the last of these out of the metric bodies and #1100 out of the // `exclude_tests` prune; the counter is what makes putting one back a // test failure rather than a silent quadratic. -crate::observation::counter!(node_resolved_sibling_lookups); +crate::observation::counter!( + node_resolved_sibling_lookups, + cfg(any(test, feature = "test-support")) +); // Child scans that built their own `TreeCursor`, on this thread. // @@ -46,30 +51,38 @@ crate::observation::counter!(node_resolved_sibling_lookups); // All five consumers are guarded: `preorder` and `act_on_node` here, // `metrics::npa::python` and the suppression DFS through a `metrics()` // / `suppression_markers` call, and `output::dump`'s renderer from that -// module's own tests. The accessor is `pub(crate)` (not `pub(super)`) +// module's own tests. The accessor is `pub` (not `pub(super)`) // precisely so the last one can be asserted from where it lives — see // `crate::observation`. -crate::observation::counter!(child_scan_cursors); +crate::observation::counter!(child_scan_cursors, cfg(any(test, feature = "test-support"))); /// A parsed source tree wrapping a [`tree_sitter::Tree`]. /// /// The "open parse seam" (see issue #251) is reached by external -/// callers through [`crate::Ast::from_tree_sitter`], which accepts a +/// callers through `big_code_analysis::Ast::from_tree_sitter`, which accepts a /// caller-built `tree_sitter::Tree` directly; this wrapper stays /// internal so the metric walker is the only thing that observes it. #[derive(Clone, Debug)] pub(crate) struct Tree(OtherTree); impl Tree { + /// # Panics + /// + /// When `T`'s per-language Cargo feature is disabled in this build. + /// + /// [`AnyParser::parse`](crate::langs::AnyParser::parse) and + /// [`AnyParser::from_tree`](crate::langs::AnyParser::from_tree) + /// cfg-gate each `LANG::*` arm and return + /// `Err(MetricsError::LanguageDisabled)` before reaching here, so + /// every path through `big_code_analysis::Ast` is safe. Since + /// #1376, though, `Parser` and [`ParserTrait::new`] are `pub` in + /// this crate, so a caller that names a `*Parser` alias directly + /// bypasses that gate — check + /// [`LANG::is_enabled`](crate::LANG::is_enabled) first, or go + /// through `AnyParser` and match on the error. pub(crate) fn new(code: &[u8]) -> Self { - // `Tree::new::` is only reachable from the `mk_action!` - // dispatchers, which themselves cfg-gate each `LANG::*` arm - // behind the matching per-language feature (see #252). When - // the feature is off the dispatcher returns - // `Err(LanguageDisabled)` before we get here, so - // `get_ts_language` is provably `Ok` at this call site. let language = T::lang().get_ts_language().expect( - "invariant: dispatcher cfg-gates this call behind the per-language Cargo feature", + "invariant: the caller checked LANG::is_enabled, or reached this through AnyParser", ); Self(parse_on_scratch_parser(&language, code)) } @@ -108,7 +121,7 @@ impl<'a> Node<'a> { /// The `tree-sitter` re-export this exposes is *value-not-stable*: /// the underlying pin may bump in any minor release, so node shape /// and node-kind ids are not part of this crate's stability - /// contract (see the [`tree_sitter`](crate::tree_sitter) re-export + /// contract (see the [`tree_sitter`] re-export /// note in the crate root). #[must_use] #[inline] @@ -123,45 +136,87 @@ impl<'a> Node<'a> { self.0.has_error() } - pub(crate) fn id(&self) -> usize { + /// An id unique to this node within its tree, stable for the tree's + /// lifetime. Suitable as a map key for per-node walk state. + #[inline] + #[must_use] + pub fn id(&self) -> usize { self.0.id() } - pub(crate) fn kind(&self) -> &'static str { + /// The grammar's name for this node's kind (`"function_item"`, + /// `"identifier"`, …). Several distinct [`kind_id`](Self::kind_id)s + /// can share one name — see the aliasing note on `kind_id`. + #[inline] + #[must_use] + pub fn kind(&self) -> &'static str { self.0.kind() } - pub(crate) fn kind_id(&self) -> u16 { + /// The numeric kind of this node in its grammar's symbol table. + /// + /// This is what the per-language token enums in [`languages`] + /// (`Rust::FunctionItem as u16`, …) and every classifier match on. + /// The value is an index into the pinned grammar and moves whenever + /// that grammar is bumped; it is not stable across versions. + /// + /// [`languages`]: crate::languages + #[inline] + #[must_use] + pub fn kind_id(&self) -> u16 { self.0.kind_id() } - pub(crate) fn utf8_text(&self, data: &'a [u8]) -> Option<&'a str> { + /// The source text this node spans, or `None` when those bytes are + /// not valid UTF-8. + #[inline] + #[must_use] + pub fn utf8_text(&self, data: &'a [u8]) -> Option<&'a str> { self.0.utf8_text(data).ok() } - pub(crate) fn start_byte(&self) -> usize { + /// The 0-based byte offset where this node starts. + #[inline] + #[must_use] + pub fn start_byte(&self) -> usize { self.0.start_byte() } - pub(crate) fn end_byte(&self) -> usize { + /// The 0-based byte offset one past this node's last byte. + #[inline] + #[must_use] + pub fn end_byte(&self) -> usize { self.0.end_byte() } - pub(crate) fn start_position(&self) -> (usize, usize) { + /// The 0-based `(row, column)` where this node starts. + #[inline] + #[must_use] + pub fn start_position(&self) -> (usize, usize) { let temp = self.0.start_position(); (temp.row, temp.column) } - pub(crate) fn end_position(&self) -> (usize, usize) { + /// The 0-based `(row, column)` one past this node's end. + #[inline] + #[must_use] + pub fn end_position(&self) -> (usize, usize) { let temp = self.0.end_position(); (temp.row, temp.column) } - pub(crate) fn start_row(&self) -> usize { + /// The 0-based row this node starts on. + #[inline] + #[must_use] + pub fn start_row(&self) -> usize { self.0.start_position().row } - pub(crate) fn end_row(&self) -> usize { + /// The 0-based row of this node's end position. A node ending at + /// column 0 does not occupy that row; see [`end_line`](Self::end_line). + #[inline] + #[must_use] + pub fn end_row(&self) -> usize { self.0.end_position().row } @@ -181,7 +236,9 @@ impl<'a> Node<'a> { /// the file root ends, and a blanket `+ 1` then reported it ending a /// line past both its enclosing space and EOF (#1163) — the shape /// behind the release `usize` underflow in #1051. - pub(crate) fn end_line(&self) -> usize { + #[inline] + #[must_use] + pub fn end_line(&self) -> usize { let end = self.0.end_position(); if end.column == 0 { end.row @@ -209,7 +266,8 @@ impl<'a> Node<'a> { /// comment-removal walks reach calls this; the remaining callers are /// `Ancestors` itself (the no-chain fallback), the one-off start node /// of a `dump`, and tests. `rg '\.parent\(\)' src/` re-checks that. - pub(crate) fn parent(&self) -> Option> { + #[inline] + pub fn parent(&self) -> Option> { self.0.parent().map(Node) } @@ -225,7 +283,8 @@ impl<'a> Node<'a> { /// /// [`wraps_any`]: Self::wraps_any #[inline] - pub(crate) fn has_sibling(&self, ancestors: Ancestors<'a, '_>, id: u16) -> bool { + #[must_use] + pub fn has_sibling(&self, ancestors: Ancestors<'a, '_>, id: u16) -> bool { ancestors .parent(self) .is_some_and(|parent| parent.is_child(id)) @@ -236,7 +295,8 @@ impl<'a> Node<'a> { /// **`O(depth)`, not `O(1)`**, for [`Node::parent`]'s reason: /// `ts_node__prev_sibling` opens with `ts_node_parent`. Callers on a /// walk should use [`Ancestors::previous_sibling`] instead (#1096). - pub(crate) fn previous_sibling(&self) -> Option> { + #[inline] + pub fn previous_sibling(&self) -> Option> { node_resolved_sibling_lookups::record(); self.0.prev_sibling().map(Node) } @@ -245,7 +305,8 @@ impl<'a> Node<'a> { /// `kind_id`. See #217 for the motivating perf finding from the /// JS/TS template-literal hot path. #[inline] - pub(crate) fn is_child(&self, id: u16) -> bool { + #[must_use] + pub fn is_child(&self, id: u16) -> bool { self.wraps_any(&[id]) } @@ -276,19 +337,24 @@ impl<'a> Node<'a> { /// /// [`is_child`]: Self::is_child #[inline] - pub(crate) fn wraps_any(&self, ids: &[u16]) -> bool { + #[must_use] + pub fn wraps_any(&self, ids: &[u16]) -> bool { self.children().any(|c| ids.contains(&c.kind_id())) } - pub(crate) fn child_count(&self) -> usize { + /// The number of direct children, named and anonymous alike. + #[inline] + #[must_use] + pub fn child_count(&self) -> usize { self.0.child_count() } - // Returns `true` if this node is a named grammar production - // (as opposed to an anonymous token such as a punctuation or - // keyword literal). Used to skip anonymous tokens like the - // leading `|` in an or-pattern. - pub(crate) fn is_named(&self) -> bool { + /// Whether this node is a named grammar production, as opposed to an + /// anonymous token such as a punctuation or keyword literal. Walks + /// use it to skip tokens like the leading `|` in an or-pattern. + #[inline] + #[must_use] + pub fn is_named(&self) -> bool { self.0.is_named() } @@ -302,11 +368,17 @@ impl<'a> Node<'a> { /// [`child`]: Self::child /// [`parent`]: Self::parent /// [`children`]: Self::children - pub(crate) fn child_by_field_name(&self, name: &str) -> Option> { + #[inline] + pub fn child_by_field_name(&self, name: &str) -> Option> { self.0.child_by_field_name(name).map(Node) } - pub(crate) fn child(&self, pos: usize) -> Option> { + /// The direct child at index `pos`, counting named and anonymous + /// children alike. Prefer [`child_by_field_name`](Self::child_by_field_name) + /// for any slot the grammar can precede with an optional preamble. + #[inline] + #[must_use] + pub fn child(&self, pos: usize) -> Option> { self.0.child(pos as u32).map(Node) } @@ -314,7 +386,9 @@ impl<'a> Node<'a> { /// node reaches the child at `child_index`, if any. Used by the /// AST builder to thread the parent's `field_name` into each child /// without a parallel cursor walk. - pub(crate) fn field_name_for_child(&self, child_index: u32) -> Option<&'static str> { + #[inline] + #[must_use] + pub fn field_name_for_child(&self, child_index: u32) -> Option<&'static str> { self.0.field_name_for_child(child_index) } @@ -326,7 +400,9 @@ impl<'a> Node<'a> { /// plumbing. /// /// [`children_with`]: Self::children_with - pub(crate) fn children(&self) -> Children<'a> { + #[inline] + #[must_use] + pub fn children(&self) -> Children<'a> { child_scan_cursors::record(); // `descend`, not `seed`: `ts_node_walk` already ran the // `ts_tree_cursor_init` that `Cursor::reset` would run again. @@ -360,12 +436,18 @@ impl<'a> Node<'a> { /// [`preorder`]: Self::preorder /// /// [`children`]: Self::children - pub(crate) fn children_with<'c>(&self, cursor: &'c mut Cursor<'a>) -> ChildrenWith<'c, 'a> { + #[inline] + pub fn children_with<'c>(&self, cursor: &'c mut Cursor<'a>) -> ChildrenWith<'c, 'a> { let scan = ChildScan::seed(self, cursor); ChildrenWith { cursor, scan } } - pub(crate) fn cursor(&self) -> Cursor<'a> { + /// A fresh cursor positioned on this node, for the traversals that + /// reuse one cursor across many child scans + /// ([`children_with`](Self::children_with)). + #[inline] + #[must_use] + pub fn cursor(&self) -> Cursor<'a> { Cursor(self.0.walk()) } @@ -378,7 +460,7 @@ impl<'a> Node<'a> { /// `ancestors` is the chain the caller descended through. Passing /// [`Ancestors::unknown`] is always correct and answers identically; /// it just pays `O(depth)` per step instead of `O(1)`. - pub(crate) fn count_specific_ancestors( + pub fn count_specific_ancestors( &self, ancestors: Ancestors<'a, '_>, check: fn(&Node) -> bool, @@ -407,7 +489,7 @@ impl<'a> Node<'a> { /// identically; it just pays [`Node::parent`]'s `O(depth)` for each /// of the two links, which on a per-node metric arm is quadratic in /// nesting depth (#1096). - pub(crate) fn parent_grandparent_match( + pub fn parent_grandparent_match( &self, ancestors: Ancestors<'a, '_>, parent_pred: fn(&Node) -> bool, @@ -454,8 +536,8 @@ impl<'a> Node<'a> { /// pre-order. /// /// Membership is an exact match against the raw grammar kind — the - /// same unaltered vocabulary [`crate::Ast::root_node`] exposes, not - /// the `Alterator`-curated kinds [`crate::Ast::dump`] emits. This is + /// same unaltered vocabulary `big_code_analysis::Ast::root_node` exposes, not + /// the `Alterator`-curated kinds `big_code_analysis::Ast::dump` emits. This is /// the Rust counterpart of the Python `Node.descendants_by_kind()` /// binding (issue #728). #[must_use] @@ -480,18 +562,22 @@ impl<'a> Node<'a> { /// [`Ancestors::unknown`], which climbs with [`Node::parent`]: the same /// answers at the original cost. #[derive(Clone, Copy, Debug)] -pub(crate) struct Ancestors<'tree, 'chain>(Option<&'chain [Node<'tree>]>); +pub struct Ancestors<'tree, 'chain>(Option<&'chain [Node<'tree>]>); impl<'tree, 'chain> Ancestors<'tree, 'chain> { /// No chain is available; every query climbs with [`Node::parent`]. - pub(crate) const fn unknown() -> Self { + #[inline] + #[must_use] + pub const fn unknown() -> Self { Self(None) } /// `chain` lists every ancestor of the node about to be queried, /// root first — so `chain.last()` is its parent and an empty chain /// means the node is the root. - pub(crate) const fn known(chain: &'chain [Node<'tree>]) -> Self { + #[inline] + #[must_use] + pub const fn known(chain: &'chain [Node<'tree>]) -> Self { Self(Some(chain)) } @@ -530,7 +616,8 @@ impl<'tree, 'chain> Ancestors<'tree, 'chain> { /// previous subtree's path, whose last entry is disjoint from the /// node that follows it. Both trip here, on the first node that /// shows them, at four integer comparisons. - pub(crate) fn checked(chain: &'chain [Node<'tree>], node: &Node<'tree>) -> Self { + #[must_use] + pub fn checked(chain: &'chain [Node<'tree>], node: &Node<'tree>) -> Self { // Opt-in only: `Node::parent` restarts at the root, so this is // the `O(nodes × depth)` walk described above. #[cfg(chain_audit)] @@ -556,12 +643,15 @@ impl<'tree, 'chain> Ancestors<'tree, 'chain> { /// How far the node this chain describes sits from the root, or /// `None` when no chain is known — deriving it then would cost the /// [`Node::parent`] climb the chain exists to remove. - pub(crate) fn depth(self) -> Option { + #[inline] + pub fn depth(self) -> Option { self.0.map(<[Node<'tree>]>::len) } /// `node`'s parent. - pub(crate) fn parent(self, node: &Node<'tree>) -> Option> { + #[inline] + #[must_use] + pub fn parent(self, node: &Node<'tree>) -> Option> { match self.0 { Some(chain) => chain.last().copied(), None => node.parent(), @@ -587,7 +677,9 @@ impl<'tree, 'chain> Ancestors<'tree, 'chain> { /// A `false` when `node` has no parent is the answer every one of /// those call sites wants: a root node's token is not inside the /// construct, so it is not suppressed. - pub(crate) fn parent_has_kind(self, node: &Node<'tree>, kind: u16) -> bool { + #[inline] + #[must_use] + pub fn parent_has_kind(self, node: &Node<'tree>, kind: u16) -> bool { self.parent(node).is_some_and(|p| p.kind_id() == kind) } @@ -599,7 +691,9 @@ impl<'tree, 'chain> Ancestors<'tree, 'chain> { /// carries the same `O(depth)` cost [`Node::parent`] does. With a /// known chain the parent is free and what remains is a cursor walk /// over the siblings. - pub(crate) fn previous_sibling(self, node: &Node<'tree>) -> Option> { + #[inline] + #[must_use] + pub fn previous_sibling(self, node: &Node<'tree>) -> Option> { let Some(chain) = self.0 else { return node.previous_sibling(); }; @@ -621,7 +715,9 @@ impl<'tree, 'chain> Ancestors<'tree, 'chain> { /// `node`'s ancestors, nearest first, each paired with *its* own /// ancestry so a predicate applied to an ancestor stays as cheap as /// one applied to `node`. - pub(crate) fn iter(self, node: &Node<'tree>) -> AncestorIter<'tree, 'chain> { + #[inline] + #[must_use] + pub fn iter(self, node: &Node<'tree>) -> AncestorIter<'tree, 'chain> { match self.0 { Some(chain) => AncestorIter::Chain(chain), None => AncestorIter::Climb(node.parent()), @@ -630,7 +726,7 @@ impl<'tree, 'chain> Ancestors<'tree, 'chain> { } /// Ancestor iterator returned by [`Ancestors::iter`], nearest first. -pub(crate) enum AncestorIter<'tree, 'chain> { +pub enum AncestorIter<'tree, 'chain> { /// The not-yet-yielded prefix of a known chain. Its last element is /// the next ancestor, and the prefix before it is that ancestor's /// own chain — so splitting from the back hands out both at once. @@ -694,22 +790,31 @@ impl<'a> Iterator for Preorder<'a> { /// An `AST` cursor. #[derive(Clone)] -pub(crate) struct Cursor<'a>(TreeCursor<'a>); +pub struct Cursor<'a>(TreeCursor<'a>); impl<'a> Cursor<'a> { - pub(crate) fn reset(&mut self, node: &Node<'a>) { + /// Re-seats the cursor on `node`. + #[inline] + pub fn reset(&mut self, node: &Node<'a>) { self.0.reset(node.0); } - pub(crate) fn goto_next_sibling(&mut self) -> bool { + /// Moves to the next sibling; `false` when there is none. + #[inline] + pub fn goto_next_sibling(&mut self) -> bool { self.0.goto_next_sibling() } - pub(crate) fn goto_first_child(&mut self) -> bool { + /// Moves to the first child; `false` when there is none. + #[inline] + pub fn goto_first_child(&mut self) -> bool { self.0.goto_first_child() } - pub(crate) fn node(&self) -> Node<'a> { + /// The node the cursor currently sits on. + #[inline] + #[must_use] + pub fn node(&self) -> Node<'a> { Node(self.0.node()) } } @@ -799,9 +904,9 @@ impl ChildScan { /// the actual sibling walk disagree. /// /// The `ExactSizeIterator` length is reported from `child_count` (tracked -/// in [`ChildScan`]). For well-formed trees the cursor walk and +/// in `ChildScan`). For well-formed trees the cursor walk and /// `child_count` agree, so the advertised length matches the data. -pub(crate) struct Children<'a> { +pub struct Children<'a> { cursor: Cursor<'a>, scan: ChildScan, } @@ -823,8 +928,8 @@ impl ExactSizeIterator for Children<'_> {} /// Iterator over a node's direct children, returned by /// [`Node::children_with`]. Borrows the caller's cursor rather than /// building one, which is the whole of the difference: it yields exactly -/// what [`Children`] yields, through the same [`ChildScan`]. -pub(crate) struct ChildrenWith<'c, 'a> { +/// what [`Children`] yields, through the same `ChildScan`. +pub struct ChildrenWith<'c, 'a> { cursor: &'c mut Cursor<'a>, scan: ChildScan, } @@ -898,94 +1003,6 @@ mod tests { use crate::langs::MozjsCode; use crate::test_support::for_each_node_with_chain; - /// Under a parent narrow enough to read forward, the - /// `exclude_tests` prune finds the run of `#[…]` siblings before an - /// item through the walker's ancestor chain, never by resolving - /// siblings from the node. - /// - /// Nothing in the output says so: the backward walk this replaced - /// returns the same answer, only `O(depth)` per step (#1100), and - /// `rust_outer_attr_scans_agree` in `checker.rs` exists precisely - /// to prove the two agree. The counter is the sole observable, so a - /// revert is a silent quadratic without this. - /// - /// Every parent in the fixture holds at most five children, which - /// keeps it under `MAX_FORWARD_ATTRIBUTE_SCAN_CHILDREN` — the - /// backward walk is still the deliberate reading above that width, - /// so a wider fixture would assert the opposite of what it looks - /// like it asserts. - /// - /// Seeding a real lookup first is what makes the assertion - /// falsifiable: compared against zero it would also pass with - /// `record()` never wired up at all. - #[cfg(feature = "rust")] - #[test] - fn the_exclude_tests_prune_resolves_no_sibling_from_a_node() { - let source = "#[cfg(test)]\nmod tests {\nfn t() {}\n}\n\ - #[inline]\nfn kept() {\n#[allow(dead_code)]\nfn nested() {}\nlet x = 1;\n}\n"; - let ast = crate::test_support::parse_named(crate::LANG::Rust, "lib.rs", source); - - let root = Node(ast.as_tree_sitter().root_node()); - let last = root.children().last().expect("the file has items"); - let _ = last.previous_sibling(); - let seeded = node_resolved_sibling_lookups::observed(); - assert!(seeded > 0, "the seed call must be counted"); - - ast.metrics(crate::MetricsOptions::default().with_exclude_tests(true)) - .expect("the walk must yield a top-level space"); - - assert_eq!( - node_resolved_sibling_lookups::observed(), - seeded, - "the metric walk resolved a sibling from a node; \ - read it off the ancestor chain instead (#1096 / #1100)" - ); - } - - /// Which arm the `exclude_tests` attribute-scan dispatch takes, at - /// the boundary in both directions and on both of its axes. - /// - /// `rust_outer_attr_scans_agree` in `checker.rs` proves the two - /// readings answer the same thing, which is exactly why it cannot - /// see which one ran — it passes at any budget, including one that - /// never reads forward. This counter is the only observable that - /// tells them apart, and it lives here, so the boundary is pinned - /// here too. - /// - /// The third case is the one #1100 got wrong: dispatching on width - /// alone sent any over-wide body to the `O(depth)` walk however deep - /// it sat, which on a nested `mod` tree is quadratic (a 3_200-deep - /// fixture measured 2.67 s against 0.045 s for the same shape one - /// child narrower). - #[cfg(feature = "rust")] - #[test] - fn the_exclude_tests_prune_reads_forward_up_to_its_depth_scaled_budget() { - // Three attributed items make a `source_file` exactly six - // children wide — the depth-1 budget. A fourth, bare item makes - // seven, one over. Wrapping that in a `mod` puts the same seven - // between two braces, so its `declaration_list` is nine wide, at - // depth 3 — where the budget is also exactly nine. - let at_budget = "#[cfg(test)]\nfn a() {}\n#[inline]\nfn b() {}\n#[cfg(test)]\nfn c() {}\n"; - let past_budget = format!("{at_budget}fn d() {{}}\n"); - let nested = format!("mod m {{\n{past_budget}}}\n"); - - for (shape, source, resolves_siblings) in [ - ("six children at depth 1", at_budget.to_string(), false), - ("seven children at depth 1", past_budget, true), - ("nine children at depth 3", nested, false), - ] { - let before = node_resolved_sibling_lookups::observed(); - crate::test_support::parse_named(crate::LANG::Rust, "lib.rs", &source) - .metrics(crate::MetricsOptions::default().with_exclude_tests(true)) - .expect("the walk must yield a top-level space"); - let resolved = node_resolved_sibling_lookups::observed() > before; - assert_eq!( - resolved, resolves_siblings, - "{shape}: the prune took the wrong dispatch arm" - ); - } - } - /// The `child(0)` + `next_sibling()` chain [`Node::wraps_any`] used /// between #217 and #1088, kept here as the reference the cursor /// walk that replaced it is checked against. @@ -1439,7 +1456,9 @@ mod tests { /// assertion in the suite holds just as well with a fresh /// `TreeCursor` built and freed per visited node. The counter is the /// only observable, which is why reverting one of these loops has to - /// be a test failure rather than a silent allocation per node. + /// be a test failure rather than a silent allocation per node. The + /// metric and suppression walks are guarded the same way from + /// `big-code-analysis`, where they live. /// /// Seeding a real scan first is what makes it falsifiable: compared /// against zero these assertions would also pass with `record()` @@ -1447,8 +1466,6 @@ mod tests { #[cfg(all(feature = "c", feature = "mozjs", feature = "python", feature = "rust"))] #[test] fn the_converted_traversals_scan_a_tree_on_one_cursor() { - use crate::traits::ParserTrait; - let seed_tree = Tree::new::(b"int main() { int a; }"); let _ = seed_tree.get_root().children().count(); assert!( @@ -1470,49 +1487,6 @@ mod tests { "preorder built a cursor per node; it holds one for the walk (#1112)" ); - // The Python instance-attribute scan walks every method body of - // a class. Before #1112 it was 92 % of the metric walk's child - // scans on the Python corpus slice — one per node under the - // class. It is not the only scan a `metrics()` call makes, so - // the bound is a fraction of the node count rather than zero. - // Measured on this fixture: 18 scans over 81 nodes with the - // cursor hoisted, 91 without, so the bound separates the two - // with room on both sides. - let source = "class C:\n def a(self):\n self.x = 1\n self.y = [1, 2]\n\ - \n def b(self):\n self.z, self.w = 1, 2\n \ - if self.x:\n self.v = self.y\n"; - let ast = crate::test_support::parse_named(crate::LANG::Python, "c.py", source); - let nodes = ast.root_node().preorder().count(); - let before = child_scan_cursors::observed(); - ast.metrics(crate::MetricsOptions::default()) - .expect("the walk must yield a top-level space"); - let scans = child_scan_cursors::observed() - before; - assert!(nodes > 60, "fixture is too small to prove much"); - assert!( - scans < nodes / 2, - "the Python metric walk built {scans} cursors over {nodes} nodes; the \ - instance-attribute scan is meant to hold one for the subtree (#1112)" - ); - - // The suppression scan is a full-tree DFS of its own: 0 scans - // over this fixture's 29 nodes with the cursor hoisted, 29 - // without. - let parser = crate::langs::RustParser::new( - b"// bca: suppress(cognitive)\nfn f() { if a { g(1, 2); } }\n".to_vec(), - std::path::Path::new("lib.rs"), - None, - ); - let nodes = parser.root().preorder().count(); - let before = child_scan_cursors::observed(); - let markers = crate::suppression::suppression_markers(&parser); - let scans = child_scan_cursors::observed() - before; - assert_eq!(markers.len(), 1, "fixture carries one marker"); - assert!(nodes > 20, "fixture is too small to prove much"); - assert!( - scans < nodes / 2, - "the suppression scan built {scans} cursors over {nodes} nodes (#1112)" - ); - // The `Search` walk, `act_on_node`. The counter records in // `children()`, the allocating form, so a walk that hoists its // cursor records nothing at all and a per-node one records once @@ -1586,7 +1560,7 @@ mod tests { /// real tree. /// /// Checked against the raw `child(i)` walk rather than against - /// `children()`: the two iterators share [`ChildScan`], so a + /// `children()`: the two iterators share `ChildScan`, so a /// comparison between them would pass just as happily if the shared /// step were wrong. It also covers the reuse itself — one cursor /// drives every node's scan here, so a `reset` that failed to rewind diff --git a/src/node/parser_cache.rs b/big-code-analysis-ast/src/node/parser_cache.rs similarity index 88% rename from src/node/parser_cache.rs rename to big-code-analysis-ast/src/node/parser_cache.rs index 99cc7fef8..3382a17c0 100644 --- a/src/node/parser_cache.rs +++ b/big-code-analysis-ast/src/node/parser_cache.rs @@ -58,7 +58,7 @@ crate::observation::counter!(parsers_built); /// sets a timeout, a cancellation flag, or included ranges: unlike the /// parse state, `set_language` does not clear those, so anything that /// starts setting one must clear it before the parser goes back. -pub(crate) fn parse_on_scratch_parser(language: &Language, code: &[u8]) -> Tree { +pub fn parse_on_scratch_parser(language: &Language, code: &[u8]) -> Tree { let mut parser = SCRATCH_PARSER .try_with(Cell::take) .ok() @@ -91,9 +91,10 @@ fn build_parser() -> Parser { #[cfg(all(test, feature = "rust"))] mod tests { use super::*; + use crate::LANG; + use crate::langs::AnyParser; use crate::langs::RustCode; use crate::traits::LanguageInfo; - use crate::{Ast, LANG, Source}; fn rust_language() -> Language { RustCode::lang() @@ -177,10 +178,19 @@ mod tests { /// they say nothing about whether anything *reaches* it: reverting /// `Tree::new` to the pre-#1118 `Parser::new()`-per-file body leaves /// all 3,143 lib tests and all 5 `tests/api/parser_reuse.rs` integration - /// tests passing (measured). Driving the public `Ast::parse` seam and - /// counting constructions is what fails there. + /// tests passing (measured). Driving `AnyParser::parse` and counting + /// constructions is what fails there. + /// + /// Before #1376 this drove `big_code_analysis::Ast::parse`. It stops + /// one layer lower now because `parsers_built` lives in this private + /// module and the root crate cannot read it — and widening the + /// module to host a test is the trade `container_scope_tests` + /// declines for the same reason. Nothing is lost that matters: + /// `Ast::parse` is a one-line delegate to `AnyParser::parse` and + /// owns no parser, so this is the lowest layer at which reuse is + /// observable at all. #[test] - fn repeated_parses_through_the_public_seam_share_one_parser() { + fn repeated_parses_through_the_dispatch_seam_share_one_parser() { const FILES: usize = 6; std::thread::spawn(|| { @@ -188,9 +198,9 @@ mod tests { // Distinct sources, so no caching layer above the parser // can turn the later parses into lookups. let code = format!("fn f{i}() {{ let x = {i}; }}"); - let ast = Ast::parse(Source::new(LANG::Rust, code.as_bytes())) + let parsed = AnyParser::parse(LANG::Rust, code.into_bytes(), None, None) .expect("rust is enabled for this module"); - let sexp = ast.as_tree_sitter().root_node().to_sexp(); + let sexp = parsed.ts_tree().root_node().to_sexp(); assert!( sexp.contains("function_item") && !sexp.contains("ERROR"), "file {i} must parse to a real tree, got {sexp}" @@ -199,7 +209,7 @@ mod tests { assert_eq!( parsers_built::observed(), 1, - "{FILES} files parsed through `Ast::parse` must share one parser" + "{FILES} files parsed through `AnyParser::parse` must share one parser" ); }) .join() diff --git a/big-code-analysis-ast/src/observation.rs b/big-code-analysis-ast/src/observation.rs new file mode 100644 index 000000000..eafae22ac --- /dev/null +++ b/big-code-analysis-ast/src/observation.rs @@ -0,0 +1,90 @@ +//! Thread-local counters that make an invisible optimization testable. +//! +//! Several optimizations change no output at all: reusing one +//! `tree_sitter::Parser` per thread (#1118), skipping a space-kind +//! lookup on a node that opens no space (#1110), serializing `Ops` +//! through a borrowed projection rather than an owned clone (#1110), +//! deferring the modeline scan behind a resolving extension (#1111). +//! Every assertion on the *result* of those paths holds just as well +//! once the optimization is reverted, so a revert is silent unless +//! something counts the work. +//! +//! # The invariant +//! +//! The counter and the function that bumps it are **unconditional**; +//! only the accessor is gated. Gating the counter itself would leave the +//! test observing a build production never ships — the counted branch +//! compiled under `cfg(test)` and the shipped one under nothing — which +//! is the exact failure these counters exist to catch. Four sites grew +//! that rule independently and each stated it in prose; `counter!` +//! makes it structural instead, emitting the three items together so the +//! wrong one cannot be gated. +//! +//! The cost is one `Cell` increment on a path that already does far more +//! (building a parser, classifying a node, projecting a whole tree), +//! which is why it is affordable to leave in the shipped build. +//! +//! # Two crates, one macro +//! +//! `big-code-analysis` declares counters of its own (in its `ops`, +//! `wire` and `spaces` walks) with the same macro, which is why it is +//! `#[macro_export]`ed. The accessor's gate is a parameter because the +//! two crates need different ones: a counter declared *here* must also be +//! readable by the root crate's tests, which see this crate as an +//! ordinary dependency (no `cfg(test)`), so it opens under the +//! `test-support` feature as well. + +/// Declares a thread-local observation counter as a module: a private +/// `Cell`, an unconditional `record()`, and a gated `observed()`. +/// +/// Takes the module's name and, optionally, the `cfg` predicate that +/// gates the accessor (`cfg(test)` when omitted). It deliberately +/// carries no narrative — *which* optimization a counter observes, and +/// why no assertion on the output can distinguish it, belongs in a +/// comment above each invocation. +/// +/// A module rather than three free items so a call site names the +/// counter once (`parsers_built::record()`), which keeps the invocation +/// to one line and leaves no room for the recorder and the accessor to +/// drift apart. +#[macro_export] +#[doc(hidden)] +macro_rules! observation_counter { + ($name:ident) => { + $crate::observation_counter!($name, cfg(test)); + }; + ($name:ident, $gate:meta) => { + #[doc(hidden)] + pub mod $name { + thread_local! { + static COUNT: ::std::cell::Cell = const { ::std::cell::Cell::new(0) }; + } + + /// Records one occurrence on this thread. + /// + /// `pub(super)` on purpose: only the module that owns the + /// counted path may bump it, so a counter cannot drift into + /// meaning "whatever any caller felt like recording". + #[inline] + pub(super) fn record() { + COUNT.with(|count| count.set(count.get() + 1)); + } + + /// Occurrences recorded on this thread. Only this accessor + /// is gated; see the `observation` module docs. + /// + /// Wider than `record` because the guarded path and the test + /// that guards it need not share a module: `child_scan_cursors` + /// counts a cursor hoist in `node`, but the metric dump in + /// `big-code-analysis` is one of the walks that has to hold + /// one, and a counter only reachable from its own module + /// silently leaves such a caller unguarded. + #[$gate] + pub fn observed() -> usize { + COUNT.with(::std::cell::Cell::get) + } + } + }; +} + +pub use observation_counter as counter; diff --git a/src/parser.rs b/big-code-analysis-ast/src/parser.rs similarity index 96% rename from src/parser.rs rename to big-code-analysis-ast/src/parser.rs index fd6564c45..8e7e144ab 100644 --- a/src/parser.rs +++ b/big-code-analysis-ast/src/parser.rs @@ -6,6 +6,8 @@ // function so the per-language impl blocks stay readable. #![allow(clippy::wildcard_imports, clippy::enum_glob_use)] +//! [`Parser`](Parser) and the node [`Filter`]s `count` / `find` accept. + use std::marker::PhantomData; use std::path::Path; use std::sync::Arc; @@ -29,7 +31,7 @@ use crate::traits::*; /// of the language code tags (`RustCode`, `PythonCode`, etc.) declared /// by the internal `mk_code!` macro. #[derive(Debug)] -pub(crate) struct Parser { +pub struct Parser { code: Vec, tree: Tree, phantom: PhantomData, @@ -43,7 +45,7 @@ type FilterFn<'a> = dyn Fn(&Node) -> bool + 'a; /// Collection of node-matching predicates used by the AST-walking /// metric and dump routines to decide whether to visit a node. -pub(crate) struct Filter<'a> { +pub struct Filter<'a> { filters: Vec>>, } @@ -212,9 +214,9 @@ impl Parser { /// Borrow the underlying [`tree_sitter::Tree`] for callers that /// want to drive their own traversal alongside the metric walker. /// - /// Doc-hidden because `Parser` itself is hidden from the rendered - /// surface; the stable spelling of this accessor is - /// [`crate::Ast::as_tree_sitter`]. + /// `Parser` is `pub` in this crate but carries no stability + /// promise (see the crate root); the stable spelling of this + /// accessor is `big_code_analysis::Ast::as_tree_sitter`. #[must_use] pub fn ts_tree(&self) -> &tree_sitter::Tree { self.tree.as_ts_tree() diff --git a/src/preproc.rs b/big-code-analysis-ast/src/preproc.rs similarity index 98% rename from src/preproc.rs rename to big-code-analysis-ast/src/preproc.rs index fda3a52a9..0fd35d7e9 100644 --- a/src/preproc.rs +++ b/big-code-analysis-ast/src/preproc.rs @@ -11,6 +11,9 @@ clippy::wildcard_imports )] +//! The C-family preprocessor pass: the include graph, macro collection, +//! and the [`PreprocResults`] document. + use std::collections::{HashMap, HashSet, hash_map}; use std::path::{Path, PathBuf}; @@ -256,7 +259,7 @@ pub fn get_macros( /// per-file sets to probe in turn, because it is consulted once per /// identifier run in the whole translation unit: an `O(headers)` probe /// would cost far more than the one-off merge it saves. -pub(crate) fn visible_macros<'a, S: ::std::hash::BuildHasher>( +pub fn visible_macros<'a, S: ::std::hash::BuildHasher>( file: &Path, files: &'a HashMap, ) -> HashSet<&'a str> { @@ -445,7 +448,10 @@ fn collapse_one_component( (replacement, paths) } -crate::observation::counter!(include_graph_walks); +crate::observation::counter!( + include_graph_walks, + cfg(any(test, feature = "test-support")) +); /// What one include-graph node contributes to every closure that reaches it. enum NodeContribution<'a> { @@ -744,11 +750,7 @@ pub fn preprocess(source: Vec, path: &Path, results: &mut PreprocResults) { /// Walk an already-built [`PreprocParser`] tree, accumulating its /// preprocessor data into `results`. Internal core shared by the public /// [`preprocess`] seam and the crate's own preprocessor tests. -pub(crate) fn preprocess_with_parser( - parser: &PreprocParser, - path: &Path, - results: &mut PreprocResults, -) { +pub fn preprocess_with_parser(parser: &PreprocParser, path: &Path, results: &mut PreprocResults) { let node = parser.root(); let mut cursor = node.cursor(); let code = parser.code(); diff --git a/src/preproc_tests.rs b/big-code-analysis-ast/src/preproc_tests.rs similarity index 97% rename from src/preproc_tests.rs rename to big-code-analysis-ast/src/preproc_tests.rs index 9491f5409..52dd0aa4e 100644 --- a/src/preproc_tests.rs +++ b/big-code-analysis-ast/src/preproc_tests.rs @@ -876,30 +876,29 @@ fn parsing_a_cpp_file_never_owns_the_macro_set() { "a fresh thread must not have owned a set yet" ); - let space = crate::Ast::parse( - crate::Source::new( - crate::LANG::Cpp, - b"int f(int x) { return DBG ? FOO : x; }".as_slice(), - ) - .with_preproc_path(Some(&path)) - .with_preproc(Some(pr)), + let parsed = crate::langs::AnyParser::parse( + crate::LANG::Cpp, + b"int f(int x) { return DBG ? FOO : x; }".to_vec(), + Some(&path), + Some(pr), ) - .expect("cpp feature enabled") - .metrics(crate::MetricsOptions::default()) - .expect("walker succeeds"); - // Both macros are three bytes, so the masking pass rewrites - // each to the same `$$$` run and the two operands collapse - // into one: `f`, `x`, `$$$`. Unmasked the count is four - // (`f`, `x`, `DBG`, `FOO`), so deleting the C-family arm of - // `get_fake_code` fails here. - // - // The name lengths are load-bearing. `$` is an identifier byte - // in tree-sitter-cpp, so masking a pair of *differently* named - // macros moves no metric at all — measured: the `DBG` / - // `FROM_DEP` pair this fixture used to carry left cyclomatic, - // both Halstead vocabularies, and lloc identical either way, - // and the assertion here was decorative. - assert_eq!(space.metrics.halstead.unique_operands(), 3); + .expect("cpp feature enabled"); + // The masking pass rewrites each macro name to a `$` run of the + // same length, so the parsed bytes carry two `$$$` runs and + // neither name. Deleting the C-family arm of `get_fake_code` + // leaves the names in place and fails here. + let code = parsed.code(); + assert!( + !code.windows(3).any(|w| w == b"DBG" || w == b"FOO"), + "macro names must be masked before parsing: {}", + String::from_utf8_lossy(code) + ); + assert_eq!( + code.windows(3).filter(|w| *w == b"$$$").count(), + 2, + "each three-byte macro name masks to one `$$$` run: {}", + String::from_utf8_lossy(code) + ); assert_eq!( owned_macro_sets::observed(), diff --git a/src/recursion.rs b/big-code-analysis-ast/src/recursion.rs similarity index 95% rename from src/recursion.rs rename to big-code-analysis-ast/src/recursion.rs index 230d90493..174233495 100644 --- a/src/recursion.rs +++ b/big-code-analysis-ast/src/recursion.rs @@ -1,7 +1,7 @@ //! Stack-depth bounds for the crate's recursive types. //! -//! [`FuncSpace`](crate::FuncSpace), [`Ops`](crate::Ops), and -//! [`AstNode`](crate::AstNode) are trees whose nesting depth is +//! `big-code-analysis`'s `FuncSpace` and `Ops`, and this crate's +//! [`AstNode`](crate::AstNode), are trees whose nesting depth is //! caller-controlled: nested functions, nested closures, and nested //! expressions are ordinary legal constructs in every supported language. //! Issues #700 / #709 converted every AST *traversal* to an explicit work @@ -55,7 +55,8 @@ impl Drop for DepthGuard { /// error. A deeper tree fails with a serializer error — reported by the /// caller like any other output failure — rather than recursing far enough /// to overflow the thread stack. -pub(crate) fn serialize_bounded( +#[doc(hidden)] +pub fn serialize_bounded( children: &[T], limit: usize, type_name: &str, @@ -92,6 +93,8 @@ where /// only after its own children have been moved out of it and its /// compiler-generated glue finds an empty list — the recursion is one /// level deep regardless of the tree's shape. +#[macro_export] +#[doc(hidden)] macro_rules! impl_iterative_drop { ($ty:ty, $children:ident) => { impl Drop for $ty { @@ -105,7 +108,7 @@ macro_rules! impl_iterative_drop { }; } -pub(crate) use impl_iterative_drop; +pub use impl_iterative_drop; #[cfg(test)] mod tests { diff --git a/src/space_kind.rs b/big-code-analysis-ast/src/space_kind.rs similarity index 98% rename from src/space_kind.rs rename to big-code-analysis-ast/src/space_kind.rs index f2ad84c0b..fb78fff96 100644 --- a/src/space_kind.rs +++ b/big-code-analysis-ast/src/space_kind.rs @@ -87,7 +87,7 @@ impl SpaceKind { /// another callable, and an extra roll-up is a milder wrong answer /// than a silently missing one. #[must_use] - pub(crate) fn is_member_scope(self) -> bool { + pub fn is_member_scope(self) -> bool { !matches!(self, Self::Function | Self::Unknown) } } diff --git a/big-code-analysis-ast/src/test_support.rs b/big-code-analysis-ast/src/test_support.rs new file mode 100644 index 000000000..d7e5a83a5 --- /dev/null +++ b/big-code-analysis-ast/src/test_support.rs @@ -0,0 +1,63 @@ +//! Parse-and-inspect helpers shared by this crate's unit tests and, behind +//! the `test-support` feature, by `big-code-analysis`'s metric tests. +//! +//! Kept out of any production file so the self-scan gate does not spend a +//! shipping module's metric budget on test-only code (#1066). + +use crate::node::{Node, Tree}; +use crate::traits::{LanguageInfo, ParserTrait}; + +/// Visits `code`'s tree in pre-order, maintaining the ancestor chain +/// exactly as `big_code_analysis::spaces::compute::metrics_inner` does, +/// and hands each node to `check` together with that chain. +/// +/// Keeping the bookkeeping identical to the walker's is the point: a +/// test that built the chain some other way would prove +/// [`Ancestors`](crate::node::Ancestors) self-consistent without proving +/// the walker feeds it the right slice. The walker lives in the other +/// crate, so nothing links the two but this sentence — a change to its +/// truncate/push discipline must be mirrored here by hand +/// (`.claude/rules/testing.md`, lesson #82). +/// +/// # Panics +/// +/// When `L`'s per-language Cargo feature is disabled in this build (see +/// [`ParserTrait::new`]), or when the fixture does not parse cleanly, so +/// the walk cannot be measuring error recovery by accident. +pub fn for_each_node_with_chain( + code: &[u8], + mut check: impl FnMut(&Node<'_>, &[Node<'_>]), +) -> usize { + let tree = Tree::new::(code); + let root = tree.get_root(); + assert!( + !root.has_error(), + "fixture must parse cleanly, else the walk covers error recovery" + ); + + let mut chain: Vec> = Vec::new(); + let mut stack = vec![(root, 0_usize)]; + let mut visited = 0; + while let Some((node, depth)) = stack.pop() { + chain.truncate(depth); + check(&node, &chain); + visited += 1; + chain.push(node); + let first = stack.len(); + stack.extend(node.children().map(|child| (child, depth + 1))); + stack[first..].reverse(); + } + visited +} + +/// Walks `parser`'s tree and reports whether any node has `kind_id == +/// target`. +/// +/// The drift marker behind lesson 34. A passing `!ast_has_kind_id(…)` +/// proves an enum variant is unreachable at the pinned grammar, so a +/// defensive dispatch arm listing it is an explicit promise rather than +/// silent dead code; a bump that starts emitting the kind fails the +/// assertion instead of quietly changing a metric. +pub fn ast_has_kind_id(parser: &P, target: u16) -> bool { + parser.root().preorder().any(|n| n.kind_id() == target) +} diff --git a/big-code-analysis-ast/src/token_role.rs b/big-code-analysis-ast/src/token_role.rs new file mode 100644 index 000000000..406f2e5fd --- /dev/null +++ b/big-code-analysis-ast/src/token_role.rs @@ -0,0 +1,26 @@ +//! The role a node plays in an expression: operator, operand, or +//! neither. +//! +//! This is a syntactic classification, not a metric one. Whether a node +//! is an operator or an operand is decided by the grammar — `+` and a +//! call expression are operators, an identifier and a string literal +//! are operands — and each language's answer lives in its +//! [`Getter::get_op_type`](crate::getter::Getter::get_op_type) table +//! beside the rest of its classifiers. +//! +//! Halstead is the metric that consumes it today, and until #1376 the +//! type was named after that consumer. It is not Halstead-specific: +//! anything that reasons about operator/operand structure can use it, +//! which is why it sits in the parse layer rather than the metric. +//! `metrics::halstead` re-exports it, and keeps the old `HalsteadType` +//! spelling as a deprecated alias. + +/// The role a node plays in an expression. +pub enum TokenRole { + /// The node acts as an operator (`+`, `&&`, a call, an index). + Operator, + /// The node acts as an operand (an identifier, a literal). + Operand, + /// The node plays neither role, so it is not counted. + Unknown, +} diff --git a/src/tools.rs b/big-code-analysis-ast/src/tools.rs similarity index 98% rename from src/tools.rs rename to big-code-analysis-ast/src/tools.rs index aa940e719..118b1cd4b 100644 --- a/src/tools.rs +++ b/big-code-analysis-ast/src/tools.rs @@ -17,6 +17,8 @@ clippy::cast_sign_loss )] +//! Language detection and the file readers. + use std::borrow::Cow; use std::cmp::Ordering; use std::collections::HashMap; @@ -47,7 +49,7 @@ use crate::langs::*; /// ``` /// use std::path::Path; /// -/// use big_code_analysis::read_file; +/// use big_code_analysis_ast::read_file; /// /// let path = Path::new("Cargo.toml"); /// read_file(&path).unwrap(); @@ -201,7 +203,7 @@ impl std::fmt::Display for SkipReason { /// ``` /// use std::path::Path; /// -/// use big_code_analysis::read_file_with_eol; +/// use big_code_analysis_ast::read_file_with_eol; /// /// let path = Path::new("Cargo.toml"); /// read_file_with_eol(&path).unwrap(); @@ -230,7 +232,7 @@ pub fn read_file_with_eol(path: &Path) -> std::io::Result>> { /// ``` /// use std::path::Path; /// -/// use big_code_analysis::{SkipReason, read_file_with_eol_classified}; +/// use big_code_analysis_ast::{SkipReason, read_file_with_eol_classified}; /// /// let dir = tempfile::tempdir().unwrap(); /// let path = dir.path().join("tiny.rs"); @@ -343,7 +345,7 @@ fn read_gated(path: &Path) -> Result, ReadStop> { /// # Examples /// /// ``` -/// use big_code_analysis::normalize_eol; +/// use big_code_analysis_ast::normalize_eol; /// /// // CRLF endings collapse to LF and a missing final newline is added. /// assert_eq!(normalize_eol(b"a\r\nb".to_vec()), b"a\nb\n"); @@ -367,7 +369,7 @@ pub fn normalize_eol(mut data: Vec) -> Vec { /// ```no_run /// use std::path::Path; /// -/// use big_code_analysis::write_file; +/// use big_code_analysis_ast::write_file; /// /// let path = Path::new("foo.txt"); /// let data: [u8; 4] = [0; 4]; @@ -388,7 +390,7 @@ pub fn write_file(path: &Path, data: &[u8]) -> std::io::Result<()> { /// ``` /// use std::path::Path; /// -/// use big_code_analysis::get_language_for_file; +/// use big_code_analysis_ast::get_language_for_file; /// /// let path = Path::new("build.rs"); /// get_language_for_file(&path).unwrap(); @@ -473,7 +475,7 @@ const GENERATED_SCAN_LINES: usize = 50; /// # Examples /// /// ``` -/// use big_code_analysis::is_generated; +/// use big_code_analysis_ast::is_generated; /// /// assert!(is_generated(b"// @generated\nfn x() {}\n")); /// assert!(is_generated( @@ -662,7 +664,7 @@ fn get_emacs_mode(buf: &[u8]) -> Option { /// ``` /// use std::path::PathBuf; /// -/// use big_code_analysis::guess_language; +/// use big_code_analysis_ast::guess_language; /// /// let source_code = "int a = 42;"; /// @@ -690,7 +692,7 @@ pub fn guess_language>(buf: &[u8], path: P) -> (Option, &'s /// Normalises all CR-only and CRLF line endings to LF throughout the buffer, /// then ensures the buffer ends with exactly one `\n`. -pub(crate) fn normalize_line_endings(data: &mut Vec) { +pub fn normalize_line_endings(data: &mut Vec) { // In-place compaction: write pointer stays ≤ read pointer, so no extra allocation. let mut w = 0; let mut r = 0; diff --git a/src/tools_tests.rs b/big-code-analysis-ast/src/tools_tests.rs similarity index 100% rename from src/tools_tests.rs rename to big-code-analysis-ast/src/tools_tests.rs diff --git a/src/traits.rs b/big-code-analysis-ast/src/traits.rs similarity index 51% rename from src/traits.rs rename to big-code-analysis-ast/src/traits.rs index 1b803b4a6..582682eaf 100644 --- a/src/traits.rs +++ b/big-code-analysis-ast/src/traits.rs @@ -6,6 +6,8 @@ // function so the per-language impl blocks stay readable. #![allow(clippy::wildcard_imports, clippy::enum_glob_use)] +//! [`ParserTrait`], [`LanguageInfo`], and [`Search`]. + use std::path::Path; use std::sync::Arc; @@ -22,8 +24,8 @@ use crate::preproc::PreprocResults; /// /// Implemented by every `XxxCode` type generated by the internal /// `mk_code!` macro. Crate-internal: reached only through the -/// `pub(crate)` `Parser` plumbing behind the [`crate::Ast`] seam. -pub(crate) trait LanguageInfo { +/// `pub` `Parser` plumbing behind the `big_code_analysis::Ast` seam. +pub trait LanguageInfo { /// Tree-sitter base language enum carried by this code tag. type BaseLang; @@ -31,32 +33,53 @@ pub(crate) trait LanguageInfo { fn lang() -> LANG; } -// Internal language-dispatch trait reached only by the macro-generated -// `Parser` impls in `src/parser.rs` and the `AnyParser` dispatch in -// `src/macros/mod.rs`. It carries the *parse* half of a language: the -// tree plus its `Checker` / `Getter` classifiers. The per-metric -// associated types live on the separate [`crate::MetricSuite`] supertrait -// (#1376), so this half has no dependency on the metric modules. Not -// part of any documented extension contract — metric extraction is -// driven through the public [`crate::Ast`] / [`crate::analyze`] seam, -// not by implementing this trait. See STABILITY.md. -pub(crate) trait ParserTrait { +/// A parsed source in one language: the tree, its bytes, and the +/// `Checker` / `Getter` classifiers that read it. +/// +/// Implemented once, generically, by [`Parser`](crate::parser::Parser) +/// for every `*Code` tag; the per-language `*Parser` aliases in +/// [`langs`](crate::langs) are its concrete forms. This is the *parse* +/// half of a language: `big-code-analysis` layers the per-metric +/// implementations on top through its own supertrait, so nothing here +/// depends on a metric (#1376). Not an extension contract — implement +/// nothing; match on [`AnyParser`] to reach one. +pub trait ParserTrait { + /// The language's classification predicates. type Checker: Alterator + Checker; + /// The language's naming / space-kind / Halstead-class accessors. type Getter: Getter; + /// Parses `code` read from `path`. `pr` carries the C-family + /// preprocessor results whose `#define`s are expanded before parsing; + /// every other language ignores it. + /// + /// # Panics + /// + /// When the language's Cargo feature is disabled in this build — + /// there is no grammar to parse with and the signature is + /// infallible. Check [`LANG::is_enabled`](crate::LANG::is_enabled) + /// first, or reach a parser through + /// [`AnyParser::parse`](crate::langs::AnyParser::parse), which + /// returns `Err(MetricsError::LanguageDisabled)` instead. fn new(code: Vec, path: &Path, pr: Option>) -> Self; + /// The root node of the parsed tree. fn root(&self) -> Node<'_>; + /// The bytes the tree was parsed from (after macro expansion, for + /// the C family). fn code(&self) -> &[u8]; /// The returned [`Filter`] borrows `self` — the `"function"` /// predicate reads the source bytes (#1162). fn filters(&self, requested: &[String]) -> Filter<'_>; } -pub(crate) trait Search<'a> { +/// Subtree traversal helpers implemented by [`Node`]. +pub trait Search<'a> { /// Visits every node of the subtree in source order, handing the /// action each node's ancestor chain alongside it so a predicate it /// applies stays off `Node::parent` (#1088). fn act_on_node(&self, action: &mut dyn FnMut(&Node<'a>, Ancestors<'a, '_>)); + /// The first direct child whose `kind_id` satisfies `pred`. fn first_child(&self, pred: fn(u16) -> bool) -> Option>; + /// Visits each direct child in source order. fn act_on_child(&self, action: &mut dyn FnMut(&Node<'a>)); } diff --git a/big-code-analysis-cli/src/html_report/styling.rs b/big-code-analysis-cli/src/html_report/styling.rs index e115617c9..cabde2fd3 100644 --- a/big-code-analysis-cli/src/html_report/styling.rs +++ b/big-code-analysis-cli/src/html_report/styling.rs @@ -237,8 +237,9 @@ table.hotspot tr:nth-child(even) td.risk-heat-4,table.hotspot td.risk-heat-4\ /// suite. `"other"` is the neutral fallback for any name not listed. /// /// Names match production output of [`big_code_analysis::LANG::name`] -/// (see `src/langs.rs`), which since #540 is the canonical lowercase -/// slug for every variant (`"cpp"`, `"csharp"`, `"tsx"`). `LANG::Tsx` +/// (see `big-code-analysis-ast/src/langs.rs`), which since #540 is +/// the canonical lowercase slug for every variant (`"cpp"`, +/// `"csharp"`, `"tsx"`). `LANG::Tsx` /// (`"tsx"`) reuses the `"typescript"` tint (it is TypeScript + JSX), /// and the Mozilla-fork `"mozjs"` reuses the `"javascript"` tint (it /// is JavaScript, just a different grammar), and likewise `"mozcpp"` diff --git a/docs/development/lessons_learned.md b/docs/development/lessons_learned.md index 2f64a62e0..c0618f3dd 100644 --- a/docs/development/lessons_learned.md +++ b/docs/development/lessons_learned.md @@ -296,7 +296,8 @@ you change Halstead classification, add a `kind_id` to `is_primitive`, or touch finalize / parent-merge, add a regression test that runs both `metrics()` and `operands_and_operators()` on the same input and asserts it. When auditing a new language, also check no kind_id is classified as -*both* operator and operand — `HalsteadType` is exhaustive but the +*both* operator and operand — `TokenRole` (then `HalsteadType`) is +exhaustive but the routing in `getter.rs` is not, and a copy-paste can land one kind_id in two arms. @@ -1412,7 +1413,7 @@ of (node kind, predicates that classify it). `TemplateString`, so a `String2` node — the `string` type-keyword alias — is counted by `find string` and contributes to Halstead string-operand totals. But the TS `impl_js_family_get_op_type!` invocation's -`operand_extras` omits `String2`, so the same node is `HalsteadType:: +`operand_extras` omits `String2`, so the same node is `TokenRole:: Unknown` to the Halstead walker. JS, MozJS, and TSX all include it; only TS does not. The drift predates #299 — the four pre-refactor impls had the same asymmetry — but the macro consolidation made the parity table diff --git a/enums/templates/rust.rs b/enums/templates/rust.rs index 10a4aea08..017550714 100644 --- a/enums/templates/rust.rs +++ b/enums/templates/rust.rs @@ -1,6 +1,9 @@ // See `src/languages/mod.rs` for the rationale behind the per-file // pedantic carve-outs below. #![allow(clippy::match_same_arms, clippy::too_many_lines)] +// One variant per grammar kind id, named after the rule; the name is +// the documentation and there is nothing per-variant to add. +#![allow(missing_docs)] // Code generated; DO NOT EDIT. diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 2f35baead..5b94f5a73 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -77,6 +77,22 @@ dependencies = [ [[package]] name = "big-code-analysis" version = "2.2.1" +dependencies = [ + "big-code-analysis-ast", + "crossbeam", + "csv", + "hmac", + "regex", + "serde", + "serde_json", + "sha2", + "termcolor", + "tree-sitter", +] + +[[package]] +name = "big-code-analysis-ast" +version = "2.2.1" dependencies = [ "aho-corasick", "bca-tree-sitter-ccomment", @@ -84,19 +100,13 @@ dependencies = [ "bca-tree-sitter-mozjs", "bca-tree-sitter-preproc", "bca-tree-sitter-tcl", - "crossbeam", - "csv", "dekobon-tree-sitter-groovy", - "hmac", "num-derive", "num-format", "num-traits", "petgraph", "regex", "serde", - "serde_json", - "sha2", - "termcolor", "tree-sitter", "tree-sitter-bash", "tree-sitter-c", diff --git a/fuzz/fuzz_targets/preproc_macro.rs b/fuzz/fuzz_targets/preproc_macro.rs index d50e2df9f..59dc19e17 100644 --- a/fuzz/fuzz_targets/preproc_macro.rs +++ b/fuzz/fuzz_targets/preproc_macro.rs @@ -4,7 +4,8 @@ //! # Why this target exists //! //! It is the direct residue of #1152. Adopting -//! `clippy::indexing_slicing` on `src/c_macro.rs` left nine per-function +//! `clippy::indexing_slicing` on `big-code-analysis-ast/src/c_macro.rs` +//! left nine per-function //! `#[allow]`s — every one a computed index into attacker-controlled //! bytes whose bound is asserted by a human comment rather than by the //! compiler. That is precisely the population a fuzzer is for, and #126 diff --git a/recreate-grammars.sh b/recreate-grammars.sh index a127fe96e..02f8f22ec 100755 --- a/recreate-grammars.sh +++ b/recreate-grammars.sh @@ -10,10 +10,10 @@ cargo clean --manifest-path ./enums/Cargo.toml # Recreate all grammars -cargo run --manifest-path ./enums/Cargo.toml -- -lrust -o ./src/languages +cargo run --manifest-path ./enums/Cargo.toml -- -lrust -o ./big-code-analysis-ast/src/languages # Recreate C macros -cargo run --manifest-path ./enums/Cargo.toml -- -lc_macros -o ./src/c_langs_macros +cargo run --manifest-path ./enums/Cargo.toml -- -lc_macros -o ./big-code-analysis-ast/src/c_langs_macros # Format the code of the recreated grammars cargo fmt diff --git a/src/c_declarator.rs b/src/c_declarator.rs deleted file mode 100644 index 2e4607b9a..000000000 --- a/src/c_declarator.rs +++ /dev/null @@ -1,641 +0,0 @@ -//! The C-family declarator-chain walk, shared by the two surfaces that -//! need it. -//! -//! C declarator syntax nests outward from the declared name, so neither -//! a function's parameter list nor its name is reliably a child of the -//! function node itself. Both are found from the one node -//! [`innermost_declarator`] returns — the innermost link on the chain -//! that is the function's own declarator rather than its return type's: -//! -//! - [`crate::metrics::nargs`] reads its `parameters` field (#1200). -//! - The `Getter::get_func_space_name` impls for C, C++, mozcpp and -//! Objective-C read its name side through [`declarator_name`] (#1208). -//! -//! Keeping one walk keeps the two answers about the same function from -//! disagreeing, which is how #1208 arose: the arity came from this -//! chain and the name came from a leftmost pre-order search that stopped -//! one level too early. The invariant is one *function*, one walk — -//! not one node: an unexpanded function-like macro puts the arity and -//! the name on two links of the chain, which each function's own doc -//! below explains (#1213). - -use crate::checker::Checker; -use crate::node::Node; - -/// The innermost declarator along a C-family function's declarator -/// chain: the node whose `parameters` field holds the function's *own* -/// formal arguments. [`declarator_name`] takes the name from the same -/// node's `declarator` field. -/// -/// C declarator syntax nests outward from the declared name, so a -/// function node's `declarator` field is only the function's parameter -/// list when the return type is plain. Anything the return type -/// contributes — a `*`, a `&`, a parenthesised group — wraps the -/// `function_declarator` that owns the real list, and the outermost -/// `parameters` a chain carries can belong to the *return type* rather -/// than to the function (`int (*f(int a))(int b)` returns a pointer to a -/// one-argument function and itself takes one argument). Taking the -/// innermost is what makes both of those come out right (#1200), and -/// the same node carries the name `f` that a leftmost search misses -/// (#1208). -/// -/// "Innermost" has one exception, and it is the one shape where the -/// grammar's reading and the preprocessor's disagree. An unexpanded -/// function-like macro — `RUN_STATS_METHOD(allocate)(JNIEnv *env, -/// jclass clazz)`, which is what every JNI shim looks like — parses as -/// a `function_declarator` sitting in another one's `declarator` field, -/// so the innermost list is the macro's `(allocate)` and the function's -/// own arguments are discarded. Neither language permits that chain: a -/// function may not return a function type (C11 6.7.6.3p1, C++ -/// `[dcl.fct]`), so a legitimate function returning a function pointer -/// always interposes a `parenthesized_declarator`, and the direct -/// nesting can only be a macro (or an `ERROR`, below). The walk stops -/// at the outer link there and reports the function's arity (#1213). -/// -/// The walk is by field name, per `.claude/rules/grammar-dispatch.md` -/// §3, which also sidesteps §1: `PointerDeclarator2`, -/// `FunctionDeclarator2`/`3` and `ReferenceDeclarator2`/`3`/`4` are -/// numeric-suffix aliases that a `kind_id` match would have to -/// enumerate and would silently regress on the next grammar bump. -/// -/// Three of the rules on the chain expose no field at all, which is why -/// the field alone is not enough. Every entry below is from the pinned -/// grammars' `node-types.json` (`tree-sitter-cpp` 0.23.4, -/// `tree-sitter-c` 0.24.2, `tree-sitter-objc` 3.0.2, vendored -/// `tree-sitter-mozcpp`), and the fieldless list is the complete set of -/// `*_declarator` rules with no fields that a *function definition's* -/// name side can reach — the rest (`variadic_declarator`, -/// `structured_binding_declarator`, Objective-C's `keyword_declarator` -/// and `struct_declarator`, and the `abstract_*` family) sit in -/// parameter, binding or type position, never here. -/// -/// | rule | `declarator` field | -/// | --- | --- | -/// | `pointer_declarator` | required | -/// | `function_declarator` | required | -/// | `abstract_function_declarator` | **optional** | -/// | `reference_declarator` | **absent — no fields** | -/// | `parenthesized_declarator` | **absent — no fields** | -/// | `attributed_declarator` | **absent — no fields** | -/// -/// In all three fieldless rules the inner declarator is the last -/// *named* child once attributes are set aside, so that is the -/// fallback: -/// -/// - `reference_declarator` is `seq(choice('&', '&&'), _declarator)`. -/// - `parenthesized_declarator` is `seq('(', -/// optional(ms_call_modifier), _declarator, ')')` — last rather than -/// sole, so `int (__cdecl *f(int a))(int b)` does not defeat it. -/// - `attributed_declarator` is `seq(_declarator, -/// repeat1(attribute_declaration))`, the one rule that puts the -/// declarator **first**. Excluding `attribute_declaration` — its only -/// non-declarator child type in all four grammars — restores "last" -/// as the right answer, and without that exclusion -/// `int f(int a, int b) [[deprecated]]` reports 0. -/// -/// `template_argument_list` is excluded for the same reason as -/// `attribute_declaration`, and it is the one exclusion the fallback -/// needs beyond the three rules above. The fallback also runs on the -/// *name* forms, which have no `declarator` field either, and two of -/// them — `template_function` and `template_method` — put their argument -/// list last: `void f(int a)`. A type argument -/// spelling a function type carries a `parameters` field of its own, so -/// descending into it made that function read as taking two arguments -/// and made its name resolve to nothing at all, the abstract declarator -/// the chain landed on spelling no identifier. Excluding the argument -/// list leaves the name itself as the last named child, which terminates -/// the chain where it should. -/// -/// Comments are excluded for the same reason, tree-sitter admitting one -/// anywhere. -/// -/// The fallback stops at a node that already carries `parameters`, -/// which is the C++ lambda: `abstract_function_declarator`'s -/// `declarator` field is optional, so `[](int a, int (*cb)(int x))` -/// would otherwise descend into the `parameter_list` and return `cb`'s -/// `(int x)` — one argument instead of two. -/// -/// Every step strictly descends a finite tree, so the walk terminates -/// without a depth cap. -/// -/// # ERROR-recovery trees are outside this contract -/// -/// Every rule above is the grammar's, and none of them holds once -/// tree-sitter starts recovering. An unexpanded macro in declarator -/// position — `T *f() TF_ATTRIBUTE_NOINLINE { … }` — puts the real -/// `function_declarator` inside an `ERROR` node and leaves the macro's -/// `field_identifier` as the `pointer_declarator`'s last named child, -/// so the fallback follows the macro and the walk answers `None`. -/// -/// Give that macro an argument — `T *f() TF_LOCKS_EXCLUDED(mu_) { … }`, -/// which is the spelling the TensorFlow / Abseil annotations actually -/// take — and it is a `function_declarator` carrying `parameters`, so -/// the walk answers with the *macro's* name rather than with nothing. -/// That is the one shape this change made worse: the leftmost pre-order -/// search it replaced descended into the `ERROR` and got `f` right. -/// `a_parenthesised_macro_takes_the_name_of_the_function_it_annotates` -/// pins it. -/// -/// Recovery also manufactures the direct `function_declarator` nesting -/// the macro rule keys on, from source containing no macro-obscured -/// declarator at all. A *statement* macro followed by an `if` — -/// TensorFlow's `TF_ASSIGN_OR_RETURN(bool ok, Try(x)); if (ok) { … }` — -/// recovers into a `function_declarator` whose `declarator` field is the -/// macro call and whose `parameters` field is the `if` **condition**. So -/// the rule changes the answer for 19 of the 46 corpus spaces it -/// touches, from the macro's argument count to the condition's, neither -/// of which is an arity. There is no fixture for it: whether the -/// grammar recovers this way depends on where the line breaks fall, -/// tree-sitter costing a recovery by the extent it skips, so any pinned -/// spelling would be a claim about whitespace (#1213). -/// -/// Whatever any strategy returns there is arbitrary, and the walk does -/// not try to be clever about it. Measured over `DeepSpeech` and -/// `pdf.js` (14,269 files), moving the four getters onto this walk -/// named 46 previously-nameless function spaces, un-named 2 and renamed -/// 4 — 354 nameless spaces down to 310, a net 44. All six of the latter -/// sit inside recovery subtrees: one of the un-named had been reporting -/// an `if` statement's callee as a function name, and one of the renamed -/// is the `TF_LOCKS_EXCLUDED` case above (#1208). -pub(crate) fn innermost_declarator<'tree, T: Checker>(node: &Node<'tree>) -> Option> { - // The chain starts at the `declarator` field rather than at `node` - // so the last-named-child fallback can never fire on the function - // node itself and walk into its body. The walk runs outside-in, so - // the innermost qualifying link is the last one it yields. - std::iter::successors( - node.child_by_field_name("declarator"), - |current| match current.child_by_field_name("declarator") { - // An unexpanded function-like macro standing in for the - // declarator, which is the shape JNI shims take. Neither C - // nor C++ lets a function return a function type (C11 - // 6.7.6.3p1, C++ `[dcl.fct]`), so a `function_declarator` - // directly inside another one's `declarator` field is not a - // declarator chain at all: the outer list is the function's - // own and the inner one holds the macro's arguments. Both - // links have to be tested — a pointer return puts a - // `function_declarator` in a `pointer_declarator`'s - // `declarator` field, and stopping *there* would end the - // chain on a node carrying no `parameters` and report 0 - // (#1213). - Some(inner) - if current.kind() == FUNCTION_DECLARATOR && inner.kind() == FUNCTION_DECLARATOR => - { - None - } - Some(declarator) => Some(declarator), - None if current.child_by_field_name("parameters").is_some() => None, - None => current - .children() - .filter(|child| { - child.is_named() - && !T::is_comment(child) - && !matches!(child.kind(), ATTRIBUTE | TEMPLATE_ARGUMENTS) - }) - .last(), - }, - ) - // A conversion operator's `declarator` field is the type it converts - // *to*, not its name side: `operator int (*)(int x)` takes no - // arguments, and everything from here inward describes that - // function-pointer type. Cutting the chain restores the 0 the - // pre-#1200 code reported by never finding `parameters` at all. - .take_while(|link| link.kind() != CONVERSION_OPERATOR) - .filter(|link| link.child_by_field_name("parameters").is_some()) - .last() -} - -/// The node spelling a C-family function's name. -/// -/// It is the `declarator` field of [`innermost_declarator`], and it is a -/// separate function only so the four `get_func_space_name` impls state -/// that pairing once instead of four times. Each caller still gates the -/// result on its own grammar's identifier kinds: what counts as a name -/// is where C, C++ and Objective-C differ (`destructor_name`, -/// `qualified_identifier`, `operator_name`, `template_function`), and a -/// kind this module accepted on their behalf would be a claim about -/// four grammars made in a module that reads none of them. -/// -/// The macro shape [`innermost_declarator`] stops at is the one place -/// the name and the arity come off different nodes. There that -/// `declarator` field is the macro *invocation* — itself a -/// `function_declarator`, which no getter's identifier gate accepts — so -/// the walk descends through it to the identifier the macro spells. -/// `RUN_STATS_METHOD` is the only name in the source: the real -/// `Java_…_allocate` exists only after `##` pasting, and it is the token -/// a reader greps for. This is why the module doc states the invariant -/// per *function* rather than per node (#1213). -pub(crate) fn declarator_name<'tree, T: Checker>(node: &Node<'tree>) -> Option> { - // A run of them rather than one: `A(b)(c)(int x)` is two nested - // invocations and the name is still `A`. Each step descends a finite - // tree, so this terminates for the same reason the walk above does. - // - // Written as a chain rather than a `while` with `?` inside it - // deliberately. The loop form needs an early return for "this - // `function_declarator` has no `declarator` field", which the - // grammars declare required and only an `ERROR` could violate — two - // arms no test can reach, and coverage counts them. - // - // The two forms are not identical on that unreachable input, which is - // worth stating rather than leaving for the next reader to rediscover - // (#1220). On an `ERROR` tree where a `function_declarator` lacks its - // `declarator` field, the loop returned `None` and this chain yields - // that `function_declarator` itself — the whole declarator span in - // place of a name. - // - // Nothing observes the difference, and the reason is external to this - // module: every caller gates the result on its own grammar's - // identifier kinds (`TypeIdentifier | Identifier | FieldIdentifier`, - // plus the C++ name forms in `getter/cpp.rs` and `getter/mozcpp.rs`), - // and `function_declarator` is in none of those lists. The `matches!` - // falls through and `get_func_space_name` returns `None` — the same - // answer the loop gave. That dependency is the thing to preserve: a - // getter that widened its gate to accept `function_declarator` would - // start naming functions after their whole declarator span on - // malformed input, and this comment is the only place that says so. - std::iter::successors( - innermost_declarator::(node)?.child_by_field_name("declarator"), - |link| { - if link.kind() == FUNCTION_DECLARATOR { - link.child_by_field_name("declarator") - } else { - None - } - }, - ) - .last() -} - -/// Compared by `kind()` string rather than `kind_id`, per -/// `.claude/rules/grammar-dispatch.md` §1: every rule below carries -/// numeric-suffix aliases across the four C-family grammars, and a -/// `kind_id` match would have to enumerate every one of them and would -/// regress silently on the next grammar bump. C and Objective-C simply -/// never emit `operator_cast`. -const CONVERSION_OPERATOR: &str = "operator_cast"; -const ATTRIBUTE: &str = "attribute_declaration"; -const TEMPLATE_ARGUMENTS: &str = "template_argument_list"; -/// Carries the most aliases of the four — `FunctionDeclarator2` through -/// `FunctionDeclarator5` in `tree-sitter-c` alone — so it is the one the -/// `kind()`-string rule above most needs to cover. -const FUNCTION_DECLARATOR: &str = "function_declarator"; - -#[cfg(test)] -mod space_name_tests { - use crate::test_support::space_verbatim; - use crate::{FuncSpace, LANG, MetricsOptions, SpaceKind}; - - /// Declarator shapes all four C-family grammars parse alike, with - /// the name each one's function space must carry. - /// - /// The first three are #1208 itself: every one resolved to `None` - /// before the getters moved onto [`super::innermost_declarator`]. - /// The macro spelling is the shape that dominates the corpora — 354 - /// nameless C-family function spaces across `DeepSpeech` and - /// `pdf.js`, clustered in TensorFlow's JNI shims — and it carries no - /// `parenthesized_declarator` at all, so a fix keyed on that kind - /// would pass the first row and miss the population. - /// `RUN_STATS_METHOD` is the macro's name, not the function's, which - /// after `##` pasting is not in the source at all; it is kept - /// because it is the token a reader greps for (#1213). - /// - /// The two macro rows after it are #1213: the arity moved to the - /// outer declarator there, so the name is the only thing still read - /// from inside the invocation, and `A` additionally pins the descent - /// through a *run* of nested invocations. - /// - /// The next three are controls. `g` in particular resolved - /// correctly *before* this change — its outer declarator is an - /// `array_declarator`, so the old leftmost pre-order search happened - /// to reach the right `function_declarator` — and would regress - /// silently if the walk stopped one link too early. - /// - /// The last row expects no name at all; its comment says why. - const SHARED_SHAPES: &[(&str, Option<&str>)] = &[ - ("int (*fp(int a, int b))(int c) { return 0; }", Some("fp")), - ( - "int (__cdecl *w(int a, int b))(int c) { return 0; }", - Some("w"), - ), - ( - "void RUN_STATS_METHOD(allocate)(int a) { }", - Some("RUN_STATS_METHOD"), - ), - ("void MACRO(a, b)(int x) { }", Some("MACRO")), - ("void A(b, c)(d)(int x, int y, int z) { }", Some("A")), - ("int (*g(void))[4] { return 0; }", Some("g")), - ("int plain(int a, int b) { return a; }", Some("plain")), - ("FILE *ptr(int a) { return 0; }", Some("ptr")), - // Why the fallback may not simply require each link to be a - // `*_declarator`, which is the tidier-looking rule. Every - // grammar recovers this TensorFlow C-API signature into a - // `qualified_identifier` holding a **zero-width** `::` and the - // real `pointer_type_declarator`, so the chain has to descend - // through a link that is not a declarator at all to reach the - // name. Gating the fallback on the kind suffix loses this and - // two more names in the corpora: a non-declarator link is not - // always a name. - ( - "TF_CAPI_EXPORT extern TF_ConcreteFunction* TF_GetFn(TF_SavedModel* m) { return 0; }", - Some("TF_GetFn"), - ), - // The one row the gate must *reject*. Redundant parentheses - // around the name are legal C and put a - // `parenthesized_declarator` where every grammar's name kinds - // would be, so each getter's `matches!` falls through and the - // space stays nameless — emitting no name rather than whatever - // text happens to sit there is what that gate is for, and no - // other row reaches its `false` branch. - // - // A boundary, not a bug-lock: the shape has zero occurrences - // across `DeepSpeech` and `pdf.js`, so there is nothing to fix - // and no issue to open. Teach the walk to unwrap the - // parentheses and this is the row to update. - ("int (fp)(int a) { return a; }", None), - ]; - - /// C++ name forms C and Objective-C have no syntax for. None of - /// these is a #1208 shape; they are here because the rewrite - /// replaced the `child(0)` the identifier-kind `match` used to read - /// with the `declarator` field, and each of these rows is a - /// different kind arriving in that slot — `destructor_name`, - /// `qualified_identifier`, `operator_name`, `template_function`. - /// The conversion operator additionally pins the `OperatorCast` - /// early return, which the shared walk cannot answer for: a - /// conversion operator's declarator field is the type it converts - /// *to*, so [`super::innermost_declarator`] deliberately cuts the - /// chain there and returns `None`. - const CPP_ONLY_SHAPES: &[(&str, Option<&str>)] = &[ - ("struct S { ~S() { } };", Some("~S")), - ("void Foo::bar(int a) { }", Some("Foo::bar")), - ( - "struct S { operator int() const { return 0; } };", - Some("operator int() const"), - ), - ( - "struct S { int operator+(int o) const { return o; } };", - Some("operator+"), - ), - ( - "Foo &Bar::get(int a) { static Foo f; return f; }", - Some("Bar::get"), - ), - ( - "template T tfree(T a) { return a; }", - Some("tfree"), - ), - // The two shapes the fallback's `template_argument_list` - // exclusion exists for. Both spell an explicit template argument - // of function type, so the chain would otherwise leave the name - // side entirely — down `template_function` into its - // `template_argument_list` — and settle on the argument's own - // `abstract_function_declarator`. That node spells no - // identifier, so the name came back `None`, and `nargs` read the - // argument's two parameters instead of the function's one. - // - // Both parse without an `ERROR` node, so neither is covered by - // the recovery caveat on [`super::innermost_declarator`]. The - // second is the more reachable of the two: an out-of-line - // member with explicit template arguments needs no `template <>` - // preamble. - ( - "template <> void tspec(int a) { }", - Some("tspec"), - ), - ( - "void Foo::tmem(int a) { }", - Some("Foo::tmem"), - ), - ]; - - /// Each fixture is padded with a leading and a trailing comment - /// line, so the asserted span is `(2, 2)` — a value a - /// default-constructed or off-by-one span does not also satisfy, - /// unlike the `(1, 1)` a bare one-line fixture would produce. - const FIXTURE_LINE: usize = 2; - - fn pad(source: &str) -> String { - format!("// leading\n{source}\n// trailing\n") - } - - /// Every `Function` space in the tree, in source order. - /// - /// The C++ rows nest their function inside a `struct` space, so the - /// assertion cannot read `root.spaces[0]`; collecting the whole - /// subtree also lets each row assert that the fixture opened - /// *exactly one* function space, which is `get_space_kind` and - /// `is_func_space` agreeing with the name — `.claude/rules/ - /// grammar-dispatch.md` §6. - fn function_spaces(space: &FuncSpace, found: &mut Vec<(Option, usize, usize)>) { - if space.kind == SpaceKind::Function { - found.push((space.name.clone(), space.start_line, space.end_line)); - } - for child in &space.spaces { - function_spaces(child, found); - } - } - - fn check(lang: LANG, shapes: &[(&str, Option<&str>)], failures: &mut Vec) { - for (source, expected) in shapes { - let root = space_verbatim(lang, pad(source).as_bytes(), MetricsOptions::default()); - let mut found = Vec::new(); - function_spaces(&root, &mut found); - let want = vec![(expected.map(str::to_owned), FIXTURE_LINE, FIXTURE_LINE)]; - if found != want { - failures.push(format!( - "{lang:?}: {source:?}\n want {want:?}\n got {found:?}" - )); - } - } - } - - /// Fail with every mismatched row, and fail *differently* when a - /// feature set left the loop empty. - /// - /// Shared so the failure formatting exists once: it is by - /// construction unreachable while the suite is green, so a second - /// copy is coverage the tests can never earn. - #[track_caller] - fn assert_all_matched(failures: &[String], checked: usize, what: &str) { - assert!( - failures.is_empty(), - "{}/{checked} {what}:\n{}", - failures.len(), - failures.join("\n") - ); - // Non-vacuity: a feature set that disabled all four languages - // would otherwise leave every assertion above unrun. - assert!(checked > 0, "no C-family language was enabled"); - } - - /// A C-family function's name comes off the declarator walk its - /// arity comes off (#1208) — from the innermost declarator itself - /// for most shapes, and from inside the macro invocation that - /// declarator wraps for the three macro rows (#1213). "Same walk" - /// rather than "same node" is why this is not named for the - /// innermost declarator alone. - #[test] - fn the_declarator_walk_names_the_function_space() { - let mut failures = Vec::new(); - let mut checked = 0; - for lang in [LANG::C, LANG::Cpp, LANG::Mozcpp, LANG::Objc] - .into_iter() - .filter(LANG::is_enabled) - { - check(lang, SHARED_SHAPES, &mut failures); - checked += SHARED_SHAPES.len(); - if matches!(lang, LANG::Cpp | LANG::Mozcpp) { - check(lang, CPP_ONLY_SHAPES, &mut failures); - checked += CPP_ONLY_SHAPES.len(); - } - } - assert_all_matched( - &failures, - checked, - "declarator shapes named the wrong space", - ); - } - - /// [`check`] must be *able* to fail. - /// - /// It collects rather than asserts, so nothing in the table above - /// would notice if `function_spaces` selected no space at all — the - /// comparison would just find two empty expectations equal, and - /// every row would pass vacuously - /// (`.claude/rules/testing.md`, "Review the selector as carefully as - /// the assertion"). Feeding it a name that is deliberately wrong is - /// the cheapest proof that the selector reaches a real space and the - /// comparison discriminates. - #[cfg(feature = "c")] - #[test] - fn the_table_reports_a_name_that_does_not_match() { - let mut failures = Vec::new(); - check( - LANG::C, - &[( - "int plain(int a, int b) { return a; }", - Some("deliberately_wrong"), - )], - &mut failures, - ); - let [only] = failures.as_slice() else { - panic!("one wrong expectation must produce one failure, got {failures:?}"); - }; - // `Some("plain")` rather than a bare `plain`: the message echoes - // the fixture source, which contains the word too, so the bare - // substring passes even when the selector found *nothing* — - // measured, by filtering `function_spaces` on `SpaceKind::Class`. - // Matching the rendered `Option` is what ties the assertion to - // the space rather than to the input. - assert!( - only.contains("deliberately_wrong") && only.contains("Some(\"plain\")"), - "the failure must name both the expectation and the space found: {only}" - ); - } - - /// An unexpanded macro where a trailing attribute belongs, which is - /// the one input in this module that reaches - /// [`super::declarator_name`]'s `?` — the arm taken when *no* link - /// on the chain carries a `parameters` field, so there is no - /// declarator to read a name from at all. Every other row resolves - /// an owner and is answered by the identifier-kind gate instead. - /// - /// The grammars split two-two on it, which is the reason this is its - /// own test rather than a table row — and the split is not the one - /// the language families would suggest: - /// - /// | grammar | parse | name | - /// | --- | --- | --- | - /// | C, **mozcpp** | clean — `function_declarator` admits the trailing identifier | `f` | - /// | C++, Objective-C | `ERROR` around the declarator | none | - /// - /// Where the parse is clean the chain follows a real `declarator` - /// field and the name resolves. Where it is not, the declarator sits - /// inside an `ERROR` and the macro is left as the - /// `pointer_declarator`'s last named child, so the fallback follows - /// the macro into a dead end. - /// - /// mozcpp siding with C rather than with the upstream `tree-sitter-cpp` - /// it forked from is the finding worth keeping here: it owns no file - /// extension, so nothing routes to it and only a unit test can see - /// it at all (`.claude/rules/grammar-dispatch.md`, "when you fix one - /// language, sweep the rest"). - /// - /// Recovery trees are outside the walk's contract — see - /// [`super::innermost_declarator`], which measured this exact shape - /// as one of the two corpus spaces #1208 un-named. This pins what - /// the walk does there, not a claim that it is the right answer. - #[test] - fn a_macro_where_an_attribute_belongs_divides_the_grammars() { - const SOURCE: &str = "int *f() TF_ATTRIBUTE_NOINLINE { return 0; }"; - - let mut failures = Vec::new(); - let mut checked = 0; - for lang in [LANG::C, LANG::Cpp, LANG::Mozcpp, LANG::Objc] - .into_iter() - .filter(LANG::is_enabled) - { - let expected = matches!(lang, LANG::C | LANG::Mozcpp).then_some("f"); - check(lang, &[(SOURCE, expected)], &mut failures); - checked += 1; - } - assert_all_matched( - &failures, - checked, - "grammars disagreed about the recovery shape", - ); - } - - /// The same macro carrying an argument, which is the spelling the - /// annotation idiom actually takes — `TF_LOCKS_EXCLUDED(mu_)`, - /// `TF_GUARDED_BY(mu_)`, `ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_)`. - /// - /// Where the grammar recovers, it is worse than the parameterless - /// spelling above rather than the same. A bare trailing identifier - /// carries no `parameters`, so the chain dead-ends and the space - /// merely goes nameless; a parenthesised one is a - /// `function_declarator` that *does*, so it becomes the walk's - /// answer and the space is named after the macro. `nargs` reads the - /// macro's argument off the same node, and two members of one class - /// sharing an annotation collapse onto a single `bca check` offender - /// key (`K::TF_LOCKS_EXCLUDED` twice). - /// - /// This is the one shape #1208 made worse: the leftmost pre-order - /// search it replaced descended into the `ERROR` and got `f` right. - /// One corpus space is affected — `resource()` in TensorFlow's - /// `resource_op_kernel_test.cc`, which #1208 renamed to - /// `TF_LOCKS_EXCLUDED`. Pinned rather than fixed, like its sibling - /// above: reaching into a recovery subtree is a strategy decision of - /// its own, and every rule the walk follows is void inside an - /// `ERROR`. Teach the walk to unwrap one and this is a row to - /// update, not a row to delete. - #[test] - fn a_parenthesised_macro_takes_the_name_of_the_function_it_annotates() { - const SOURCE: &str = "int *f() TF_LOCKS_EXCLUDED(mu_) { return 0; }"; - - let mut failures = Vec::new(); - let mut checked = 0; - for lang in [LANG::C, LANG::Cpp, LANG::Mozcpp, LANG::Objc] - .into_iter() - .filter(LANG::is_enabled) - { - // Only C parses this cleanly, and there the chain follows a - // real `declarator` field past the macro to `f`. The split - // is *not* the two-two of the parameterless spelling: - // mozcpp sides with C there and with C++ here, so a fixture - // in either spelling alone would misreport what the other - // does. - let expected = if lang == LANG::C { - "f" - } else { - "TF_LOCKS_EXCLUDED" - }; - check(lang, &[(SOURCE, Some(expected))], &mut failures); - checked += 1; - } - assert_all_matched( - &failures, - checked, - "grammars disagreed about the annotated recovery shape", - ); - } -} diff --git a/src/c_family_space_names_tests.rs b/src/c_family_space_names_tests.rs new file mode 100644 index 000000000..709cf86b8 --- /dev/null +++ b/src/c_family_space_names_tests.rs @@ -0,0 +1,365 @@ +//! Function-space names for the C-family declarator shapes (#1208). +//! +//! The declarator unwrapping these exercise lives in +//! `big-code-analysis-ast`, but what they assert is the name the metric +//! walk puts on a `FuncSpace` — a whole-`analyze` outcome — so the tests +//! belong on this side. Keeping them here is also what lets the parse +//! layer carry no dependency on this crate at all (#1376). + +use crate::test_support::space_verbatim; +use crate::{FuncSpace, LANG, MetricsOptions, SpaceKind}; + +/// Declarator shapes all four C-family grammars parse alike, with +/// the name each one's function space must carry. +/// +/// The first three are #1208 itself: every one resolved to `None` +/// before the getters moved onto [`super::innermost_declarator`]. +/// The macro spelling is the shape that dominates the corpora — 354 +/// nameless C-family function spaces across `DeepSpeech` and +/// `pdf.js`, clustered in TensorFlow's JNI shims — and it carries no +/// `parenthesized_declarator` at all, so a fix keyed on that kind +/// would pass the first row and miss the population. +/// `RUN_STATS_METHOD` is the macro's name, not the function's, which +/// after `##` pasting is not in the source at all; it is kept +/// because it is the token a reader greps for (#1213). +/// +/// The two macro rows after it are #1213: the arity moved to the +/// outer declarator there, so the name is the only thing still read +/// from inside the invocation, and `A` additionally pins the descent +/// through a *run* of nested invocations. +/// +/// The next three are controls. `g` in particular resolved +/// correctly *before* this change — its outer declarator is an +/// `array_declarator`, so the old leftmost pre-order search happened +/// to reach the right `function_declarator` — and would regress +/// silently if the walk stopped one link too early. +/// +/// The last row expects no name at all; its comment says why. +const SHARED_SHAPES: &[(&str, Option<&str>)] = &[ + ("int (*fp(int a, int b))(int c) { return 0; }", Some("fp")), + ( + "int (__cdecl *w(int a, int b))(int c) { return 0; }", + Some("w"), + ), + ( + "void RUN_STATS_METHOD(allocate)(int a) { }", + Some("RUN_STATS_METHOD"), + ), + ("void MACRO(a, b)(int x) { }", Some("MACRO")), + ("void A(b, c)(d)(int x, int y, int z) { }", Some("A")), + ("int (*g(void))[4] { return 0; }", Some("g")), + ("int plain(int a, int b) { return a; }", Some("plain")), + ("FILE *ptr(int a) { return 0; }", Some("ptr")), + // Why the fallback may not simply require each link to be a + // `*_declarator`, which is the tidier-looking rule. Every + // grammar recovers this TensorFlow C-API signature into a + // `qualified_identifier` holding a **zero-width** `::` and the + // real `pointer_type_declarator`, so the chain has to descend + // through a link that is not a declarator at all to reach the + // name. Gating the fallback on the kind suffix loses this and + // two more names in the corpora: a non-declarator link is not + // always a name. + ( + "TF_CAPI_EXPORT extern TF_ConcreteFunction* TF_GetFn(TF_SavedModel* m) { return 0; }", + Some("TF_GetFn"), + ), + // The one row the gate must *reject*. Redundant parentheses + // around the name are legal C and put a + // `parenthesized_declarator` where every grammar's name kinds + // would be, so each getter's `matches!` falls through and the + // space stays nameless — emitting no name rather than whatever + // text happens to sit there is what that gate is for, and no + // other row reaches its `false` branch. + // + // A boundary, not a bug-lock: the shape has zero occurrences + // across `DeepSpeech` and `pdf.js`, so there is nothing to fix + // and no issue to open. Teach the walk to unwrap the + // parentheses and this is the row to update. + ("int (fp)(int a) { return a; }", None), +]; + +/// C++ name forms C and Objective-C have no syntax for. None of +/// these is a #1208 shape; they are here because the rewrite +/// replaced the `child(0)` the identifier-kind `match` used to read +/// with the `declarator` field, and each of these rows is a +/// different kind arriving in that slot — `destructor_name`, +/// `qualified_identifier`, `operator_name`, `template_function`. +/// The conversion operator additionally pins the `OperatorCast` +/// early return, which the shared walk cannot answer for: a +/// conversion operator's declarator field is the type it converts +/// *to*, so [`super::innermost_declarator`] deliberately cuts the +/// chain there and returns `None`. +const CPP_ONLY_SHAPES: &[(&str, Option<&str>)] = &[ + ("struct S { ~S() { } };", Some("~S")), + ("void Foo::bar(int a) { }", Some("Foo::bar")), + ( + "struct S { operator int() const { return 0; } };", + Some("operator int() const"), + ), + ( + "struct S { int operator+(int o) const { return o; } };", + Some("operator+"), + ), + ( + "Foo &Bar::get(int a) { static Foo f; return f; }", + Some("Bar::get"), + ), + ( + "template T tfree(T a) { return a; }", + Some("tfree"), + ), + // The two shapes the fallback's `template_argument_list` + // exclusion exists for. Both spell an explicit template argument + // of function type, so the chain would otherwise leave the name + // side entirely — down `template_function` into its + // `template_argument_list` — and settle on the argument's own + // `abstract_function_declarator`. That node spells no + // identifier, so the name came back `None`, and `nargs` read the + // argument's two parameters instead of the function's one. + // + // Both parse without an `ERROR` node, so neither is covered by + // the recovery caveat on [`super::innermost_declarator`]. The + // second is the more reachable of the two: an out-of-line + // member with explicit template arguments needs no `template <>` + // preamble. + ( + "template <> void tspec(int a) { }", + Some("tspec"), + ), + ( + "void Foo::tmem(int a) { }", + Some("Foo::tmem"), + ), +]; + +/// Each fixture is padded with a leading and a trailing comment +/// line, so the asserted span is `(2, 2)` — a value a +/// default-constructed or off-by-one span does not also satisfy, +/// unlike the `(1, 1)` a bare one-line fixture would produce. +const FIXTURE_LINE: usize = 2; + +fn pad(source: &str) -> String { + format!("// leading\n{source}\n// trailing\n") +} + +/// Every `Function` space in the tree, in source order. +/// +/// The C++ rows nest their function inside a `struct` space, so the +/// assertion cannot read `root.spaces[0]`; collecting the whole +/// subtree also lets each row assert that the fixture opened +/// *exactly one* function space, which is `get_space_kind` and +/// `is_func_space` agreeing with the name — `.claude/rules/ +/// grammar-dispatch.md` §6. +fn function_spaces(space: &FuncSpace, found: &mut Vec<(Option, usize, usize)>) { + if space.kind == SpaceKind::Function { + found.push((space.name.clone(), space.start_line, space.end_line)); + } + for child in &space.spaces { + function_spaces(child, found); + } +} + +fn check(lang: LANG, shapes: &[(&str, Option<&str>)], failures: &mut Vec) { + for (source, expected) in shapes { + let root = space_verbatim(lang, pad(source).as_bytes(), MetricsOptions::default()); + let mut found = Vec::new(); + function_spaces(&root, &mut found); + let want = vec![(expected.map(str::to_owned), FIXTURE_LINE, FIXTURE_LINE)]; + if found != want { + failures.push(format!( + "{lang:?}: {source:?}\n want {want:?}\n got {found:?}" + )); + } + } +} + +/// Fail with every mismatched row, and fail *differently* when a +/// feature set left the loop empty. +/// +/// Shared so the failure formatting exists once: it is by +/// construction unreachable while the suite is green, so a second +/// copy is coverage the tests can never earn. +#[track_caller] +fn assert_all_matched(failures: &[String], checked: usize, what: &str) { + assert!( + failures.is_empty(), + "{}/{checked} {what}:\n{}", + failures.len(), + failures.join("\n") + ); + // Non-vacuity: a feature set that disabled all four languages + // would otherwise leave every assertion above unrun. + assert!(checked > 0, "no C-family language was enabled"); +} + +/// A C-family function's name comes off the declarator walk its +/// arity comes off (#1208) — from the innermost declarator itself +/// for most shapes, and from inside the macro invocation that +/// declarator wraps for the three macro rows (#1213). "Same walk" +/// rather than "same node" is why this is not named for the +/// innermost declarator alone. +#[test] +fn the_declarator_walk_names_the_function_space() { + let mut failures = Vec::new(); + let mut checked = 0; + for lang in [LANG::C, LANG::Cpp, LANG::Mozcpp, LANG::Objc] + .into_iter() + .filter(LANG::is_enabled) + { + check(lang, SHARED_SHAPES, &mut failures); + checked += SHARED_SHAPES.len(); + if matches!(lang, LANG::Cpp | LANG::Mozcpp) { + check(lang, CPP_ONLY_SHAPES, &mut failures); + checked += CPP_ONLY_SHAPES.len(); + } + } + assert_all_matched( + &failures, + checked, + "declarator shapes named the wrong space", + ); +} + +/// [`check`] must be *able* to fail. +/// +/// It collects rather than asserts, so nothing in the table above +/// would notice if `function_spaces` selected no space at all — the +/// comparison would just find two empty expectations equal, and +/// every row would pass vacuously +/// (`.claude/rules/testing.md`, "Review the selector as carefully as +/// the assertion"). Feeding it a name that is deliberately wrong is +/// the cheapest proof that the selector reaches a real space and the +/// comparison discriminates. +#[cfg(feature = "c")] +#[test] +fn the_table_reports_a_name_that_does_not_match() { + let mut failures = Vec::new(); + check( + LANG::C, + &[( + "int plain(int a, int b) { return a; }", + Some("deliberately_wrong"), + )], + &mut failures, + ); + let [only] = failures.as_slice() else { + panic!("one wrong expectation must produce one failure, got {failures:?}"); + }; + // `Some("plain")` rather than a bare `plain`: the message echoes + // the fixture source, which contains the word too, so the bare + // substring passes even when the selector found *nothing* — + // measured, by filtering `function_spaces` on `SpaceKind::Class`. + // Matching the rendered `Option` is what ties the assertion to + // the space rather than to the input. + assert!( + only.contains("deliberately_wrong") && only.contains("Some(\"plain\")"), + "the failure must name both the expectation and the space found: {only}" + ); +} + +/// An unexpanded macro where a trailing attribute belongs, which is +/// the one input in this module that reaches +/// [`super::declarator_name`]'s `?` — the arm taken when *no* link +/// on the chain carries a `parameters` field, so there is no +/// declarator to read a name from at all. Every other row resolves +/// an owner and is answered by the identifier-kind gate instead. +/// +/// The grammars split two-two on it, which is the reason this is its +/// own test rather than a table row — and the split is not the one +/// the language families would suggest: +/// +/// | grammar | parse | name | +/// | --- | --- | --- | +/// | C, **mozcpp** | clean — `function_declarator` admits the trailing identifier | `f` | +/// | C++, Objective-C | `ERROR` around the declarator | none | +/// +/// Where the parse is clean the chain follows a real `declarator` +/// field and the name resolves. Where it is not, the declarator sits +/// inside an `ERROR` and the macro is left as the +/// `pointer_declarator`'s last named child, so the fallback follows +/// the macro into a dead end. +/// +/// mozcpp siding with C rather than with the upstream `tree-sitter-cpp` +/// it forked from is the finding worth keeping here: it owns no file +/// extension, so nothing routes to it and only a unit test can see +/// it at all (`.claude/rules/grammar-dispatch.md`, "when you fix one +/// language, sweep the rest"). +/// +/// Recovery trees are outside the walk's contract — see +/// [`super::innermost_declarator`], which measured this exact shape +/// as one of the two corpus spaces #1208 un-named. This pins what +/// the walk does there, not a claim that it is the right answer. +#[test] +fn a_macro_where_an_attribute_belongs_divides_the_grammars() { + const SOURCE: &str = "int *f() TF_ATTRIBUTE_NOINLINE { return 0; }"; + + let mut failures = Vec::new(); + let mut checked = 0; + for lang in [LANG::C, LANG::Cpp, LANG::Mozcpp, LANG::Objc] + .into_iter() + .filter(LANG::is_enabled) + { + let expected = matches!(lang, LANG::C | LANG::Mozcpp).then_some("f"); + check(lang, &[(SOURCE, expected)], &mut failures); + checked += 1; + } + assert_all_matched( + &failures, + checked, + "grammars disagreed about the recovery shape", + ); +} + +/// The same macro carrying an argument, which is the spelling the +/// annotation idiom actually takes — `TF_LOCKS_EXCLUDED(mu_)`, +/// `TF_GUARDED_BY(mu_)`, `ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_)`. +/// +/// Where the grammar recovers, it is worse than the parameterless +/// spelling above rather than the same. A bare trailing identifier +/// carries no `parameters`, so the chain dead-ends and the space +/// merely goes nameless; a parenthesised one is a +/// `function_declarator` that *does*, so it becomes the walk's +/// answer and the space is named after the macro. `nargs` reads the +/// macro's argument off the same node, and two members of one class +/// sharing an annotation collapse onto a single `bca check` offender +/// key (`K::TF_LOCKS_EXCLUDED` twice). +/// +/// This is the one shape #1208 made worse: the leftmost pre-order +/// search it replaced descended into the `ERROR` and got `f` right. +/// One corpus space is affected — `resource()` in TensorFlow's +/// `resource_op_kernel_test.cc`, which #1208 renamed to +/// `TF_LOCKS_EXCLUDED`. Pinned rather than fixed, like its sibling +/// above: reaching into a recovery subtree is a strategy decision of +/// its own, and every rule the walk follows is void inside an +/// `ERROR`. Teach the walk to unwrap one and this is a row to +/// update, not a row to delete. +#[test] +fn a_parenthesised_macro_takes_the_name_of_the_function_it_annotates() { + const SOURCE: &str = "int *f() TF_LOCKS_EXCLUDED(mu_) { return 0; }"; + + let mut failures = Vec::new(); + let mut checked = 0; + for lang in [LANG::C, LANG::Cpp, LANG::Mozcpp, LANG::Objc] + .into_iter() + .filter(LANG::is_enabled) + { + // Only C parses this cleanly, and there the chain follows a + // real `declarator` field past the macro to `f`. The split + // is *not* the two-two of the parameterless spelling: + // mozcpp sides with C there and with C++ here, so a fixture + // in either spelling alone would misreport what the other + // does. + let expected = if lang == LANG::C { + "f" + } else { + "TF_LOCKS_EXCLUDED" + }; + check(lang, &[(SOURCE, Some(expected))], &mut failures); + checked += 1; + } + assert_all_matched( + &failures, + checked, + "grammars disagreed about the annotated recovery shape", + ); +} diff --git a/src/halstead_type.rs b/src/halstead_type.rs deleted file mode 100644 index be3b50613..000000000 --- a/src/halstead_type.rs +++ /dev/null @@ -1,17 +0,0 @@ -//! The operator / operand / unknown classification a `Getter` assigns -//! to each node for the Halstead metric. -//! -//! Defined beside the classifiers rather than in `metrics::halstead` -//! because `Getter::get_op_type` returns it: the parse layer names the -//! type, the metric consumes it (#1376). `metrics::halstead` re-exports -//! it under its historical public path. - -/// Specifies the type of nodes accepted by the `Halstead` metric. -pub enum HalsteadType { - /// The node is an `Halstead` operator - Operator, - /// The node is an `Halstead` operand - Operand, - /// The node is unknown to the `Halstead` metric - Unknown, -} diff --git a/src/lib.rs b/src/lib.rs index 2e8c5af9d..12e1fcffa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -124,16 +124,16 @@ // Cargo lint (#1227). #![cfg_attr(not(test), warn(clippy::unwrap_used))] -// Internal-only modules. Nothing is re-exported from these. -mod c_declarator; -mod c_langs_macros; -mod c_macro; -mod cfg_predicate; -mod checker; -mod getter; -mod halstead_type; -mod lang_helpers; -mod space_kind; +// The parse and classification layer lives in `big-code-analysis-ast` +// (#1376). Its public names — the generated token enums, the `*Code` / +// `*Parser` tags, `Node`, `Ancestors`, `Checker`, `Getter`, the language +// helpers — are the working vocabulary every metric module reaches +// through `use crate::*`, so the whole crate root is glob-imported here +// at `pub(crate)`. Only the explicit `pub use` lines further down widen +// the published surface. +#[doc(hidden)] +pub(crate) use big_code_analysis_ast::*; + // Fast hashing for the walk's integer-keyed maps. Shared by `spaces` // (node ids) and `metrics::halstead` (grammar `kind_id`s); `metrics::loc` // was the third until #1109 moved its line sets to a bitset. The module @@ -141,59 +141,24 @@ mod space_kind; // collections are excluded — extend it, not this line, when a third // arrives. mod int_hash; -#[cfg(test)] -mod language_enum_roundtrip; -mod languages; +// The metric-side macros (`implement_metric_trait!`) plus re-exports of +// the kind-set and dispatch macros defined in `big-code-analysis-ast`. mod macros; -// One declaration form for the thread-local counters that make an -// output-invisible optimization testable. The module doc states the -// shared invariant (the counter is unconditional, only its accessor is -// test-gated) once; each invocation carries its own narrative. -mod observation; // Parse-and-inspect shims shared by the per-metric test modules. Kept out // of any production file so the self-scan gate does not spend a shipping // module's metric budget on test-only code (#1066). #[cfg(test)] mod test_support; +// Drift guard for the crate-level `## Supported Languages` list above. +#[cfg(test)] +mod c_family_space_names_tests; +#[cfg(test)] +mod lib_docs_tests; +#[cfg(test)] +mod observation_tests; -// `langs` hosts the `mk_langs!` macro expansion. `LANG` is the only -// public name; the per-language `Code` tags and `Parser` -// aliases are `pub(crate)` parser machinery reached only through the -// [`Ast`] seam. -mod langs; -pub use crate::langs::{LANG, get_from_emacs_mode, get_from_ext}; -// `Code` tags are reached crate-internally through `use crate::*` -// in the per-language `Checker` / `Getter` / `Alterator` / metric impls. -pub(crate) use crate::langs::{ - BashCode, CCode, CcommentCode, CppCode, CsharpCode, ElixirCode, GoCode, GroovyCode, IrulesCode, - JavaCode, JavascriptCode, KotlinCode, LuaCode, MozcppCode, MozjsCode, ObjcCode, PerlCode, - PhpCode, PreprocCode, PythonCode, RubyCode, RustCode, TclCode, TsxCode, TypescriptCode, -}; -// The `Parser` aliases are the concrete `Parser<Code>` types -// driven by the `AnyParser` dispatch in `crate::langs`; at the crate root -// they are reached only from `#[cfg(test)]` modules, so the re-export is -// `unused` in a non-test build. -#[allow(unused_imports)] -pub(crate) use crate::langs::{ - BashParser, CParser, CcommentParser, CppParser, CsharpParser, ElixirParser, GoParser, - GroovyParser, IrulesParser, JavaParser, JavascriptParser, KotlinParser, LuaParser, - MozcppParser, MozjsParser, ObjcParser, PerlParser, PhpParser, PreprocParser, PythonParser, - RubyParser, RustParser, TclParser, TsxParser, TypescriptParser, -}; -// `ParseLangError` is the `FromStr` error for `LANG`; it is defined in -// the `mk_lang!` macro layer (`crate::macros`) rather than `crate::langs`. -pub use crate::macros::ParseLangError; - -// Internal crate-root re-exports. Hand-written per-language modules -// (`src/getter.rs`, `src/checker.rs`, `src/alterator.rs`, the -// per-language metric impls) use `use crate::*` to bring the -// macro-generated `Code` token enums and per-language helper -// types into scope; the per-language token enums in -// `src/languages/language_*.rs` are also reached through the crate -// root. Re-exporting these as `pub(crate)` keeps internal compilation -// working without widening the published surface. -pub(crate) use crate::checker::*; -pub(crate) use crate::languages::*; +// --- Language identification (defined in `big-code-analysis-ast`) --- +pub use big_code_analysis_ast::{LANG, ParseLangError, get_from_emacs_mode, get_from_ext}; // Hand-written modules (`src/spaces.rs`, `src/output/dump_metrics.rs`, // the metric macros) refer to per-metric submodules by their short @@ -253,8 +218,12 @@ pub mod vcs; mod diag; // --- Errors --- -mod error; -pub use crate::error::MetricsError; +// +// `MetricsError` is the parse layer's error (every dispatch entry point +// returns it, including `LANG::tree_sitter_language`), so it is defined +// in `big-code-analysis-ast`; `FromPathError` wraps it for the +// file-backed `Ast::from_path` and stays here. +pub use big_code_analysis_ast::MetricsError; mod from_path_error; pub use crate::from_path_error::FromPathError; @@ -295,13 +264,10 @@ pub use crate::output::{ }; // --- AST plumbing (Node) --- -mod node; -pub(crate) use crate::node::Ancestors; -pub use crate::node::Node; +pub use big_code_analysis_ast::Node; // --- Language detection / I/O helpers --- -mod tools; -pub use crate::tools::{ +pub use big_code_analysis_ast::{ SkipReason, get_language_for_file, guess_language, is_generated, normalize_eol, read_file, read_file_with_eol, read_file_with_eol_classified, write_file, }; @@ -312,61 +278,36 @@ pub use crate::concurrent_files::{ ConcurrentErrors, ConcurrentRunner, FilesData, NumJobs, ParseNumJobsError, }; -// --- Comment removal --- -// -// `rm_comments` is the internal walk core reached only through the -// [`Ast::strip_comments`] seam (`with_any_parser!` in `spaces/ast.rs`). -mod comment_rm; - -// --- Per-file node counting / finding (reached via the `Ast` seam) --- -mod count; -pub use crate::count::{Count, CountCollector}; - -mod find; +// --- Per-file node counting (reached via the `Ast` seam) --- +pub use big_code_analysis_ast::{Count, CountCollector}; mod function; pub use crate::function::{FunctionSpan, dump_function_spans, dump_function_spans_with_color}; // --- AST dump --- -mod ast; -pub use crate::ast::{AstCfg, AstNode, AstPayload, AstResponse, MAX_AST_SERIALIZE_DEPTH, Span}; - -// --- Stack-depth bounds shared by the crate's recursive types --- -mod recursion; +pub use big_code_analysis_ast::{ + AstCfg, AstNode, AstPayload, AstResponse, MAX_AST_SERIALIZE_DEPTH, Span, +}; // --- Halstead operator/operand result type --- mod ops; pub use crate::ops::Ops; // --- Preprocessor handling (C/C++) --- -mod preproc; -pub use crate::preproc::{ +pub use big_code_analysis_ast::{ PreprocDiagnostic, PreprocFile, PreprocResults, fix_includes, get_macros, preprocess, }; -// --- Alterator trait (per-language AST simplification) --- +// --- Generic parser plumbing --- // -// Crate-internal: an extension trait over the `pub(crate)` `Checker` -// machinery, used only by the per-language `Parser` impls behind the -// [`Ast`] seam. -mod alterator; -pub(crate) use crate::alterator::Alterator; - -// --- Generic parser plumbing (crate-internal) --- +// `Parser`, `ParserTrait`, `Filter`, `LanguageInfo`, `Checker`, +// `Getter` and `Alterator` are defined in `big-code-analysis-ast` and +// reach this crate through the glob import above. They are not +// re-exported: the single public analysis seam is [`Ast`], which wraps +// the language-dispatched `AnyParser` carrier. See STABILITY.md. // -// `Parser`, `ParserTrait`, `Filter`, and `LanguageInfo` are the -// internal parser machinery driving every metric walk. They are -// `pub(crate)` only: the single public analysis seam is [`Ast`], -// which wraps the language-dispatched `AnyParser` carrier. See -// STABILITY.md. -mod parser; -pub(crate) use crate::parser::Parser; - -mod traits; -pub(crate) use crate::traits::{LanguageInfo, ParserTrait, Search}; - -// The metric half of the parser contract; `ParserTrait` above is the -// parse half. See `src/metric_suite.rs`. +// The metric half of the parser contract; `ParserTrait` is the parse +// half. See `src/metric_suite.rs`. mod metric_suite; pub(crate) use crate::metric_suite::MetricSuite; diff --git a/src/lib_docs_tests.rs b/src/lib_docs_tests.rs new file mode 100644 index 000000000..970bd0fdf --- /dev/null +++ b/src/lib_docs_tests.rs @@ -0,0 +1,49 @@ +//! Drift guards on the crate-level rustdoc in `src/lib.rs`. + +use crate::LANG; + +// Drift guard for the crate-level `## Supported Languages` rustdoc +// list in `src/lib.rs` (#769): every LANG variant's canonical slug +// — the single source of truth from `name()` — must appear in that +// list as a backtick-delimited token. Without this guard, adding a +// language (or renaming a slug) silently desyncs the docs.rs +// landing page, which is exactly how Objective-C went missing for a +// full release after #724 shipped it. +// +// The slug set is derived from `LANG::into_enum_iter()`, which is +// compiled unconditionally (the enum surface is feature-independent; +// only the grammar crates are gated), so this test is robust under +// `--no-default-features` and any per-language feature subset — no +// `all-languages` gate needed. Distinct slugs are deduplicated, so +// shared-slug families do not require one bullet per variant. +#[test] +fn supported_languages_rustdoc_lists_every_slug() { + // Bound the search to the `## Supported Languages` section so an + // incidental backtick match elsewhere in the module docs (e.g. + // a slug named in the metrics section) cannot mask a real + // omission from the list itself. + const SECTION_HEADER: &str = "## Supported Languages"; + const NEXT_HEADER: &str = "## Supported Metrics"; + + let lib_rs = std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/src/lib.rs")) + .expect("src/lib.rs is readable from CARGO_MANIFEST_DIR"); + + let section_start = lib_rs + .find(SECTION_HEADER) + .expect("rustdoc must contain a `## Supported Languages` section"); + let section_end = lib_rs[section_start..] + .find(NEXT_HEADER) + .map(|offset| section_start + offset) + .expect("`## Supported Languages` must be followed by `## Supported Metrics`"); + let section = &lib_rs[section_start..section_end]; + + for lang in LANG::into_enum_iter() { + let slug_token = format!("`{}`", lang.name()); + assert!( + section.contains(&slug_token), + "LANG::{lang:?} slug {slug_token} is missing from the \ + `## Supported Languages` rustdoc list in src/lib.rs — \ + add an entry there (see #769)", + ); + } +} diff --git a/src/macros/mod.rs b/src/macros/mod.rs index 8cfdb3eb4..f93621937 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -1,23 +1,5 @@ -// `get_language!` is invoked only from feature-gated arms in `mk_lang!` -// (one arm per `LANG::*` variant whose per-language Cargo feature is -// enabled). A build with `--no-default-features` and no language -// feature has no remaining call sites; suppress the lint for that -// pathological-but-valid configuration. -#[allow(unused_macros)] -macro_rules! get_language { - (tree_sitter_typescript) => { - tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into() - }; - (tree_sitter_tsx) => { - tree_sitter_typescript::LANGUAGE_TSX.into() - }; - (tree_sitter_php) => { - tree_sitter_php::LANGUAGE_PHP.into() - }; - ($name:ident) => { - $name::LANGUAGE.into() - }; -} +//! The metric-side macros, plus re-exports of the kind-set macros the +//! metric modules share with the classifiers in `big-code-analysis-ast`. // `implement_metric_trait!` emits no-op `compute` bodies for every // metric / language pair listed. Every named-trait arm below @@ -156,505 +138,17 @@ macro_rules! implement_metric_trait { ) } -macro_rules! mk_lang { - ( $( ($feature:literal, $camel:ident, $name:ident, $display: expr, $description:expr, $version:literal) ),* ) => { - /// The list of supported languages. - /// - /// Every variant is always defined regardless of the Cargo - /// feature set: per-language features only gate the grammar - /// crate references, never the enum surface itself. Disabled - /// variants surface at runtime as - /// [`crate::MetricsError::LanguageDisabled`] from every entry - /// point that returns a `Result`. - #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] - pub enum LANG { - $( - #[doc = $description] - $camel, - )* - } - impl LANG { - /// Return an iterator over the supported languages. - /// - /// # Examples - /// - /// ``` - /// use big_code_analysis::LANG; - /// - /// for lang in LANG::into_enum_iter() { - /// println!("{:?}", lang); - /// } - /// ``` - pub fn into_enum_iter() -> impl Iterator { - use LANG::*; - [$( $camel, )*].into_iter() - } - - /// Returns the name of a language as a `&str`. - /// - /// # Examples - /// - /// ``` - /// use big_code_analysis::LANG; - /// - /// println!("{}", LANG::Rust.name()); - /// ``` - pub fn name(&self) -> &'static str { - match self { - $( - LANG::$camel => $display, - )* - } - } - - /// Returns the pinned tree-sitter grammar crate version that - /// backs this variant (e.g. `"0.25.1"` for [`LANG::Bash`]). - /// - /// The value mirrors the `=X.Y.Z` pin in the workspace - /// `Cargo.toml` and is independent of the per-language Cargo - /// feature: it is returned even for a variant whose feature is - /// disabled in the current build (a build-time constant, no - /// grammar crate reference). A drift test in `src/langs.rs` - /// asserts every value here matches the manifest pin. - /// - /// # Grammars vs. forks - /// - /// For languages backed by an upstream crates.io grammar - /// (`bash`, `rust`, `python`, `typescript`, …) this is the - /// exact upstream grammar version, so a consumer migrating - /// matchers off py-tree-sitter can line node-kind vocabularies - /// up against the same pin. For the vendored big-code-analysis - /// forks (`mozcpp`, `mozjs`, `tcl`, `ccomment`, `preproc`, - /// `kotlin`) the value is the **fork crate's** version - /// (published as `bca-tree-sitter-*` / `tree-sitter-kotlin-ng`), - /// not an upstream tree-sitter grammar semver — there is no - /// upstream release to compare against. - /// - /// This is part of the value-not-stable surface: the returned - /// version changes whenever the grammar pin is bumped. - #[must_use] - pub fn grammar_version(&self) -> &'static str { - match self { - $( - LANG::$camel => $version, - )* - } - } - - /// Reports whether this variant's grammar crate is - /// compiled into the current build. - /// - /// Returns `false` for variants whose per-language Cargo - /// feature is disabled; calling - /// [`Self::tree_sitter_language`], [`crate::analyze`], - /// or any other dispatcher with such a variant will - /// return [`crate::MetricsError::LanguageDisabled`]. - #[must_use] - pub fn is_enabled(&self) -> bool { - match self { - $( - #[cfg(feature = $feature)] - LANG::$camel => true, - #[cfg(not(feature = $feature))] - LANG::$camel => false, - )* - } - } - - // Returns a tree-sitter language paired with this variant, - // or `Err(LanguageDisabled)` when the matching Cargo - // feature is off. This is the internal entry point used - // by `Tree::new` to construct a parser; the public - // counterpart is `tree_sitter_language`. - pub(crate) fn get_ts_language(&self) -> Result { - match self { - $( - #[cfg(feature = $feature)] - LANG::$camel => Ok(get_language!($name)), - #[cfg(not(feature = $feature))] - LANG::$camel => Err(crate::MetricsError::LanguageDisabled(*self)), - )* - } - } - - /// Returns the [`tree_sitter::Language`] grammar used by - /// this variant. - /// - /// Useful when feeding a caller-built - /// [`tree_sitter::Parser`] into the - /// [`crate::Ast::from_tree_sitter`] entry point — the - /// language returned here is the one the metric walker - /// expects for `kind_id` matching, so the trees agree - /// structurally. - /// - /// This method is part of the value-not-stable surface: - /// the underlying `tree-sitter-*` grammar pin may bump - /// in any minor release, which can change `Language` - /// equality on the caller side. - /// - /// # Errors - /// - /// Returns [`crate::MetricsError::LanguageDisabled`] when - /// the variant's per-language Cargo feature is not - /// enabled in the current build (see the `[features]` - /// table in the root `Cargo.toml`). - /// - /// # Examples - /// - /// ``` - /// use big_code_analysis::LANG; - /// - /// let _lang = LANG::Rust.tree_sitter_language().expect("rust feature enabled"); - /// ``` - pub fn tree_sitter_language(&self) -> Result<::tree_sitter::Language, crate::MetricsError> { - self.get_ts_language() - } - } - - /// Renders the language's canonical lowercase slug, identical to - /// [`LANG::name`]. - /// - /// Every variant has a distinct slug, so `Display` is injective - /// and a `Display` → [`FromStr`](std::str::FromStr) round-trip - /// returns the original variant (see the round-trip test in - /// `src/langs.rs`). The slug is the single canonical identifier - /// used across every surface (CLI JSON, web `/metrics`, the - /// Python bindings): it contains no punctuation and is always a - /// valid `FromStr` lookup token. - impl ::std::fmt::Display for LANG { - fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { - f.write_str(self.name()) - } - } - - /// Parses a [`LANG`] from its [`Display`](std::fmt::Display) - /// spelling (the canonical lowercase [`LANG::name`] slug, e.g. - /// `"rust"`, `"cpp"`, `"csharp"`, `"tsx"`). - /// - /// Matching is case-sensitive and exact, mirroring - /// [`Metric`](crate::Metric)'s `FromStr`: only the canonical - /// lowercase slug is accepted. File extensions and emacs modes - /// are deliberately *not* accepted here — use - /// [`get_from_ext`](crate::get_from_ext) / - /// [`get_from_emacs_mode`](crate::get_from_emacs_mode) for those. - /// - /// Every variant has a distinct slug, so this is the exact - /// inverse of [`Display`](std::fmt::Display): the round-trip - /// `LANG::from_str(&lang.to_string())` returns the original - /// variant for every `LANG`. - impl ::std::str::FromStr for LANG { - type Err = $crate::macros::ParseLangError; - - fn from_str(s: &str) -> Result { - LANG::into_enum_iter() - .find(|lang| lang.name() == s) - .ok_or_else(|| $crate::macros::ParseLangError::new(s)) - } - } - }; -} - -/// Error returned by [`LANG`](crate::LANG)'s -/// [`FromStr`](std::str::FromStr) impl when the input is not a -/// recognised language name. -/// -/// Holds the offending input verbatim so wrapper layers can format -/// their own user-facing message; mirrors -/// [`ParseMetricError`](crate::ParseMetricError). -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ParseLangError(String); - -impl ParseLangError { - // Constructor kept `pub(crate)` so the macro-generated `FromStr` - // impl in `src/langs.rs` can build the error without exposing the - // private field across module boundaries. - pub(crate) fn new(input: &str) -> Self { - Self(input.to_owned()) - } - - /// The rejected input that failed to parse as a language name. - /// - /// Lets callers recover the offending string programmatically - /// rather than scraping it out of the [`Display`](std::fmt::Display) - /// output. Mirrors - /// [`ParseMetricError::input`](crate::ParseMetricError::input). - #[must_use] - pub fn input(&self) -> &str { - &self.0 - } -} - -impl ::std::fmt::Display for ParseLangError { - fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { - write!(f, "unknown language: {}", self.0) - } -} - -impl ::std::error::Error for ParseLangError {} - -macro_rules! mk_action { - ( $( ($feature:literal, $camel:ident, $parser:ident) ),* ) => { - /// A parsed tree plus its source bytes for a language chosen at - /// runtime — one variant per [`LANG`], each holding that - /// language's `Parser`. The public seam is - /// [`crate::Ast`]; this enum is the language-dispatched carrier - /// it wraps, and the value a caller matches on to reach a - /// concrete parser (`with_any_parser!`). - /// - /// Every variant exists regardless of the Cargo feature set: a - /// `Parser` *type* needs no grammar crate, only - /// [`Self::parse`] / [`Self::from_tree`] do, and those are the - /// arms that are feature-gated. A disabled language is therefore - /// never *constructed* — the constructors return - /// `Err(LanguageDisabled)` for it — but it can always be *named*, - /// which is what lets `with_any_parser!` be written once without - /// any `cfg` of its own (#1376). - pub(crate) enum AnyParser { - $( - #[doc = concat!("The `", stringify!($camel), "` parser.")] - $camel($parser), - )* - } - - impl AnyParser { - /// Parse `source` as `lang`. - /// - /// `Parser::new` keys the C-family macro-expansion lookup off - /// the caller-supplied path; callers analysing in-memory - /// snippets pass `None` and get the empty `Path` (`""`), - /// which the lookup ignores. That path never leaks into a - /// display name — `Ast` carries the name separately. - /// `source` is taken by value so an owned buffer moves - /// straight into the parser instead of being copied. - /// - /// # Errors - /// - /// `MetricsError::LanguageDisabled` when `lang`'s Cargo - /// feature is not enabled in this build. - pub(crate) fn parse( - lang: LANG, - source: Vec, - preproc_path: Option<&Path>, - preproc: Option>, - ) -> Result { - let preproc_path = preproc_path.unwrap_or(Path::new("")); - match lang { - $( - #[cfg(feature = $feature)] - LANG::$camel => Ok(AnyParser::$camel($parser::new(source, preproc_path, preproc))), - #[cfg(not(feature = $feature))] - LANG::$camel => { - let _ = (source, preproc_path, preproc); - Err(MetricsError::LanguageDisabled(lang)) - }, - )* - } - } - - /// Adopt a caller-built [`tree_sitter::Tree`] produced from - /// `source` with `lang`'s grammar. - /// - /// # Errors - /// - /// `MetricsError::LanguageDisabled` when `lang`'s Cargo - /// feature is not enabled in this build. - pub(crate) fn from_tree( - lang: LANG, - tree: ::tree_sitter::Tree, - source: Vec, - ) -> Result { - match lang { - $( - #[cfg(feature = $feature)] - LANG::$camel => Ok(AnyParser::$camel($parser::from_tree(tree, source))), - #[cfg(not(feature = $feature))] - LANG::$camel => { - let _ = (tree, source); - Err(MetricsError::LanguageDisabled(lang)) - }, - )* - } - } - - /// The language this parser was built for. - #[must_use] - pub(crate) fn language(&self) -> LANG { - match self { - $( AnyParser::$camel(_) => LANG::$camel, )* - } - } - } - }; -} - -/// Dispatches over every [`AnyParser`] variant, binding the concrete -/// `Parser` to `$p` and evaluating `$body` once per arm. -/// -/// Written out by hand rather than generated inside `mk_action!` so the -/// arm list stays a plain match: a variant missing here is a -/// non-exhaustive-match compile error, which is the whole guarantee. -/// Every arm is unconditional — see the [`AnyParser`] docs for why no -/// `cfg` is needed. Add a line here when `mk_langs!` gains a language. -/// -/// [`AnyParser`]: crate::langs::AnyParser -macro_rules! with_any_parser { - ($any:expr, |$p:ident| $body:expr) => { - match $any { - $crate::langs::AnyParser::Javascript($p) => $body, - $crate::langs::AnyParser::Mozjs($p) => $body, - $crate::langs::AnyParser::Java($p) => $body, - $crate::langs::AnyParser::Go($p) => $body, - $crate::langs::AnyParser::Kotlin($p) => $body, - $crate::langs::AnyParser::Lua($p) => $body, - $crate::langs::AnyParser::Rust($p) => $body, - $crate::langs::AnyParser::Tcl($p) => $body, - $crate::langs::AnyParser::Irules($p) => $body, - $crate::langs::AnyParser::C($p) => $body, - $crate::langs::AnyParser::Cpp($p) => $body, - $crate::langs::AnyParser::Mozcpp($p) => $body, - $crate::langs::AnyParser::Objc($p) => $body, - $crate::langs::AnyParser::Csharp($p) => $body, - $crate::langs::AnyParser::Elixir($p) => $body, - $crate::langs::AnyParser::Python($p) => $body, - $crate::langs::AnyParser::Tsx($p) => $body, - $crate::langs::AnyParser::Typescript($p) => $body, - $crate::langs::AnyParser::Bash($p) => $body, - $crate::langs::AnyParser::Ccomment($p) => $body, - $crate::langs::AnyParser::Preproc($p) => $body, - $crate::langs::AnyParser::Perl($p) => $body, - $crate::langs::AnyParser::Php($p) => $body, - $crate::langs::AnyParser::Ruby($p) => $body, - $crate::langs::AnyParser::Groovy($p) => $body, - } - }; -} - -macro_rules! mk_extensions { - ( $( ($camel:ident, [ $( $ext:ident ),* ]) ),* ) => { - /// Detects the language associated to the input file extension. - /// - /// # Examples - /// - /// ``` - /// use big_code_analysis::get_from_ext; - /// - /// let ext = "rs"; - /// - /// get_from_ext(ext).unwrap(); - /// ``` - pub fn get_from_ext(ext: &str) -> Option{ - match ext { - $( - $( - stringify!($ext) => Some(LANG::$camel), - )* - )* - _ => None, - } - } - - impl LANG { - /// Returns the file extensions recognised for this language. - /// - /// The returned list is the same one consulted by - /// [`get_from_ext`] and [`crate::get_language_for_file`]. - /// Helper variants without user-facing files (`Ccomment`, - /// `Preproc`) return an empty slice. - /// - /// # Examples - /// - /// ``` - /// use big_code_analysis::LANG; - /// - /// assert!(LANG::Rust.extensions().contains(&"rs")); - /// ``` - #[must_use] - pub fn extensions(&self) -> &'static [&'static str] { - match self { - $( - LANG::$camel => &[ $( stringify!($ext), )* ], - )* - } - } - } - }; -} - -macro_rules! mk_emacs_mode { - ( $( ($camel:ident, [ $( $emacs_mode:expr ),* ]) ),* ) => { - /// Detects the language associated to the input `Emacs` mode. - /// - /// An `Emacs` mode is used to detect a language according to - /// particular text-information contained in a file. - /// - /// # Examples - /// - /// ``` - /// use big_code_analysis::get_from_emacs_mode; - /// - /// let emacs_mode = "rust"; - /// - /// get_from_emacs_mode(emacs_mode).unwrap(); - /// ``` - pub fn get_from_emacs_mode(mode: &str) -> Option{ - match mode { - $( - $( - $emacs_mode => Some(LANG::$camel), - )* - )* - _ => None, - } - } - }; -} - -macro_rules! mk_code { - ( $( ($camel:ident, $code:ident, $parser:ident, $name:ident, $docname:expr) ),* ) => { - $( - #[doc = concat!("Per-language code type tag for ", $docname, "; carries no data.")] - pub(crate) struct $code { _guard: (), } - - impl LanguageInfo for $code { - type BaseLang = $camel; - - fn lang() -> LANG { - LANG::$camel - } - } - - #[doc = "The `"] - #[doc = $docname] - #[doc = "` language parser."] - pub(crate) type $parser = Parser<$code>; - )* - }; -} - -macro_rules! mk_langs { - ( $( ($feature:literal, $camel:ident, $description: expr, $display: expr, $code:ident, $parser:ident, $name:ident, [ $( $ext:ident ),* ], [ $( $emacs_mode:expr ),* ], $version:literal) ),* ) => { - mk_lang!($( ($feature, $camel, $name, $display, $description, $version) ),*); - mk_action!($( ($feature, $camel, $parser) ),*); - mk_extensions!($( ($camel, [ $( $ext ),* ]) ),*); - mk_emacs_mode!($( ($camel, [ $( $emacs_mode ),* ]) ),*); - mk_code!($( ($camel, $code, $parser, $name, stringify!($camel)) ),*); - }; -} - -mod kind_sets; - pub(crate) use implement_metric_trait; -pub(crate) use kind_sets::{ - cpp_bool_terminal_kinds, csharp_bool_terminal_kinds, csharp_invocation_expr_kinds, - csharp_paren_expr_kinds, csharp_prefix_unary_expr_kinds, csharp_var_decl_kinds, - csharp_var_declarator_kinds, elixir_bool_terminal_kinds, go_bool_terminal_kinds, - groovy_bool_terminal_kinds, irules_bool_terminal_kinds, java_bool_terminal_kinds, - javascript_bool_terminal_kinds, kotlin_bool_terminal_kinds, lua_bool_terminal_kinds, - mozjs_bool_terminal_kinds, perl_bool_terminal_kinds, php_bool_terminal_kinds, - python_bool_terminal_kinds, ruby_bool_terminal_kinds, rust_bool_terminal_kinds, - tcl_bool_terminal_kinds, tsx_bool_terminal_kinds, typescript_bool_terminal_kinds, -}; -pub(crate) use { - get_language, mk_action, mk_code, mk_emacs_mode, mk_extensions, mk_lang, mk_langs, - with_any_parser, +// Kind-set aliases and the parser dispatch macro are defined in +// `big-code-analysis-ast`; the metric modules keep reaching them as +// `crate::macros::`. +pub(crate) use big_code_analysis_ast::{ + cpp_bool_terminal_kinds, csharp_bool_terminal_kinds, csharp_paren_expr_kinds, + csharp_prefix_unary_expr_kinds, csharp_var_decl_kinds, csharp_var_declarator_kinds, + elixir_bool_terminal_kinds, go_bool_terminal_kinds, groovy_bool_terminal_kinds, + irules_bool_terminal_kinds, java_bool_terminal_kinds, javascript_bool_terminal_kinds, + kotlin_bool_terminal_kinds, lua_bool_terminal_kinds, mozjs_bool_terminal_kinds, + perl_bool_terminal_kinds, php_bool_terminal_kinds, python_bool_terminal_kinds, + ruby_bool_terminal_kinds, rust_bool_terminal_kinds, tcl_bool_terminal_kinds, + tsx_bool_terminal_kinds, typescript_bool_terminal_kinds, with_any_parser, }; diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 5f50ed4d9..0c5b9f37c 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -28,9 +28,23 @@ use std::collections::HashMap; use std::fmt; -// Re-exported here so the public `metrics::halstead::HalsteadType` path -// survives the enum's move beside `Getter`, whose `get_op_type` returns it. -pub use crate::halstead_type::HalsteadType; +// The operator / operand classification is syntactic and lives beside +// the per-language `Getter` tables that produce it; re-exported here +// because this metric is what consumes it. +pub use crate::token_role::TokenRole; + +/// The former name of [`TokenRole`], kept so the +/// `metrics::halstead::HalsteadType` path still resolves. +/// +/// The classification never was Halstead-specific — it answers whether +/// a node acts as an operator or an operand, which the grammar decides +/// and other consumers can use (#1376). Renaming it outright would be a +/// SemVer break, so the old spelling stays until the next major. +#[deprecated( + since = "2.3.0", + note = "renamed to `TokenRole`: the classification is syntactic, not Halstead-specific" +)] +pub type HalsteadType = TokenRole; use crate::checker::Checker; use crate::getter::Getter; @@ -370,7 +384,7 @@ fn compute_halstead<'a, T: Getter + Checker>( halstead_maps: &mut HalsteadMaps<'a>, ) { match T::get_op_type_with_code(node, code, ancestors) { - HalsteadType::Operator => { + TokenRole::Operator => { if T::is_primitive(node) { // Store primitive-type operators by text so distinct // primitives (e.g. `int` vs `double`) that share a @@ -383,7 +397,7 @@ fn compute_halstead<'a, T: Getter + Checker>( *halstead_maps.operators.entry(node.kind_id()).or_insert(0) += 1; } } - HalsteadType::Operand => { + TokenRole::Operand => { *halstead_maps .operands .entry(T::get_operand_id(node, code, ancestors)) @@ -396,9 +410,10 @@ fn compute_halstead<'a, T: Getter + Checker>( // Every language's `Halstead::compute` is the same forward to // `compute_halstead`, which classifies each node through the language's // own `Getter` / `Checker`. Nothing per-language lives here — it lives -// in `src/getter/.rs` — so writing the impls out was 23 copies of -// one signature. (This is the only metric whose per-language impls are -// all identical; every other trait has real per-language bodies.) +// in `big-code-analysis-ast/src/getter/.rs` — so writing the +// impls out was 23 copies of one signature. (This is the only metric +// whose per-language impls are all identical; every other trait has +// real per-language bodies.) macro_rules! impl_halstead_forwarding { ($($code:ty),+ $(,)?) => { $( @@ -466,6 +481,27 @@ mod tests { check_metrics_only_shim!(check_metrics, Halstead); + // `HalsteadType` is the pre-#1376 name for `TokenRole`, kept as a + // deprecated alias because removing it before `3.0` would be a + // SemVer break (STABILITY.md). Nothing in this repository uses the + // old spelling any more, so only this guard would notice it being + // dropped — and dropping it silently is exactly what the promise + // rules out. + // + // The `matches!` arms are the assertion: a value typed through the + // alias can only be matched against `TokenRole`'s variants if the + // alias still names that enum, so this fails to *compile* if the + // alias is removed or re-pointed, rather than failing at runtime. + // `TokenRole` derives nothing, so there is no `assert_eq!` to reach + // for here. + #[test] + #[allow(deprecated)] + fn halstead_type_alias_still_names_token_role() { + let via_alias: HalsteadType = HalsteadType::Operand; + assert!(matches!(via_alias, TokenRole::Operand)); + assert!(!matches!(via_alias, TokenRole::Operator)); + } + // Pins the lesson-4 invariant `n2 == len(dedupe(ops.operands))` by // running `operands_and_operators` (the text-keyed `--ops` store) // on the same source and comparing its deduplicated operand count @@ -946,10 +982,11 @@ mod tests { // Regression for issue #95 (lesson #2): the Rust grammar emits 17 // distinct `kind_id`s for `primitive_type` (one base plus 16 // numeric-suffixed alias variants). `RustCode::is_primitive` in - // `src/checker.rs` must list every variant; if a future regression - // omits one, primitive type names emitted in that aliased position - // silently drop into the kind_id-keyed operators bucket instead of - // the text-keyed primitive_operators map, miscounting Halstead n1. + // `big-code-analysis-ast/src/checker.rs` must list every variant; + // if a future regression omits one, primitive type names emitted + // in that aliased position silently drop into the kind_id-keyed + // operators bucket instead of the text-keyed primitive_operators + // map, miscounting Halstead n1. // // The snippet exercises every primitive scalar type across many // syntactic positions (function parameter types, return types, @@ -1013,7 +1050,7 @@ mod tests { fn rust_field_identifier_is_operand() { // Regression for issue #390: prior to the fix, `FieldIdentifier` // (e.g. the `x` / `y` in `p.x`, `p.y`) fell through to - // `HalsteadType::Unknown`, so the field names were not counted + // `TokenRole::Unknown`, so the field names were not counted // as operands. Both C++ and Go already classify FieldIdentifier // as an operand. After the fix: // unique operators: fn, (), {}, let, =, +, ;, . @@ -1062,7 +1099,7 @@ mod tests { fn rust_type_identifier_is_operand() { // Regression for issue #390: `TypeIdentifier` (e.g. `Vec`, // `HashMap`, `String` when used as a path name) was dropped to - // `HalsteadType::Unknown` for Rust. C++ and Go classify them as + // `TokenRole::Unknown` for Rust. C++ and Go classify them as // operands. After the fix, u_operands = 8: // main, v, m, Vec, HashMap, new, K, V // (`i32` is a primitive type, classified as an operator.) @@ -1118,7 +1155,7 @@ mod tests { // Java, C#, and Kotlin all classify it as an operator. Path- // heavy code (`std::collections::HashMap`, `Vec::new`, // `T::method`) had every `::` silently dropped into - // HalsteadType::Unknown. + // TokenRole::Unknown. // // Snippet has three `::` tokens (`std::collections::HashMap`, // counted as two `::` separators, plus `HashMap::new`). @@ -1350,7 +1387,7 @@ mod tests { // Regression: issue #192. A backtick-delimited `` `hello` `` // without `${...}` is semantically identical to `"hello"` / // `'hello'` and must contribute exactly one operand — before - // the fix `TemplateString` fell through to `HalsteadType::Unknown` + // the fix `TemplateString` fell through to `TokenRole::Unknown` // and contributed zero. expected: operands are `f` (function // name) and the wrapping `` `hello` `` template literal → // u_operands = 2, N2 = 2 (matches the equivalent @@ -3078,7 +3115,7 @@ mod tests { #[test] fn perl_plain_heredoc_counts_as_one_operand() { // Regression: issue #287. A plain (non-interpolating) Perl - // heredoc body used to be classified `HalsteadType::Unknown`, + // heredoc body used to be classified `TokenRole::Unknown`, // so its visible `HeredocBodyStatement` node contributed // nothing to N2 even though it is a string literal. The fix // adds `HeredocBodyStatement` to the interpolation-aware @@ -5161,7 +5198,7 @@ f() { // Regression for #277. Before the fix, `"$x is $y"` produced an // extra operand for the wrapping `QuotedWord` on top of the two // inner `VariableSubstitution` operands (`$x`, `$y`), giving 7. - // After the fix, the wrapper is `HalsteadType::Unknown` whenever + // After the fix, the wrapper is `TokenRole::Unknown` whenever // it carries an interpolation child, so operand attribution // belongs solely to the inner substitutions. check_metrics::( @@ -5876,8 +5913,9 @@ f() { } /// The iRules twin. The tables are shared, so a fix that reached - /// only `src/getter/tcl.rs` fails every row here — the two getters - /// are deliberate clones and #1354 names both. + /// only `big-code-analysis-ast/src/getter/tcl.rs` fails every row + /// here — the two getters are deliberate clones and #1354 names + /// both. #[test] fn irules_braced_word_bills_its_content_once_1354() { let mut witnessed = check_braced_word_cases::( @@ -6969,7 +7007,8 @@ f() { // A bare string literal contributes exactly one operand. The // counterpart to `ruby_halstead_interpolated_string_no_double_count` // — verifies the "no interpolation" branch of the same arm - // (see `src/getter.rs::get_op_type`'s `R::String | …` case). + // (see `get_op_type`'s `R::String | …` case in + // `big-code-analysis-ast/src/getter.rs`). // expected: operators = {def, end} = 2; operands = {f, "hello"} = 2. check_metrics::("def f\n \"hello\"\nend\n", "foo.rb", |metric| { assert_eq!(metric.halstead.unique_operators(), 2); @@ -8169,7 +8208,7 @@ f() { source.as_bytes(), Ancestors::known(chain) ), - HalsteadType::Unknown + TokenRole::Unknown ), "{label}: `{}` inside a character literal is classified, so the \ literal now double-counts against its wrapper", @@ -8330,7 +8369,7 @@ f() { CPP_THIS_POSITIONS.as_bytes(), Ancestors::known(chain) ), - HalsteadType::Operand + TokenRole::Operand ), "{label}: a `this` under `{}` is not an operand", chain.last().map_or("", Node::kind) @@ -8406,7 +8445,7 @@ f() { source.as_bytes(), Ancestors::known(&chain[..i]) ), - HalsteadType::Unknown + TokenRole::Unknown ), "{label}: `{}` contains a `this` and is itself classified, so the \ reference now counts twice", diff --git a/src/metrics/loc/bash.rs b/src/metrics/loc/bash.rs index 496b78935..2d59ecd2e 100644 --- a/src/metrics/loc/bash.rs +++ b/src/metrics/loc/bash.rs @@ -42,9 +42,9 @@ impl Loc for BashCode { // `HeredocBody2` is the parser-node symbol observed parse trees // actually carry; the duplicate `HeredocBody` entry is a // defensive arm that tree-sitter-bash 0.25.1 does not surface — - // `src/checker/bash.rs` records the same finding for `is_string` - // and omits it there (`.claude/rules/grammar-dispatch.md` - // sections 1 and 2). + // `big-code-analysis-ast/src/checker/bash.rs` records the + // same finding for `is_string` and omits it there + // (`.claude/rules/grammar-dispatch.md` sections 1 and 2). // // This arm owns its opening row, which is why it calls // `add_string_interior_ploc` rather than the parent-gated diff --git a/src/metrics/npa.rs b/src/metrics/npa.rs index b869eeacc..1a8f3c1ec 100644 --- a/src/metrics/npa.rs +++ b/src/metrics/npa.rs @@ -20,7 +20,6 @@ use std::fmt; use crate::checker::Checker; -use crate::langs::*; use crate::macros::{csharp_var_decl_kinds, csharp_var_declarator_kinds, implement_metric_trait}; use crate::node::Node; use crate::*; diff --git a/src/metrics/npm.rs b/src/metrics/npm.rs index 26ea73238..a4c50f347 100644 --- a/src/metrics/npm.rs +++ b/src/metrics/npm.rs @@ -21,7 +21,6 @@ use std::fmt; use crate::checker::{Checker, csharp_accessor_count}; use crate::lang_helpers::python::python_is_block; -use crate::langs::*; use crate::macros::implement_metric_trait; use crate::metrics::npa::{accessibility_ratio, ts_member_is_public}; use crate::node::Node; diff --git a/src/observation.rs b/src/observation.rs deleted file mode 100644 index 691c135db..000000000 --- a/src/observation.rs +++ /dev/null @@ -1,74 +0,0 @@ -//! Thread-local counters that make an invisible optimization testable. -//! -//! Several of this crate's optimizations change no output at all: -//! reusing one `tree_sitter::Parser` per thread (#1118), skipping a -//! space-kind lookup on a node that opens no space (#1110), serializing -//! `Ops` through a borrowed projection rather than an owned clone -//! (#1110), deferring the modeline scan behind a resolving extension -//! (#1111). Every assertion on the *result* of those paths holds just as -//! well once the optimization is reverted, so a revert is silent unless -//! something counts the work. -//! -//! # The invariant -//! -//! The counter and the function that bumps it are **unconditional**; -//! only the accessor is `#[cfg(test)]`. Gating the counter itself would -//! leave the test observing a build production never ships — the counted -//! branch compiled under `cfg(test)` and the shipped one under nothing — -//! which is the exact failure these counters exist to catch. Four sites -//! grew that rule independently and each stated it in prose; [`counter`] -//! makes it structural instead, emitting the three items together so the -//! wrong one cannot be gated. -//! -//! The cost is one `Cell` increment on a path that already does far more -//! (building a parser, classifying a node, projecting a whole tree), -//! which is why it is affordable to leave in the shipped build. - -/// Declares a thread-local observation counter as a module: a private -/// `Cell`, an unconditional `record()`, and a `#[cfg(test)] observed()`. -/// -/// Takes the module's name and nothing else. It deliberately carries no -/// narrative — *which* optimization a counter observes, and why no -/// assertion on the output can distinguish it, belongs in a comment -/// above each invocation. -/// -/// A module rather than three free items so a call site names the -/// counter once (`parsers_built::record()`), which keeps the invocation -/// to one line and leaves no room for the recorder and the accessor to -/// drift apart. -macro_rules! counter { - ($name:ident) => { - pub(crate) mod $name { - thread_local! { - static COUNT: ::std::cell::Cell = const { ::std::cell::Cell::new(0) }; - } - - /// Records one occurrence on this thread. - /// - /// `pub(super)` on purpose: only the module that owns the - /// counted path may bump it, so a counter cannot drift into - /// meaning "whatever any caller felt like recording". - #[inline] - pub(super) fn record() { - COUNT.with(|count| count.set(count.get() + 1)); - } - - /// Occurrences recorded on this thread. Only this accessor - /// is test-gated; see [`crate::observation`]. - /// - /// `pub(crate)`, unlike [`record`], because the guarded path - /// and the test that guards it need not share a module. That - /// asymmetry is the point: `child_scan_cursors` counts a - /// cursor hoist in `node`, but `output::dump` is one of the - /// walks that has to hold one, and a counter only reachable - /// from its own module silently leaves such a caller - /// unguarded. - #[cfg(test)] - pub(crate) fn observed() -> usize { - COUNT.with(::std::cell::Cell::get) - } - } - }; -} - -pub(crate) use counter; diff --git a/src/observation_tests.rs b/src/observation_tests.rs new file mode 100644 index 000000000..0643ce8fd --- /dev/null +++ b/src/observation_tests.rs @@ -0,0 +1,170 @@ +//! Guards on the parse layer's observation counters for the walks that +//! live in this crate. +//! +//! The counters (`node_resolved_sibling_lookups`, `child_scan_cursors`) +//! are declared in `big-code-analysis-ast`, which this crate's tests see +//! as an ordinary dependency; the `test-support` feature is what opens +//! their `observed()` accessors here. The walks under guard — the +//! `exclude_tests` prune and the metric / suppression scans — are this +//! crate's, so the assertions are too (#1376). + +/// Under a parent narrow enough to read forward, the +/// `exclude_tests` prune finds the run of `#[…]` siblings before an +/// item through the walker's ancestor chain, never by resolving +/// siblings from the node. +/// +/// Nothing in the output says so: the backward walk this replaced +/// returns the same answer, only `O(depth)` per step (#1100), and +/// `rust_outer_attr_scans_agree` in `checker.rs` exists precisely +/// to prove the two agree. The counter is the sole observable, so a +/// revert is a silent quadratic without this. +/// +/// Every parent in the fixture holds at most five children, which +/// keeps it under `MAX_FORWARD_ATTRIBUTE_SCAN_CHILDREN` — the +/// backward walk is still the deliberate reading above that width, +/// so a wider fixture would assert the opposite of what it looks +/// like it asserts. +/// +/// Seeding a real lookup first is what makes the assertion +/// falsifiable: compared against zero it would also pass with +/// `record()` never wired up at all. +#[cfg(feature = "rust")] +#[test] +fn the_exclude_tests_prune_resolves_no_sibling_from_a_node() { + let source = "#[cfg(test)]\nmod tests {\nfn t() {}\n}\n\ + #[inline]\nfn kept() {\n#[allow(dead_code)]\nfn nested() {}\nlet x = 1;\n}\n"; + let ast = crate::test_support::parse_named(crate::LANG::Rust, "lib.rs", source); + + let root = ast.root_node(); + let last = root.children().last().expect("the file has items"); + let _ = last.previous_sibling(); + let seeded = crate::node::node_resolved_sibling_lookups::observed(); + assert!(seeded > 0, "the seed call must be counted"); + + ast.metrics(crate::MetricsOptions::default().with_exclude_tests(true)) + .expect("the walk must yield a top-level space"); + + assert_eq!( + crate::node::node_resolved_sibling_lookups::observed(), + seeded, + "the metric walk resolved a sibling from a node; \ + read it off the ancestor chain instead (#1096 / #1100)" + ); +} + +/// Which arm the `exclude_tests` attribute-scan dispatch takes, at +/// the boundary in both directions and on both of its axes. +/// +/// `rust_outer_attr_scans_agree` in `checker.rs` proves the two +/// readings answer the same thing, which is exactly why it cannot +/// see which one ran — it passes at any budget, including one that +/// never reads forward. This counter is the only observable that +/// tells them apart, and it lives here, so the boundary is pinned +/// here too. +/// +/// The third case is the one #1100 got wrong: dispatching on width +/// alone sent any over-wide body to the `O(depth)` walk however deep +/// it sat, which on a nested `mod` tree is quadratic (a 3_200-deep +/// fixture measured 2.67 s against 0.045 s for the same shape one +/// child narrower). +#[cfg(feature = "rust")] +#[test] +fn the_exclude_tests_prune_reads_forward_up_to_its_depth_scaled_budget() { + // Three attributed items make a `source_file` exactly six + // children wide — the depth-1 budget. A fourth, bare item makes + // seven, one over. Wrapping that in a `mod` puts the same seven + // between two braces, so its `declaration_list` is nine wide, at + // depth 3 — where the budget is also exactly nine. + let at_budget = "#[cfg(test)]\nfn a() {}\n#[inline]\nfn b() {}\n#[cfg(test)]\nfn c() {}\n"; + let past_budget = format!("{at_budget}fn d() {{}}\n"); + let nested = format!("mod m {{\n{past_budget}}}\n"); + + for (shape, source, resolves_siblings) in [ + ("six children at depth 1", at_budget.to_string(), false), + ("seven children at depth 1", past_budget, true), + ("nine children at depth 3", nested, false), + ] { + let before = crate::node::node_resolved_sibling_lookups::observed(); + crate::test_support::parse_named(crate::LANG::Rust, "lib.rs", &source) + .metrics(crate::MetricsOptions::default().with_exclude_tests(true)) + .expect("the walk must yield a top-level space"); + let resolved = crate::node::node_resolved_sibling_lookups::observed() > before; + assert_eq!( + resolved, resolves_siblings, + "{shape}: the prune took the wrong dispatch arm" + ); + } +} + +/// The metric and suppression walks #1112 moved onto +/// `Node::children_with` must scan a whole tree on one cursor, not one +/// per node. The `preorder` and `act_on_node` halves of this guard sit +/// beside the counter in `big-code-analysis-ast`. +/// +/// Seeding a real scan first is what makes it falsifiable: compared +/// against zero these assertions would also pass with `record()` never +/// wired up at all. +// Gated on the union of the languages the body parses — `c` for the +// seed, `python` for the instance-attribute scan, `rust` for the +// suppression walk. Dropping `c` here would leave `CParser::new` +// reaching `Tree::new`'s disabled-language `expect` under +// `--features python,rust` (`.claude/rules/testing.md`). +#[cfg(all(feature = "c", feature = "python", feature = "rust"))] +#[test] +fn the_metric_and_suppression_walks_scan_a_tree_on_one_cursor() { + use crate::ParserTrait; + + let seed = crate::CParser::new( + b"int main() { int a; }".to_vec(), + std::path::Path::new("a.c"), + None, + ); + let _ = seed.root().children().count(); + assert!( + crate::node::child_scan_cursors::observed() > 0, + "the seed scan must be counted" + ); + + // The Python instance-attribute scan walks every method body of + // a class. Before #1112 it was 92 % of the metric walk's child + // scans on the Python corpus slice — one per node under the + // class. It is not the only scan a `metrics()` call makes, so + // the bound is a fraction of the node count rather than zero. + // Measured on this fixture: 18 scans over 81 nodes with the + // cursor hoisted, 91 without, so the bound separates the two + // with room on both sides. + let source = "class C:\n def a(self):\n self.x = 1\n self.y = [1, 2]\n\ + \n def b(self):\n self.z, self.w = 1, 2\n \ + if self.x:\n self.v = self.y\n"; + let ast = crate::test_support::parse_named(crate::LANG::Python, "c.py", source); + let nodes = ast.root_node().preorder().count(); + let before = crate::node::child_scan_cursors::observed(); + ast.metrics(crate::MetricsOptions::default()) + .expect("the walk must yield a top-level space"); + let scans = crate::node::child_scan_cursors::observed() - before; + assert!(nodes > 60, "fixture is too small to prove much"); + assert!( + scans < nodes / 2, + "the Python metric walk built {scans} cursors over {nodes} nodes; the \ + instance-attribute scan is meant to hold one for the subtree (#1112)" + ); + + // The suppression scan is a full-tree DFS of its own: 0 scans + // over this fixture's 29 nodes with the cursor hoisted, 29 + // without. + let parser = crate::langs::RustParser::new( + b"// bca: suppress(cognitive)\nfn f() { if a { g(1, 2); } }\n".to_vec(), + std::path::Path::new("lib.rs"), + None, + ); + let nodes = parser.root().preorder().count(); + let before = crate::node::child_scan_cursors::observed(); + let markers = crate::suppression::suppression_markers(&parser); + let scans = crate::node::child_scan_cursors::observed() - before; + assert_eq!(markers.len(), 1, "fixture carries one marker"); + assert!(nodes > 20, "fixture is too small to prove much"); + assert!( + scans < nodes / 2, + "the suppression scan built {scans} cursors over {nodes} nodes (#1112)" + ); +} diff --git a/src/output/dump.rs b/src/output/dump.rs index 75831a425..8098e3b5d 100644 --- a/src/output/dump.rs +++ b/src/output/dump.rs @@ -683,7 +683,8 @@ mod tests { #[test] fn dump_output_depth_limits_recursion() { - // `bca find` dumps with depth=1 (src/find.rs) to show only the + // `bca find` dumps with depth=1 + // (`big-code-analysis-ast/src/find.rs`) to show only the // matched node, not its subtree. depth=1 renders the node and stops // before its children; depth=0 renders nothing. This is the only // positive-depth path in production, and it is what the `depth - 1` diff --git a/src/spaces.rs b/src/spaces.rs index b1e83c94b..7671ca52d 100644 --- a/src/spaces.rs +++ b/src/spaces.rs @@ -88,8 +88,6 @@ use crate::npm::{self, Npm}; use crate::tokens::{self, Tokens}; use crate::wmc::{self, Wmc}; -use crate::traits::*; - mod compute; // Inherent / trait impl blocks for the public types defined below live @@ -108,9 +106,12 @@ pub use compute::analyze; // it) and re-exported here so `crate::spaces::SpaceKind` keeps resolving. pub use crate::space_kind::SpaceKind; // `metrics_inner` and `push_children` are `pub(crate)` — `metrics_inner` -// is re-exported from `lib.rs` and `push_children` is consumed by -// `crate::ops`, so both must stay reachable at their `crate::spaces::` -// paths. +// is called from `crate::spaces::ast` (through `with_any_parser!`), +// `crate::test_support` and `crate::spaces_tests`, and `push_children` +// is consumed by `crate::ops`, so both must stay reachable at their +// `crate::spaces::` paths. (Before #1376 `metrics_inner` was also +// re-exported from `lib.rs` for the feature-gated `mk_action!` arms; +// that re-export is gone with the arms.) pub(crate) use compute::{metrics_inner, push_children}; // The inline `mod tests` drives `apply_suppression` via // `super::apply_suppression`; re-import the name into this module diff --git a/src/spaces/ast.rs b/src/spaces/ast.rs index e0a1e0a0b..b0b559d52 100644 --- a/src/spaces/ast.rs +++ b/src/spaces/ast.rs @@ -187,7 +187,7 @@ impl Ast { #[must_use] #[inline] pub fn source(&self) -> &[u8] { - with_any_parser!(&self.inner, |p| p.code()) + self.inner.code() } /// Display name carried through to [`FuncSpace::name`] by every @@ -208,7 +208,7 @@ impl Ast { #[must_use] #[inline] pub fn as_tree_sitter(&self) -> &tree_sitter::Tree { - with_any_parser!(&self.inner, |p| p.ts_tree()) + self.inner.ts_tree() } /// Strip non-doc comments from the held parse, returning the source @@ -228,7 +228,7 @@ impl Ast { /// ``` #[must_use] pub fn strip_comments(&self) -> Option> { - with_any_parser!(&self.inner, |p| crate::comment_rm::rm_comments(p)) + self.inner.strip_comments() } /// Detect the span of every function in the held parse. Safe to call @@ -271,7 +271,7 @@ impl Ast { /// ``` #[must_use] pub fn dump(&self, cfg: crate::AstCfg) -> crate::AstResponse { - with_any_parser!(&self.inner, |p| crate::ast::dump_inner(p, cfg)) + self.inner.dump(cfg) } /// Count `(matching, total)` nodes in the held parse, where a node @@ -293,7 +293,7 @@ impl Ast { /// ``` #[must_use] pub fn count(&self, filters: &[String]) -> (usize, usize) { - with_any_parser!(&self.inner, |p| crate::count::count(p, filters)) + self.inner.count(filters) } /// Find every node in the held parse whose kind is named in @@ -306,7 +306,7 @@ impl Ast { /// Currently infallible; the [`Result`] wrapper is reserved for a /// future strict-parsing mode (matching the other `Ast` walkers). pub fn find(&self, filters: &[String]) -> Result>, MetricsError> { - with_any_parser!(&self.inner, |p| crate::find::find(p, filters)) + self.inner.find(filters) } /// Collect every in-source suppression marker (`// bca: suppress …`) @@ -323,6 +323,6 @@ impl Ast { #[must_use] #[inline] pub fn root_node(&self) -> Node<'_> { - with_any_parser!(&self.inner, |p| p.root()) + self.inner.root_node() } } diff --git a/src/spaces_tests.rs b/src/spaces_tests.rs index e879eacd1..806434cba 100644 --- a/src/spaces_tests.rs +++ b/src/spaces_tests.rs @@ -41,8 +41,9 @@ fn space_kind_non_exhaustive_serde_roundtrip_unchanged() { /// `tree-sitter-mozcpp` currently emits. The structural /// `FunctionDefinition*` contract for the aliased kind_ids /// (489/491/494) that no observed input parses to is documented -/// at the predicate call sites in `src/checker.rs` and -/// `src/getter.rs` — see issue #285. +/// at the predicate call sites in +/// `big-code-analysis-ast/src/checker.rs` and +/// `big-code-analysis-ast/src/getter.rs` — see issue #285. #[test] fn cpp_function_definition_is_classified_as_function() { use crate::Cpp; @@ -2531,3 +2532,50 @@ fn walk_frees_every_cognitive_nesting_slot() { ); } } + +#[cfg(test)] +mod empty_root_contract { + use crate::{LANG, MetricsOptions, Source, SpaceKind, analyze}; + + // Regression guard for issue #262: the `MetricsError::EmptyRoot` + // variant is documented as "Reserved — not produced today". + // `metrics_with_options` pushes a synthetic top-level Unit + // `FuncSpace` before walking, so every parse — including empty, + // whitespace-only, and comment-only input — currently returns + // `Ok(FuncSpace { kind: Unit, .. })`. If the walker is ever + // changed to legitimately drain its state stack (e.g. by + // dropping the synthetic root), this test will start failing + // and the variant docs must be revisited. + #[test] + fn empty_and_comment_only_input_never_returns_empty_root() { + // Pair every enabled language with sources that would, by + // the old (false) variant doc, surface `EmptyRoot`. The + // comment syntaxes cover line and block forms across the + // supported language families. + let inputs: &[&[u8]] = &[b"", b" \n\t\n", b"// just a comment\n", b"/* block */\n"]; + + for lang in LANG::into_enum_iter() { + if !lang.is_enabled() { + continue; + } + for src in inputs { + let space = analyze(Source::new(lang, src), MetricsOptions::default()) + .unwrap_or_else(|err| { + panic!( + "{} on input {:?} unexpectedly returned {err:?}; \ + EmptyRoot is documented as not produced today", + lang.name(), + String::from_utf8_lossy(src), + ) + }); + assert_eq!( + space.kind, + SpaceKind::Unit, + "{} on input {:?} produced a non-Unit top-level FuncSpace", + lang.name(), + String::from_utf8_lossy(src), + ); + } + } + } +} diff --git a/src/test_support.rs b/src/test_support.rs index c16215f04..168eafe44 100644 --- a/src/test_support.rs +++ b/src/test_support.rs @@ -10,12 +10,9 @@ use std::path::PathBuf; -use crate::node::{Node, Tree}; use crate::spaces::metrics_inner; -use crate::traits::LanguageInfo; use crate::{ - CodeMetrics, FuncSpace, LANG, Metric, MetricSuite, MetricsOptions, ParserTrait, Source, - SpaceKind, analyze, + CodeMetrics, FuncSpace, LANG, Metric, MetricSuite, MetricsOptions, Source, SpaceKind, analyze, }; /// Parses `source` as `filename` under `options` and hands the resulting @@ -275,53 +272,6 @@ pub(crate) fn function_space<'a>(func_space: &'a FuncSpace, name: &str) -> &'a F } } -/// Visits `code`'s tree in pre-order, maintaining the ancestor chain -/// exactly as `spaces::compute::metrics_inner` does, and hands each -/// node to `check` together with that chain. -/// -/// Keeping the bookkeeping identical to the walker's is the point: a -/// test that built the chain some other way would prove -/// [`crate::node::Ancestors`] self-consistent without proving the -/// walker feeds it the right slice. -pub(crate) fn for_each_node_with_chain( - code: &[u8], - mut check: impl FnMut(&Node<'_>, &[Node<'_>]), -) -> usize { - let tree = Tree::new::(code); - let root = tree.get_root(); - assert!( - !root.has_error(), - "fixture must parse cleanly, else the walk covers error recovery" - ); - - let mut chain: Vec> = Vec::new(); - let mut stack = vec![(root, 0_usize)]; - let mut visited = 0; - while let Some((node, depth)) = stack.pop() { - chain.truncate(depth); - check(&node, &chain); - visited += 1; - chain.push(node); - let first = stack.len(); - stack.extend(node.children().map(|child| (child, depth + 1))); - stack[first..].reverse(); - } - visited -} - -/// Walks `parser`'s tree and reports whether any node has `kind_id == -/// target`. -/// -/// The drift marker behind lesson 34. A passing `!ast_has_kind_id(…)` -/// proves an enum variant is unreachable at the pinned grammar, so a -/// defensive dispatch arm listing it is an explicit promise rather than -/// silent dead code; a bump that starts emitting the kind fails the -/// assertion instead of quietly changing a metric. -/// -/// `preorder` rather than a hand-rolled index walk: three modules each -/// carried their own copy pairing `node.child(i)` with an `if let -/// Some(..)` whose `None` arm is unreachable for `i < child_count()`, -/// which Codecov reported as a partial branch in each of them. -pub(crate) fn ast_has_kind_id(parser: &P, target: u16) -> bool { - parser.root().preorder().any(|n| n.kind_id() == target) -} +// The parse-only helpers live beside the parse layer and are shared with +// its own tests through the `test-support` feature. +pub(crate) use big_code_analysis_ast::test_support::{ast_has_kind_id, for_each_node_with_chain}; diff --git a/tests/api/ast_seam_test.rs b/tests/api/ast_seam_test.rs index 93aceff1d..f7416f12c 100644 --- a/tests/api/ast_seam_test.rs +++ b/tests/api/ast_seam_test.rs @@ -604,6 +604,53 @@ fn find_returns_named_nodes() { assert!(none.is_empty()); } +#[cfg(feature = "rust")] +#[test] +fn node_accessors_are_reachable_through_the_root_crate() { + // #1376 moved `Node` into `big-code-analysis-ast` and made the + // accessors the metric walk uses public; they are part of this + // crate's escape-hatch surface now (STABILITY.md), so pin that a + // caller of `big_code_analysis` alone can reach every one of them. + // The values are the pinned grammar's and are not asserted beyond + // what any tree-sitter-rust release must agree on. + let source = b"fn a(x: u8) {} +"; + let ast = Ast::parse(Source::new(LANG::Rust, source)).expect("rust feature enabled"); + let root = ast.root_node(); + assert_eq!(root.kind(), "source_file"); + assert!(root.is_named()); + assert_eq!(root.start_byte(), 0); + assert_eq!(root.end_byte(), source.len()); + assert_eq!(root.start_position(), (0, 0)); + assert_eq!(root.start_row(), 0); + assert!(root.parent().is_none()); + + let item = root.child(0).expect("the file has one item"); + assert_eq!(item.kind(), "function_item"); + assert_eq!( + item.kind_id(), + root.children().next().expect("same child").kind_id() + ); + assert_eq!( + item.child_count(), + root.children_with(&mut root.cursor()) + .next() + .expect("same child") + .child_count() + ); + assert_ne!(item.id(), root.id()); + assert!(item.previous_sibling().is_none()); + assert_eq!(item.end_line(), 1); + + let name = item + .child_by_field_name("name") + .expect("function_item has a name field"); + assert_eq!(name.utf8_text(source), Some("a")); + assert_eq!(name.parent().map(|p| p.id()), Some(item.id())); + assert_eq!((name.start_row(), name.end_row()), (0, 0)); + assert_eq!(name.end_position(), (0, 4)); +} + #[cfg(feature = "rust")] #[test] fn suppressions_collects_in_source_markers() { diff --git a/tests/grammars/alterator_string_flattening.rs b/tests/grammars/alterator_string_flattening.rs index 2939c5ef9..66d168e23 100644 --- a/tests/grammars/alterator_string_flattening.rs +++ b/tests/grammars/alterator_string_flattening.rs @@ -1,5 +1,5 @@ //! Behavioural coverage for the per-language `Alterator::alterate` -//! string-flattening arms in `src/alterator.rs`. +//! string-flattening arms in `big-code-analysis-ast/src/alterator.rs`. //! //! `alterate` collapses a string-like literal into a single leaf //! [`AstNode`] holding its verbatim source text, so the AST dump (and the diff --git a/tests/parity/halstead_set_target_parity.rs b/tests/parity/halstead_set_target_parity.rs index dab0af0b1..4ef1c9b74 100644 --- a/tests/parity/halstead_set_target_parity.rs +++ b/tests/parity/halstead_set_target_parity.rs @@ -10,7 +10,8 @@ //! every assigned variable from `n2`/`N2` (#1294). The two dialects had //! drifted in opposite directions — this fixture keeps them agreeing. //! -//! `src/getter/tcl.rs` and `src/getter/irules.rs` are deliberate clones +//! `big-code-analysis-ast/src/getter/tcl.rs` and its `irules.rs` +//! sibling are deliberate clones //! and a defect in one is a defect in both, so a fix landed in only one //! of them is what these tests exist to catch. Each drives the same //! source through both dialects and asserts the same operand list. diff --git a/utils/check-diagnostic-prefix-test.py b/utils/check-diagnostic-prefix-test.py index f33ff158e..f9ee8ec2a 100644 --- a/utils/check-diagnostic-prefix-test.py +++ b/utils/check-diagnostic-prefix-test.py @@ -150,8 +150,9 @@ def test_an_escaped_cr_does_not_open_a_raw_string(self) -> None: # without being one. Reading either as an open skips every line # until the next quote, and the offender hiding in there is a # false *clean* — the one outcome this gate exists to prevent. - # Both shapes are live in this tree (`src/tools.rs:409`, - # `src/languages/language_ruby.rs:481`). + # Both shapes are live in this tree + # (`big-code-analysis-ast/src/tools.rs:409`, + # `big-code-analysis-ast/src/languages/language_ruby.rs:481`). text = 'let l = s.strip_suffix(b"\\r").unwrap_or(s);\neprintln!("Error: x");\n' self.assertEqual([(2, "Error")], [(n, w) for n, w, _ in GATE.scan_text(text)]) diff --git a/utils/check-enums-codegen-drift-test.py b/utils/check-enums-codegen-drift-test.py index cca583f2f..e62b15207 100755 --- a/utils/check-enums-codegen-drift-test.py +++ b/utils/check-enums-codegen-drift-test.py @@ -4,8 +4,9 @@ Each test stages a synthetic mini-repo in a tempdir: the `enums/` crate is symlinked to the live repo (path-dep resolution depends on a real sibling layout, and the data files -are read verbatim), but `src/c_langs_macros/` and -`src/languages/` are deep-copied so per-test mutations stay +are read verbatim), but `big-code-analysis-ast/src/c_langs_macros/` +and `big-code-analysis-ast/src/languages/` are deep-copied so +per-test mutations stay isolated. The drift script is then invoked from the tempdir. The shared cargo target cache (`enums/target/`) is warmed once @@ -29,6 +30,8 @@ # or writes is anchored at the repository root one level above. UTILS_DIR = pathlib.Path(__file__).resolve().parent REPO_ROOT = UTILS_DIR.parent +# Where the generated files live since #1376 (relative to the repo root). +AST_SRC = "big-code-analysis-ast/src" SCRIPT_SRC = UTILS_DIR / "check-enums-codegen-drift.sh" @@ -42,9 +45,7 @@ def _staged(tmpdir: pathlib.Path) -> pathlib.Path: return tmpdir / "utils" / SCRIPT_SRC.name -def _run( - tmpdir: pathlib.Path, *args: str -) -> subprocess.CompletedProcess[str]: +def _run(tmpdir: pathlib.Path, *args: str) -> subprocess.CompletedProcess[str]: """Run the drift script from `tmpdir` (its $ROOT).""" return subprocess.run( ["bash", str(_staged(tmpdir)), *args], @@ -69,8 +70,7 @@ def setUpClass(cls) -> None: ) if result.returncode != 0: raise RuntimeError( - f"warm-up cargo build failed (rc={result.returncode}):\n" - f"{result.stderr}" + f"warm-up cargo build failed (rc={result.returncode}):\n{result.stderr}" ) def setUp(self) -> None: @@ -93,11 +93,11 @@ def setUp(self) -> None: (self.tmpdir / ts_crate).symlink_to(REPO_ROOT / ts_crate) # Copy the mutable artifact dirs; per-test mutations # land here and are torn down with the tempdir. - (self.tmpdir / "src").mkdir() + (self.tmpdir / AST_SRC).mkdir(parents=True) for sub in ("c_langs_macros", "languages"): shutil.copytree( - REPO_ROOT / "src" / sub, - self.tmpdir / "src" / sub, + REPO_ROOT / AST_SRC / sub, + self.tmpdir / AST_SRC / sub, ) # Copy the script itself so `dirname "$BASH_SOURCE"/..` # resolves to the tempdir. (`git rev-parse @@ -126,7 +126,7 @@ def test_mutated_c_macros_fails_with_drift_message(self) -> None: # diverges. The script must report drift, the specific # filename, AND the remediation block (defended by the # pipefail-safe diff pipeline). - target = self.tmpdir / "src" / "c_langs_macros" / "c_macros.rs" + target = self.tmpdir / AST_SRC / "c_langs_macros" / "c_macros.rs" text = target.read_text(encoding="utf-8") target.write_text( text.replace('"INT16_C",', '"FAKE_INT16_C",', 1), @@ -134,9 +134,7 @@ def test_mutated_c_macros_fails_with_drift_message(self) -> None: ) result = _run(self.tmpdir) self.assertEqual(result.returncode, 1) - self.assertIn( - "drift: src/c_langs_macros/c_macros.rs", result.stderr - ) + self.assertIn(f"drift: {AST_SRC}/c_langs_macros/c_macros.rs", result.stderr) # Remediation block must print despite the diff # truncation pipeline — this is the regression test # for the `diff | head -40` pipefail abort that the @@ -145,9 +143,9 @@ def test_mutated_c_macros_fails_with_drift_message(self) -> None: self.assertIn("Regenerate the checked-in files", result.stderr) def test_mutated_language_file_fails_with_drift_message(self) -> None: - # Same as above but on the src/languages side, to + # Same as above but on the languages side, to # exercise both diff_dir invocations. - target = self.tmpdir / "src" / "languages" / "language_rust.rs" + target = self.tmpdir / AST_SRC / "languages" / "language_rust.rs" text = target.read_text(encoding="utf-8") target.write_text( text.replace("pub enum Rust", "pub enum RustFake", 1), @@ -155,9 +153,7 @@ def test_mutated_language_file_fails_with_drift_message(self) -> None: ) result = _run(self.tmpdir) self.assertEqual(result.returncode, 1) - self.assertIn( - "drift: src/languages/language_rust.rs", result.stderr - ) + self.assertIn(f"drift: {AST_SRC}/languages/language_rust.rs", result.stderr) self.assertIn("Codegen drift detected", result.stderr) # --- orphan detection --- @@ -165,7 +161,7 @@ def test_mutated_language_file_fails_with_drift_message(self) -> None: def test_orphan_language_file_fails_with_stale_message(self) -> None: # A `language_zombie.rs` that the codegen doesn't emit # must trip the reverse-direction (orphan) check. - orphan = self.tmpdir / "src" / "languages" / "language_zombie.rs" + orphan = self.tmpdir / AST_SRC / "languages" / "language_zombie.rs" orphan.write_text( "// orphan generated file no codegen produces\n", encoding="utf-8", @@ -179,7 +175,7 @@ def test_orphan_language_file_fails_with_stale_message(self) -> None: ) def test_orphan_c_langs_macros_file_fails(self) -> None: - orphan = self.tmpdir / "src" / "c_langs_macros" / "c_extra.rs" + orphan = self.tmpdir / AST_SRC / "c_langs_macros" / "c_extra.rs" orphan.write_text("// orphan\n", encoding="utf-8") result = _run(self.tmpdir) self.assertEqual(result.returncode, 1) @@ -203,13 +199,11 @@ def test_large_diff_prints_truncation_footer(self) -> None: # exceed the 40-line head cap. The footer must report # how many lines were hidden so the reviewer knows the # output is incomplete. - target = self.tmpdir / "src" / "c_langs_macros" / "c_macros.rs" + target = self.tmpdir / AST_SRC / "c_langs_macros" / "c_macros.rs" text = target.read_text(encoding="utf-8") - fake_block = "\n".join( - f' "FAKE_ENTRY_{i:03d}",' for i in range(50) - ) + fake_block = "\n".join(f' "FAKE_ENTRY_{i:03d}",' for i in range(50)) target.write_text( - text.replace('"INT16_C",', f"{fake_block}\n \"INT16_C\",", 1), + text.replace('"INT16_C",', f'{fake_block}\n "INT16_C",', 1), encoding="utf-8", ) result = _run(self.tmpdir) @@ -222,9 +216,7 @@ def test_large_diff_prints_truncation_footer(self) -> None: # --- script failure-propagation path (exit 2) --- - def _run_with_cargo_stub( - self, stub_body: str - ) -> subprocess.CompletedProcess[str]: + def _run_with_cargo_stub(self, stub_body: str) -> subprocess.CompletedProcess[str]: """Run the drift script with a fake `cargo` shadowing PATH. Mirrors `test_independent_of_fd`'s stub-on-PATH technique: @@ -335,7 +327,7 @@ def test_script_re_passes_after_drift_revert(self) -> None: # Mutate, run (expect failure), revert, re-run (expect OK). # Pins that the script doesn't leave state behind that # would make subsequent invocations fail. - target = self.tmpdir / "src" / "c_langs_macros" / "c_macros.rs" + target = self.tmpdir / AST_SRC / "c_langs_macros" / "c_macros.rs" original = target.read_text(encoding="utf-8") target.write_text( original.replace('"INT16_C",', '"FAKE",', 1), @@ -411,9 +403,7 @@ def test_output_is_independent_of_prior_contents(self) -> None: self._run_generator() from_empty = self._emitted_names() - self.data_file.write_text( - "EXTRA_ONE\nEXTRA_TWO\n", encoding="utf-8" - ) + self.data_file.write_text("EXTRA_ONE\nEXTRA_TWO\n", encoding="utf-8") self._run_generator() from_polluted = self._emitted_names() diff --git a/utils/check-enums-codegen-drift.sh b/utils/check-enums-codegen-drift.sh index 395410c43..689acdf41 100755 --- a/utils/check-enums-codegen-drift.sh +++ b/utils/check-enums-codegen-drift.sh @@ -2,8 +2,10 @@ # check-enums-codegen-drift # # Runs the `enums/` codegen into a tempdir, formats the output, -# and diffs against the checked-in `src/c_langs_macros/*.rs` and -# `src/languages/language_*.rs` files. Any divergence fails. +# and diffs against the checked-in +# `big-code-analysis-ast/src/c_langs_macros/*.rs` and +# `big-code-analysis-ast/src/languages/language_*.rs` files. Any +# divergence fails. # # Closes the failure mode from #405: running any grammar regen # silently regenerated `c_macros.rs` / `c_specials.rs` to a @@ -49,7 +51,7 @@ if ! cargo build --manifest-path "$MANIFEST" --quiet; then fi # Each codegen mode pairs an `enums -l` flag with the -# target subdir under `src/`. Parallel arrays (rather than a +# target subdir under `big-code-analysis-ast/src/`. Parallel arrays (rather than a # `:`-separated single array) keep this safe if a future mode # name ever contains `:` and stay bash-3 compatible (no # associative-array dependency for macOS contributors). @@ -131,7 +133,7 @@ diff_dir() { # Reverse: checked-in → codegen output. Skip `mod.rs` # (hand-maintained module index, not generated). If a # future hand-maintained file is added to either target - # subdir (e.g., a `src/languages/shared.rs`), extend the + # subdir (e.g., a `languages/shared.rs`), extend the # skip list — flagged orphans would otherwise look like # real drift to a confused reviewer. for f in "$checked_in_dir"/*.rs; do @@ -147,8 +149,8 @@ diff_dir() { done } -diff_dir "$WORK_DIR/languages" "src/languages" -diff_dir "$WORK_DIR/c_langs_macros" "src/c_langs_macros" +diff_dir "$WORK_DIR/languages" "big-code-analysis-ast/src/languages" +diff_dir "$WORK_DIR/c_langs_macros" "big-code-analysis-ast/src/c_langs_macros" if [ "$fail" -ne 0 ]; then { @@ -156,9 +158,9 @@ if [ "$fail" -ne 0 ]; then echo "Codegen drift detected. Either:" echo " - Regenerate the checked-in files:" echo " cargo run --manifest-path ./enums/Cargo.toml -- \\" - echo " -lrust -o ./src/languages" + echo " -lrust -o ./big-code-analysis-ast/src/languages" echo " cargo run --manifest-path ./enums/Cargo.toml -- \\" - echo " -lc_macros -o ./src/c_langs_macros" + echo " -lc_macros -o ./big-code-analysis-ast/src/c_langs_macros" echo " cargo fmt" echo " - Or update enums/templates/ to match the checked-in form." echo " - Or, for stale generated files in repo but not produced" diff --git a/utils/check-grammar-crate-test.py b/utils/check-grammar-crate-test.py index eff67c94c..344a853b8 100644 --- a/utils/check-grammar-crate-test.py +++ b/utils/check-grammar-crate-test.py @@ -2,7 +2,7 @@ """Tests for check-grammar-crate.py. The headline test re-derives the grammar -> extension mapping from the -single source of truth (`src/langs.rs` `mk_langs!`) and asserts the +single source of truth (`big-code-analysis-ast/src/langs.rs` `mk_langs!`) and asserts the hand-maintained `EXTENSIONS` table matches it exactly. This is the anti-drift guard for #869: the old table had gone stale across the #507 / #720 / #721 / #724 refactors (wrong files, a wrong crate-name @@ -24,7 +24,7 @@ UTILS_DIR = pathlib.Path(__file__).resolve().parent REPO_ROOT = UTILS_DIR.parent SCRIPT_SRC = UTILS_DIR / "check-grammar-crate.py" -LANGS_RS = REPO_ROOT / "src" / "langs.rs" +LANGS_RS = REPO_ROOT / "big-code-analysis-ast" / "src" / "langs.rs" # The tree-sitter function token in `mk_langs!` maps 1:1 to the grammar # crate name (underscores -> hyphens) for every variant EXCEPT the Tsx @@ -112,12 +112,12 @@ def _mk_langs_tuples(block: str) -> list[str]: def _derive_extensions_from_langs_rs() -> dict[str, list[str]]: - """Re-derive the grammar -> extension-glob table from src/langs.rs.""" + """Re-derive the grammar -> extension-glob table from langs.rs.""" text = "\n".join( _strip_line_comment(line) for line in LANGS_RS.read_text().splitlines() ) opener = re.search(r"mk_langs!\s*\(", text) - assert opener is not None, "mk_langs! macro not found in src/langs.rs" + assert opener is not None, f"mk_langs! macro not found in {LANGS_RS}" start = opener.end() - 1 depth = 0 end = start @@ -157,7 +157,7 @@ def test_table_matches_langs_rs(self) -> None: table, derived, "check-grammar-crate.py EXTENSIONS has drifted from " - "src/langs.rs mk_langs!; update the table (see #869)", + "big-code-analysis-ast/src/langs.rs mk_langs!; update the table (see #869)", ) def test_every_key_is_a_real_grammar_crate(self) -> None: diff --git a/utils/check-grammar-crate.py b/utils/check-grammar-crate.py index 683acba2d..dea7884ab 100755 --- a/utils/check-grammar-crate.py +++ b/utils/check-grammar-crate.py @@ -42,7 +42,7 @@ # File-extension globs each tree-sitter grammar crate owns. # -# SOURCE OF TRUTH: `src/langs.rs` `mk_langs!` — the per-variant +# SOURCE OF TRUTH: `big-code-analysis-ast/src/langs.rs` `mk_langs!` — the per-variant # extension lists routed by `get_from_ext`. Keys here are the grammar # *crate* names from the root `Cargo.toml` (e.g. `tree-sitter-kotlin-ng`, # `tree-sitter-c-sharp`), because a grammar bump names the crate. Each @@ -51,7 +51,7 @@ # # Drift here means a bump tests the wrong files (or none). The # `check-grammar-crate-test.py` sync test re-derives this table from -# `src/langs.rs` and fails on any divergence (#869). When `mk_langs!` +# that file and fails on any divergence (#869). When `mk_langs!` # changes (new language, moved extension), update both together. # # Notes on the non-obvious entries: diff --git a/utils/check-publish-metadata-test.py b/utils/check-publish-metadata-test.py index dd2609f0b..7eb7d8c8c 100644 --- a/utils/check-publish-metadata-test.py +++ b/utils/check-publish-metadata-test.py @@ -153,7 +153,9 @@ def test_workspace_inheritance_is_resolved(self) -> None: def test_inheritance_from_a_workspace_without_the_key_is_none(self) -> None: manifest = {"package": {"include": {"workspace": True}}} - self.assertIsNone(GATE.resolve_include(manifest, {"workspace": {"package": {}}})) + self.assertIsNone( + GATE.resolve_include(manifest, {"workspace": {"package": {}}}) + ) self.assertIsNone(GATE.resolve_include(manifest, {})) def test_unrecognised_include_table_is_a_hard_error(self) -> None: @@ -238,13 +240,20 @@ def test_sums_only_files_present_on_disk(self) -> None: with tempfile.TemporaryDirectory() as raw: root = pathlib.Path(raw) self._tree(root) - listing = ["src/lib.rs", "README.md", "Cargo.toml.orig", ".cargo_vcs_info.json"] + listing = [ + "src/lib.rs", + "README.md", + "Cargo.toml.orig", + ".cargo_vcs_info.json", + ] self.assertEqual(GATE.measure_listing(listing, root), 123) def test_every_generated_entry_is_skipped(self) -> None: with tempfile.TemporaryDirectory() as raw: root = pathlib.Path(raw) - self.assertEqual(GATE.measure_listing(sorted(GATE.GENERATED_ENTRIES), root), 0) + self.assertEqual( + GATE.measure_listing(sorted(GATE.GENERATED_ENTRIES), root), 0 + ) def test_an_unexpected_absent_entry_is_a_hard_error(self) -> None: # Without this the wrong-base-directory case totals zero bytes @@ -377,7 +386,9 @@ def test_stdout_is_returned_on_success(self) -> None: self.assertEqual(GATE.run_cargo(["metadata"], REPO_ROOT), "out") def test_non_zero_exit_is_a_hard_error_carrying_stderr(self) -> None: - completed = subprocess.CompletedProcess(["cargo"], 101, stdout="", stderr="boom") + completed = subprocess.CompletedProcess( + ["cargo"], 101, stdout="", stderr="boom" + ) with ( mock.patch.object(subprocess, "run", return_value=completed), self.assertRaises(SystemExit) as caught, @@ -495,7 +506,10 @@ def test_a_dropped_include_at_the_workspace_root_is_reported(self) -> None: self.assertIn("include", problems[0]) def test_an_oversized_crate_is_reported(self) -> None: - with self.stubbed_listings(), mock.patch.object(GATE, "MAX_PACKAGED_BYTES", 600): + with ( + self.stubbed_listings(), + mock.patch.object(GATE, "MAX_PACKAGED_BYTES", 600), + ): problems = GATE.audit(self.metadata()) self.assertEqual(len(problems), 1, problems) self.assertIn("root:", problems[0]) @@ -544,7 +558,9 @@ def test_a_clean_workspace_exits_zero_and_reports_the_count(self) -> None: self.assertEqual(err, "") def test_findings_exit_one_and_reach_stderr_with_remediation(self) -> None: - code, out, err = self._run([" demo: [package].description is missing or empty"]) + code, out, err = self._run( + [" demo: [package].description is missing or empty"] + ) self.assertEqual(code, 1) self.assertEqual(out, "") self.assertIn("demo: [package].description", err) @@ -585,16 +601,22 @@ def test_cargo_packages_the_declared_readme_without_being_asked(self) -> None: parent["readme"], GATE.package_listing("big-code-analysis", REPO_ROOT) ) - def test_the_three_top_level_crates_are_the_ones_checked(self) -> None: + def test_the_four_top_level_crates_are_the_ones_checked(self) -> None: # Pins the discovery rule, not just that discovery found # something: `publish = false` on the bench and Python crates is # what keeps them out, and a change there must be deliberate. names = sorted( - entry["name"] for entry in GATE.publishable_packages(GATE.cargo_metadata(REPO_ROOT)) + entry["name"] + for entry in GATE.publishable_packages(GATE.cargo_metadata(REPO_ROOT)) ) self.assertEqual( names, - ["big-code-analysis", "big-code-analysis-cli", "big-code-analysis-web"], + [ + "big-code-analysis", + "big-code-analysis-ast", + "big-code-analysis-cli", + "big-code-analysis-web", + ], ) diff --git a/utils/check-publish-metadata.py b/utils/check-publish-metadata.py index 8cb015d9c..0c7c57bae 100644 --- a/utils/check-publish-metadata.py +++ b/utils/check-publish-metadata.py @@ -9,7 +9,7 @@ ``cargo publish --dry-run`` is the natural pre-tag gate: it rejects a missing ``description`` / ``license``, a ``readme`` pointing outside the package, and an ``include`` whitelist that stopped covering what the -crate needs. For the three top-level crates it cannot run before the +crate needs. For the four top-level crates it cannot run before the tag, and the workaround that was in place did not run *ever*. ``big-code-analysis`` pins each vendored grammar leaf at diff --git a/utils/check-rustfmt-bail-test.py b/utils/check-rustfmt-bail-test.py index 3c6490f39..6e83c0557 100755 --- a/utils/check-rustfmt-bail-test.py +++ b/utils/check-rustfmt-bail-test.py @@ -195,9 +195,33 @@ def test_vendored_grammar_crates_resolve_to_their_own_edition(self) -> None: "2021", ) self.assertEqual( - gate.edition_for(REPO_ROOT / "src/getter/lua.rs", REPO_ROOT), "2024" + gate.edition_for(REPO_ROOT / "src/lib.rs", REPO_ROOT), "2024" ) + def test_a_member_crate_inheriting_the_edition_resolves_to_2024(self) -> None: + # The third crate shape, and since #1376 the one that owns 19 of + # the 25 baselined bail files: a workspace *member* whose + # manifest spells `edition.workspace = true` rather than a + # literal. `edition_for` has no inheritance branch and reaches + # DEFAULT_EDITION for it, which is right today only because the + # workspace edition is also 2024 -- pin it so a workspace bump + # that leaves the fallback behind fails here rather than as + # "rustfmt refused to parse these files". + # + # The paths are checked for existence first: an `edition_for` + # lookup never stats its argument (it walks parents looking for + # a Cargo.toml), so a stale path resolves through the repo root + # and asserts nothing -- which is how `src/getter/lua.rs` + # survived this file's move. + for relative in ( + "big-code-analysis-ast/src/getter/lua.rs", + "big-code-analysis-cli/src/lib.rs", + ): + with self.subTest(path=relative): + path = REPO_ROOT / relative + self.assertTrue(path.is_file(), f"{relative} must exist") + self.assertEqual(gate.edition_for(path, REPO_ROOT), "2024") + @unittest.skipUnless(HAVE_RUSTFMT, "rustfmt not installed") def test_a_2024_keyword_as_an_identifier_parses_under_2021(self) -> None: # `gen` became a keyword in 2024. This is the concrete shape the diff --git a/utils/check-rustfmt-bail.py b/utils/check-rustfmt-bail.py index 94cb9fbd1..782645e2a 100755 --- a/utils/check-rustfmt-bail.py +++ b/utils/check-rustfmt-bail.py @@ -22,7 +22,8 @@ Probing every arm is load-bearing. The bail is scoped to the enclosing ``match``, so a file whose first match formats cleanly still hides a later one that does not; the first-arm-only version of this probe gave -11 false verdicts out of 18 bailing modules in ``src/getter/`` (#1136). +11 false verdicts out of 18 bailing modules in +``big-code-analysis-ast/src/getter/`` (#1136). ## Three causes, one measurement @@ -304,9 +305,10 @@ def probe_arms( The text is fed to rustfmt on **stdin**. That matters: given a path, rustfmt resolves and recurses into ``mod`` declarations, so probing - ``src/getter.rs`` or ``src/metrics/cognitive.rs`` standalone errors - out with "file not found for module" — an error that reads exactly - like a bail if stderr is discarded, and the reason + ``big-code-analysis-ast/src/getter.rs`` or + ``src/metrics/cognitive.rs`` standalone errors out with "file not + found for module" — an error that reads exactly like a bail if + stderr is discarded, and the reason ``.claude/rules/formatting.md`` calls those two files unprobeable. On stdin there is nothing to resolve, so they probe like any other. """ @@ -361,9 +363,10 @@ def discover_targets(root: pathlib.Path) -> list[pathlib.Path]: The sweep is workspace-wide on purpose. The site list in ``.claude/rules/formatting.md`` was scoped to ``src/metrics/`` for - two revisions, which is exactly why ``src/getter/`` — where the bail - is close to universal — went unmentioned: a directory nobody swept - reads as clean. + two revisions, which is exactly why + ``big-code-analysis-ast/src/getter/`` — where the bail is close to + universal — went unmentioned: a directory nobody swept reads as + clean. """ listed = subprocess.run( ["git", "ls-files", "-z", "*.rs"], diff --git a/utils/check-versions-test.py b/utils/check-versions-test.py index 63c95f3ac..8e3fbd782 100644 --- a/utils/check-versions-test.py +++ b/utils/check-versions-test.py @@ -85,6 +85,18 @@ def test_consumer_pin_still_recognized(self) -> None: line = 'big-code-analysis = { path = "..", version = "=1.1.0" }' self.assertEqual(_scan_internal_pins(line), [("big-code-analysis", "1.1.0")]) + def test_sub_crate_consumer_pin_is_recognized(self) -> None: + # #1376: the root pins `big-code-analysis-ast` at `=X.Y.Z` in + # both `[dependencies]` and `[dev-dependencies]`. The pre-#1376 + # key alternative was a bare `big-code-analysis` under + # `fullmatch`, so neither pin was seen and the gate still + # reported every owned crate in lockstep. + line = ( + 'big-code-analysis-ast = { path = "big-code-analysis-ast", ' + 'version = "=1.1.0", default-features = false }' + ) + self.assertEqual(_scan_internal_pins(line), [("big-code-analysis-ast", "1.1.0")]) + def test_external_grammar_table_not_treated_as_internal(self) -> None: # A non-vendored grammar declared as an inline table (no # bca-* package alias) must NOT be swept into the internal pin @@ -108,13 +120,15 @@ def test_old_key_only_pattern_would_have_missed_vendored(self) -> None: def test_real_manifests_expose_all_vendored_pins(self) -> None: # Across the real INTERNAL_PIN_MANIFESTS the scan must find the - # 10 vendored grammar pins plus the 2 consumer pins (12 total), - # all at the canonical workspace version. + # 10 vendored grammar pins, the 2 `big-code-analysis` consumer + # pins, and the 2 `big-code-analysis-ast` pins the root carries + # in `[dependencies]` and `[dev-dependencies]` since #1376 + # (14 total), all at the canonical workspace version. canonical = cv.workspace_version(REPO_ROOT) pins: list[tuple[str, str]] = [] for manifest_path in cv.INTERNAL_PIN_MANIFESTS: pins += _scan_internal_pins(cv.read(REPO_ROOT / manifest_path)) - self.assertEqual(len(pins), 12, pins) + self.assertEqual(len(pins), 14, pins) self.assertTrue(all(ver == canonical for _, ver in pins), pins) diff --git a/utils/check-versions.py b/utils/check-versions.py index b55b3b9cd..6343e4cec 100755 --- a/utils/check-versions.py +++ b/utils/check-versions.py @@ -52,6 +52,7 @@ INTERNAL_PIN_MANIFESTS = ( "Cargo.toml", "enums/Cargo.toml", + "big-code-analysis-ast/Cargo.toml", "big-code-analysis-cli/Cargo.toml", "big-code-analysis-web/Cargo.toml", ) @@ -145,9 +146,17 @@ INTERNAL_TABLE_RE = re.compile(r"(?P[\w-]+)\s*=\s*\{(?P[^}]*?)\}") INTERNAL_VERSION_PIN_RE = re.compile(r"\bversion\s*=\s*\"=([^\"]+)\"") # An internal crate is identified by the dependency table KEY being -# `big-code-analysis` / `bca-tree-sitter-*`, OR by the table body -# aliasing a `bca-tree-sitter-*` package (the vendored grammar form). -_INTERNAL_KEY_RE = re.compile(r"bca-tree-sitter-[\w-]+|big-code-analysis") +# `big-code-analysis` / `big-code-analysis-*` / `bca-tree-sitter-*`, OR +# by the table body aliasing a `bca-tree-sitter-*` package (the vendored +# grammar form). +# +# The `-*` suffix group is what makes `big-code-analysis-ast` (#1376) +# match: the pre-#1376 spelling was a bare `big-code-analysis` +# alternative under `fullmatch`, so the root's two `=X.Y.Z` pins on the +# parse layer were silently skipped while the gate still reported +# "versions OK: every owned crate at …". Any future owned +# `big-code-analysis-` is covered by construction. +_INTERNAL_KEY_RE = re.compile(r"bca-tree-sitter-[\w-]+|big-code-analysis(?:-[\w-]+)?") _INTERNAL_PACKAGE_RE = re.compile(r"\bpackage\s*=\s*\"bca-tree-sitter-[\w-]+\"") # Match: `big-code-analysis = "X.Y.Z"`, `bca-tree-sitter-* = "X.Y"`, # or `big-code-analysis = "= X.Y.Z"` style snippets in doc prose. From 9b010847a61ab6f8a2a329219cdfbaab21164ae9 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Thu, 10 Sep 2026 09:09:37 -0700 Subject: [PATCH 4/9] docs: point at big-code-analysis-ast --- .claude/rules/formatting.md | 13 +-- .claude/rules/grammar-dispatch.md | 15 +-- .claude/rules/testing.md | 2 +- .claude/rules/tool-output.md | 2 +- .claude/skills/add-lang/SKILL.md | 101 ++++++++++++------ .claude/skills/audit-crate/SKILL.md | 8 +- .claude/skills/audit-file/SKILL.md | 4 +- .claude/skills/audit-naming/SKILL.md | 8 +- .claude/skills/audit-tests/SKILL.md | 2 +- .claude/skills/batch-fix/SKILL.md | 18 ++-- .claude/skills/cleanup-crate/SKILL.md | 6 +- .claude/skills/fix-issue/SKILL.md | 6 +- .claude/skills/improve-crate/SKILL.md | 10 +- .claude/skills/issue-plan/SKILL.md | 4 +- .claude/skills/lessons-learned/SKILL.md | 2 +- .claude/skills/review/SKILL.md | 2 +- .claude/skills/rust-optimize/SKILL.md | 2 +- .claude/skills/scan-project/SKILL.md | 92 ++++++++-------- .claude/skills/simplify-rust/SKILL.md | 2 +- AGENTS.md | 38 ++++--- CONTRIBUTING.md | 2 +- README.ja.md | 4 +- README.md | 2 +- RELEASING.md | 50 +++++---- big-code-analysis-book/src/commands/check.md | 2 +- .../src/developers/new-language.md | 14 +-- big-code-analysis-book/src/languages.md | 2 +- big-code-analysis-book/src/metrics.md | 2 +- docs/development/benchmarking.md | 8 +- docs/development/fuzzing.md | 4 +- docs/development/mutation_testing.md | 10 +- docs/file-detection.md | 17 +-- enums/src/lib.rs | 2 +- tests/README.md | 4 +- 34 files changed, 263 insertions(+), 197 deletions(-) diff --git a/.claude/rules/formatting.md b/.claude/rules/formatting.md index 78fddaf09..ec40bc03e 100644 --- a/.claude/rules/formatting.md +++ b/.claude/rules/formatting.md @@ -66,7 +66,7 @@ Three things that version gets right and the hand-rolled probe below did not: - **It feeds rustfmt on stdin.** Given a *path*, rustfmt resolves and - recurses into `mod` declarations, so `src/getter.rs` and + recurses into `mod` declarations, so `big-code-analysis-ast/src/getter.rs` and `src/metrics/cognitive.rs` error out with "file not found for module" — which reads exactly like a bail if stderr is discarded. On stdin there is nothing to resolve, so those two probe like any other file. @@ -81,10 +81,11 @@ did not: literals**. Do not act on a count from a probe that skips this step. - **It probes every arm, not the first.** The bail is match-scoped, so a file whose first `match` formats cleanly still hides a later one that - does not (`src/getter/c.rs`), and a module whose arms are all + does not (`big-code-analysis-ast/src/getter/c.rs`), and a module whose arms are all expression-bodied (`… => HalsteadType::Operator,` in - `src/getter/go.rs`) has no `=> {` to probe at all. Over `src/getter` - the first-arm-only version gave 11 false verdicts out of 18. + `big-code-analysis-ast/src/getter/go.rs`) has no `=> {` to probe at + all. Over `big-code-analysis-ast/src/getter` the first-arm-only + version gave 11 false verdicts out of 18. ## Two causes, one measurement @@ -114,7 +115,7 @@ never go away. That is a deliberate change from how this section used to read. It carried a hand-maintained list of files, and that list was wrong twice: first by naming only `src/metrics/`, which hid the largest cluster -(`src/getter/`, 18 of 25 modules) for two revisions of this file, and +(`big-code-analysis-ast/src/getter/`, 18 of 25 modules) for two revisions of this file, and then by going stale the moment #1136 hoisted seven comments out of `src/metrics/cognitive/`. A stale list of bailing files reads exactly like a clean tree — the failure this whole rule exists to prevent — so @@ -132,7 +133,7 @@ reported clean, and every one was found by reading the diff instead. ## How to apply -- After any bulk, scripted, or regex edit under `src/getter/` or +- After any bulk, scripted, or regex edit under `big-code-analysis-ast/src/getter/` or `src/metrics/`, read the resulting diff rather than trusting the fmt gate. Check indentation and line length by eye. - Line length is worth a direct check, since it is mechanical: diff --git a/.claude/rules/grammar-dispatch.md b/.claude/rules/grammar-dispatch.md index 938515254..a80bfa204 100644 --- a/.claude/rules/grammar-dispatch.md +++ b/.claude/rules/grammar-dispatch.md @@ -25,13 +25,13 @@ Confirm every numeric-suffix variant of every matched rule is either listed or excluded with a comment: ```bash -rg 'Lang::([A-Za-z]+)\b' src/getter/ src/checker/ src/alterator.rs \ +rg 'Lang::([A-Za-z]+)\b' big-code-analysis-ast/src/getter/ big-code-analysis-ast/src/checker/ big-code-analysis-ast/src/alterator.rs \ src/spaces.rs src/metrics/ ``` The bug class reaches every match on a grammar rule — `alterator.rs`, `spaces.rs`, and `src/metrics/*` are as susceptible as `getter.rs` and -`checker.rs`. Centralised alias sets live in `src/macros/kind_sets.rs`; +`checker.rs`. Centralised alias sets live in `big-code-analysis-ast/src/macros/kind_sets.rs`; prefer extending those over open-coding a list. When a rule has many aliases, prefer one `node.kind()` string comparison over enumerating seventeen variants — pay the small runtime cost for forward @@ -42,7 +42,8 @@ manifests in lockstep (root `Cargo.toml` and `enums/Cargo.toml` — the excluded crate cannot inherit the workspace pin), then: ```bash -cargo run --manifest-path ./enums/Cargo.toml -- -lrust -o ./src/languages +cargo run --manifest-path ./enums/Cargo.toml -- \ + -lrust -o ./big-code-analysis-ast/src/languages ``` Stable *named*-node ids are not evidence the ids held. Inserting one @@ -55,7 +56,7 @@ anonymous terminal renumbers the whole anonymous block after it, so A rule whose name begins with `_` (`_string`, `_multiline_string_literal`) is hidden: the variant exists in the enum and the parser never emits it. Check the `Lang::Variant => "name"` arm in -`src/languages/language_.rs` before listing a "looks like an alias" +`big-code-analysis-ast/src/languages/language_.rs` before listing a "looks like an alias" variant. Keep the defensive arm *and* pin its hidden status with a `!ast_has_kind_id(&parser, Lang::HiddenVariant as u16)` assertion naming the hidden rule — otherwise a future grammar that promotes the rule @@ -92,8 +93,8 @@ new one. **Re-derive it rather than trusting the list below** — it moves whenever a language is added: ```bash -rg -o 'impl_is_else_if_(\w+)!\(\s*(\w+)' -r '$1 $2' src/checker/ --no-filename | sort -rg -l 'fn is_else_if' src/checker/ # hand-written impls +rg -o 'impl_is_else_if_(\w+)!\(\s*(\w+)' -r '$1 $2' big-code-analysis-ast/src/checker/ --no-filename | sort +rg -l 'fn is_else_if' big-code-analysis-ast/src/checker/ # hand-written impls ``` | Strategy | Macro | Languages | @@ -280,7 +281,7 @@ that descends — and test-via-revert that arm alone per ## When you fix one language, sweep the rest Every item above is a per-language failure that almost always exists in -siblings. `src/languages/` modules are deliberate clones, so the fix for +siblings. `big-code-analysis-ast/src/languages/` modules are deliberate clones, so the fix for one is the audit table for the other twenty. Build that table in the issue, land the sibling fixes in one commit so the symmetry is visible to a reviewer, and anchor any known-wrong-but-unfixed case with an diff --git a/.claude/rules/testing.md b/.claude/rules/testing.md index 0fdf247c5..c06f9bfae 100644 --- a/.claude/rules/testing.md +++ b/.claude/rules/testing.md @@ -220,7 +220,7 @@ whenever many tests reach a line while all supplying the same value to the part that matters, and the tool cannot see the difference because which-inputs-varied is not what it measures. -`CommaIndex::splits` (`src/cfg_predicate.rs`) measured 11 of 11 regions +`CommaIndex::splits` (`big-code-analysis-ast/src/cfg_predicate.rs`) measured 11 of 11 regions covered and was entered 150,200 times in one run. Replacing its `region.start` lower bound with `0` panics on ordinary input — and before #1105 that perturbation failed **none** of the 3,969 tests then diff --git a/.claude/rules/tool-output.md b/.claude/rules/tool-output.md index de3c6f428..014184413 100644 --- a/.claude/rules/tool-output.md +++ b/.claude/rules/tool-output.md @@ -16,7 +16,7 @@ it: - `sort | uniq -c` — the counts you are hunting are the largest, and they sort **last** unless you passed `-rn`. - `rg` over a tree — hits arrive in path order, so a sweep across - `src/languages/` or `src/getter/` shows the alphabetically early + `big-code-analysis-ast/src/languages/` or `big-code-analysis-ast/src/getter/` shows the alphabetically early languages and hides every one after them. - `cargo test` / `make pre-commit` — the failure summary is at the end, behind all the passing output. diff --git a/.claude/skills/add-lang/SKILL.md b/.claude/skills/add-lang/SKILL.md index 7a00aa2db..f9b14f08c 100644 --- a/.claude/skills/add-lang/SKILL.md +++ b/.claude/skills/add-lang/SKILL.md @@ -25,7 +25,7 @@ Parse `$ARGUMENTS` as: ` = [...]` - `` (required): PascalCase enum variant name, e.g. `Go`, `Ruby`, `Swift`. Must not collide with existing variants in - `src/langs.rs` or `enums/src/languages.rs`. + `big-code-analysis-ast/src/langs.rs` or `enums/src/languages.rs`. - `=` (required): the tree-sitter crate name and pinned version, e.g. `tree-sitter-ruby=0.23.1`. The version MUST be pinned with `=X.Y.Z` (project convention — see `AGENTS.md`). @@ -45,8 +45,10 @@ continuing. - **No public-API breaks** in unrelated crates. The new language variant is itself a public-API addition (acceptable; minor bump); do not change other variants or trait signatures. -- **Pin the grammar version** with `=X.Y.Z` in both `Cargo.toml` and - `enums/Cargo.toml`. Never use a range without explicit user +- **Pin the grammar version** with `=X.Y.Z` in the root `Cargo.toml` + (`[workspace.dependencies]`) and in `enums/Cargo.toml`; the + `big-code-analysis-ast` manifest inherits the pin with + `workspace = true`. Never use a range without explicit user approval. - **Cross-language parity.** All 12 metric trait impls (`Abc`, `Cognitive`, `Cyclomatic`, `Exit`, `Halstead`, `Loc`, `Mi`, @@ -77,7 +79,7 @@ the user's behalf. ### 0b: Validate name and version pin - Confirm `` is not already a variant in - `src/langs.rs` (search for `Lang::`). + `big-code-analysis-ast/src/langs.rs` (search for `Lang::`). - Confirm `=` parses cleanly and that `crates.io/crates/` actually publishes `` — fetch the crate page or `cargo search` to verify. @@ -99,7 +101,7 @@ symbol-level navigation/editing is the default for all `.rs` edits. ## Step 1: Wire up the `enums` codegen helper The `enums` crate is excluded from the default workspace and exists -solely to regenerate `src/languages/language_.rs` from a tree-sitter +solely to regenerate `big-code-analysis-ast/src/languages/language_.rs` from a tree-sitter grammar's node-kind table. Wire it up first so we can produce the enum file before touching the main crate. @@ -182,7 +184,8 @@ empty kind name at one position. If it is missing, add it. From the repo root, mirroring `recreate-grammars.sh`: ```bash -cargo run --manifest-path ./enums/Cargo.toml -- -l rust -o ./src/languages +cargo run --manifest-path ./enums/Cargo.toml -- \ + -l rust -o ./big-code-analysis-ast/src/languages cargo fmt --all ``` @@ -197,8 +200,8 @@ one. The `enums` binary iterates `Lang::into_enum_iter()` and writes one file per registered variant. After running, inspect the diff: ```bash -git status -- src/languages/ -git diff src/languages/ +git status -- big-code-analysis-ast/src/languages/ +git diff big-code-analysis-ast/src/languages/ ``` The new `language_.rs` should appear as a new file. Existing @@ -212,14 +215,15 @@ no longer matching what the workspace clippy gate expects); fix the template — not the emitted output — and re-run codegen. See lesson 17 in `lessons_learned.md`. -Confirm the new file exists at `src/languages/language_.rs` and +Confirm the new file exists at `big-code-analysis-ast/src/languages/language_.rs` and that it begins with `// Code generated; DO NOT EDIT.`. If the project also depends on C-macro tables for the new language (only relevant for C/C++ family preprocessor work), also run: ```bash -cargo run --manifest-path ./enums/Cargo.toml -- -l c_macros -o ./src/c_langs_macros +cargo run --manifest-path ./enums/Cargo.toml -- \ + -l c_macros -o ./big-code-analysis-ast/src/c_langs_macros ``` Most languages do not need this step. @@ -228,16 +232,44 @@ Most languages do not need this step. ## Step 2: Wire the grammar into the main crate -### 2a: Add the grammar to root `Cargo.toml` +### 2a: Wire the grammar into two manifests -Same pinned-version line as 1a, inserted alphabetically among the -other `tree-sitter-*` deps: +Since #1376 the grammar crates are dependencies of +`big-code-analysis-ast`, not of the root crate, so a language needs +three edits across two manifests. Missing any of them leaves +`LANG::` returning `LanguageDisabled` in every build, or leaves an +unused dependency that `cargo +nightly udeps` fails on. -```toml -tree-sitter- = "=" -``` +1. Root `Cargo.toml`, `[workspace.dependencies]` — the version pin, + inserted alphabetically among the other `tree-sitter-*` entries + (same pinned-version line as 1a): + + ```toml + tree-sitter- = "=" + ``` + +2. `big-code-analysis-ast/Cargo.toml` — the optional dependency and the + feature that enables it, plus an entry in `all-languages`: + + ```toml + [dependencies] + tree-sitter- = { workspace = true, optional = true } + + [features] + all-languages = [..., "", ...] + = ["dep:tree-sitter-"] + ``` + +3. Root `Cargo.toml`, `[features]` — the forwarding feature and the + matching `all-languages` entry, so the root's feature set stays a + superset of the sub-crate's: + + ```toml + all-languages = [..., "", ...] + = ["big-code-analysis-ast/"] + ``` -### 2b: Export the generated module from `src/languages/mod.rs` +### 2b: Export the generated module from `big-code-analysis-ast/src/languages/mod.rs` ```rust pub mod language_; @@ -246,7 +278,7 @@ pub use language_::*; Insert alphabetically. -### 2c: Add the language definition to `src/langs.rs` +### 2c: Add the language definition to `big-code-analysis-ast/src/langs.rs` Append a `mk_langs!` tuple alphabetically: @@ -306,7 +338,7 @@ test that parses a deliberately malformed fixture and asserts `blank ≥ 0` and `kind == Unit` at the file level. See lesson 9 in `lessons_learned.md` (issue #80, `dc09eb3`). -### 3a: `Checker` impl in `src/checker.rs` +### 3a: `Checker` impl in `big-code-analysis-ast/src/checker.rs` Append an `impl Checker for Code` block. Required methods: @@ -346,7 +378,7 @@ Append an `impl Checker for Code` block. Required methods: - `is_primitive` — usually `false` unless the grammar emits a primitive-type kind (most don't). -### 3b: `Getter` impl in `src/getter.rs` +### 3b: `Getter` impl in `big-code-analysis-ast/src/getter.rs` Append an `impl Getter for Code` block. Required methods: @@ -374,7 +406,7 @@ clashed with `use Go::*` in pattern position; the fix was `use Go as G;`. Detect the collision proactively after Step 1e: ```bash -rg "^\s*\s*=" src/languages/language_.rs +rg "^\s*\s*=" big-code-analysis-ast/src/languages/language_.rs ``` If the search returns a hit, alias the import at the top of the @@ -392,7 +424,7 @@ assert the load-bearing invariants from lesson 4: run both assert that `len(dedupe(ops.operators)) == n1` and `len(dedupe(ops.operands)) == n2`. -### 3c: `Alterator` impl in `src/alterator.rs` +### 3c: `Alterator` impl in `big-code-analysis-ast/src/alterator.rs` If the language has string/raw-string/char-literal node kinds whose default text representation should be preserved verbatim (no whitespace @@ -451,7 +483,7 @@ rule-name root for which both an unsuffixed variant and one or more numbered siblings exist in the generated enum: ```bash -LANG_FILE="src/languages/language_.rs" +LANG_FILE="big-code-analysis-ast/src/languages/language_.rs" comm -12 \ <(rg -o '^\s+([A-Z][A-Za-z]*)\d+\s*=' -r '$1' "$LANG_FILE" | sort -u) \ <(rg -o '^\s+([A-Z][A-Za-z]*)\s*=' -r '$1' "$LANG_FILE" | sort -u) @@ -465,8 +497,8 @@ names whose every numbered variant is a potential aliasing risk. For each printed base, confirm that EVERY numbered variant in its group holds one of the following in EVERY file that does a `match` -on the underlying rule (`src/checker.rs`, `src/getter.rs`, -`src/alterator.rs`, `src/metrics/*.rs`, `src/spaces.rs`): +on the underlying rule (`big-code-analysis-ast/src/checker.rs`, `big-code-analysis-ast/src/getter.rs`, +`big-code-analysis-ast/src/alterator.rs`, `src/metrics/*.rs`, `src/spaces.rs`): 1. The variant is explicitly listed in the relevant arm (typically alongside its unsuffixed sibling: @@ -561,7 +593,7 @@ whose name suggests the construct: ```bash rg 'For[A-Z]|While[A-Z]|If[A-Z]|Switch[A-Z]|Conditional|Ternary|Try[A-Z]|Catch[A-Z]|Match[A-Z]|Case[A-Z]' \ - src/languages/language_.rs + big-code-analysis-ast/src/languages/language_.rs ``` Confirm each hit is either explicitly matched, or explicitly excluded @@ -1059,16 +1091,17 @@ Before exiting, print a one-screen summary: Added language support. Files changed: - Cargo.toml + Cargo.toml ([workspace.dependencies] pin + forwarding feature) + big-code-analysis-ast/Cargo.toml (optional dep + feature) enums/Cargo.toml enums/src/languages.rs enums/src/macros.rs - src/langs.rs - src/languages/mod.rs - src/languages/language_.rs (generated) - src/checker.rs - src/getter.rs - src/alterator.rs (if applicable) + big-code-analysis-ast/src/langs.rs + big-code-analysis-ast/src/languages/mod.rs + big-code-analysis-ast/src/languages/language_.rs (generated) + big-code-analysis-ast/src/checker.rs + big-code-analysis-ast/src/getter.rs + big-code-analysis-ast/src/alterator.rs (if applicable) src/metrics/{abc,cognitive,cyclomatic,nexits,halstead,loc,mi,nargs,nom,npa,npm,wmc}.rs big-code-analysis-book/src/languages.md *.snap (new insta snapshots) @@ -1083,7 +1116,7 @@ manually. **Heads up: mutation testing.** Quarterly mutation testing (`.github/workflows/mutation-test.yml`, see `docs/development/mutation_testing.md`) runs against -`src/metrics/`, `src/checker.rs`, and `src/getter.rs`. Within one +`src/metrics/`, `big-code-analysis-ast/src/checker.rs`, and `big-code-analysis-ast/src/getter.rs`. Within one cycle, expect auto-filed issues labelled `mutation-testing` against the new language's impls; treat them as standard fix-issue work. Escapes mean the per-language test set under-specified the new diff --git a/.claude/skills/audit-crate/SKILL.md b/.claude/skills/audit-crate/SKILL.md index edd93e824..90f35c5b5 100644 --- a/.claude/skills/audit-crate/SKILL.md +++ b/.claude/skills/audit-crate/SKILL.md @@ -19,7 +19,7 @@ titles). Never let an unresolved or empty `$ARGUMENTS` reach a template like `audit-state-$ARGUMENTS` — that would write to a malformed memory key. **Memory-key sanitization**: when `$ARGUMENTS` is a directory path (e.g., -`src/languages`), replace `/` with `-` before composing memory keys so the +`big-code-analysis-ast/src/languages`), replace `/` with `-` before composing memory keys so the key is a single flat token (e.g., `audit-state-src-languages`, not `audit-state-src/languages`). This avoids backend interpretation of slashes as path separators. Apply the same rule to every memory-key reference @@ -249,7 +249,7 @@ Group every file into one of these categories before auditing: | Group | Contents | |-------|----------| -| A — Library core | `src/lib.rs` and the modules it exports (`src/languages/`, `src/metrics/`, `src/output/`, `src/spaces.rs`, `src/parser.rs`, `src/checker.rs`, `src/getter.rs`, `src/alterator.rs`, `src/node.rs`, `src/traits.rs`, etc.) | +| A — Library core | `src/lib.rs` and the modules it exports (`big-code-analysis-ast/src/languages/`, `src/metrics/`, `src/output/`, `src/spaces.rs`, `big-code-analysis-ast/src/parser.rs`, `big-code-analysis-ast/src/checker.rs`, `big-code-analysis-ast/src/getter.rs`, `big-code-analysis-ast/src/alterator.rs`, `big-code-analysis-ast/src/node.rs`, `big-code-analysis-ast/src/traits.rs`, etc.) | | B — Binaries | `src/bin/` entries plus the workspace crates `big-code-analysis-cli` and `big-code-analysis-web` when those are the audit target | | C — Tests | `tests/` directory | | D — Supporting files | `README.md`, examples, `Cargo.toml`, `big-code-analysis-book/`, helper scripts, `.claude/rules/` if present | @@ -353,7 +353,7 @@ will apply, so the vocabulary must match end-to-end. Mapping rules: ### Project-Specific (big-code-analysis) -27. Per-language modules under `src/languages/` deliberately mirror each +27. Per-language modules under `big-code-analysis-ast/src/languages/` deliberately mirror each other. Does any change introduce a discrepancy that one language exhibits and another does not (different metric formula, different node-type handling, different operator/operand classification) without justification? @@ -554,7 +554,7 @@ last_model: ## File Coverage src/lib.rs | full | 2026-04-25 | 3 findings | claude-opus-4-7 -src/languages/language_rust.rs | partial | 2026-04-25 | 1 finding | claude-opus-4-7 +big-code-analysis-ast/src/languages/language_rust.rs | partial | 2026-04-25 | 1 finding | claude-opus-4-7 src/metrics/halstead.rs | none | - | - | - tests/parser.rs | full | 2026-04-25 | 0 findings | claude-sonnet-4-6 ``` diff --git a/.claude/skills/audit-file/SKILL.md b/.claude/skills/audit-file/SKILL.md index 93fe98f0f..3bfe73e9d 100644 --- a/.claude/skills/audit-file/SKILL.md +++ b/.claude/skills/audit-file/SKILL.md @@ -248,7 +248,7 @@ Read the target file entirely. Then briefly establish: - Does it export items re-exported from `lib.rs`? (If so, public-API stability rules apply — see checklist Q28.) - Does it belong to a family that mirrors other language modules under - `src/languages/`? (If so, cross-language consistency rules apply — + `big-code-analysis-ast/src/languages/`? (If so, cross-language consistency rules apply — see checklist Q27.) - Does it contain metric computation or AST traversal? (If so, security checklist items Q7-Q10 apply with higher weight.) @@ -342,7 +342,7 @@ will apply. Mapping rules: ### Project-Specific (big-code-analysis) -27. Per-language modules under `src/languages/` deliberately mirror each +27. Per-language modules under `big-code-analysis-ast/src/languages/` deliberately mirror each other. Does any change introduce a discrepancy that one language exhibits and another does not (different metric formula, different node-type handling, different operator/operand classification) without justification? diff --git a/.claude/skills/audit-naming/SKILL.md b/.claude/skills/audit-naming/SKILL.md index 65a2dc830..162a02731 100644 --- a/.claude/skills/audit-naming/SKILL.md +++ b/.claude/skills/audit-naming/SKILL.md @@ -21,7 +21,7 @@ resolved value for every subsequent reference (memory keys, `cargo -p`, issue titles). **Memory-key sanitization**: when `$ARGUMENTS` is a directory path (e.g., -`src/languages`), replace `/` with `-` before composing memory keys so the +`big-code-analysis-ast/src/languages`), replace `/` with `-` before composing memory keys so the key is a single flat token (e.g., `naming-audit-state-src-languages`, not `naming-audit-state-src/languages`). Apply this sanitization to every memory-key reference below. @@ -208,11 +208,11 @@ detect. | Group | Contents | |-------|----------| -| A — Library core | `src/lib.rs`, `src/languages/`, `src/metrics/`, `src/output/`, `src/parser.rs`, `src/checker.rs`, `src/getter.rs`, `src/alterator.rs`, `src/spaces.rs`, `src/node.rs`, `src/traits.rs`, etc. | +| A — Library core | `src/lib.rs`, `big-code-analysis-ast/src/languages/`, `src/metrics/`, `src/output/`, `big-code-analysis-ast/src/parser.rs`, `big-code-analysis-ast/src/checker.rs`, `big-code-analysis-ast/src/getter.rs`, `big-code-analysis-ast/src/alterator.rs`, `src/spaces.rs`, `big-code-analysis-ast/src/node.rs`, `big-code-analysis-ast/src/traits.rs`, etc. | | B — Tests | `tests/` directory and `#[cfg(test)]` modules | | C — Workspace binaries | `big-code-analysis-cli/src/`, `big-code-analysis-web/src/` (when those are the audit target) | -Per-language modules under `src/languages/` deliberately mirror each other — +Per-language modules under `big-code-analysis-ast/src/languages/` deliberately mirror each other — naming inconsistency *between* languages (same concept named differently) is a primary target for this audit. @@ -411,7 +411,7 @@ last_model: ## File Coverage src/lib.rs | full | 2026-04-26 | 3 findings | claude-opus-4-7 -src/languages/language_rust.rs | partial | 2026-04-26 | 1 finding | claude-opus-4-7 +big-code-analysis-ast/src/languages/language_rust.rs | partial | 2026-04-26 | 1 finding | claude-opus-4-7 src/metrics/halstead.rs | none | - | - | - ``` diff --git a/.claude/skills/audit-tests/SKILL.md b/.claude/skills/audit-tests/SKILL.md index e47fd8136..07f575366 100644 --- a/.claude/skills/audit-tests/SKILL.md +++ b/.claude/skills/audit-tests/SKILL.md @@ -135,7 +135,7 @@ These tests verify something different from what their name implies. 16. **Wrong language tested**: Does a test named for one language (e.g., `test_python_function_count`) actually parse a different language's source? Easy to introduce when tests are copy-pasted across the - `src/languages/` modules. + `big-code-analysis-ast/src/languages/` modules. ### Incidental coupling diff --git a/.claude/skills/batch-fix/SKILL.md b/.claude/skills/batch-fix/SKILL.md index 10fe497a2..649a0dbcf 100644 --- a/.claude/skills/batch-fix/SKILL.md +++ b/.claude/skills/batch-fix/SKILL.md @@ -9,7 +9,7 @@ Fix multiple GitHub issues on a single integration branch. Issues are classified by affected crate(s) and triaged for quick-win priority and cross-issue dependencies, then scheduled into waves where issues touching different crates run in parallel. Quick wins are front-loaded for fast -feedback. Issues sharing a crate, or any issue that touches `src/languages/` +feedback. Issues sharing a crate, or any issue that touches `big-code-analysis-ast/src/languages/` (per-language modules deliberately mirror each other), are serialized to avoid merge conflicts. Each issue goes through the full pipeline: investigate, fix, simplify, review, remediate, validate, commit. Successful @@ -132,7 +132,7 @@ Use these signals in priority order: `big-code-analysis` - "checker", "getter", "alterator", "spaces" -> `big-code-analysis` - "language X", a specific language name (rust, python, javascript, c, - cpp, java, kotlin, typescript, etc.), or `src/languages/` path -> + cpp, java, kotlin, typescript, etc.), or `big-code-analysis-ast/src/languages/` path -> `big-code-analysis` with `cross_lang: true` (see special case below) - "CLI", "command-line", "argument", "output format" (JSON/YAML/TOML/CBOR CLI flags) -> `big-code-analysis-cli` @@ -146,8 +146,8 @@ Use these signals in priority order: 3. **Ambiguous**: If the crate cannot be determined from labels or keywords, classify as `unknown`. -**Special case — `src/languages/` and cross-language code**: The -per-language modules under `src/languages/` deliberately mirror each other; +**Special case — `big-code-analysis-ast/src/languages/` and cross-language code**: The +per-language modules under `big-code-analysis-ast/src/languages/` deliberately mirror each other; a bug in one language often exists in several. Issues that touch this directory or any metric implementation that walks the AST should be flagged `cross_lang: true`. These are NOT cross-crate (they all live in @@ -214,7 +214,7 @@ For each issue, record: - `crate`: the primary affected crate name, or `unknown` - `cross_crate`: `true` if the issue clearly spans multiple crates, `false` otherwise -- `cross_lang`: `true` if the issue touches `src/languages/` or otherwise +- `cross_lang`: `true` if the issue touches `big-code-analysis-ast/src/languages/` or otherwise requires changes mirrored across language modules - `quick_win`: `true` if the issue matches the quick-win criteria above - `depends_on`: list of issue numbers this issue depends on (empty if none) @@ -848,12 +848,12 @@ Follow the `/fix-issue` workflow: any lessons relevant to this issue's domain. 3. Investigate the codebase to understand root cause. For tree-sitter grammar / language-specific behavior, examine the corresponding module - under `src/languages/` and confirm whether the bug is in our wrapper or + under `big-code-analysis-ast/src/languages/` and confirm whether the bug is in our wrapper or upstream in the grammar crate. If the bug is upstream, scope the fix accordingly (workaround locally, file an issue against the grammar repo, or both — do NOT silently paper over an upstream grammar bug). 4. **Check for the same bug pattern across sibling languages.** The - `src/languages/` modules deliberately mirror each other; a bug in one + `big-code-analysis-ast/src/languages/` modules deliberately mirror each other; a bug in one language's metric implementation often exists in several. If the root cause is repeated, fix all instances. Similarly, if metric code under `src/metrics/` has the same anti-pattern in multiple metrics, fix all of @@ -912,7 +912,7 @@ Follow the `/fix-issue` workflow: `cargo build --workspace` before integration tests so they exercise the new binaries — never test against a stale binary. - **Per-language coverage**: if the fix touches metric computation, AST - traversal, or any code under `src/languages/`, exercise **every** + traversal, or any code under `big-code-analysis-ast/src/languages/`, exercise **every** language affected. - **Snapshot tests** (`insta`): if existing snapshots changed, run `cargo insta test --review` and accept each diff individually rather @@ -968,7 +968,7 @@ directly: - Helper functions that duplicate standard library or crate functionality - Duplicate logic across sibling language modules that should live in a shared helper, trait method, or macro (the project already uses - `src/c_langs_macros/`, `src/macros/`, and `src/c_macro.rs` for shared + `big-code-analysis-ast/src/c_langs_macros/`, `big-code-analysis-ast/src/macros/`, and `big-code-analysis-ast/src/c_macro.rs` for shared structure; follow `.claude/rules/macro-comments.md` when consolidating) **Clarity**: diff --git a/.claude/skills/cleanup-crate/SKILL.md b/.claude/skills/cleanup-crate/SKILL.md index 5fd956f3c..3ab41cf4e 100644 --- a/.claude/skills/cleanup-crate/SKILL.md +++ b/.claude/skills/cleanup-crate/SKILL.md @@ -170,7 +170,7 @@ The agent must: - **Without Serena**: use Grep across the crate (`pub(crate)`/private) or workspace (`pub`). -3. Per-language modules under `src/languages/` deliberately mirror each +3. Per-language modules under `big-code-analysis-ast/src/languages/` deliberately mirror each other. A symbol that *appears* unused in one language may be present so the language modules expose the same shape — verify by checking sibling modules before flagging. @@ -310,7 +310,7 @@ Before deleting any code, confirm the finding is still valid (re-run 3. Stay within scope — only remove items in your removal area. 4. Never remove items in `#[cfg(test)]` blocks or `tests/` directories. 5. **Per-language consistency**: if you remove a symbol from - `src/languages/language_.rs`, verify the same symbol is also unused + `big-code-analysis-ast/src/languages/language_.rs`, verify the same symbol is also unused in every sibling `language_*.rs`. If a sibling still uses it, do NOT remove — flag as inconsistent and SKIP this area. @@ -476,6 +476,6 @@ If approval-required items exist, suggest `--aggressive`. - Do NOT re-examine files marked clean unless they have new git changes - Do NOT use `git push --force` or destructive git operations - Do NOT delete worktrees -- If a removal would create cross-language inconsistency in `src/languages/`, +- If a removal would create cross-language inconsistency in `big-code-analysis-ast/src/languages/`, SKIP it - When in doubt, leave it alone diff --git a/.claude/skills/fix-issue/SKILL.md b/.claude/skills/fix-issue/SKILL.md index c7f453e90..6253ebd76 100644 --- a/.claude/skills/fix-issue/SKILL.md +++ b/.claude/skills/fix-issue/SKILL.md @@ -16,11 +16,11 @@ description: Complete workflow for fixing GitHub issues including investigation, directly relevant so it can be cited in the fix. 4. Investigate the codebase to understand the root cause. For tree-sitter grammar / language-specific behavior, examine the corresponding module under - `src/languages/` and confirm whether the bug is in our wrapper or upstream + `big-code-analysis-ast/src/languages/` and confirm whether the bug is in our wrapper or upstream in the grammar crate. If the bug is upstream, scope the fix accordingly (workaround locally, file an issue against the grammar repo, or both). 5. Check for the same bug pattern elsewhere in the codebase. The - `src/languages/` modules deliberately mirror each other; a bug in one + `big-code-analysis-ast/src/languages/` modules deliberately mirror each other; a bug in one language's metric implementation often exists in several. Fix all instances — do not leave known-broken siblings for a follow-up. 6. **Plan the fix with explicit step-by-step reasoning.** If the @@ -60,7 +60,7 @@ description: Complete workflow for fixing GitHub issues including investigation, `cargo build --workspace` before integration tests so they exercise the new binaries — never test against a stale binary. - **Per-language coverage**: if the fix touches metric computation, AST - traversal, or any code under `src/languages/`, exercise **every** + traversal, or any code under `big-code-analysis-ast/src/languages/`, exercise **every** language affected. A regression in one language is not caught by passing tests in another. - **Regression check**: `cargo test --workspace` and diff --git a/.claude/skills/improve-crate/SKILL.md b/.claude/skills/improve-crate/SKILL.md index 73894e2db..f18e64548 100644 --- a/.claude/skills/improve-crate/SKILL.md +++ b/.claude/skills/improve-crate/SKILL.md @@ -35,7 +35,7 @@ Parse `$ARGUMENTS` as: ` [--dry-run]` re-exports, public traits (`ParserTrait`, `LanguageInfo`, etc.), and public types (`Metrics`, `FuncSpace`, language enums) are off-limits unless the user explicitly authorizes a version bump -- **Cross-language parity**: per-language modules under `src/languages/` +- **Cross-language parity**: per-language modules under `big-code-analysis-ast/src/languages/` deliberately mirror each other; any change to one usually requires the same change to all sibling language modules - **Do not merge to main**: leave the integration branch for the user @@ -165,7 +165,7 @@ The agent must: - Touches a cohesive set of symbols (ideally within one file) - Can be described in a single conventional commit message - Is independent of other change areas - - For changes under `src/languages/`, ALL affected sibling language + - For changes under `big-code-analysis-ast/src/languages/`, ALL affected sibling language modules are included in the same area (you cannot improve one language module without bringing the rest along) 6. Return as structured list: @@ -279,7 +279,7 @@ Read, Edit, Grep, Glob. `src/lib.rs`, it is part of the published API surface. Do NOT change its signature or behavior. Limit changes to internal implementation. 3. **Cross-language parity**: if your change area includes a symbol in - one `src/languages/language_.rs`, apply the equivalent change in + one `big-code-analysis-ast/src/languages/language_.rs`, apply the equivalent change in every sibling `language_*.rs` that defines the same symbol. 4. Apply improvements: - **With Serena**: `replace_symbol_body`, `insert_before_symbol`, @@ -299,7 +299,7 @@ full. Apply fixes directly: - Manual error-mapping chains replaceable by a single `From` impl - Identical match arms that can be consolidated - Helpers duplicated across `language_*.rs` that could move to - `src/macros/` / `src/c_langs_macros/` / a shared module (follow + `big-code-analysis-ast/src/macros/` / `big-code-analysis-ast/src/c_langs_macros/` / a shared module (follow `.claude/rules/macro-comments.md` when consolidating into macros) **Clarity**: @@ -509,7 +509,7 @@ review. Merge to `main` when satisfied." - Do NOT merge `improve/` into `main` - Do NOT change public APIs, public traits, or data models - Do NOT change items re-exported from `src/lib.rs` without authorization -- Do NOT introduce per-language inconsistency in `src/languages/` +- Do NOT introduce per-language inconsistency in `big-code-analysis-ast/src/languages/` - Do NOT touch code outside the target crate - Do NOT loosen tree-sitter grammar version pins in `Cargo.toml` - Do NOT re-examine symbols marked clean or changed unless the file has diff --git a/.claude/skills/issue-plan/SKILL.md b/.claude/skills/issue-plan/SKILL.md index c202185ca..45d3ef7e4 100644 --- a/.claude/skills/issue-plan/SKILL.md +++ b/.claude/skills/issue-plan/SKILL.md @@ -39,7 +39,7 @@ Before planning, understand the relevant code: `big-code-analysis-cli`, `big-code-analysis-web`, `big-code-analysis-py` (PyO3 bindings), `xtask` (man-page generation), and `enums` (language-enum codegen). Per-language - logic lives under `src/languages/` (one `language_.rs` per + logic lives under `big-code-analysis-ast/src/languages/` (one `language_.rs` per supported language) and metric implementations under `src/metrics/`. 3. Use Serena LSP tools (`find_symbol`, `get_symbols_overview`, `find_referencing_symbols`) — or Grep / Glob if Serena is unavailable — @@ -47,7 +47,7 @@ Before planning, understand the relevant code: 4. Note the scope: how many files, which crate(s), which language modules, whether public API is affected, whether tree-sitter grammar versions are involved. -5. **Cross-language sweep**: if the issue is in `src/languages/` or +5. **Cross-language sweep**: if the issue is in `big-code-analysis-ast/src/languages/` or `src/metrics/`, check whether the same defect exists in sibling language modules. A bug in one usually exists in several. diff --git a/.claude/skills/lessons-learned/SKILL.md b/.claude/skills/lessons-learned/SKILL.md index d4d5b1b54..a9ff6c474 100644 --- a/.claude/skills/lessons-learned/SKILL.md +++ b/.claude/skills/lessons-learned/SKILL.md @@ -294,7 +294,7 @@ edit them unprompted. **Renumbering is a breaking change and is never in scope.** Roughly sixty files cite lessons by number, including production source -(`src/checker/*.rs`, `src/macros/kind_sets.rs`), the cross-language +(`big-code-analysis-ast/src/checker/*.rs`, `big-code-analysis-ast/src/macros/kind_sets.rs`), the cross-language parity tests, `AGENTS.md`, `CONTRIBUTING.md`, the book, and the `Makefile`; #2, #11, #19, #4 and #6 carry the most references. Verify before assuming a number is free: diff --git a/.claude/skills/review/SKILL.md b/.claude/skills/review/SKILL.md index e41d93fd2..1f0457124 100644 --- a/.claude/skills/review/SKILL.md +++ b/.claude/skills/review/SKILL.md @@ -159,7 +159,7 @@ EFFORT: trivial | small | medium 41. Is this the simplest implementation that solves the problem? No over-engineering, premature abstraction, or speculative generality. 42. Are there duplicated logic blocks that should share a helper or macro? - (Per-language modules in `src/languages/` deliberately use macros for + (Per-language modules in `big-code-analysis-ast/src/languages/` deliberately use macros for shared structure — extend them rather than copy-pasting.) 43. Are there functions longer than ~50 lines of logic that should be decomposed? diff --git a/.claude/skills/rust-optimize/SKILL.md b/.claude/skills/rust-optimize/SKILL.md index b03c7fd37..9051b84ed 100644 --- a/.claude/skills/rust-optimize/SKILL.md +++ b/.claude/skills/rust-optimize/SKILL.md @@ -324,7 +324,7 @@ Eliminates custom `Serialize`/`Deserialize` impls on single-field wrappers. ### E3. Replace repetitive `impl` blocks with declarative macros If 3+ types share identical method implementations differing only in type name, extract a `macro_rules!` to generate them. (This pattern is already used heavily -across `src/languages/` — prefer extending the existing macros over inventing new +across `big-code-analysis-ast/src/languages/` — prefer extending the existing macros over inventing new ones.) When consolidating, follow `.claude/rules/macro-comments.md`: keep the macro body minimal and hoist per-call rationale comments above each invocation, never into the macro definition. diff --git a/.claude/skills/scan-project/SKILL.md b/.claude/skills/scan-project/SKILL.md index cfd992051..17ed68c31 100644 --- a/.claude/skills/scan-project/SKILL.md +++ b/.claude/skills/scan-project/SKILL.md @@ -249,9 +249,9 @@ Files: - `src/metrics/npm.rs` - `src/metrics/tokens.rs` - `src/metrics/wmc.rs` -- `src/checker.rs` -- `src/getter.rs` -- `src/alterator.rs` +- `big-code-analysis-ast/src/checker.rs` +- `big-code-analysis-ast/src/getter.rs` +- `big-code-analysis-ast/src/alterator.rs` Checklist focus: all 50 questions. Special attention to Q31–Q36, Q39–Q42, Q44–Q46, Q50 (metrics-specific section). @@ -259,10 +259,10 @@ Q44–Q46, Q50 (metrics-specific section). ### Partition B — JS-family language modules Files: -- `src/languages/language_mozjs.rs` -- `src/languages/language_javascript.rs` -- `src/languages/language_typescript.rs` -- `src/languages/language_tsx.rs` +- `big-code-analysis-ast/src/languages/language_mozjs.rs` +- `big-code-analysis-ast/src/languages/language_javascript.rs` +- `big-code-analysis-ast/src/languages/language_typescript.rs` +- `big-code-analysis-ast/src/languages/language_tsx.rs` Checklist focus: all 50 questions. Special attention to Q32–Q36, Q38–Q45 (aliased variants, sibling parity, Halstead, dispatch gaps). @@ -281,12 +281,12 @@ Report any discrepancy as a separate FINDING. ### Partition C — C-family language modules Files: -- `src/languages/language_c.rs` -- `src/languages/language_cpp.rs` -- `src/languages/language_mozcpp.rs` -- `src/languages/language_csharp.rs` -- `src/languages/language_java.rs` -- `src/languages/language_kotlin.rs` +- `big-code-analysis-ast/src/languages/language_c.rs` +- `big-code-analysis-ast/src/languages/language_cpp.rs` +- `big-code-analysis-ast/src/languages/language_mozcpp.rs` +- `big-code-analysis-ast/src/languages/language_csharp.rs` +- `big-code-analysis-ast/src/languages/language_java.rs` +- `big-code-analysis-ast/src/languages/language_kotlin.rs` Checklist focus: all 50 questions. Special attention to Q32, Q37–Q42 (grammar root, else-if structural model, cross-language parity, dispatch gaps). @@ -297,24 +297,24 @@ the JVM/managed side (`csharp`, `java`, `kotlin`). ### Partition D — Other language modules -Files (every `src/languages/language_*.rs` not in B or C): -- `src/languages/language_python.rs` -- `src/languages/language_rust.rs` -- `src/languages/language_go.rs` -- `src/languages/language_bash.rs` -- `src/languages/language_php.rs` -- `src/languages/language_ruby.rs` -- `src/languages/language_lua.rs` -- `src/languages/language_perl.rs` -- `src/languages/language_tcl.rs` -- `src/languages/language_elixir.rs` -- `src/languages/language_groovy.rs` -- `src/languages/language_objc.rs` -- `src/languages/language_irules.rs` -- `src/languages/language_ccomment.rs` -- `src/languages/language_preproc.rs` - -If `ls src/languages/language_*.rs` reveals a module not in this list +Files (every `big-code-analysis-ast/src/languages/language_*.rs` not in B or C): +- `big-code-analysis-ast/src/languages/language_python.rs` +- `big-code-analysis-ast/src/languages/language_rust.rs` +- `big-code-analysis-ast/src/languages/language_go.rs` +- `big-code-analysis-ast/src/languages/language_bash.rs` +- `big-code-analysis-ast/src/languages/language_php.rs` +- `big-code-analysis-ast/src/languages/language_ruby.rs` +- `big-code-analysis-ast/src/languages/language_lua.rs` +- `big-code-analysis-ast/src/languages/language_perl.rs` +- `big-code-analysis-ast/src/languages/language_tcl.rs` +- `big-code-analysis-ast/src/languages/language_elixir.rs` +- `big-code-analysis-ast/src/languages/language_groovy.rs` +- `big-code-analysis-ast/src/languages/language_objc.rs` +- `big-code-analysis-ast/src/languages/language_irules.rs` +- `big-code-analysis-ast/src/languages/language_ccomment.rs` +- `big-code-analysis-ast/src/languages/language_preproc.rs` + +If `ls big-code-analysis-ast/src/languages/language_*.rs` reveals a module not in this list (or in Partitions B/C), add it to this partition and flag the omission in the Step 8 summary so the file list can be refreshed. @@ -326,11 +326,11 @@ dispatch gaps). Files: - `src/spaces.rs` -- `src/node.rs` -- `src/parser.rs` -- `src/traits.rs` -- `src/macros/` (all files) -- `src/c_macro.rs` +- `big-code-analysis-ast/src/node.rs` +- `big-code-analysis-ast/src/parser.rs` +- `big-code-analysis-ast/src/traits.rs` +- `big-code-analysis-ast/src/macros/` (all files) +- `big-code-analysis-ast/src/c_macro.rs` - `src/lib.rs` Checklist focus: all 50 questions. Special attention to Q5–Q7, Q28, Q41, @@ -432,7 +432,7 @@ Track depth per file: `full` | `partial` | `skimmed`. ### Section F — Project-Specific Baseline (Q27–Q30) -27. Per-language modules under `src/languages/` deliberately mirror each other. +27. Per-language modules under `big-code-analysis-ast/src/languages/` deliberately mirror each other. Does any change introduce a discrepancy that one language exhibits and another does not (different metric formula, different node-type handling, different operator/operand classification) without justification? @@ -471,8 +471,8 @@ Partitions A–D. numeric-suffix variants (`Kind2`, `Kind3`, … `Kind17`) of every matched rule either explicitly listed or explicitly excluded with a comment? Run: ```bash - rg 'Lang::([A-Za-z]+)\b' src/getter.rs src/checker.rs \ - src/alterator.rs src/spaces.rs src/metrics/ + rg 'Lang::([A-Za-z]+)\b' big-code-analysis-ast/src/getter.rs big-code-analysis-ast/src/checker.rs \ + big-code-analysis-ast/src/alterator.rs src/spaces.rs src/metrics/ ``` then cross-reference against the regenerated `language_.rs` to confirm every suffixed variant is accounted for. @@ -484,7 +484,7 @@ Partitions A–D. arm present and correct in all three siblings? Run: ```bash rg '' \ - src/languages/language_{javascript,mozjs,typescript,tsx}.rs \ + big-code-analysis-ast/src/languages/language_{javascript,mozjs,typescript,tsx}.rs \ src/{getter,checker}.rs ``` Apply the same check to C-family siblings (`c`, `cpp`, `mozcpp`) and to @@ -587,15 +587,15 @@ Partitions A–D. ```bash # Loops that might be missing from cyclomatic/cognitive dispatch: - rg 'For[A-Z]' src/languages/ + rg 'For[A-Z]' big-code-analysis-ast/src/languages/ # Ternary/conditional that might be missing: - rg 'Conditional|Ternary' src/languages/ + rg 'Conditional|Ternary' big-code-analysis-ast/src/languages/ # Enhanced/range-based loop variants: - rg 'Enhanced|Range|For[A-Z]' src/languages/ + rg 'Enhanced|Range|For[A-Z]' big-code-analysis-ast/src/languages/ # Pattern matching / structural match: - rg 'Match|Case[A-Z]|Pattern' src/languages/ + rg 'Match|Case[A-Z]|Pattern' big-code-analysis-ast/src/languages/ # Nullish / short-circuit operators: - rg 'Nullish|NullCoal' src/languages/ + rg 'Nullish|NullCoal' big-code-analysis-ast/src/languages/ ``` For each candidate kind found, confirm it is either: @@ -827,7 +827,7 @@ Also list: - Any partitions skipped (priority 5 — recently scanned and unchanged). - Any partitions whose scope listed files that did not exist on disk — name the missing files. This catches drift between the skill's - partition tables and `src/languages/`. + partition tables and `big-code-analysis-ast/src/languages/`. If `/tmp/scan-metrics.json` from Step 2b is missing or empty, prefix the summary with a `DEGRADED: metric hotspots unavailable — ` banner diff --git a/.claude/skills/simplify-rust/SKILL.md b/.claude/skills/simplify-rust/SKILL.md index 54db080de..ef17da7ad 100644 --- a/.claude/skills/simplify-rust/SKILL.md +++ b/.claude/skills/simplify-rust/SKILL.md @@ -57,7 +57,7 @@ Look for duplicated logic and missing abstractions. - Manual error mapping chains replaceable by a single `From` impl - Identical match arms that can be consolidated - Helper functions that duplicate standard library or crate functionality -- Per-language duplicated logic in `src/languages/` that could be expressed via +- Per-language duplicated logic in `big-code-analysis-ast/src/languages/` that could be expressed via a trait method or macro instead of being copied across language modules (when consolidating into a macro, follow `.claude/rules/macro-comments.md`: hoist per-language rationale comments above each invocation, not into the diff --git a/AGENTS.md b/AGENTS.md index d9e192402..108eddb96 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,8 @@ The repository is a Cargo workspace: | Crate | Path | Purpose | |-------|------|---------| -| `big-code-analysis` | `./` (root) | Library: parsers, AST traversal, metric computation | +| `big-code-analysis` | `./` (root) | Library: the metric walk, output formats, suppression, VCS metrics; the published API surface | +| `big-code-analysis-ast` | `big-code-analysis-ast/` | Parse and classification layer (#1376): generated kind enums, `LANG`, `Node`, `Checker` / `Getter` / `Alterator`, the C preprocessor pass. Internal plumbing the root pins at `=X.Y.Z`; depend on the root, not on it | | `big-code-analysis-cli` | `big-code-analysis-cli/` | CLI for invoking the library on files / trees | | `big-code-analysis-py` | `big-code-analysis-py/` (excluded from default-members; needs Python headers + maturin) | PyO3 Python bindings | | `big-code-analysis-web` | `big-code-analysis-web/` | REST API server wrapping the library | @@ -40,19 +41,30 @@ and `cargo run -p big-code-analysis-web --`. ## Project layout - `src/lib.rs` — public re-exports; this is the published API surface. -- `src/languages/` — one `language_.rs` per supported language. These - modules deliberately mirror each other; macros under - `src/c_langs_macros/`, `src/macros/`, and `src/c_macro.rs` generate the - shared structure. A - bug in one language module typically exists in several — fix all - affected siblings together. + Everything the parse layer defines reaches the metric modules through + one `pub(crate) use big_code_analysis_ast::*` here, so `use crate::*` + in a metric file still names every token enum, tag and classifier. +- `big-code-analysis-ast/src/languages/` — one `language_.rs` per + supported language, generated by `enums/`. These modules deliberately + mirror each other, as do the per-language `checker/`, `getter/` and + `src/metrics//` siblings. A bug in one language module + typically exists in several — fix all affected siblings together. +- `big-code-analysis-ast/src/` — `langs.rs` (`LANG`, `AnyParser`), + `node.rs`, `parser.rs`, `traits.rs` (`ParserTrait`), `checker.rs`, + `getter.rs`, `alterator.rs`, `lang_helpers/`, `preproc.rs`, + `c_macro.rs`, `comment_rm.rs`, `ast.rs`, `count.rs`, `find.rs`, + `tools.rs` — everything that turns bytes into a classified tree and + computes no metric. `macros/mod.rs` there holds `mk_langs!` and the + `with_any_parser!` dispatch macro. - `src/metrics/` — individual metric implementations: `abc.rs`, `cognitive.rs`, `cyclomatic.rs`, `nexits.rs`, `halstead.rs`, `loc.rs`, `mi.rs`, `nargs.rs`, `nom.rs`, `npa.rs`, `npm.rs`, `tokens.rs`, `wmc.rs`. - `src/output/` — JSON / YAML / TOML / CBOR serializers for metric output. -- `src/parser.rs`, `src/node.rs`, `src/spaces.rs`, `src/checker.rs`, - `src/getter.rs`, `src/alterator.rs`, `src/traits.rs` — core AST plumbing. +- `src/spaces.rs` / `src/spaces/` — `Ast`, `analyze`, `FuncSpace`, and + the metric walk; `src/metric_suite.rs` — the `MetricSuite` supertrait + that keys the 13 per-metric impls on a parser; `src/macros/mod.rs` — + `implement_metric_trait!`. - `tests/` — integration tests, including `insta` snapshot tests (`*.snap` / `*.snap.new`). - `big-code-analysis-book/` — mdBook documentation source. @@ -130,7 +142,7 @@ and `cargo run -p big-code-analysis-web --`. (`find_referencing_symbols` if an LSP tool is available, otherwise a workspace-wide search). Cross-crate breakage is silent until CI. - When a change touches metric computation, AST traversal, or anything - under `src/languages/`, exercise **every** language affected — passing + under `big-code-analysis-ast/src/languages/`, exercise **every** language affected — passing tests in one language do not catch regressions in another. Per-language modules deliberately mirror each other; a bug in one typically exists in several. @@ -451,13 +463,15 @@ exact form is `Node::parent`'s `O(depth)` per node and made every debug-build walk quadratic (#1122). The approximation misses a chain that is short by exactly one, so run this around any change to a walk's truncate/push bookkeeping — `src/spaces/compute.rs`, `src/ops.rs`, -`src/comment_rm.rs`, `src/suppression.rs`, `Search::act_on_node`. It is +`src/suppression.rs`, and in `big-code-analysis-ast`, `comment_rm.rs` and +`Search::act_on_node`. It is not part of `make pre-commit`; the `chain-audit` CI job runs it per PR. See [Benchmarking](docs/development/benchmarking.md#chain-audit). **Mutation testing** runs out-of-band on a quarterly cron via `.github/workflows/mutation-test.yml` against `src/metrics/`, -`src/checker.rs`, and `src/getter.rs`. It is intentionally not part of +`big-code-analysis-ast/src/checker.rs`, and +`big-code-analysis-ast/src/getter.rs`. It is intentionally not part of the per-PR gate (a full run is tens of minutes per file). Escapes auto-file a GitHub issue labelled `mutation-testing`. See [`docs/development/mutation_testing.md`](docs/development/mutation_testing.md) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a7ca4f472..539dc6b0e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -347,7 +347,7 @@ for why this matters. paths used as identifiers (map keys, JSON output, error correlation); use `to_str()` with explicit error handling. - **Per-language modules mirror each other**: a bug in one - `src/languages/language_.rs` typically exists in several. Fix + `big-code-analysis-ast/src/languages/language_.rs` typically exists in several. Fix every affected sibling together. - **Public API**: this is a published library on crates.io. Treat `lib.rs` re-exports, public traits (`ParserTrait`, `LanguageInfo`, diff --git a/README.ja.md b/README.ja.md index 3806deabb..4e057773a 100644 --- a/README.ja.md +++ b/README.ja.md @@ -146,6 +146,6 @@ make pre-commit # CI と同等のローカルゲート一式 - 同梱の文法クレート（`tree-sitter-ccomment`、`tree-sitter-mozcpp`、`tree-sitter-mozjs`、 `tree-sitter-preproc`、`tree-sitter-tcl`）は MIT ライセンスで公開されています。 -- **big-code-analysis**、**big-code-analysis-cli**、**big-code-analysis-web**、 - **big-code-analysis-py** は [Mozilla Public License v2.0](https://www.mozilla.org/MPL/2.0/) +- **big-code-analysis**、**big-code-analysis-ast**、**big-code-analysis-cli**、 + **big-code-analysis-web**、**big-code-analysis-py** は [Mozilla Public License v2.0](https://www.mozilla.org/MPL/2.0/) のもとで公開されています。 diff --git a/README.md b/README.md index edbe83b17..abceb7d2f 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,7 @@ metric, and updating grammars. `tree-sitter-mozcpp`, `tree-sitter-mozjs`, `tree-sitter-preproc`, `tree-sitter-tcl`) are released under the MIT license. -- **big-code-analysis**, **big-code-analysis-cli**, +- **big-code-analysis**, **big-code-analysis-ast**, **big-code-analysis-cli**, **big-code-analysis-web**, and **big-code-analysis-py** are released under the [Mozilla Public License v2.0](https://www.mozilla.org/MPL/2.0/). diff --git a/RELEASING.md b/RELEASING.md index 47ef2f46d..fa2fc08a9 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -17,7 +17,8 @@ Rationale: - Edition 2024 is the active edition for every crate; `let-else`, let-chains, and the relaxed lifetime-elision rules used across - `src/languages/` require Rust 1.85+, but several individual + `big-code-analysis-ast/src/languages/` require Rust 1.85+, but + several individual improvements rely on later releases (e.g. const slice indexing stabilizations, refined drop-order semantics). - Treating 1.94 as the floor avoids "works on my machine" reports @@ -65,8 +66,9 @@ One push of a `v*` tag will run this end-to-end: 7. **publish-crates**: for non pre-releases, **subject to the gating variables below**, runs `cargo publish` for each publishable workspace crate in dependency order: the five `bca-tree-sitter-*` - grammar leaves first, then `big-code-analysis` (library), then - `big-code-analysis-cli` and `big-code-analysis-web`. Skips + grammar leaves first, then `big-code-analysis-ast`, then + `big-code-analysis` (library), then `big-code-analysis-cli` and + `big-code-analysis-web`. Skips idempotently if the version is already on crates.io. 8. **verify**: downloads the published musl tarball back out of the release, verifies the minisign signature, checksum, and SLSA @@ -161,10 +163,11 @@ only resolve once each leaf is on crates.io. The sparse-index existence check in each step makes the job idempotent across re-runs of the same tag. -**The three top-level crates cannot be dry-run before the tag.** -`big-code-analysis` pins each leaf at `bca-tree-sitter- = -"="`, and `big-code-analysis-cli` / `big-code-analysis-web` -pin `big-code-analysis` the same way. Packaging resolves those +**The four top-level crates cannot be dry-run before the tag.** +`big-code-analysis-ast` pins each leaf at `bca-tree-sitter- = +"="`, `big-code-analysis` pins `big-code-analysis-ast` and +the leaves, and `big-code-analysis-cli` / `big-code-analysis-web` pin +`big-code-analysis` the same way. Packaging resolves those requirements against the registry, and the Lockstep version policy below makes the pinned version the one this tag is releasing — which is, by definition, not yet published. So it fails, every time, not @@ -207,14 +210,16 @@ release-check`, and the `preflight` job of `release.yml`. The five vendored leaves are not in its scope: they carry no internal pins, so `release-check` and `preflight` dry-run them for real. -**The leaves still publish first.** That ordering is what lets the -parent resolve during `publish-crates`, and it is unchanged. It also +**The leaves still publish first, then `big-code-analysis-ast`.** That +ordering is what lets each crate resolve its pins during +`publish-crates`, and it is unchanged in kind. It also means a metadata regression in the parent or the binaries would fail *after* the five leaves are irrevocably on crates.io — the split release the gate above exists to make unreachable. **Lockstep version policy.** Every crate in this repository (the -library, the CLI, the web crate, the Python crate, the `enums` / +library, the `big-code-analysis-ast` parse layer, the CLI, the web +crate, the Python crate, the `enums` / `xtask` helpers, and the five `bca-tree-sitter-*` vendored grammar leaves) shares one version number. There is no per-crate version drift. A version bump touches: @@ -231,7 +236,9 @@ drift. A version bump touches: matching block in `enums/Cargo.toml`. 5. The `version = "="` pin on the `big-code-analysis` path-dep in `big-code-analysis-cli/Cargo.toml` and - `big-code-analysis-web/Cargo.toml`. + `big-code-analysis-web/Cargo.toml`, and the two pins on the + `big-code-analysis-ast` path-dep in the root `Cargo.toml` + (`[dependencies]` and `[dev-dependencies]`). 6. Only when the bump is the release-prep commit for the version being cut: the hard-coded version references in user-facing docs (`README.md`, `STABILITY.md`, the book's `quick-start.md` and @@ -365,8 +372,9 @@ Stable releases push to (subject to the gating variables above): commits only `bucket/big-code-analysis.json` and leaves the other manifests in the bucket untouched. - crates.io, leaf-first: the five `bca-tree-sitter-*` grammar - crates, then `big-code-analysis` (library), then - `big-code-analysis-cli` and `big-code-analysis-web`. See + crates, then `big-code-analysis-ast`, then `big-code-analysis` + (library), then `big-code-analysis-cli` and `big-code-analysis-web`. + See [crates.io ownership](#cratesio-ownership) for the publish loop and rate-limit details. @@ -374,8 +382,8 @@ Both tap and bucket repos must exist and accept the configured PAT. ### crates.io ownership -Before the first automated publish you must manually claim **all eight -crate names**: the five `bca-tree-sitter-*` leaves plus the three +Before the first automated publish you must manually claim **all nine +crate names**: the five `bca-tree-sitter-*` leaves plus the four top-level crates. The `publish-crates` job in `release.yml` uses Trusted Publishing which requires the crate to exist before TP can be registered, so the very first publish has to be a hand-rolled @@ -386,6 +394,7 @@ registered, so the very first publish has to be a hand-rolled - `bca-tree-sitter-ccomment`, `…-mozcpp`, `…-mozjs`, `…-preproc`, `…-tcl` + - `big-code-analysis-ast` - `big-code-analysis` - `big-code-analysis-cli` - `big-code-analysis-web` @@ -444,6 +453,7 @@ registered, so the very first publish has to be a hand-rolled # Parent + binaries. These will hit the new-crate rate limit on # the first try; the until-loop retries every 60s until cargo # exits 0. + until cargo publish -p big-code-analysis-ast --locked; do sleep 60; done until cargo publish -p big-code-analysis --locked; do sleep 60; done until cargo publish -p big-code-analysis-cli --locked; do sleep 60; done until cargo publish -p big-code-analysis-web --locked; do sleep 60; done @@ -479,9 +489,9 @@ one-time setup steps are required on top of the The name must match the TP registration exactly; a typo here is the most common self-inflicted failure mode. -2. **Register a Trusted Publisher for each of the eight crates.** +2. **Register a Trusted Publisher for each of the nine crates.** On crates.io, open the settings page for each of the five - `bca-tree-sitter-*` leaves, `big-code-analysis`, + `bca-tree-sitter-*` leaves, `big-code-analysis-ast`, `big-code-analysis`, `big-code-analysis-cli`, and `big-code-analysis-web`. In the **Trusted Publishing** section, add a GitHub publisher with: @@ -535,8 +545,8 @@ cargo update --workspace cargo metadata --format-version 1 --no-deps \ | python3 -c "import json,sys; d=json.load(sys.stdin); \ print({p['name']: p['version'] for p in d['packages']})" -# Expect big-code-analysis, big-code-analysis-cli, and -# big-code-analysis-web at the target version. +# Expect big-code-analysis-ast, big-code-analysis, big-code-analysis-cli, +# and big-code-analysis-web at the target version. ``` The `cargo update --workspace` step is **mandatory**, not @@ -754,7 +764,7 @@ Before tagging, on `main`: `grep '^untrusted comment: placeholder' minisign.pub`; it should print nothing). - [ ] `make check-publish-metadata` passes. It is the only pre-tag - gate on the three top-level crates' publish metadata — none of + gate on the four top-level crates' publish metadata — none of them can be `cargo publish --dry-run`-ed before the tag — and it is what catches an `[package].include` block that has regressed or stopped covering a newly-added directory. See diff --git a/big-code-analysis-book/src/commands/check.md b/big-code-analysis-book/src/commands/check.md index 4b8a0922e..a8c767169 100644 --- a/big-code-analysis-book/src/commands/check.md +++ b/big-code-analysis-book/src/commands/check.md @@ -437,7 +437,7 @@ In `bca.toml`: [check] exclude = [ "tests/**", - "src/languages/language_*.rs", + "big-code-analysis-ast/src/languages/language_*.rs", "xtask/**", ] ``` diff --git a/big-code-analysis-book/src/developers/new-language.md b/big-code-analysis-book/src/developers/new-language.md index e1c540d79..5bb5ccf05 100644 --- a/big-code-analysis-book/src/developers/new-language.md +++ b/big-code-analysis-book/src/developers/new-language.md @@ -52,9 +52,9 @@ this project to evaluate metrics. At this point we should have a new grammar file for the new language in -[/src/languages/](https://github.com/dekobon/big-code-analysis/tree/main/src/languages). +[/big-code-analysis-ast/src/languages/](https://github.com/dekobon/big-code-analysis/tree/main/big-code-analysis-ast/src/languages). See -[/src/languages/language_rust.rs](https://github.com/dekobon/big-code-analysis/blob/main/src/languages/language_rust.rs) +[/big-code-analysis-ast/src/languages/language_rust.rs](https://github.com/dekobon/big-code-analysis/blob/main/big-code-analysis-ast/src/languages/language_rust.rs) as an example of the generated enum. ## Adding the new grammar to big-code-analysis @@ -66,7 +66,7 @@ as an example of the generated enum. [Cargo.toml](https://github.com/dekobon/big-code-analysis/blob/main/Cargo.toml) is `tree-sitter-rust = "=0.24.2"`. 1. Next we add the new `tree-sitter` language namespace to - [/src/languages/mod.rs](https://github.com/dekobon/big-code-analysis/blob/main/src/languages/mod.rs) + [/big-code-analysis-ast/src/languages/mod.rs](https://github.com/dekobon/big-code-analysis/blob/main/big-code-analysis-ast/src/languages/mod.rs) eg. ```rust @@ -76,7 +76,7 @@ pub use language_rust::*; 1. Lastly, we add a definition of the language to the arguments of `mk_langs!` macro in - [/src/langs.rs](https://github.com/dekobon/big-code-analysis/blob/main/src/langs.rs). + [/big-code-analysis-ast/src/langs.rs](https://github.com/dekobon/big-code-analysis/blob/main/big-code-analysis-ast/src/langs.rs). ```rust // 1) Cargo feature name that enables this variant's grammar @@ -111,15 +111,15 @@ must also implement the AST plumbing and every metric trait the workspace defines: - **`Checker`** in - [/src/checker.rs](https://github.com/dekobon/big-code-analysis/blob/main/src/checker.rs) + [/big-code-analysis-ast/src/checker.rs](https://github.com/dekobon/big-code-analysis/blob/main/big-code-analysis-ast/src/checker.rs) — comment, function, closure, call, string-literal, and `else-if` predicates over the grammar's `kind_id`s. - **`Getter`** in - [/src/getter.rs](https://github.com/dekobon/big-code-analysis/blob/main/src/getter.rs) + [/big-code-analysis-ast/src/getter.rs](https://github.com/dekobon/big-code-analysis/blob/main/big-code-analysis-ast/src/getter.rs) — `get_space_kind` plus the Halstead operator/operand classification table. - **`Alterator`** in - [/src/alterator.rs](https://github.com/dekobon/big-code-analysis/blob/main/src/alterator.rs) + [/big-code-analysis-ast/src/alterator.rs](https://github.com/dekobon/big-code-analysis/blob/main/big-code-analysis-ast/src/alterator.rs) — usually only string-literal preservation; the default impl works for most languages. - **All thirteen metric traits**: `Abc`, `Cognitive`, `Cyclomatic`, diff --git a/big-code-analysis-book/src/languages.md b/big-code-analysis-book/src/languages.md index d0f6a4d95..4c28ea1e7 100644 --- a/big-code-analysis-book/src/languages.md +++ b/big-code-analysis-book/src/languages.md @@ -2,7 +2,7 @@ This is the list of programming languages parsed by **big-code-analysis**. Each entry below is a real `LANG` variant -(defined by the `mk_langs!` invocation in `src/langs.rs`) and is +(defined by the `mk_langs!` invocation in `big-code-analysis-ast/src/langs.rs`) and is gated behind the matching per-language Cargo feature documented in [Per-language Cargo features](./library/cargo-features.md). diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index aa340b6e1..62df965d7 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -230,7 +230,7 @@ against, which equals `magnitude` at a leaf space), the per-component averages (`assignments_average`, `branches_average`, `conditions_average`), and per-component `*_min` / `*_max` at the file scope, for fourteen fields total. The metric is specialised per -language in `src/languages/language_*.rs`. +language in `big-code-analysis-ast/src/languages/language_*.rs`. ### How to read it diff --git a/docs/development/benchmarking.md b/docs/development/benchmarking.md index 934c329c2..6a717de33 100644 --- a/docs/development/benchmarking.md +++ b/docs/development/benchmarking.md @@ -159,7 +159,7 @@ predicate in the walk that asked a node for its parent was therefore `O(depth)` per node and `O(depth^2)` over a deeply nested file, however few steps it took. [#1084][parent-walk] fixed three of them by having the metric walk carry the ancestor chain down with it (`Ancestors` in -`src/node.rs`), so a predicate reads an ancestor as a slice index. Their +`big-code-analysis-ast/src/node.rs`), so a predicate reads an ancestor as a slice index. Their bounds moved to the linear bound in that same change, which is what now catches a relapse. `cognitive/nested-fn` is the fourth of the same family: `increment_function_depth` was deferred out of #1084 and fixed @@ -239,7 +239,7 @@ not have caught why. A forward pass over the parent's children is `O(children)` and flat in depth; the backward walk is the reverse. The measured costs behind that trade, and the break-even they put it at, live on `MAX_FORWARD_ATTRIBUTE_SCAN_CHILDREN` and -`FORWARD_ATTRIBUTE_SCAN_CHILDREN_PER_DEPTH` in `src/checker.rs` — one +`FORWARD_ATTRIBUTE_SCAN_CHILDREN_PER_DEPTH` in `big-code-analysis-ast/src/checker.rs` — one copy, so re-measuring updates one place. Reading forward unconditionally fixed the depth axis and broke the width one: a generated file of 2 000 top-level attributed items went from 6.0 ms to @@ -262,7 +262,7 @@ covers what it claims and that the depth probes do not. The unit suite still pins the *dispatch* separately: `the_exclude_tests_prune_reads_forward_up_to_its_depth_scaled_budget` -in `src/node.rs` asserts which arm each boundary shape takes, so +in `big-code-analysis-ast/src/node.rs` asserts which arm each boundary shape takes, so widening the budget past a shallow parent fails a test rather than slipping through. The break-even *numbers* the budget is derived from remain unguarded — the gate sees the complexity class, not the @@ -336,7 +336,7 @@ which a `metrics()` call runs, so they are absent from the figures above. `Node::children_with` lets all five hoist one cursor out of their loop, -and the counter `child_scan_cursors` in `src/node.rs` is what keeps +and the counter `child_scan_cursors` in `big-code-analysis-ast/src/node.rs` is what keeps them there, since the change moves no metric value. All five are asserted: the walks reachable from `node.rs` in `the_converted_traversals_scan_a_tree_on_one_cursor`, and the renderer diff --git a/docs/development/fuzzing.md b/docs/development/fuzzing.md index 9ca795c2c..80cbbbb7e 100644 --- a/docs/development/fuzzing.md +++ b/docs/development/fuzzing.md @@ -15,11 +15,11 @@ The target set is deliberately narrow. Two pieces of evidence bound it. **The static lints already cover the known class.** #1152 adopted `clippy::arithmetic_side_effects` on the `loc` metric module and -`clippy::indexing_slicing` on `src/c_macro.rs`, and would have caught the +`clippy::indexing_slicing` on `big-code-analysis-ast/src/c_macro.rs`, and would have caught the five-byte input that panicked the library (#1051) at compile time. What a lint cannot check is the residue it could not discharge: the nine per-function `#[allow(clippy::indexing_slicing)]` sites in -`src/c_macro.rs`, each a computed index into attacker-controlled bytes +`big-code-analysis-ast/src/c_macro.rs`, each a computed index into attacker-controlled bytes whose bound is asserted by a human comment. That population is what `preproc_macro` exists for. diff --git a/docs/development/mutation_testing.md b/docs/development/mutation_testing.md index a912adc1a..6ad3dfeba 100644 --- a/docs/development/mutation_testing.md +++ b/docs/development/mutation_testing.md @@ -2,7 +2,7 @@ `big-code-analysis` runs [cargo-mutants][cm] on a quarterly schedule against the highest-leverage modules: every metric implementation -under `src/metrics/`, plus `src/checker.rs` and `src/getter.rs`. +under `src/metrics/`, plus `big-code-analysis-ast/src/checker.rs` and `big-code-analysis-ast/src/getter.rs`. Mutation testing complements the regular test suite by mechanically mutating production code (e.g. flipping `>` to `>=`, replacing a function body with `Default::default()`) and re-running the tests. @@ -32,8 +32,8 @@ The job: 1. Checks out the repo with submodules. 2. Installs `cargo-mutants` via `taiki-e/install-action@v2`. -3. Runs `cargo mutants` against `src/metrics/`, `src/checker.rs`, - and `src/getter.rs`. +3. Runs `cargo mutants` against `src/metrics/`, `big-code-analysis-ast/src/checker.rs`, + and `big-code-analysis-ast/src/getter.rs`. 4. Uploads `target/mutants/` as the `cargo-mutants-report` artifact (90-day retention). 5. On non-zero exit, opens a GitHub issue labelled @@ -64,7 +64,9 @@ cargo mutants -f src/metrics/cognitive.rs To exercise the same surface as CI: ```bash -cargo mutants -f src/metrics/ -f src/checker.rs -f src/getter.rs +cargo mutants --package big-code-analysis --package big-code-analysis-ast \ + -f src/metrics/ -f big-code-analysis-ast/src/checker.rs \ + -f big-code-analysis-ast/src/getter.rs ``` Plan on tens of minutes per file on a laptop. Use `-j N` to bound diff --git a/docs/file-detection.md b/docs/file-detection.md index 7428c1a2a..8216e497e 100644 --- a/docs/file-detection.md +++ b/docs/file-detection.md @@ -2,8 +2,11 @@ How `big-code-analysis` decides which language a file is written in, and what it reads off disk before parsing. All of the logic lives in -[`src/tools.rs`](../src/tools.rs), [`src/langs.rs`](../src/langs.rs), -and the macros in [`src/macros/mod.rs`](../src/macros/mod.rs). +[`big-code-analysis-ast/src/tools.rs`](../big-code-analysis-ast/src/tools.rs), +[`big-code-analysis-ast/src/langs.rs`](../big-code-analysis-ast/src/langs.rs), +and the macros in +[`big-code-analysis-ast/src/macros/mod.rs`](../big-code-analysis-ast/src/macros/mod.rs) +(the `big-code-analysis-ast` crate, re-exported by the root). ## Reading the file @@ -30,7 +33,7 @@ would shift line numbers and break LoC counts. ## Detecting the language There are two public entry points, both returning a -[`LANG`](../src/langs.rs) variant: +[`LANG`](../big-code-analysis-ast/src/langs.rs) variant: ### `get_language_for_file(path)`: extension only @@ -140,7 +143,7 @@ the #724 change: `.m` now reports `"objc"` natively and `.mm` reports The per-language extension list and Emacs mode list are declared as the last two tuple fields of each `mk_langs!` entry in -[`src/langs.rs`](../src/langs.rs): +[`big-code-analysis-ast/src/langs.rs`](../big-code-analysis-ast/src/langs.rs): ```rust ( @@ -156,7 +159,8 @@ last two tuple fields of each `mk_langs!` entry in ``` The `mk_extensions!` and `mk_emacs_mode!` macros in -[`src/macros/mod.rs`](../src/macros/mod.rs) expand these into the public +[`big-code-analysis-ast/src/macros/mod.rs`](../big-code-analysis-ast/src/macros/mod.rs) +expand these into the public `get_from_ext(ext) -> Option` and `get_from_emacs_mode(mode) -> Option` lookup functions. Both are plain `match` arms: no fuzzy matching, no fallback. @@ -195,5 +199,6 @@ If `guess_language` returns `(None, _)`: Beyond the shebang scan described above, there is no content-based heuristic and no MIME sniffing. Add a missing extension or Emacs mode to `mk_langs!` rather than working around it at the call site, and -extend the shebang interpreter table in `src/tools.rs` if a new +extend the shebang interpreter table in +`big-code-analysis-ast/src/tools.rs` if a new script interpreter needs to be recognised. diff --git a/enums/src/lib.rs b/enums/src/lib.rs index 9d07ca963..9a3ab4490 100644 --- a/enums/src/lib.rs +++ b/enums/src/lib.rs @@ -3,7 +3,7 @@ //! Every grammar in [`Lang`] is loaded through its tree-sitter crate, //! its node kinds are enumerated, and the result is rendered through //! one of the `templates/` files into a Rust, Go, or JSON module. The -//! Rust output is what lands in the parent crate's `src/languages/`; +//! Rust output is what lands in `big-code-analysis-ast/src/languages/`; //! `make enums-codegen-drift` fails when the checked-in files no longer //! match what this crate emits. diff --git a/tests/README.md b/tests/README.md index 53c69d9f0..00bcbeb37 100644 --- a/tests/README.md +++ b/tests/README.md @@ -195,9 +195,9 @@ Per-metric unit-test counts across `src/metrics/*.rs`. Numbers combine Plus module-level tests not counted above: -- `src/checker.rs` — 4 `bash_*` tests +- `big-code-analysis-ast/src/checker.rs` — 4 `bash_*` tests - `src/spaces.rs` — 1 `c_*`, 1 `cpp_*` test -- `src/alterator.rs` — 1 each for `javascript`, `typescript`, `tsx` +- `big-code-analysis-ast/src/alterator.rs` — 1 each for `javascript`, `typescript`, `tsx` Best-covered: **C#** (110 unit tests, 11 of 13 metrics, plus Pattern-B corpus). Close runner-up: **Java** (104, 12 of 13, plus cross-language From 7e0cc49959a9dda48d6d6aba468eee24904f6a5d Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Thu, 10 Sep 2026 09:22:09 -0700 Subject: [PATCH 5/9] refactor(metrics): own the member-scope rule in this crate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `SpaceKind::is_member_scope` answered whether `wmc`, `npm` and `npa` roll up on a space kind — a question about three metrics, asked from a crate that is meant to know nothing about metrics. Its only callers were those three. It becomes a `pub(crate)` `MemberScopeExt` extension trait beside `average`, the other helper the metric modules share. `SpaceKind` keeps its name, its variants and its public paths; the parse layer now carries no metric vocabulary at all. Doing this before the release matters: the split had widened the predicate to `pub` so the walk could reach it across the crate boundary, and that widening would have been a public item this crate was then stuck with until 3.0. It never ships. The method takes `&self` because a trait cannot tell clippy::wrong_self_convention that every implementor is `Copy`, and a carve-out for a by-value `Copy` receiver would buy nothing. --- .claude/rules/grammar-dispatch.md | 3 +- CHANGELOG.md | 12 +++---- big-code-analysis-ast/src/space_kind.rs | 25 --------------- src/metrics/container_scope_tests.rs | 5 +-- src/metrics/mod.rs | 42 +++++++++++++++++++++++++ src/metrics/npa.rs | 1 + src/metrics/npm.rs | 1 + src/metrics/wmc.rs | 1 + 8 files changed, 56 insertions(+), 34 deletions(-) diff --git a/.claude/rules/grammar-dispatch.md b/.claude/rules/grammar-dispatch.md index a80bfa204..258392546 100644 --- a/.claude/rules/grammar-dispatch.md +++ b/.claude/rules/grammar-dispatch.md @@ -184,7 +184,8 @@ Minimum cross-walk when you edit one: into a **wrong count**, which a snapshot diff shows you. This one disagrees into an **absent key**: a node the walker promoted but the getter left `Unknown` is not a member scope - (`SpaceKind::is_member_scope`), so its space serializes no `npm` / + (`MemberScopeExt::is_member_scope`, `src/metrics/mod.rs`), so its + space serializes no `npm` / `npa` block at all — which looks exactly like a language that legitimately has no containers. Nothing diffs. §6's "gate all three on the same predicate" is this bullet's fix. diff --git a/CHANGELOG.md b/CHANGELOG.md index 376e67a26..c844ffc98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,12 +47,12 @@ for historical reference. (see the escape-hatches section of `STABILITY.md`). - `SpaceKind` and the operator/operand classification are now defined in `big-code-analysis-ast` (they are `Getter` return types) and - re-exported at their existing paths; `MetricsError` likewise. One - consequence is additive on the published surface: `SpaceKind` gains a - public `is_member_scope()` — the walk consults it across the new crate - boundary, so it can no longer be `pub(crate)`. It answers "does this - kind roll up `npm` / `npa` members", i.e. anything but `Function` and - `Unknown`. + re-exported at their existing paths; `MetricsError` likewise. The + public surface is unchanged in shape. `SpaceKind::is_member_scope`, + which answers whether `wmc` / `npm` / `npa` roll up on a kind, is not + part of that move: it is a question only this crate can ask, so it + became a `pub(crate)` extension trait here rather than an inherent + method on a type the parse layer owns. - Per-space *own* value for `nargs` in the serialized wire shape: `nargs.value` (#1236). `nargs.total` remains the subtree sum; the new diff --git a/big-code-analysis-ast/src/space_kind.rs b/big-code-analysis-ast/src/space_kind.rs index fb78fff96..a9a716b84 100644 --- a/big-code-analysis-ast/src/space_kind.rs +++ b/big-code-analysis-ast/src/space_kind.rs @@ -65,31 +65,6 @@ impl SpaceKind { _ => Self::Unknown, } } - - /// Whether the object-oriented member metrics — `wmc`, `npm`, `npa` — - /// are meaningful on a space of this kind. - /// - /// True for every container (which owns methods and attributes) and - /// for the file [`Unit`](SpaceKind::Unit) (which aggregates its - /// containers' counts into a whole-file roll-up). False for a - /// function space, which owns neither, and for - /// [`Unknown`](SpaceKind::Unknown). - /// - /// This is the single definition the three metrics share. `wmc` - /// carried it alone as an inline `matches!`; `npm` and `npa` enabled - /// themselves from `Checker::is_func_space` instead, which admits - /// function spaces and so gave a Kotlin `` or a JavaScript method - /// an all-zero block its sibling method did not have (#1197). - /// - /// Phrased as an exclusion so that a future [`SpaceKind`] variant — - /// the enum is `#[non_exhaustive]` — defaults to *carrying* the - /// metrics. A new kind is far likelier to be another container than - /// another callable, and an extra roll-up is a milder wrong answer - /// than a silently missing one. - #[must_use] - pub fn is_member_scope(self) -> bool { - !matches!(self, Self::Function | Self::Unknown) - } } impl fmt::Display for SpaceKind { diff --git a/src/metrics/container_scope_tests.rs b/src/metrics/container_scope_tests.rs index e8fb3cfb8..2863919e4 100644 --- a/src/metrics/container_scope_tests.rs +++ b/src/metrics/container_scope_tests.rs @@ -15,7 +15,8 @@ //! #1184 added Kotlin property accessors and `init` / `static` blocks to //! the list, next to sibling methods that had none. //! -//! The rule is [`SpaceKind::is_member_scope`], which `wmc` already used: +//! The rule is `MemberScopeExt::is_member_scope`, which `wmc` already +//! used: //! containers and the file unit carry the block, a function space never //! does. Both directions are asserted below, because narrowing too far //! would silently delete the whole-file roll-up rather than the all-zero @@ -581,7 +582,7 @@ fn the_file_root_keeps_its_rollup() { /// node the walker promoted — via /// `Checker::promotes_to_func_space_with_code` — whose classifier /// answered `Unknown` becomes a space that is not a -/// [`SpaceKind::is_member_scope`], and `note_member_scope` then records +/// `MemberScopeExt::is_member_scope`, and `note_member_scope` then records /// a kind that suppresses `npm` / `npa` outright. That is an *absent /// key*, not a wrong count: it reads the same as a language with no /// containers, so no snapshot diff and no value assertion can see it. diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 2cd2ac6a6..0beedfb03 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -4,6 +4,8 @@ //! traits, and its `Stats` accumulator. See the crate-level docs for an //! overview of the metric suite. +use crate::SpaceKind; + /// Assignment / Branch / Condition counts. pub mod abc; /// Cognitive complexity. @@ -69,3 +71,43 @@ mod container_scope_tests; pub(crate) fn average(sum: f64, count: usize) -> f64 { sum / count.max(1) as f64 } + +/// Whether the object-oriented member metrics — `wmc`, `npm`, `npa` — +/// are meaningful on a space of a given [`SpaceKind`]. +/// +/// An extension trait rather than an inherent method because +/// `SpaceKind` is defined in `big-code-analysis-ast`, which knows +/// nothing about metrics: "does `wmc` apply here" is a question only +/// this crate can ask, so it is answered here (#1376). +pub(crate) trait MemberScopeExt { + /// True for every container (which owns methods and attributes) and + /// for the file [`SpaceKind::Unit`] (which aggregates its + /// containers' counts into a whole-file roll-up). False for a + /// function space, which owns neither, and for + /// [`SpaceKind::Unknown`]. + /// + /// This is the single definition the three metrics share. `wmc` + /// carried it alone as an inline `matches!`; `npm` and `npa` enabled + /// themselves from `Checker::is_func_space` instead, which admits + /// function spaces and so gave a Kotlin `` or a JavaScript + /// method an all-zero block its sibling method did not have (#1197). + /// + /// Phrased as an exclusion so that a future [`SpaceKind`] variant — + /// the enum is `#[non_exhaustive]` — defaults to *carrying* the + /// metrics. A new kind is far likelier to be another container than + /// another callable, and an extra roll-up is a milder wrong answer + /// than a silently missing one. + /// + /// Takes `&self` rather than a by-value `Copy`: the receiver is + /// cost-free either way, and a trait cannot tell + /// `clippy::wrong_self_convention` that every implementor is `Copy`, + /// so by-value would need a carve-out that buys nothing. + fn is_member_scope(&self) -> bool; +} + +impl MemberScopeExt for SpaceKind { + #[inline] + fn is_member_scope(&self) -> bool { + !matches!(*self, SpaceKind::Function | SpaceKind::Unknown) + } +} diff --git a/src/metrics/npa.rs b/src/metrics/npa.rs index 1a8f3c1ec..8a98bc03a 100644 --- a/src/metrics/npa.rs +++ b/src/metrics/npa.rs @@ -21,6 +21,7 @@ use std::fmt; use crate::checker::Checker; use crate::macros::{csharp_var_decl_kinds, csharp_var_declarator_kinds, implement_metric_trait}; +use crate::metrics::MemberScopeExt; use crate::node::Node; use crate::*; diff --git a/src/metrics/npm.rs b/src/metrics/npm.rs index a4c50f347..095cbce45 100644 --- a/src/metrics/npm.rs +++ b/src/metrics/npm.rs @@ -22,6 +22,7 @@ use std::fmt; use crate::checker::{Checker, csharp_accessor_count}; use crate::lang_helpers::python::python_is_block; use crate::macros::implement_metric_trait; +use crate::metrics::MemberScopeExt; use crate::metrics::npa::{accessibility_ratio, ts_member_is_public}; use crate::node::Node; use crate::*; diff --git a/src/metrics/wmc.rs b/src/metrics/wmc.rs index 9287d0df1..2166f15f7 100644 --- a/src/metrics/wmc.rs +++ b/src/metrics/wmc.rs @@ -18,6 +18,7 @@ use std::fmt; use crate::checker::Checker; use crate::macros::implement_metric_trait; +use crate::metrics::MemberScopeExt; use crate::*; /// The `Wmc` metric. From 4549552746758f54c5761b9d25dd33bfb49bf655 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Thu, 10 Sep 2026 10:16:05 -0700 Subject: [PATCH 6/9] docs: finish the crate-split sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AGENTS.md's layout bullet now names `space_kind.rs` and `token_role.rs` and states the rule they follow: nothing under the parse crate names a metric, which is why `HalsteadType` became `TokenRole` and the member-scope predicate lives in the root. A baseline example still pointed at `src/count.rs`, which moved. The book gains three corrections. Its feature page explains that the grammar crates now arrive through `big-code-analysis-ast`, so they sit one level down in `cargo tree` while nothing changes for a consumer. The developer guide lists the new member among the per-crate builds. The stability summary picks up a bullet saying the parse crate is not a stability surface. That page also carried a claim that predates this work: it said `Node` exposes its `tree_sitter::Node` "through `.0`", which #556 made private in favour of `as_tree_sitter()`. Corrected, and extended to cover the walk accessors this branch made public — shape-stable signatures over values that follow the grammar pin. --- AGENTS.md | 13 +++++++---- .../src/developers/README.md | 1 + .../src/library/cargo-features.md | 8 +++++++ .../src/library/stability.md | 23 ++++++++++++++----- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 108eddb96..68aefe753 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,9 +53,13 @@ and `cargo run -p big-code-analysis-web --`. `node.rs`, `parser.rs`, `traits.rs` (`ParserTrait`), `checker.rs`, `getter.rs`, `alterator.rs`, `lang_helpers/`, `preproc.rs`, `c_macro.rs`, `comment_rm.rs`, `ast.rs`, `count.rs`, `find.rs`, - `tools.rs` — everything that turns bytes into a classified tree and - computes no metric. `macros/mod.rs` there holds `mk_langs!` and the - `with_any_parser!` dispatch macro. + `tools.rs`, plus `space_kind.rs` and `token_role.rs` (the two + classifications a `Getter` returns) — everything that turns bytes + into a classified tree and computes no metric. `macros/mod.rs` there + holds `mk_langs!` and the `with_any_parser!` dispatch macro. Nothing + under this crate names a metric: `TokenRole` was `HalsteadType` until + #1376, and the "do the member metrics apply" predicate lives in the + root as `metrics::MemberScopeExt`. - `src/metrics/` — individual metric implementations: `abc.rs`, `cognitive.rs`, `cyclomatic.rs`, `nexits.rs`, `halstead.rs`, `loc.rs`, `mi.rs`, `nargs.rs`, `nom.rs`, `npa.rs`, `npm.rs`, `tokens.rs`, @@ -406,7 +410,8 @@ runtime shapes for the gate to pass. metric past its recorded `.bca-baseline.toml` value must refresh the baseline in the **same PR**. The baseline filter only suppresses a violation while the live measurement stays at or below the recorded -value; once a file grows past it (e.g., #445 grew `src/count.rs`'s +value; once a file grows past it (e.g., #445 grew what is now +`big-code-analysis-ast/src/count.rs`'s `halstead.effort` from ~103k to ~191k), the filter no longer covers the offender and `make self-scan` goes red on a clean checkout — for everyone, not just the author (#449). The existing `bca-self-scan` diff --git a/big-code-analysis-book/src/developers/README.md b/big-code-analysis-book/src/developers/README.md index 9da01ba07..b4315c0f0 100644 --- a/big-code-analysis-book/src/developers/README.md +++ b/big-code-analysis-book/src/developers/README.md @@ -63,6 +63,7 @@ For an individual crate, invoke `cargo` directly: ```console cargo build # library only +cargo build -p big-code-analysis-ast # parse + classification layer only cargo build -p big-code-analysis-cli # CLI only cargo build -p big-code-analysis-web # web server only ``` diff --git a/big-code-analysis-book/src/library/cargo-features.md b/big-code-analysis-book/src/library/cargo-features.md index 3ad82423d..9bf7984d4 100644 --- a/big-code-analysis-book/src/library/cargo-features.md +++ b/big-code-analysis-book/src/library/cargo-features.md @@ -63,6 +63,14 @@ per-language pipeline depends on). | `tcl` | `bca-tree-sitter-tcl` | | `typescript` | `tree-sitter-typescript` (used by both the `Typescript` and `Tsx` variants) | +Since #1376 the grammar crates are dependencies of +`big-code-analysis-ast`, the parse and classification layer this crate +is built on, so every feature above forwards to the same-named feature +there and the grammars appear one level down in `cargo tree`. Nothing +changes for a consumer: the feature names, the `LANG` enum and the +whole analysis surface are unaffected, and enabling `rust` still links +exactly one grammar. + The umbrella `all-languages` feature enables every entry in this table. The `bca-tree-sitter-*` crates are in-tree forks of the upstream Mozilla / community grammars; the Rust import path remains diff --git a/big-code-analysis-book/src/library/stability.md b/big-code-analysis-book/src/library/stability.md index a668cb591..8eddafb79 100644 --- a/big-code-analysis-book/src/library/stability.md +++ b/big-code-analysis-book/src/library/stability.md @@ -27,12 +27,23 @@ The headlines for library consumers: - **MSRV is `1.94`.** Bumping the MSRV is treated as a minor-bump event and is flagged in the changelog under **(breaking)** — see [STABILITY.md § MSRV policy][stability-msrv]. -- **Escape hatches.** The [`Node`][Node] wrapper exposes - `tree_sitter::Node` through `.0`, and the `tree_sitter` crate is - re-exported as `big_code_analysis::tree_sitter`. Anything reached - through those seams follows the pinned `tree-sitter` version, not - our own [SemVer]. See [STABILITY.md § Escape hatches][stability-escape] - before depending on them. +- **Escape hatches.** The [`Node`][Node] wrapper exposes its + `tree_sitter::Node` through `Node::as_tree_sitter()`, and the + `tree_sitter` crate is re-exported as + `big_code_analysis::tree_sitter`. `Node` also carries the accessors + the metric walk itself uses (`kind`, `kind_id`, `child`, `children`, + `child_by_field_name`, `utf8_text`, the position accessors). Their + signatures are shape-stable, but the *values* they return — node + kinds and `kind_id` numbers — belong to the pinned grammar and move + when it does, so anything reached through those seams follows the + pinned `tree-sitter` version rather than our own [SemVer]. See + [STABILITY.md § Escape hatches][stability-escape] before depending on + them. +- **`big-code-analysis-ast` is not a stability surface.** Since #1376 + the parse and classification layer lives in that separate published + crate, which this one pins exactly and releases in lockstep. It will + appear in your `cargo tree`. Nothing in it carries a promise of its + own: depend on `big-code-analysis` and use what it re-exports. [stability-shape]: https://github.com/dekobon/big-code-analysis/blob/main/STABILITY.md#what-is-stable-in-shape [stability-msrv]: https://github.com/dekobon/big-code-analysis/blob/main/STABILITY.md#msrv-policy From 8c06a5b7fb1d3721ab1ef0580211c36cdc942ec7 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Thu, 10 Sep 2026 10:34:04 -0700 Subject: [PATCH 7/9] ci: make the sarif tool install idempotent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The clippy job aborted with "binary `clippy-sarif` already exists in destination", taking the aggregate `ci` gate down with it, and it did so on three consecutive attempts. Two cache layers own `~/.cargo/bin`. `Swatinem/rust-cache` runs with `cache-bin: true` and restores both binaries; the dedicated `actions/cache` step keyed on `sarif-tools--0.8.0` can still report a miss, because its key is scoped separately and a branch that has never populated it misses while rust-cache restores from the default branch. The install step then runs against a directory that already has the binaries and refuses to overwrite. `--force` makes it idempotent in both states. It is still reached only on a cache miss, so a warm run skips the build as before. Not part of #1376 — the failure surfaced on that branch because it is new, but the conflict is repo-wide and independent of it. --- .github/workflows/ci.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c23cd2afe..0eda05daa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,11 +121,21 @@ jobs: ~/.cargo/bin/clippy-sarif ~/.cargo/bin/sarif-fmt key: sarif-tools-${{ runner.os }}-0.8.0 + # `--force` because two cache layers own `~/.cargo/bin`: the + # `Swatinem/rust-cache` step above runs with `cache-bin: true` and + # restores these two binaries, while the dedicated cache below + # them can still report a miss (its key is scoped separately, so a + # branch that has never populated it misses while rust-cache + # restores from the default branch). `cargo install` then aborts + # with "binary `clippy-sarif` already exists in destination" and + # takes the whole clippy job with it. `--force` makes the install + # idempotent in both states; it is reached only on a miss, so the + # warm path still skips the build entirely. - name: Install clippy-sarif and sarif-fmt if: steps.cache-sarif-tools.outputs.cache-hit != 'true' run: | - cargo install clippy-sarif --version 0.8.0 --locked - cargo install sarif-fmt --version 0.8.0 --locked + cargo install clippy-sarif --version 0.8.0 --locked --force + cargo install sarif-fmt --version 0.8.0 --locked --force # Capture clippy's JSON diagnostics to a file (not a pipe) so the SARIF # is generated from a complete document — a hard compile-error that # aborts clippy mid-stream cannot truncate a piped report. From 1daf397cb7cb0982d031b4213720b97804ab68e0 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Thu, 10 Sep 2026 11:40:32 -0700 Subject: [PATCH 8/9] docs(ast): record that AnyParser's total dispatch is decided MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The unconditional variants make every generic walk monomorphise for all 25 languages under any feature subset — measured at 25 instantiations of metrics_inner and ops_inner in a --features rust build, against one before the split. That is now written next to the enum along with why it stays: the fix needs the dispatch macro in the root crate, which buys an eighth site enumerating every language, and a visitor trait cannot substitute because one defined here can only bound its type parameter by traits this crate can name. Nothing in the workspace pays the cost, and what it costs is compile time rather than shipped binary size. Written as a DECIDED note, matching .rustfmt-bail-baseline.txt's, so the next reader prices it from the record instead of re-deriving it. --- big-code-analysis-ast/src/macros/mod.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/big-code-analysis-ast/src/macros/mod.rs b/big-code-analysis-ast/src/macros/mod.rs index 34fdadffe..f34de1201 100644 --- a/big-code-analysis-ast/src/macros/mod.rs +++ b/big-code-analysis-ast/src/macros/mod.rs @@ -276,6 +276,30 @@ macro_rules! mk_action { /// `Err(LanguageDisabled)` for it — but it can always be *named*, /// which is what lets `with_any_parser!` be written once without /// any `cfg` of its own (#1376). + /// + /// The cost is real and was priced: because the match is total + /// over all 25 variants, every generic walk is monomorphised for + /// every language whatever features were selected. Measured on a + /// `--no-default-features --features rust` build, `metrics_inner` + /// and `ops_inner` each get 25 instantiations where the + /// pre-split crate got one. + /// + /// **DECIDED (#1376): it stays.** Reversing it means moving the + /// dispatch macro into `big-code-analysis`, so the `cfg`s are + /// evaluated in the crate whose features they name — a + /// `#[cfg]` inside a `#[macro_export]` macro is evaluated in the + /// *invoking* crate, which is the whole reason this shape exists + /// — and that buys an eighth site enumerating every language, + /// permanently. A visitor trait does not help: one defined here + /// can only bound its type parameter by traits this crate can + /// name, so it hands back a `ParserTrait` where the metric walks + /// need the root's `MetricSuite`, and Rust cannot let a caller + /// supply that bound. Nothing in this workspace pays the cost — + /// the CLI, the server and the Python bindings all build + /// `all-languages` — and what it costs is compile time and rlib + /// size, not shipped binary size, since unreferenced + /// instantiations are dropped at link time. Revisit only if a + /// consumer of the narrow feature set actually appears. pub enum AnyParser { $( #[doc = concat!("The `", stringify!($camel), "` parser.")] From 68e49a6df6860fd898a13064cd2e803c2c516876 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Thu, 10 Sep 2026 12:04:20 -0700 Subject: [PATCH 9/9] ci(coverage): report the whole workspace, not just the root crate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `cargo llvm-cov report` scopes its output to the current package even when the run that produced the profile data was `--workspace`: it builds an `--ignore-filename-regex` naming every other member directory. The run and the report had drifted apart, so the report only ever described the root crate. That was invisible while the root crate held everything. Moving the parse layer out in #1376 made it visible: 103 files and ~15k lines left the report, which Codecov read as the project shrinking. Those files are not untested — measured at 97.99% — they were simply no longer being described. Repeating `--workspace` on the report also restores two crates that had never been measured at all: the CLI (62 files, 97.26%) and the web server (12 files, 92.30%). Measured lines go from 61,720 to 91,075. The reported percentage moves 99.29% -> 98.67%, which is the denominator growing rather than anything getting worse — the covered count rises from 61,280 to 89,866. Compare covered counts, not the percent column. The benchmark harness joins xtask in codecov's ignore list, and so does the PyO3 crate's Rust layer: pytest drives it and `cargo llvm-cov nextest` cannot see that, so it measures ~52% while being far better tested than that implies. codecov.yml already documented that gap and warned against papering over it with redundant cargo tests; reporting the number would have invited exactly those. --- .github/workflows/ci.yml | 12 ++++++++++-- codecov.yml | 11 +++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0eda05daa..2cc0a6d84 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -348,8 +348,16 @@ jobs: run: cargo llvm-cov --no-report nextest --all-features --workspace --locked # Codecov's native format preserves region coverage (lcov would drop to # line-only). + # + # `--workspace` has to be repeated here: the scope flags on the *run* + # above only decide which tests execute, while `report` scopes the + # emitted file set to the current package on its own. Without it + # cargo-llvm-cov builds an `--ignore-filename-regex` naming every other + # workspace member directory, so the CLI and the web crate were never + # reported, and after #1376 the whole parse layer left the report too — + # 103 files that are in fact ~98% covered. Keep the two in step. - name: Generate Codecov report - run: cargo llvm-cov report --codecov --output-path codecov.json + run: cargo llvm-cov report --workspace --codecov --output-path codecov.json # Per-file coverage table in the run's Summary tab, so reviewers can # read coverage without leaving GitHub or waiting on the Codecov PR # comment. @@ -359,7 +367,7 @@ jobs: echo '### Coverage' echo '' echo '```' - cargo llvm-cov report + cargo llvm-cov report --workspace echo '```' } >> "$GITHUB_STEP_SUMMARY" # token authenticates same-repo runs; fork PRs fall back to Codecov's diff --git a/codecov.yml b/codecov.yml index 41e8d3910..23be30d85 100644 --- a/codecov.yml +++ b/codecov.yml @@ -67,5 +67,16 @@ ignore: # libFuzzer, not by the test suite. - "fuzz/**" - "xtask/**" + # The benchmark harness, for the same reason as xtask: dev tooling this + # repo authors but never ships, and criterion drives it rather than the + # test suite. + - "big-code-analysis-bench/**" + # The PyO3 crate's *Rust* layer, for the reason the measurement-gap note + # above already gives: pytest exercises it and `cargo llvm-cov nextest` + # cannot see that, so it measures ~52% while being far better tested than + # that implies. Reporting the misleading number would only invite the + # redundant cargo tests that note warns against. The pure-Python surface + # is still measured under the `python` flag. + - "big-code-analysis-py/src/**" - "**/tests/**" - "**/build.rs"