diff --git a/Cargo.toml b/Cargo.toml index cb853ba..9bcbb98 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,8 @@ members = [ "crates/scheduler", "crates/refineable", "crates/refineable/derive_refineable", + "crates/perf", + "crates/util_macros", ] [workspace.package] @@ -37,6 +39,8 @@ util = { path = "crates/util" } scheduler = { path = "crates/scheduler" } refineable = { path = "crates/refineable" } derive_refineable = { path = "crates/refineable/derive_refineable" } +perf = { path = "crates/perf" } +util_macros = { path = "crates/util_macros" } # # External crates diff --git a/crates/perf/Cargo.toml b/crates/perf/Cargo.toml new file mode 100644 index 0000000..d4acad1 --- /dev/null +++ b/crates/perf/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "perf" +version = "0.1.0" +publish = false +edition.workspace = true +license = "Apache-2.0" +description = "A tool for measuring Zed test performance, with too many Clippy lints" + +[lib] + +# Some personal lint preferences :3 +[lints.rust] +missing_docs = "warn" + +[lints.clippy] +needless_continue = "allow" # For a convenience macro +all = "warn" +pedantic = "warn" +style = "warn" +missing_docs_in_private_items = "warn" +as_underscore = "deny" +allow_attributes = "deny" +allow_attributes_without_reason = "deny" # This covers `expect` also, since we deny `allow` +let_underscore_must_use = "forbid" +undocumented_unsafe_blocks = "forbid" +missing_safety_doc = "forbid" +disallowed_methods = { level = "allow", priority = 1} + +[dependencies] +collections.workspace = true +serde.workspace = true +serde_json.workspace = true diff --git a/crates/perf/LICENSE-APACHE b/crates/perf/LICENSE-APACHE new file mode 100644 index 0000000..461a0fe --- /dev/null +++ b/crates/perf/LICENSE-APACHE @@ -0,0 +1,222 @@ +Copyright 2022 - 2025 Zed Industries, Inc. + + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + + http://www.apache.org/licenses/LICENSE-2.0 + + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + + +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + + 1. Definitions. + + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + + END OF TERMS AND CONDITIONS diff --git a/crates/perf/src/implementation.rs b/crates/perf/src/implementation.rs new file mode 100644 index 0000000..c151dda --- /dev/null +++ b/crates/perf/src/implementation.rs @@ -0,0 +1,450 @@ +//! The implementation of the this crate is kept in a separate module +//! so that it is easy to publish this crate as part of GPUI's dependencies + +use collections::HashMap; +use serde::{Deserialize, Serialize}; +use std::{num::NonZero, time::Duration}; + +pub mod consts { + //! Preset identifiers and constants so that the profiler and proc macro agree + //! on their communication protocol. + + /// The suffix on the actual test function. + pub const SUF_NORMAL: &str = "__ZED_PERF_FN"; + /// The suffix on an extra function which prints metadata about a test to stdout. + pub const SUF_MDATA: &str = "__ZED_PERF_MDATA"; + /// The env var in which we pass the iteration count to our tests. + pub const ITER_ENV_VAR: &str = "ZED_PERF_ITER"; + /// The prefix printed on all benchmark test metadata lines, to distinguish it from + /// possible output by the test harness itself. + pub const MDATA_LINE_PREF: &str = "ZED_MDATA_"; + /// The version number for the data returned from the test metadata function. + /// Increment on non-backwards-compatible changes. + pub const MDATA_VER: u32 = 0; + /// The default weight, if none is specified. + pub const WEIGHT_DEFAULT: u8 = 50; + /// How long a test must have run to be assumed to be reliable-ish. + pub const NOISE_CUTOFF: std::time::Duration = std::time::Duration::from_millis(250); + + /// Identifier for the iteration count of a test metadata. + pub const ITER_COUNT_LINE_NAME: &str = "iter_count"; + /// Identifier for the weight of a test metadata. + pub const WEIGHT_LINE_NAME: &str = "weight"; + /// Identifier for importance in test metadata. + pub const IMPORTANCE_LINE_NAME: &str = "importance"; + /// Identifier for the test metadata version. + pub const VERSION_LINE_NAME: &str = "version"; + + /// Where to save json run information. + pub const RUNS_DIR: &str = ".perf-runs"; +} + +/// How relevant a benchmark is. +#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub enum Importance { + /// Regressions shouldn't be accepted without good reason. + Critical = 4, + /// Regressions should be paid extra attention. + Important = 3, + /// No extra attention should be paid to regressions, but they might still + /// be indicative of something happening. + #[default] + Average = 2, + /// Unclear if regressions are likely to be meaningful, but still worth keeping + /// an eye on. Lowest level that's checked by default by the profiler. + Iffy = 1, + /// Regressions are likely to be spurious or don't affect core functionality. + /// Only relevant if a lot of them happen, or as supplemental evidence for a + /// higher-importance benchmark regressing. Not checked by default. + Fluff = 0, +} + +impl std::fmt::Display for Importance { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Importance::Critical => f.write_str("critical"), + Importance::Important => f.write_str("important"), + Importance::Average => f.write_str("average"), + Importance::Iffy => f.write_str("iffy"), + Importance::Fluff => f.write_str("fluff"), + } + } +} + +/// Why or when did this test fail? +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum FailKind { + /// Failed while triaging it to determine the iteration count. + Triage, + /// Failed while profiling it. + Profile, + /// Failed due to an incompatible version for the test. + VersionMismatch, + /// Could not parse metadata for a test. + BadMetadata, + /// Skipped due to filters applied on the perf run. + Skipped, +} + +impl std::fmt::Display for FailKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + FailKind::Triage => f.write_str("errored in triage"), + FailKind::Profile => f.write_str("errored while profiling"), + FailKind::VersionMismatch => f.write_str("test version mismatch"), + FailKind::BadMetadata => f.write_str("bad test metadata"), + FailKind::Skipped => f.write_str("skipped"), + } + } +} + +/// Information about a given perf test. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TestMdata { + /// A version number for when the test was generated. If this is greater + /// than the version this test handler expects, one of the following will + /// happen in an unspecified manner: + /// - The test is skipped silently. + /// - The handler exits with an error message indicating the version mismatch + /// or inability to parse the metadata. + /// + /// INVARIANT: If `version` <= `MDATA_VER`, this tool *must* be able to + /// correctly parse the output of this test. + pub version: u32, + /// How many iterations to pass this test if this is preset, or how many + /// iterations a test ended up running afterwards if determined at runtime. + pub iterations: Option>, + /// The importance of this particular test. See the docs on `Importance` for + /// details. + pub importance: Importance, + /// The weight of this particular test within its importance category. Used + /// when comparing across runs. + pub weight: u8, +} + +/// The actual timings of a test, as measured by Hyperfine. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Timings { + /// Mean runtime for `self.iter_total` runs of this test. + pub mean: Duration, + /// Standard deviation for the above. + pub stddev: Duration, +} + +impl Timings { + /// How many iterations does this test seem to do per second? + #[expect( + clippy::cast_precision_loss, + reason = "We only care about a couple sig figs anyways" + )] + #[must_use] + pub fn iters_per_sec(&self, total_iters: NonZero) -> f64 { + (1000. / self.mean.as_millis() as f64) * total_iters.get() as f64 + } +} + +/// Aggregate results, meant to be used for a given importance category. Each +/// test name corresponds to its benchmark results, iteration count, and weight. +type CategoryInfo = HashMap, u8)>; + +/// Aggregate output of all tests run by this handler. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct Output { + /// A list of test outputs. Format is `(test_name, mdata, timings)`. + /// The latter being `Ok(_)` indicates the test succeeded. + /// + /// INVARIANT: If the test succeeded, the second field is `Some(mdata)` and + /// `mdata.iterations` is `Some(_)`. + tests: Vec<(String, Option, Result)>, +} + +impl Output { + /// Instantiates an empty "output". Useful for merging. + #[must_use] + pub fn blank() -> Self { + Output { tests: Vec::new() } + } + + /// Reports a success and adds it to this run's `Output`. + pub fn success( + &mut self, + name: impl AsRef, + mut mdata: TestMdata, + iters: NonZero, + timings: Timings, + ) { + mdata.iterations = Some(iters); + self.tests + .push((name.as_ref().to_string(), Some(mdata), Ok(timings))); + } + + /// Reports a failure and adds it to this run's `Output`. If this test was tried + /// with some number of iterations (i.e. this was not a version mismatch or skipped + /// test), it should be reported also. + /// + /// Using the `fail!()` macro is usually more convenient. + pub fn failure( + &mut self, + name: impl AsRef, + mut mdata: Option, + attempted_iters: Option>, + kind: FailKind, + ) { + if let Some(ref mut mdata) = mdata { + mdata.iterations = attempted_iters; + } + self.tests + .push((name.as_ref().to_string(), mdata, Err(kind))); + } + + /// True if no tests executed this run. + #[must_use] + pub fn is_empty(&self) -> bool { + self.tests.is_empty() + } + + /// Sorts the runs in the output in the order that we want them printed. + pub fn sort(&mut self) { + self.tests.sort_unstable_by(|a, b| match (a, b) { + // Tests where we got no metadata go at the end. + ((_, Some(_), _), (_, None, _)) => std::cmp::Ordering::Greater, + ((_, None, _), (_, Some(_), _)) => std::cmp::Ordering::Less, + // Then sort by importance, then weight. + ((_, Some(a_mdata), _), (_, Some(b_mdata), _)) => { + let c = a_mdata.importance.cmp(&b_mdata.importance); + if matches!(c, std::cmp::Ordering::Equal) { + a_mdata.weight.cmp(&b_mdata.weight) + } else { + c + } + } + // Lastly by name. + ((a_name, ..), (b_name, ..)) => a_name.cmp(b_name), + }); + } + + /// Merges the output of two runs, appending a prefix to the results of the new run. + /// To be used in conjunction with `Output::blank()`, or else only some tests will have + /// a prefix set. + pub fn merge<'a>(&mut self, other: Self, pref_other: impl Into>) { + let pref = if let Some(pref) = pref_other.into() { + "crates/".to_string() + pref + "::" + } else { + String::new() + }; + self.tests = std::mem::take(&mut self.tests) + .into_iter() + .chain( + other + .tests + .into_iter() + .map(|(name, md, tm)| (pref.clone() + &name, md, tm)), + ) + .collect(); + } + + /// Evaluates the performance of `self` against `baseline`. The latter is taken + /// as the comparison point, i.e. a positive resulting `PerfReport` means that + /// `self` performed better. + /// + /// # Panics + /// `self` and `baseline` are assumed to have the iterations field on all + /// `TestMdata`s set to `Some(_)` if the `TestMdata` is present itself. + #[must_use] + pub fn compare_perf(self, baseline: Self) -> PerfReport { + let self_categories = self.collapse(); + let mut other_categories = baseline.collapse(); + + let deltas = self_categories + .into_iter() + .filter_map(|(cat, self_data)| { + // Only compare categories where both meow + // runs have data. / + let mut other_data = other_categories.remove(&cat)?; + let mut max = f64::MIN; + let mut min = f64::MAX; + + // Running totals for averaging out tests. + let mut r_total_numerator = 0.; + let mut r_total_denominator = 0; + // Yeah this is O(n^2), but realistically it'll hardly be a bottleneck. + for (name, (s_timings, s_iters, weight)) in self_data { + // Only use the new weights if they conflict. + let Some((o_timings, o_iters, _)) = other_data.remove(&name) else { + continue; + }; + let shift = + (o_timings.iters_per_sec(o_iters) / s_timings.iters_per_sec(s_iters)) - 1.; + if shift > max { + max = shift; + } + if shift < min { + min = shift; + } + r_total_numerator += shift * f64::from(weight); + r_total_denominator += u32::from(weight); + } + // There were no runs here! + if r_total_denominator == 0 { + None + } else { + let mean = r_total_numerator / f64::from(r_total_denominator); + // TODO: also aggregate standard deviation? That's harder to keep + // meaningful, though, since we dk which tests are correlated. + Some((cat, PerfDelta { max, mean, min })) + } + }) + .collect(); + + PerfReport { deltas } + } + + /// Collapses the `PerfReport` into a `HashMap` over `Importance`, with + /// each importance category having its tests contained. + fn collapse(self) -> HashMap { + let mut categories = HashMap::>::default(); + for entry in self.tests { + if let Some(mdata) = entry.1 + && let Ok(timings) = entry.2 + { + if let Some(handle) = categories.get_mut(&mdata.importance) { + handle.insert(entry.0, (timings, mdata.iterations.unwrap(), mdata.weight)); + } else { + let mut new = HashMap::default(); + new.insert(entry.0, (timings, mdata.iterations.unwrap(), mdata.weight)); + categories.insert(mdata.importance, new); + } + } + } + + categories + } +} + +impl std::fmt::Display for Output { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Don't print the header for an empty run. + if self.tests.is_empty() { + return Ok(()); + } + + // We want to print important tests at the top, then alphabetical. + let mut sorted = self.clone(); + sorted.sort(); + // Markdown header for making a nice little table :> + writeln!( + f, + "| Command | Iter/sec | Mean [ms] | SD [ms] | Iterations | Importance (weight) |", + )?; + writeln!(f, "|:---|---:|---:|---:|---:|---:|")?; + for (name, metadata, timings) in &sorted.tests { + match metadata { + Some(metadata) => match timings { + // Happy path. + Ok(timings) => { + // If the test succeeded, then metadata.iterations is Some(_). + writeln!( + f, + "| {} | {:.2} | {} | {:.2} | {} | {} ({}) |", + name, + timings.iters_per_sec(metadata.iterations.unwrap()), + { + // Very small mean runtimes will give inaccurate + // results. Should probably also penalise weight. + let mean = timings.mean.as_secs_f64() * 1000.; + if mean < consts::NOISE_CUTOFF.as_secs_f64() * 1000. / 8. { + format!("{mean:.2} (unreliable)") + } else { + format!("{mean:.2}") + } + }, + timings.stddev.as_secs_f64() * 1000., + metadata.iterations.unwrap(), + metadata.importance, + metadata.weight, + )?; + } + // We have (some) metadata, but the test errored. + Err(err) => writeln!( + f, + "| ({}) {} | N/A | N/A | N/A | {} | {} ({}) |", + err, + name, + metadata + .iterations + .map_or_else(|| "N/A".to_owned(), |i| format!("{i}")), + metadata.importance, + metadata.weight + )?, + }, + // No metadata, couldn't even parse the test output. + None => writeln!( + f, + "| ({}) {} | N/A | N/A | N/A | N/A | N/A |", + timings.as_ref().unwrap_err(), + name + )?, + } + } + Ok(()) + } +} + +/// The difference in performance between two runs within a given importance +/// category. +struct PerfDelta { + /// The biggest improvement / least bad regression. + max: f64, + /// The weighted average change in test times. + mean: f64, + /// The worst regression / smallest improvement. + min: f64, +} + +/// Shim type for reporting all performance deltas across importance categories. +pub struct PerfReport { + /// Inner (group, diff) pairing. + deltas: HashMap, +} + +impl std::fmt::Display for PerfReport { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.deltas.is_empty() { + return write!(f, "(no matching tests)"); + } + let sorted = self.deltas.iter().collect::>(); + writeln!(f, "| Category | Max | Mean | Min |")?; + // We don't want to print too many newlines at the end, so handle newlines + // a little jankily like this. + write!(f, "|:---|---:|---:|---:|")?; + for (cat, delta) in sorted.into_iter().rev() { + const SIGN_POS: &str = "↑"; + const SIGN_NEG: &str = "↓"; + const SIGN_NEUTRAL_POS: &str = "±↑"; + const SIGN_NEUTRAL_NEG: &str = "±↓"; + + let prettify = |time: f64| { + let sign = if time > 0.05 { + SIGN_POS + } else if time > 0. { + SIGN_NEUTRAL_POS + } else if time > -0.05 { + SIGN_NEUTRAL_NEG + } else { + SIGN_NEG + }; + format!("{} {:.1}%", sign, time.abs() * 100.) + }; + + // Pretty-print these instead of just using the float display impl. + write!( + f, + "\n| {cat} | {} | {} | {} |", + prettify(delta.max), + prettify(delta.mean), + prettify(delta.min) + )?; + } + Ok(()) + } +} diff --git a/crates/perf/src/lib.rs b/crates/perf/src/lib.rs new file mode 100644 index 0000000..7933e66 --- /dev/null +++ b/crates/perf/src/lib.rs @@ -0,0 +1,7 @@ +//! Some constants and datatypes used in the Zed perf profiler. Should only be +//! consumed by the crate providing the matching macros. +//! +//! For usage documentation, see the docs on this crate's binary. + +mod implementation; +pub use implementation::*; diff --git a/crates/perf/src/main.rs b/crates/perf/src/main.rs new file mode 100644 index 0000000..243658e --- /dev/null +++ b/crates/perf/src/main.rs @@ -0,0 +1,582 @@ +//! Perf profiler for Zed tests. Outputs timings of tests marked with the `#[perf]` +//! attribute to stdout in Markdown. See the documentation of `util_macros::perf` +//! for usage details on the actual attribute. +//! +//! # Setup +//! Make sure `hyperfine` is installed and in the shell path. +//! +//! # Usage +//! Calling this tool rebuilds the targeted crate(s) with some cfg flags set for the +//! perf proc macro *and* enables optimisations (`release-fast` profile), so expect +//! it to take a little while. +//! +//! To test an individual crate, run: +//! ```sh +//! cargo perf-test -p $CRATE +//! ``` +//! +//! To test everything (which will be **VERY SLOW**), run: +//! ```sh +//! cargo perf-test --workspace +//! ``` +//! +//! Some command-line parameters are also recognised by this profiler. To filter +//! out all tests below a certain importance (e.g. `important`), run: +//! ```sh +//! cargo perf-test $WHATEVER -- --important +//! ``` +//! +//! Similarly, to skip outputting progress to the command line, pass `-- --quiet`. +//! These flags can be combined. +//! +//! ## Comparing runs +//! Passing `--json=ident` will save per-crate run files in `.perf-runs`, e.g. +//! `cargo perf-test -p gpui -- --json=blah` will result in `.perf-runs/blah.gpui.json` +//! being created (unless no tests were run). These results can be automatically +//! compared. To do so, run `cargo perf-compare new-ident old-ident`. +//! +//! To save the markdown output to a file instead, run `cargo perf-compare --save=$FILE +//! new-ident old-ident`. +//! +//! NB: All files matching `.perf-runs/ident.*.json` will be considered when +//! doing this comparison, so ensure there aren't leftover files in your `.perf-runs` +//! directory that might match that! +//! +//! # Notes +//! This should probably not be called manually unless you're working on the profiler +//! itself; use the `cargo perf-test` alias (after building this crate) instead. + +mod implementation; + +use implementation::{FailKind, Importance, Output, TestMdata, Timings, consts}; + +use std::{ + fs::OpenOptions, + io::{Read, Write}, + num::NonZero, + path::{Path, PathBuf}, + process::{Command, Stdio}, + sync::atomic::{AtomicBool, Ordering}, + time::{Duration, Instant}, +}; + +/// How many iterations to attempt the first time a test is run. +const DEFAULT_ITER_COUNT: NonZero = NonZero::new(3).unwrap(); +/// Multiplier for the iteration count when a test doesn't pass the noise cutoff. +const ITER_COUNT_MUL: NonZero = NonZero::new(4).unwrap(); + +/// Do we keep stderr empty while running the tests? +static QUIET: AtomicBool = AtomicBool::new(false); + +/// Report a failure into the output and skip an iteration. +macro_rules! fail { + ($output:ident, $name:expr, $kind:expr) => {{ + $output.failure($name, None, None, $kind); + continue; + }}; + ($output:ident, $name:expr, $mdata:expr, $kind:expr) => {{ + $output.failure($name, Some($mdata), None, $kind); + continue; + }}; + ($output:ident, $name:expr, $mdata:expr, $count:expr, $kind:expr) => {{ + $output.failure($name, Some($mdata), Some($count), $kind); + continue; + }}; +} + +/// How does this perf run return its output? +enum OutputKind<'a> { + /// Print markdown to the terminal. + Markdown, + /// Save JSON to a file. + Json(&'a Path), +} + +impl OutputKind<'_> { + /// Logs the output of a run as per the `OutputKind`. + fn log(&self, output: &Output, t_bin: &str) { + match self { + OutputKind::Markdown => println!("{output}"), + OutputKind::Json(ident) => { + // We're going to be in tooling/perf/$whatever. + let wspace_dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()) + .join("..") + .join(".."); + let runs_dir = PathBuf::from(&wspace_dir).join(consts::RUNS_DIR); + std::fs::create_dir_all(&runs_dir).unwrap(); + assert!( + !ident.to_string_lossy().is_empty(), + "FATAL: Empty filename specified!" + ); + // Get the test binary's crate's name; a path like + // target/release-fast/deps/gpui-061ff76c9b7af5d7 + // would be reduced to just "gpui". + let test_bin_stripped = Path::new(t_bin) + .file_name() + .unwrap() + .to_str() + .unwrap() + .rsplit_once('-') + .unwrap() + .0; + let mut file_path = runs_dir.join(ident); + file_path + .as_mut_os_string() + .push(format!(".{test_bin_stripped}.json")); + let mut out_file = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&file_path) + .unwrap(); + out_file + .write_all(&serde_json::to_vec(&output).unwrap()) + .unwrap(); + if !QUIET.load(Ordering::Relaxed) { + eprintln!("JSON output written to {}", file_path.display()); + } + } + } + } +} + +/// Runs a given metadata-returning function from a test handler, parsing its +/// output into a `TestMdata`. +fn parse_mdata(t_bin: &str, mdata_fn: &str) -> Result { + let mut cmd = Command::new(t_bin); + cmd.args([mdata_fn, "--exact", "--nocapture"]); + let out = cmd + .output() + .expect("FATAL: Could not run test binary {t_bin}"); + assert!(out.status.success()); + let stdout = String::from_utf8_lossy(&out.stdout); + let mut version = None; + let mut iterations = None; + let mut importance = Importance::default(); + let mut weight = consts::WEIGHT_DEFAULT; + for line in stdout + .lines() + .filter_map(|l| l.strip_prefix(consts::MDATA_LINE_PREF)) + { + let mut items = line.split_whitespace(); + // For v0, we know the ident always comes first, then one field. + match items.next().ok_or(FailKind::BadMetadata)? { + consts::VERSION_LINE_NAME => { + let v = items + .next() + .ok_or(FailKind::BadMetadata)? + .parse::() + .map_err(|_| FailKind::BadMetadata)?; + if v > consts::MDATA_VER { + return Err(FailKind::VersionMismatch); + } + version = Some(v); + } + consts::ITER_COUNT_LINE_NAME => { + // This should never be zero! + iterations = Some( + items + .next() + .ok_or(FailKind::BadMetadata)? + .parse::() + .map_err(|_| FailKind::BadMetadata)? + .try_into() + .map_err(|_| FailKind::BadMetadata)?, + ); + } + consts::IMPORTANCE_LINE_NAME => { + importance = match items.next().ok_or(FailKind::BadMetadata)? { + "critical" => Importance::Critical, + "important" => Importance::Important, + "average" => Importance::Average, + "iffy" => Importance::Iffy, + "fluff" => Importance::Fluff, + _ => return Err(FailKind::BadMetadata), + }; + } + consts::WEIGHT_LINE_NAME => { + weight = items + .next() + .ok_or(FailKind::BadMetadata)? + .parse::() + .map_err(|_| FailKind::BadMetadata)?; + } + _ => unreachable!(), + } + } + + Ok(TestMdata { + version: version.ok_or(FailKind::BadMetadata)?, + // Iterations may be determined by us and thus left unspecified. + iterations, + // In principle this should always be set, but just for the sake of + // stability allow the potentially-breaking change of not reporting the + // importance without erroring. Maybe we want to change this. + importance, + // Same with weight. + weight, + }) +} + +/// Compares the perf results of two profiles as per the arguments passed in. +fn compare_profiles(args: &[String]) { + let mut save_to = None; + let mut ident_idx = 0; + args.first().inspect(|a| { + if a.starts_with("--save") { + save_to = Some( + a.strip_prefix("--save=") + .expect("FATAL: save param formatted incorrectly"), + ); + ident_idx = 1; + } + }); + let ident_new = args + .get(ident_idx) + .expect("FATAL: missing identifier for new run"); + let ident_old = args + .get(ident_idx + 1) + .expect("FATAL: missing identifier for old run"); + let wspace_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); + let runs_dir = PathBuf::from(&wspace_dir) + .join("..") + .join("..") + .join(consts::RUNS_DIR); + + // Use the blank outputs initially, so we can merge into these with prefixes. + let mut outputs_new = Output::blank(); + let mut outputs_old = Output::blank(); + + for e in runs_dir.read_dir().unwrap() { + let Ok(entry) = e else { + continue; + }; + let Ok(metadata) = entry.metadata() else { + continue; + }; + if metadata.is_file() { + let Ok(name) = entry.file_name().into_string() else { + continue; + }; + + // A little helper to avoid code duplication. Reads the `output` from + // a json file, then merges it into what we have so far. + let read_into = |output: &mut Output| { + let mut elems = name.split('.').skip(1); + let prefix = elems.next().unwrap(); + assert_eq!("json", elems.next().unwrap()); + assert!(elems.next().is_none()); + let mut buffer = Vec::new(); + let _ = OpenOptions::new() + .read(true) + .open(entry.path()) + .unwrap() + .read_to_end(&mut buffer) + .unwrap(); + let o_other: Output = serde_json::from_slice(&buffer).unwrap(); + output.merge(o_other, prefix); + }; + + if name.starts_with(ident_old) { + read_into(&mut outputs_old); + } else if name.starts_with(ident_new) { + read_into(&mut outputs_new); + } + } + } + + let res = outputs_new.compare_perf(outputs_old); + if let Some(filename) = save_to { + let mut file = std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(filename) + .expect("FATAL: couldn't save run results to file"); + file.write_all(format!("{res}").as_bytes()).unwrap(); + } else { + println!("{res}"); + } +} + +/// Runs a test binary, filtering out tests which aren't marked for perf triage +/// and giving back the list of tests we care about. +/// +/// The output of this is an iterator over `test_fn_name, test_mdata_name`. +fn get_tests(t_bin: &str) -> impl ExactSizeIterator { + let mut cmd = Command::new(t_bin); + // --format=json is nightly-only :( + cmd.args(["--list", "--format=terse"]); + let out = cmd + .output() + .expect("FATAL: Could not run test binary {t_bin}"); + assert!( + out.status.success(), + "FATAL: Cannot do perf check - test binary {t_bin} returned an error" + ); + if !QUIET.load(Ordering::Relaxed) { + eprintln!("Test binary ran successfully; starting profile..."); + } + // Parse the test harness output to look for tests we care about. + let stdout = String::from_utf8_lossy(&out.stdout); + let mut test_list: Vec<_> = stdout + .lines() + .filter_map(|line| { + // This should split only in two; e.g., + // "app::test::test_arena: test" => "app::test::test_arena:", "test" + let line: Vec<_> = line.split_whitespace().collect(); + match line[..] { + // Final byte of t_name is ":", which we need to ignore. + [t_name, kind] => (kind == "test").then(|| &t_name[..t_name.len() - 1]), + _ => None, + } + }) + // Exclude tests that aren't marked for perf triage based on suffix. + .filter(|t_name| { + t_name.ends_with(consts::SUF_NORMAL) || t_name.ends_with(consts::SUF_MDATA) + }) + .collect(); + + // Pulling itertools just for .dedup() would be quite a big dependency that's + // not used elsewhere, so do this on a vec instead. + test_list.sort_unstable(); + test_list.dedup(); + + // Tests should come in pairs with their mdata fn! + assert!( + test_list.len().is_multiple_of(2), + "Malformed tests in test binary {t_bin}" + ); + + let out = test_list + .chunks_exact_mut(2) + .map(|pair| { + // Be resilient against changes to these constants. + if consts::SUF_NORMAL < consts::SUF_MDATA { + (pair[0].to_owned(), pair[1].to_owned()) + } else { + (pair[1].to_owned(), pair[0].to_owned()) + } + }) + .collect::>(); + out.into_iter() +} + +/// Runs the specified test `count` times, returning the time taken if the test +/// succeeded. +#[inline] +fn spawn_and_iterate(t_bin: &str, t_name: &str, count: NonZero) -> Option { + let mut cmd = Command::new(t_bin); + cmd.args([t_name, "--exact"]); + cmd.env(consts::ITER_ENV_VAR, format!("{count}")); + // Don't let the child muck up our stdin/out/err. + cmd.stdin(Stdio::null()); + cmd.stdout(Stdio::null()); + cmd.stderr(Stdio::null()); + let pre = Instant::now(); + // Discard the output beyond ensuring success. + let out = cmd.spawn().unwrap().wait(); + let post = Instant::now(); + out.iter().find_map(|s| s.success().then_some(post - pre)) +} + +/// Triage a test to determine the correct number of iterations that it should run. +/// Specifically, repeatedly runs the given test until its execution time exceeds +/// `thresh`, calling `step(iterations)` after every failed run to determine the new +/// iteration count. Returns `None` if the test errored or `step` returned `None`, +/// else `Some(iterations)`. +/// +/// # Panics +/// This will panic if `step(usize)` is not monotonically increasing, or if the test +/// binary is invalid. +fn triage_test( + t_bin: &str, + t_name: &str, + thresh: Duration, + mut step: impl FnMut(NonZero) -> Option>, +) -> Option> { + let mut iter_count = DEFAULT_ITER_COUNT; + // It's possible that the first loop of a test might be an outlier (e.g. it's + // doing some caching), in which case we want to skip it. + let duration_once = spawn_and_iterate(t_bin, t_name, NonZero::new(1).unwrap())?; + loop { + let duration = spawn_and_iterate(t_bin, t_name, iter_count)?; + if duration.saturating_sub(duration_once) > thresh { + break Some(iter_count); + } + let new = step(iter_count)?; + assert!( + new > iter_count, + "FATAL: step must be monotonically increasing" + ); + iter_count = new; + } +} + +/// Try to find the hyperfine binary the user has installed. +fn hyp_binary() -> Option { + const HYP_PATH: &str = "hyperfine"; + const HYP_HOME: &str = "~/.cargo/bin/hyperfine"; + if Command::new(HYP_PATH).output().is_err() { + if Command::new(HYP_HOME).output().is_err() { + None + } else { + Some(Command::new(HYP_HOME)) + } + } else { + Some(Command::new(HYP_PATH)) + } +} + +/// Profiles a given test with hyperfine, returning the mean and standard deviation +/// for its runtime. If the test errors, returns `None` instead. +fn hyp_profile(t_bin: &str, t_name: &str, iterations: NonZero) -> Option { + let mut perf_cmd = hyp_binary().expect("Couldn't find the Hyperfine binary on the system"); + + // Warm up the cache and print markdown output to stdout, which we parse. + perf_cmd.args([ + "--style", + "none", + "--warmup", + "1", + "--export-markdown", + "-", + // Parse json instead... + "--time-unit", + "millisecond", + &format!("{t_bin} --exact {t_name}"), + ]); + perf_cmd.env(consts::ITER_ENV_VAR, format!("{iterations}")); + let p_out = perf_cmd.output().unwrap(); + if !p_out.status.success() { + return None; + } + + let cmd_output = String::from_utf8_lossy(&p_out.stdout); + // Can't use .last() since we have a trailing newline. Sigh. + let results_line = cmd_output.lines().nth(3).unwrap(); + // Grab the values out of the pretty-print. + // TODO: Parse json instead. + let mut res_iter = results_line.split_whitespace(); + // Durations are given in milliseconds, so account for that. + let mean = Duration::from_secs_f64(res_iter.nth(5).unwrap().parse::().unwrap() / 1000.); + let stddev = Duration::from_secs_f64(res_iter.nth(1).unwrap().parse::().unwrap() / 1000.); + + Some(Timings { mean, stddev }) +} + +fn main() { + let args = std::env::args().collect::>(); + // We get passed the test we need to run as the 1st argument after our own name. + let t_bin = args + .get(1) + .expect("FATAL: No test binary or command; this shouldn't be manually invoked!"); + + // We're being asked to compare two results, not run the profiler. + if t_bin == "compare" { + compare_profiles(&args[2..]); + return; + } + + // Minimum test importance we care about this run. + let mut thresh = Importance::Iffy; + // Where to print the output of this run. + let mut out_kind = OutputKind::Markdown; + + for arg in args.iter().skip(2) { + match arg.as_str() { + "--critical" => thresh = Importance::Critical, + "--important" => thresh = Importance::Important, + "--average" => thresh = Importance::Average, + "--iffy" => thresh = Importance::Iffy, + "--fluff" => thresh = Importance::Fluff, + "--quiet" => QUIET.store(true, Ordering::Relaxed), + s if s.starts_with("--json") => { + out_kind = OutputKind::Json(Path::new( + s.strip_prefix("--json=") + .expect("FATAL: Invalid json parameter; pass --json=ident"), + )); + } + _ => (), + } + } + if !QUIET.load(Ordering::Relaxed) { + eprintln!("Starting perf check"); + } + + let mut output = Output::default(); + + // Spawn and profile an instance of each perf-sensitive test, via hyperfine. + // Each test is a pair of (test, metadata-returning-fn), so grab both. We also + // know the list is sorted. + let i = get_tests(t_bin); + let len = i.len(); + for (idx, (ref t_name, ref t_mdata)) in i.enumerate() { + if !QUIET.load(Ordering::Relaxed) { + eprint!("\rProfiling test {}/{}", idx + 1, len); + } + // Pretty-printable stripped name for the test. + let t_name_pretty = t_name.replace(consts::SUF_NORMAL, ""); + + // Get the metadata this test reports for us. + let t_mdata = match parse_mdata(t_bin, t_mdata) { + Ok(mdata) => mdata, + Err(err) => fail!(output, t_name_pretty, err), + }; + + if t_mdata.importance < thresh { + fail!(output, t_name_pretty, t_mdata, FailKind::Skipped); + } + + // Time test execution to see how many iterations we need to do in order + // to account for random noise. This is skipped for tests with fixed + // iteration counts. + let final_iter_count = t_mdata.iterations.or_else(|| { + triage_test(t_bin, t_name, consts::NOISE_CUTOFF, |c| { + if let Some(c) = c.checked_mul(ITER_COUNT_MUL) { + Some(c) + } else { + // This should almost never happen, but maybe..? + eprintln!( + "WARNING: Ran nearly usize::MAX iterations of test {t_name_pretty}; skipping" + ); + None + } + }) + }); + + // Don't profile failing tests. + let Some(final_iter_count) = final_iter_count else { + fail!(output, t_name_pretty, t_mdata, FailKind::Triage); + }; + + // Now profile! + if let Some(timings) = hyp_profile(t_bin, t_name, final_iter_count) { + output.success(t_name_pretty, t_mdata, final_iter_count, timings); + } else { + fail!( + output, + t_name_pretty, + t_mdata, + final_iter_count, + FailKind::Profile + ); + } + } + if !QUIET.load(Ordering::Relaxed) { + if output.is_empty() { + eprintln!("Nothing to do."); + } else { + // If stdout and stderr are on the same terminal, move us after the + // output from above. + eprintln!(); + } + } + + // No need making an empty json file on every empty test bin. + if output.is_empty() { + return; + } + + out_kind.log(&output, t_bin); +} diff --git a/crates/util/Cargo.toml b/crates/util/Cargo.toml index 25b2b36..87f9b92 100644 --- a/crates/util/Cargo.toml +++ b/crates/util/Cargo.toml @@ -18,6 +18,7 @@ test-support = ["git2", "rand"] [dependencies] anyhow.workspace = true +util_macros.workspace = true async-fs.workspace = true async_zip.workspace = true collections.workspace = true diff --git a/crates/util/src/util.rs b/crates/util/src/util.rs index ca77e51..9cec9a5 100644 --- a/crates/util/src/util.rs +++ b/crates/util/src/util.rs @@ -1,3 +1,5 @@ +pub use util_macros::{line_endings, path, uri}; + pub mod arc_cow; pub mod archive; pub mod command; diff --git a/crates/util_macros/Cargo.toml b/crates/util_macros/Cargo.toml new file mode 100644 index 0000000..f72955b --- /dev/null +++ b/crates/util_macros/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "util_macros" +version = "0.1.0" +edition.workspace = true +publish = false +license = "Apache-2.0" +description = "Utility macros for Zed" + +[lints] +workspace = true + +[lib] +path = "src/util_macros.rs" +proc-macro = true +doctest = false + +[dependencies] +quote.workspace = true +syn.workspace = true +perf.workspace = true + +[features] +perf-enabled = [] diff --git a/crates/util_macros/LICENSE-APACHE b/crates/util_macros/LICENSE-APACHE new file mode 100644 index 0000000..461a0fe --- /dev/null +++ b/crates/util_macros/LICENSE-APACHE @@ -0,0 +1,222 @@ +Copyright 2022 - 2025 Zed Industries, Inc. + + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + + http://www.apache.org/licenses/LICENSE-2.0 + + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + + +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + + 1. Definitions. + + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + + END OF TERMS AND CONDITIONS diff --git a/crates/util_macros/src/util_macros.rs b/crates/util_macros/src/util_macros.rs new file mode 100644 index 0000000..4973e41 --- /dev/null +++ b/crates/util_macros/src/util_macros.rs @@ -0,0 +1,286 @@ +#![cfg_attr(not(target_os = "windows"), allow(unused))] +#![allow(clippy::test_attr_in_doctest)] + +use perf::*; +use proc_macro::TokenStream; +use quote::{ToTokens, quote}; +use syn::{ItemFn, LitStr, parse_macro_input, parse_quote}; + +/// A macro used in tests for cross-platform path string literals in tests. On Windows it replaces +/// `/` with `\\` and adds `C:` to the beginning of absolute paths. On other platforms, the path is +/// returned unmodified. +/// +/// # Example +/// ```rust +/// use util_macros::path; +/// +/// let path = path!("/Users/user/file.txt"); +/// #[cfg(target_os = "windows")] +/// assert_eq!(path, "C:\\Users\\user\\file.txt"); +/// #[cfg(not(target_os = "windows"))] +/// assert_eq!(path, "/Users/user/file.txt"); +/// ``` +#[proc_macro] +pub fn path(input: TokenStream) -> TokenStream { + let path = parse_macro_input!(input as LitStr); + let mut path = path.value(); + + #[cfg(target_os = "windows")] + { + path = path.replace("/", "\\"); + if path.starts_with("\\") { + path = format!("C:{}", path); + } + } + + TokenStream::from(quote! { + #path + }) +} + +/// This macro replaces the path prefix `file:///` with `file:///C:/` for Windows. +/// But if the target OS is not Windows, the URI is returned as is. +/// +/// # Example +/// ```rust +/// use util_macros::uri; +/// +/// let uri = uri!("file:///path/to/file"); +/// #[cfg(target_os = "windows")] +/// assert_eq!(uri, "file:///C:/path/to/file"); +/// #[cfg(not(target_os = "windows"))] +/// assert_eq!(uri, "file:///path/to/file"); +/// ``` +#[proc_macro] +pub fn uri(input: TokenStream) -> TokenStream { + let uri = parse_macro_input!(input as LitStr); + let uri = uri.value(); + + #[cfg(target_os = "windows")] + let uri = uri.replace("file:///", "file:///C:/"); + + TokenStream::from(quote! { + #uri + }) +} + +/// This macro replaces the line endings `\n` with `\r\n` for Windows. +/// But if the target OS is not Windows, the line endings are returned as is. +/// +/// # Example +/// ```rust +/// use util_macros::line_endings; +/// +/// let text = line_endings!("Hello\nWorld"); +/// #[cfg(target_os = "windows")] +/// assert_eq!(text, "Hello\r\nWorld"); +/// #[cfg(not(target_os = "windows"))] +/// assert_eq!(text, "Hello\nWorld"); +/// ``` +#[proc_macro] +pub fn line_endings(input: TokenStream) -> TokenStream { + let text = parse_macro_input!(input as LitStr); + let text = text.value(); + + #[cfg(target_os = "windows")] + let text = text.replace("\n", "\r\n"); + + TokenStream::from(quote! { + #text + }) +} + +/// Inner data for the perf macro. +#[derive(Default)] +struct PerfArgs { + /// How many times to loop a test before rerunning the test binary. If left + /// empty, the test harness will auto-determine this value. + iterations: Option, + /// How much this test's results should be weighed when comparing across runs. + /// If unspecified, defaults to `WEIGHT_DEFAULT` (50). + weight: Option, + /// How relevant a benchmark is to overall performance. See docs on the enum + /// for details. If unspecified, `Average` is selected. + importance: Importance, +} + +#[warn(clippy::all, clippy::pedantic)] +impl PerfArgs { + /// Parses attribute arguments into a `PerfArgs`. + fn parse_into(&mut self, meta: syn::meta::ParseNestedMeta) -> syn::Result<()> { + if meta.path.is_ident("iterations") { + self.iterations = Some(meta.value()?.parse()?); + } else if meta.path.is_ident("weight") { + self.weight = Some(meta.value()?.parse()?); + } else if meta.path.is_ident("critical") { + self.importance = Importance::Critical; + } else if meta.path.is_ident("important") { + self.importance = Importance::Important; + } else if meta.path.is_ident("average") { + // This shouldn't be specified manually, but oh well. + self.importance = Importance::Average; + } else if meta.path.is_ident("iffy") { + self.importance = Importance::Iffy; + } else if meta.path.is_ident("fluff") { + self.importance = Importance::Fluff; + } else { + return Err(syn::Error::new_spanned(meta.path, "unexpected identifier")); + } + Ok(()) + } +} + +/// Marks a test as perf-sensitive, to be triaged when checking the performance +/// of a build. This also automatically applies `#[test]`. +/// +/// # Usage +/// Applying this attribute to a test marks it as average importance by default. +/// There are 5 levels of importance (`Critical`, `Important`, `Average`, `Iffy`, +/// `Fluff`); see the documentation on `Importance` for details. Add the importance +/// as a parameter to override the default (e.g. `#[perf(important)]`). +/// +/// Each test also has a weight factor. This is irrelevant on its own, but is considered +/// when comparing results across different runs. By default, this is set to 50; +/// pass `weight = n` as a parameter to override this. Note that this value is only +/// relevant within its importance category. +/// +/// By default, the number of iterations when profiling this test is auto-determined. +/// If this needs to be overwritten, pass the desired iteration count as a parameter +/// (`#[perf(iterations = n)]`). Note that the actual profiler may still run the test +/// an arbitrary number times; this flag just sets the number of executions before the +/// process is restarted and global state is reset. +/// +/// This attribute should probably not be applied to tests that do any significant +/// disk IO, as locks on files may not be released in time when repeating a test many +/// times. This might lead to spurious failures. +/// +/// # Examples +/// ```rust +/// use util_macros::perf; +/// +/// #[perf] +/// fn generic_test() { +/// // Test goes here. +/// } +/// +/// #[perf(fluff, weight = 30)] +/// fn cold_path_test() { +/// // Test goes here. +/// } +/// ``` +/// +/// This also works with `#[gpui::test]`s, though in most cases it shouldn't +/// be used with automatic iterations. +/// ```rust,ignore +/// use util_macros::perf; +/// +/// #[perf(iterations = 1, critical)] +/// #[gpui::test] +/// fn oneshot_test(_cx: &mut gpui::TestAppContext) { +/// // Test goes here. +/// } +/// ``` +#[proc_macro_attribute] +#[warn(clippy::all, clippy::pedantic)] +pub fn perf(our_attr: TokenStream, input: TokenStream) -> TokenStream { + let mut args = PerfArgs::default(); + let parser = syn::meta::parser(|meta| PerfArgs::parse_into(&mut args, meta)); + parse_macro_input!(our_attr with parser); + + let ItemFn { + attrs: mut attrs_main, + vis, + sig: mut sig_main, + block, + } = parse_macro_input!(input as ItemFn); + if !attrs_main + .iter() + .any(|a| Some(&parse_quote!(test)) == a.path().segments.last()) + { + attrs_main.push(parse_quote!(#[test])); + } + attrs_main.push(parse_quote!(#[allow(non_snake_case)])); + + let fns = if cfg!(perf_enabled) { + #[allow(clippy::wildcard_imports, reason = "We control the other side")] + use consts::*; + + // Make the ident obvious when calling, for the test parser. + // Also set up values for the second metadata-returning "test". + let mut new_ident_main = sig_main.ident.to_string(); + let mut new_ident_meta = new_ident_main.clone(); + new_ident_main.push_str(SUF_NORMAL); + new_ident_meta.push_str(SUF_MDATA); + + let new_ident_main = syn::Ident::new(&new_ident_main, sig_main.ident.span()); + sig_main.ident = new_ident_main; + + // We don't want any nonsense if the original test had a weird signature. + let new_ident_meta = syn::Ident::new(&new_ident_meta, sig_main.ident.span()); + let sig_meta = parse_quote!(fn #new_ident_meta()); + let attrs_meta = parse_quote!(#[test] #[allow(non_snake_case)]); + + // Make the test loop as the harness instructs it to. + let block_main = { + // The perf harness will pass us the value in an env var. Even if we + // have a preset value, just do this to keep the code paths unified. + parse_quote!({ + let iter_count = std::env::var(#ITER_ENV_VAR).unwrap().parse::().unwrap(); + for _ in 0..iter_count { + #block + } + }) + }; + let importance = format!("{}", args.importance); + let block_meta = { + // This function's job is to just print some relevant info to stdout, + // based on the params this attr is passed. It's not an actual test. + // Since we use a custom attr set on our metadata fn, it shouldn't + // cause problems with xfail tests. + let q_iter = if let Some(iter) = args.iterations { + quote! { + println!("{} {} {}", #MDATA_LINE_PREF, #ITER_COUNT_LINE_NAME, #iter); + } + } else { + quote! {} + }; + let weight = args + .weight + .unwrap_or_else(|| parse_quote! { #WEIGHT_DEFAULT }); + parse_quote!({ + #q_iter + println!("{} {} {}", #MDATA_LINE_PREF, #WEIGHT_LINE_NAME, #weight); + println!("{} {} {}", #MDATA_LINE_PREF, #IMPORTANCE_LINE_NAME, #importance); + println!("{} {} {}", #MDATA_LINE_PREF, #VERSION_LINE_NAME, #MDATA_VER); + }) + }; + + vec![ + // The real test. + ItemFn { + attrs: attrs_main, + vis: vis.clone(), + sig: sig_main, + block: block_main, + }, + // The fake test. + ItemFn { + attrs: attrs_meta, + vis, + sig: sig_meta, + block: block_meta, + }, + ] + } else { + vec![ItemFn { + attrs: attrs_main, + vis, + sig: sig_main, + block, + }] + }; + + fns.into_iter() + .flat_map(|f| TokenStream::from(f.into_token_stream())) + .collect() +}