Auto merge of #159642 - Kobzol:bootstrap-pgo-clippy, r=<try>

Optimize Clippy with PGO

Author: bors

bootstrap.example.toml

@@ -995,6 +995,11 @@
 # Instrument cargo, so that when executed, it will gather profiles
 # to this path.
 #pgo.cargo.generate = "/tmp/profiles/foo.profraw"
+# Use the following profile to PGO optimize clippy.
+#pgo.clippy.use = "/tmp/profiles/foo.profraw"
+# Instrument clippy, so that when executed, it will gather profiles
+# to this path.
+#pgo.clippy.generate = "/tmp/profiles/foo.profraw"
 # Use the following profile to PGO optimize LLVM.
 #pgo.llvm.use = "/tmp/profiles/foo.profraw"
 # Instrument LLVM, so that when executed, it will gather profiles

src/bootstrap/src/core/build_steps/tool.rs

@@ -141,6 +141,7 @@ impl Step for ToolBuild {
         let pgo_config = match self.path {
             "src/tools/rustdoc" => Some(&builder.config.rustdoc_pgo),
             "src/tools/cargo" => Some(&builder.config.cargo_pgo),
+            "src/tools/clippy" => Some(&builder.config.clippy_pgo),
             _ => None,
         };
         if let Some(pgo_config) = pgo_config {

src/bootstrap/src/core/config/config.rs

@@ -237,6 +237,7 @@ pub struct Config {
     pub rust_pgo: PgoConfig,
     pub rustdoc_pgo: PgoConfig,
     pub cargo_pgo: PgoConfig,
+    pub clippy_pgo: PgoConfig,
 
     pub llvm_libunwind_default: Option<LlvmLibunwind>,
     pub enable_bolt_settings: bool,
@@ -658,8 +659,13 @@ impl Config {
             libgccjit_libs_dir: gcc_libgccjit_libs_dir,
         } = toml_gcc.unwrap_or_default();
 
-        let Pgo { rustc: pgo_rustc, llvm: pgo_llvm, rustdoc: pgo_rustdoc, cargo: pgo_cargo } =
-            toml_pgo.unwrap_or_default();
+        let Pgo {
+            rustc: pgo_rustc,
+            rustdoc: pgo_rustdoc,
+            cargo: pgo_cargo,
+            clippy: pgo_clippy,
+            llvm: pgo_llvm,
+        } = toml_pgo.unwrap_or_default();
 
         // Backcompat: flags have priority over config
         if flags_rust_profile_use.is_some() || flags_rust_profile_generate.is_some() {
@@ -714,6 +720,7 @@ impl Config {
 
         let pgo_rustdoc = init_pgo(pgo_rustdoc, "rustdoc");
         let pgo_cargo = init_pgo(pgo_cargo, "cargo");
+        let pgo_clippy = init_pgo(pgo_clippy, "clippy");
 
         if rust_bootstrap_override_lld.is_some() && rust_bootstrap_override_lld_legacy.is_some() {
             panic!(
@@ -1413,6 +1420,7 @@ NOTE: Please add `--stage 2` to your command line, or if you're sure you want to
             channel,
             ci_env,
             clippy_info,
+            clippy_pgo: pgo_clippy,
             cmd: flags_cmd,
             codegen_tests: rust_codegen_tests.unwrap_or(true),
             color: flags_color,

src/bootstrap/src/core/config/toml/pgo.rs

@@ -28,6 +28,7 @@ define_config! {
         rustc: Option<PgoConfig> = "rustc",
         rustdoc: Option<PgoConfig> = "rustdoc",
         cargo: Option<PgoConfig> = "cargo",
+        clippy: Option<PgoConfig> = "clippy",
         llvm: Option<PgoConfig> = "llvm",
     }
 }

src/tools/opt-dist/src/exec.rs

@@ -7,7 +7,9 @@ use camino::{Utf8Path, Utf8PathBuf};
 use crate::environment::Environment;
 use crate::metrics::{load_metrics, record_metrics};
 use crate::timer::TimerSection;
-use crate::training::{BoltProfile, LlvmPGOProfile, RustcPGOProfile, RustdocPGOProfile};
+use crate::training::{
+    BoltProfile, ClippyPGOProfile, LlvmPGOProfile, RustcPGOProfile, RustdocPGOProfile,
+};
 use crate::utils::io::normalize_path;
 
 #[derive(Default)]
@@ -129,6 +131,11 @@ impl Bootstrap {
         self
     }
 
+    pub fn with_clippy(mut self) -> Self {
+        self.cmd = self.cmd.arg("clippy");
+        self
+    }
+
     pub fn dist(env: &Environment, dist_args: &[String]) -> Self {
         let metrics_path = env.build_root().join("metrics.json");
         let args = dist_args.iter().map(|arg| arg.as_str()).collect::<Vec<_>>();
@@ -218,6 +225,22 @@ impl Bootstrap {
         self
     }
 
+    pub fn clippy_pgo_instrument(mut self, profile_dir: &Utf8Path) -> Self {
+        self.cmd = self
+            .cmd
+            .arg("--set")
+            .arg(format!(r#"pgo.clippy.generate="{}""#, normalize_path(profile_dir).as_str()));
+        self
+    }
+
+    pub fn clippy_pgo_optimize(mut self, profile: &ClippyPGOProfile) -> Self {
+        self.cmd = self
+            .cmd
+            .arg("--set")
+            .arg(format!(r#"pgo.clippy.use="{}""#, normalize_path(&profile.0).as_str()));
+        self
+    }
+
     pub fn with_llvm_bolt_ldflags(mut self) -> Self {
         self.cmd = self.cmd.arg("--set").arg("llvm.ldflags=-Wl,-q");
         self

src/tools/opt-dist/src/main.rs

@@ -10,8 +10,8 @@ use crate::exec::{Bootstrap, cmd};
 use crate::tests::run_tests;
 use crate::timer::Timer;
 use crate::training::{
-    gather_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles, gather_rustdoc_profiles,
-    llvm_benchmarks, rustc_benchmarks,
+    gather_bolt_profiles, gather_clippy_profiles, gather_llvm_profiles, gather_rustc_profiles,
+    gather_rustdoc_profiles, llvm_benchmarks, rustc_benchmarks,
 };
 use crate::utils::artifact_size::print_binary_sizes;
 use crate::utils::io::{copy_directory, reset_directory};
@@ -239,7 +239,7 @@ fn execute_pipeline(
     // Stage 1: Build PGO instrumented rustc
     // We use a normal build of LLVM, because gathering PGO profiles for LLVM and `rustc` at the
     // same time can cause issues, because the host and in-tree LLVM versions can diverge.
-    let (rustc_pgo_profile, rustdoc_pgo_profile) =
+    let (rustc_pgo_profile, rustdoc_pgo_profile, clippy_pgo_profile) =
         timer.section("Stage 1 (Rustc + rustdoc PGO)", |stage| {
             let rustc_profile_dir_root = env.artifact_dir().join("rustc-pgo");
 
@@ -250,8 +250,10 @@ fn execute_pipeline(
                 let mut builder = Bootstrap::build(env)
                     .with_rustdoc()
                     .with_cargo()
+                    .with_clippy()
                     .rustc_pgo_instrument(&rustc_profile_dir_root)
                     .cargo_pgo_instrument(&rustc_profile_dir_root)
+                    .clippy_pgo_instrument(&rustc_profile_dir_root)
                     .rustdoc_pgo_instrument(&rustc_profile_dir_root);
 
                 if env.supports_shared_llvm() {
@@ -271,23 +273,28 @@ fn execute_pipeline(
             let rustdoc_profile = stage.section("Gather rustdoc profiles", |_| {
                 gather_rustdoc_profiles(env, &rustc_profile_dir_root)
             })?;
+            let clippy_profile = stage.section("Gather clippy profiles", |_| {
+                gather_clippy_profiles(env, &rustc_profile_dir_root)
+            })?;
             print_free_disk_space()?;
 
             stage.section("Build PGO optimized rustc", |section| {
                 let mut cmd = Bootstrap::build(env)
                     .with_rustdoc()
                     .with_cargo()
+                    .with_clippy()
                     .rustc_pgo_optimize(&rustc_profile)
                     .cargo_pgo_optimize(&rustc_profile)
-                    .rustdoc_pgo_optimize(&rustdoc_profile);
+                    .rustdoc_pgo_optimize(&rustdoc_profile)
+                    .clippy_pgo_optimize(&clippy_profile);
                 if env.use_bolt() {
                     cmd = cmd.with_rustc_bolt_ldflags();
                 }
 
                 cmd.run(section)
             })?;
 
-            Ok((rustc_profile, rustdoc_profile))
+            Ok((rustc_profile, rustdoc_profile, clippy_profile))
         })?;
 
     // Stage 2: Gather LLVM PGO profiles
@@ -422,7 +429,8 @@ fn execute_pipeline(
         .llvm_pgo_optimize(llvm_pgo_profile.as_ref())
         .rustc_pgo_optimize(&rustc_pgo_profile)
         .cargo_pgo_optimize(&rustc_pgo_profile)
-        .rustdoc_pgo_optimize(&rustdoc_pgo_profile);
+        .rustdoc_pgo_optimize(&rustdoc_pgo_profile)
+        .clippy_pgo_optimize(&clippy_pgo_profile);
 
     // if LLVM is not built we'll have PGO optimized rustc
     dist = if env.supports_shared_llvm() || !env.build_llvm() {

src/tools/opt-dist/src/training.rs

@@ -115,10 +115,14 @@ pub fn rustc_benchmarks(env: &Environment) -> CmdBuilder {
     init_compiler_benchmarks(env, &["Check", "Debug", "Opt"], &["All"], RUSTC_PGO_CRATES)
 }
 
-pub fn rustdoc_benchmarks(env: &Environment) -> CmdBuilder {
+fn rustdoc_benchmarks(env: &Environment) -> CmdBuilder {
     init_compiler_benchmarks(env, &["Doc"], &["Full"], RUSTC_PGO_CRATES)
 }
 
+fn clippy_benchmarks(env: &Environment) -> CmdBuilder {
+    init_compiler_benchmarks(env, &["Clippy"], &["All"], RUSTC_PGO_CRATES)
+}
+
 pub struct LlvmPGOProfile(pub Utf8PathBuf);
 
 pub fn gather_llvm_profiles(
@@ -165,6 +169,16 @@ pub fn gather_rustdoc_profiles(
         .map(RustdocPGOProfile)
 }
 
+pub struct ClippyPGOProfile(pub Utf8PathBuf);
+
+pub fn gather_clippy_profiles(
+    env: &Environment,
+    profile_root: &Utf8Path,
+) -> anyhow::Result<ClippyPGOProfile> {
+    log::info!("Running benchmarks with PGO instrumented clippy");
+    gather_pgo_profiles(env, profile_root, "clippy", clippy_benchmarks(env)).map(ClippyPGOProfile)
+}
+
 pub fn gather_pgo_profiles(
     env: &Environment,
     profile_root: &Utf8Path,