fix(http): rebuild client-pump stdlib for node:http under PERRY_NO_AUTO_OPTIMIZE

changelog.d/10667-no-auto-http-client-pump.md

@@ -0,0 +1,9 @@
+Fixed `node:http`/`node:https` client dynamic dispatch (`res.pipe()`, `req.setHeader()`, `req.setTimeout()`, and
+the rest of the client `IncomingMessage`/`ClientRequest` fallback surface) being silently absent under
+`PERRY_NO_AUTO_OPTIMIZE=1`: the prebuilt stdlib archive is built with the default `full` feature set, which
+deliberately excludes `external-http-client-pump` (folding it into `full` would force every no-auto program to
+carry `libperry_ext_http.a`). When the program imports `http`/`https`, the no-auto path now rebuilds
+`perry-stdlib-static` with that feature on top of `full`, in the same cargo invocation as `perry-ext-http` itself
+(two archives built in separate invocations can carry different tokio compilations even off an identical
+`Cargo.lock`, which the existing link-time guard in `runtime_compat.rs` refuses to link). Mirrors the on-demand
+`wasm-host` rebuild `build_optional_runtime` already does for `WebAssembly.*` support (#10466).

crates/perry/src/commands/compile/optimized_libs/no_auto.rs

@@ -42,8 +42,9 @@ pub(crate) fn resolve_no_auto_optimized_libs(
     if matches!(format, OutputFormat::Text) && verbose > 0 {
         eprintln!("  auto-optimize: skipped; using prebuilt target/release/libperry_*.a");
     }
-    let well_known_libs = if std::env::var_os("PERRY_DISABLE_WELL_KNOWN").is_none() {
-        resolve_prebuilt_ext_libs(&well_known_iteration_set(ctx), target, format, verbose)
+    let iteration_set = well_known_iteration_set(ctx);
+    let mut well_known_libs = if std::env::var_os("PERRY_DISABLE_WELL_KNOWN").is_none() {
+        resolve_prebuilt_ext_libs(&iteration_set, target, format, verbose)
     } else {
         Vec::new()
     };
@@ -61,6 +62,46 @@ pub(crate) fn resolve_no_auto_optimized_libs(
     } else {
         (None, None)
     };
+    // #10466 — the prebuilt `libperry_stdlib.a` is built with the default
+    // `full` feature set, which deliberately excludes
+    // `external-http-client-pump` (adding it to `full` would force every
+    // no-auto program, HTTP client or not, to link `libperry_ext_http.a` —
+    // see the Cargo.toml comment on `full`, #5983/#8587). Without that
+    // feature, perry-stdlib's dynamic-dispatch fallbacks for the
+    // `node:http`/`node:https` CLIENT surface (`res.pipe()`, `req.setHeader()`,
+    // `req.setTimeout()`, …) don't exist in the linked archive at all — they
+    // read `undefined` with no compile-time warning. When the program
+    // imports `http`/`https`, rebuild perry-stdlib-static with that feature
+    // added on top of `full`, the same on-demand-rebuild shape
+    // `build_optional_runtime` uses for `wasm-host` — AND, in the SAME cargo
+    // invocation, `perry-ext-http` itself: two archives built in separate
+    // cargo invocations can bundle different tokio compilations even from an
+    // identical Cargo.lock (`runtime_compat.rs`'s link-time guard exists
+    // exactly for this), so a stdlib-only rebuild would leave the fresh
+    // stdlib archive unlinkable against whatever `libperry_ext_http.a`
+    // `resolve_prebuilt_ext_libs` found on disk. A prior wasm/native-addon
+    // rebuild above already producing a stdlib archive (Windows) takes
+    // precedence; this only fills the common case where `stdlib` is `None`.
+    let stdlib = stdlib.or_else(|| {
+        let imports_http_client = iteration_set.iter().any(|m| {
+            matches!(
+                m.strip_prefix("node:").unwrap_or(m.as_str()),
+                "http" | "https"
+            )
+        });
+        if !imports_http_client {
+            return None;
+        }
+        let (stdlib_path, ext_http_path) = build_http_client_pump_stdlib(target, format, verbose)?;
+        // Replace whatever `libperry_ext_http.a` `resolve_prebuilt_ext_libs`
+        // found (built in a different cargo invocation, so a different
+        // tokio compilation) with the one just built alongside this stdlib,
+        // in the same invocation — the pair the link-time guard requires.
+        let ext_http_name = ext_http_path.file_name().map(|n| n.to_owned());
+        well_known_libs.retain(|p| p.file_name() != ext_http_name.as_deref());
+        well_known_libs.push(ext_http_path);
+        Some(stdlib_path)
+    });
     OptimizedLibs {
         runtime,
         stdlib,
@@ -70,6 +111,141 @@ pub(crate) fn resolve_no_auto_optimized_libs(
     }
 }
 
+/// #10466 — on-demand rebuild of `perry-stdlib-static` (default `full`
+/// features plus `external-http-client-pump`) into a dedicated target dir,
+/// so the no-auto path's client-side `node:http`/`node:https` dynamic
+/// dispatch (`res.pipe()`/`req.setHeader()`/`req.setTimeout()`/…) has
+/// somewhere to link against without forcing every other no-auto program to
+/// carry `libperry_ext_http.a`. `perry-ext-http` is rebuilt **in the same
+/// cargo invocation** — two archives from separate invocations can bundle
+/// different tokio compilations even off an identical `Cargo.lock`
+/// (`runtime_compat.rs`'s link-time guard exists exactly for this pair), so
+/// a stdlib-only rebuild would leave the fresh stdlib unlinkable against
+/// whatever `libperry_ext_http.a` `resolve_prebuilt_ext_libs` found on disk.
+/// Mirrors `build_optional_runtime`'s `wasm-host` rebuild; returns `None` on
+/// any failure (no source on disk, no cargo, build error) so the caller
+/// falls back to the prebuilt full stdlib (same #10466 gap, not a new
+/// failure mode). Returns `(stdlib_archive, ext_http_archive)`.
+fn build_http_client_pump_stdlib(
+    target: Option<&str>,
+    format: OutputFormat,
+    verbose: u8,
+) -> Option<(PathBuf, PathBuf)> {
+    let workspace_root = cargo_target_dir_path(find_perry_workspace_root()?);
+    let stdlib_crate_dir = workspace_root.join("crates").join("perry-stdlib-static");
+    let ext_http_crate_dir = workspace_root.join("crates").join("perry-ext-http");
+    if !stdlib_crate_dir.is_dir() || !ext_http_crate_dir.is_dir() {
+        if matches!(format, OutputFormat::Text) && verbose > 0 {
+            eprintln!(
+                "  http-client-pump (no-auto): skipping rebuild — crate source not found at {} or {}",
+                stdlib_crate_dir.display(),
+                ext_http_crate_dir.display()
+            );
+        }
+        return None;
+    }
+
+    if matches!(format, OutputFormat::Text) {
+        println!(
+            "  http-client-pump (no-auto): rebuilding stdlib (external-http-client-pump) + perry-ext-http together"
+        );
+    }
+
+    // Dedicated target dir so the prebuilt libperry_stdlib.a in
+    // target/release is not overwritten. Cargo's incremental cache makes
+    // repeat builds a no-op.
+    let relative_target_dir = PathBuf::from("target").join("perry-no-auto-http-pump");
+    let pump_target_dir = cargo_target_dir_path(workspace_root.join(&relative_target_dir));
+    let cargo_target_dir = if cfg!(windows) {
+        relative_target_dir
+    } else {
+        pump_target_dir.clone()
+    };
+
+    let mut cargo_cmd = Command::new("cargo");
+    cargo_cmd
+        .current_dir(&workspace_root)
+        .env("CARGO_TARGET_DIR", &cargo_target_dir)
+        .arg("build")
+        .arg("--release")
+        .arg("-p")
+        .arg("perry-stdlib-static")
+        .arg("-p")
+        .arg("perry-ext-http")
+        .arg("--features")
+        .arg("perry-stdlib/external-http-client-pump");
+    if let Some(triple) = rust_target_triple(target) {
+        cargo_cmd.arg("--target").arg(triple);
+    }
+    if is_android_target(target) {
+        if let Some(ndk) = std::env::var_os("ANDROID_NDK_HOME") {
+            for (k, v) in android_cross_env(std::path::Path::new(&ndk), target) {
+                cargo_cmd.env(k, v);
+            }
+        }
+    }
+    if matches!(target, Some("harmonyos") | Some("harmonyos-simulator")) {
+        match find_harmonyos_sdk() {
+            Some(sdk) => {
+                for (k, v) in harmonyos_cross_env(&sdk, target) {
+                    cargo_cmd.env(k, v);
+                }
+            }
+            None => {
+                if matches!(format, OutputFormat::Text) && verbose > 0 {
+                    eprintln!(
+                        "  http-client-pump (no-auto): skipping rebuild — OHOS SDK not found (set OHOS_SDK_HOME)"
+                    );
+                }
+                return None;
+            }
+        }
+    }
+
+    match super::super::tool_output::run_internal_tool(&mut cargo_cmd, verbose) {
+        Ok(status) if status.success() => {}
+        Ok(status) => {
+            if matches!(format, OutputFormat::Text) {
+                eprintln!(
+                    "  http-client-pump (no-auto): cargo build for http-client-pump stdlib+ext-http failed ({status})"
+                );
+            }
+            return None;
+        }
+        Err(err) => {
+            if matches!(format, OutputFormat::Text) {
+                eprintln!("  http-client-pump (no-auto): failed to spawn cargo ({err})");
+            }
+            return None;
+        }
+    }
+
+    let (stdlib_name, ext_http_name) = if is_windows_target(target) {
+        ("perry_stdlib.lib", "perry_ext_http.lib")
+    } else {
+        ("libperry_stdlib.a", "libperry_ext_http.a")
+    };
+    let mut release_dir = pump_target_dir;
+    if let Some(triple) = rust_target_triple(target) {
+        release_dir = release_dir.join(triple);
+    }
+    let release_dir = release_dir.join("release");
+    let stdlib = release_dir.join(stdlib_name);
+    let ext_http = release_dir.join(ext_http_name);
+    for path in [&stdlib, &ext_http] {
+        if !path.exists() {
+            if matches!(format, OutputFormat::Text) && verbose > 0 {
+                eprintln!(
+                    "  http-client-pump (no-auto): cargo finished but {} was not produced",
+                    path.display()
+                );
+            }
+            return None;
+        }
+    }
+    Some((stdlib, ext_http))
+}
+
 /// Build `perry-runtime-static` with default features + `perry-runtime/wasm-host`
 /// into a dedicated target dir so the prebuilt `libperry_runtime.a` is not
 /// clobbered. Windows also builds `perry-stdlib-static` in the same graph and

crates/perry/src/commands/compile/optimized_libs/tests.rs

@@ -788,16 +788,18 @@ fn no_auto_still_resolves_prebuilt_well_known_archives() {
     let old_disable_well_known = std::env::var("PERRY_DISABLE_WELL_KNOWN").ok();
 
     let dir = tempfile::tempdir().expect("tempdir");
-    let http =
-        super::super::well_known::lookup_well_known("http").expect("http well-known binding");
+    // #10466 — deliberately NOT "http"/"https" here: importing either now
+    // triggers `build_http_client_pump_stdlib`'s on-demand rebuild (a real
+    // cargo invocation), which this test's fake `PERRY_LIB_DIR` archives
+    // (raw `!<arch>\n` placeholders, not real cargo output) can't stand in
+    // for, and which would turn this fast unit test into a slow, real build.
+    // That new behavior has its own coverage below
+    // (`no_auto_http_client_import_rebuilds_pump_stdlib_with_ext_http`).
+    // This test's job is unrelated: confirm `resolve_prebuilt_ext_libs` still
+    // finds multiple well-known archives via `PERRY_LIB_DIR` when no rebuild
+    // trigger is present.
     let net = super::super::well_known::lookup_well_known("net").expect("net well-known binding");
     let ws = super::super::well_known::lookup_well_known("ws").expect("ws well-known binding");
-    let http_lib = dir
-        .path()
-        .join(super::super::well_known::ext_staticlib_filename(
-            &http.lib,
-            rust_target_triple(None),
-        ));
     let net_lib = dir
         .path()
         .join(super::super::well_known::ext_staticlib_filename(
@@ -810,7 +812,6 @@ fn no_auto_still_resolves_prebuilt_well_known_archives() {
             &ws.lib,
             rust_target_triple(None),
         ));
-    std::fs::write(&http_lib, b"!<arch>\n").expect("write fake http archive");
     std::fs::write(&net_lib, b"!<arch>\n").expect("write fake net archive");
     std::fs::write(&ws_lib, b"!<arch>\n").expect("write fake ws archive");
 
@@ -822,7 +823,6 @@ fn no_auto_still_resolves_prebuilt_well_known_archives() {
     set_env_var("PERRY_DISABLE_WELL_KNOWN", None);
 
     let mut ctx = CompilationContext::new(dir.path().to_path_buf());
-    ctx.native_module_imports.insert("http".to_string());
     ctx.native_module_imports.insert("net".to_string());
     ctx.native_module_imports.insert("ws".to_string());
     let libs = resolve_no_auto_optimized_libs(&ctx, None, OutputFormat::Json, 0);
@@ -836,11 +836,6 @@ fn no_auto_still_resolves_prebuilt_well_known_archives() {
 
     assert_eq!(libs.runtime, None);
     assert_eq!(libs.stdlib, None);
-    assert!(
-        libs.well_known_libs.contains(&http_lib),
-        "expected no-auto well-known libs to include {http_lib:?}, got {:?}",
-        libs.well_known_libs
-    );
     assert!(
         libs.well_known_libs.contains(&net_lib),
         "expected no-auto well-known libs to include {net_lib:?}, got {:?}",
@@ -853,6 +848,43 @@ fn no_auto_still_resolves_prebuilt_well_known_archives() {
     );
 }
 
+/// #10466 — the flip side of the test above: when the program DOES import
+/// `http`, no-auto now rebuilds `perry-stdlib-static` (with
+/// `external-http-client-pump`) and `perry-ext-http` together, and the
+/// rebuilt `perry-ext-http` archive takes the place of whatever
+/// `resolve_prebuilt_ext_libs` would otherwise have found on disk for it.
+/// This does a real (if small) cargo build, so it's slower than the rest of
+/// this file — that's the trade-off for exercising the actual rebuild path
+/// rather than re-asserting the pass-through plumbing against a mock.
+#[test]
+fn no_auto_http_client_import_rebuilds_pump_stdlib_with_ext_http() {
+    let _guard = env_lock();
+    let mut ctx = CompilationContext::new(
+        find_perry_workspace_root().expect("workspace root for this checkout"),
+    );
+    ctx.native_module_imports.insert("http".to_string());
+    let libs = resolve_no_auto_optimized_libs(&ctx, None, OutputFormat::Json, 0);
+
+    let stdlib = libs
+        .stdlib
+        .as_ref()
+        .expect("http import should trigger the http-client-pump stdlib rebuild");
+    assert!(
+        stdlib.ends_with("libperry_stdlib.a") || stdlib.ends_with("perry_stdlib.lib"),
+        "unexpected stdlib archive name: {stdlib:?}"
+    );
+    let ext_http_in_well_known = libs.well_known_libs.iter().any(|p| {
+        p.file_name()
+            .and_then(|n| n.to_str())
+            .is_some_and(|n| n.contains("perry_ext_http"))
+    });
+    assert!(
+        ext_http_in_well_known,
+        "expected the freshly-rebuilt perry-ext-http archive in well_known_libs, got {:?}",
+        libs.well_known_libs
+    );
+}
+
 #[cfg(windows)]
 #[test]
 fn cargo_target_dir_strips_windows_verbatim_prefixes() {