Auto merge of #139244 - jieyouxu:exp/auto-cross-run-make, r=<try>

[WIP] Enable automatic cross-compilation in run-make tests

> [!CAUTION]
>
> Stacked on top of #142414, needs rebase.

Supersedes #138066.

Blocker for #141856.

Based on #138066 with #139242 + #139239 cherry-picked in, plus `rustdoc()` cross-compile changes.

r? `@ghost`

try-job: armhf-gnu
try-job: test-various

Author: rust-bors[bot]

src/tools/run-make-support/src/external_deps/rustc.rs

@@ -6,7 +6,7 @@ use crate::command::Command;
 use crate::env::env_var;
 use crate::path_helpers::cwd;
 use crate::util::set_host_compiler_dylib_path;
-use crate::{is_aix, is_darwin, is_msvc, is_windows, uname};
+use crate::{is_aix, is_darwin, is_msvc, is_windows, target, uname};
 
 /// Construct a new `rustc` invocation. This will automatically set the library
 /// search path as `-L cwd()`. Use [`bare_rustc`] to avoid this.
@@ -27,9 +27,15 @@ pub fn bare_rustc() -> Rustc {
 #[must_use]
 pub struct Rustc {
     cmd: Command,
+    target: Option<String>,
 }
 
-crate::macros::impl_common_helpers!(Rustc);
+// Only fill in the target just before execution, so that it can be overridden.
+crate::macros::impl_common_helpers!(Rustc, |rustc: &mut Rustc| {
+    if let Some(target) = &rustc.target {
+        rustc.cmd.arg(&format!("--target={target}"));
+    }
+});
 
 pub fn rustc_path() -> String {
     env_var("RUSTC")
@@ -46,19 +52,22 @@ impl Rustc {
     // `rustc` invocation constructor methods
 
     /// Construct a new `rustc` invocation. This will automatically set the library
-    /// search path as `-L cwd()`. Use [`bare_rustc`] to avoid this.
+    /// search path as `-L cwd()` and also the compilation target.
+    /// Use [`bare_rustc`] to avoid this.
     #[track_caller]
     pub fn new() -> Self {
         let mut cmd = setup_common();
         cmd.arg("-L").arg(cwd());
-        Self { cmd }
+
+        // Automatically default to cross-compilation
+        Self { cmd, target: Some(target()) }
     }
 
     /// Construct a bare `rustc` invocation with no flags set.
     #[track_caller]
     pub fn bare() -> Self {
         let cmd = setup_common();
-        Self { cmd }
+        Self { cmd, target: None }
     }
 
     // Argument provider methods
@@ -234,8 +243,9 @@ impl Rustc {
 
     /// Specify the target triple, or a path to a custom target json spec file.
     pub fn target<S: AsRef<str>>(&mut self, target: S) -> &mut Self {
-        let target = target.as_ref();
-        self.cmd.arg(format!("--target={target}"));
+        // We store the target as a separate field, so that it can be specified multiple times.
+        // This is in particular useful to override the default target set in Rustc::new().
+        self.target = Some(target.as_ref().to_string());
         self
     }
 

src/tools/run-make-support/src/external_deps/rustdoc.rs

@@ -3,21 +3,36 @@ use std::path::Path;
 
 use crate::command::Command;
 use crate::env::env_var;
+use crate::target;
 use crate::util::set_host_compiler_dylib_path;
 
-/// Construct a new `rustdoc` invocation. This will configure the host compiler runtime libs.
+/// Construct a new `rustdoc` invocation with target automatically set to cross-compile target and
+/// with host compiler runtime libs configured. Use [`bare_rustdoc`] to avoid automatically setting
+/// cross-compile target.
 #[track_caller]
 pub fn rustdoc() -> Rustdoc {
     Rustdoc::new()
 }
 
+/// Bare `rustdoc` invocation, no args set.
+#[track_caller]
+pub fn bare_rustdoc() -> Rustdoc {
+    Rustdoc::bare()
+}
+
 #[derive(Debug)]
 #[must_use]
 pub struct Rustdoc {
     cmd: Command,
+    target: Option<String>,
 }
 
-crate::macros::impl_common_helpers!(Rustdoc);
+// Only fill in the target just before execution, so that it can be overridden.
+crate::macros::impl_common_helpers!(Rustdoc, |rustdoc: &mut Rustdoc| {
+    if let Some(target) = &rustdoc.target {
+        rustdoc.cmd.arg(&format!("--target={target}"));
+    }
+});
 
 #[track_caller]
 fn setup_common() -> Command {
@@ -28,11 +43,20 @@ fn setup_common() -> Command {
 }
 
 impl Rustdoc {
-    /// Construct a bare `rustdoc` invocation. This will configure the host compiler runtime libs.
+    /// Construct a new `rustdoc` invocation with target automatically set to cross-compile target
+    /// and with host compiler runtime libs configured. Use [`bare_rustdoc`] to avoid automatically
+    /// setting cross-compile target.
     #[track_caller]
     pub fn new() -> Self {
         let cmd = setup_common();
-        Self { cmd }
+        Self { cmd, target: Some(target()) }
+    }
+
+    /// Bare `rustdoc` invocation, no args set.
+    #[track_caller]
+    pub fn bare() -> Self {
+        let cmd = setup_common();
+        Self { cmd, target: None }
     }
 
     /// Specify where an external library is located.
@@ -85,8 +109,9 @@ impl Rustdoc {
 
     /// Specify the target triple, or a path to a custom target json spec file.
     pub fn target<S: AsRef<str>>(&mut self, target: S) -> &mut Self {
-        let target = target.as_ref();
-        self.cmd.arg(format!("--target={target}"));
+        // We store the target as a separate field, so that it can be specified multiple times.
+        // This is in particular useful to override the default target set in `Rustdoc::new()`.
+        self.target = Some(target.as_ref().to_string());
         self
     }
 

src/tools/run-make-support/src/lib.rs

@@ -68,7 +68,7 @@ pub use llvm::{
 };
 pub use python::python_command;
 pub use rustc::{bare_rustc, rustc, rustc_path, Rustc};
-pub use rustdoc::{rustdoc, Rustdoc};
+pub use rustdoc::{bare_rustdoc, rustdoc, Rustdoc};
 
 /// [`diff`][mod@diff] is implemented in terms of the [similar] library.
 ///

src/tools/run-make-support/src/macros.rs

@@ -23,10 +23,16 @@
 /// }
 /// ```
 ///
+/// You can pass an optional second parameter which should be a function that is passed
+/// `&mut self` just before the command is executed.
+///
 /// [`Command`]: crate::command::Command
 /// [`CompletedProcess`]: crate::command::CompletedProcess
 macro_rules! impl_common_helpers {
     ($wrapper: ident) => {
+        $crate::macros::impl_common_helpers!($wrapper, |_| {});
+    };
+    ($wrapper: ident, $before_exec: expr) => {
         impl $wrapper {
             /// In very rare circumstances, you may need a e.g. `bare_rustc()` or `bare_rustdoc()`
             /// with host runtime libs configured, but want the underlying raw
@@ -130,12 +136,14 @@ macro_rules! impl_common_helpers {
             /// Run the constructed command and assert that it is successfully run.
             #[track_caller]
             pub fn run(&mut self) -> crate::command::CompletedProcess {
+                $before_exec(&mut *self);
                 self.cmd.run()
             }
 
             /// Run the constructed command and assert that it does not successfully run.
             #[track_caller]
             pub fn run_fail(&mut self) -> crate::command::CompletedProcess {
+                $before_exec(&mut *self);
                 self.cmd.run_fail()
             }
 
@@ -145,6 +153,7 @@ macro_rules! impl_common_helpers {
             /// whenever possible.
             #[track_caller]
             pub fn run_unchecked(&mut self) -> crate::command::CompletedProcess {
+                $before_exec(&mut *self);
                 self.cmd.run_unchecked()
             }
 

tests/run-make/CURRENT_RUSTC_VERSION/rmake.rs

@@ -1,3 +1,4 @@
+//@ needs-target-std
 // ignore-tidy-linelength
 
 // Check that the `CURRENT_RUSTC_VERSION` placeholder is correctly replaced by the current

tests/run-make/allow-warnings-cmdline-stability/rmake.rs

@@ -1,3 +1,4 @@
+//@ needs-target-std
 // Test that `-Awarnings` suppresses warnings for unstable APIs.
 
 use run_make_support::rustc;

tests/run-make/apple-deployment-target/rmake.rs

@@ -41,7 +41,6 @@ fn main() {
 
     // Remove env vars to get `rustc`'s default
     let output = rustc()
-        .target(target())
         .env_remove("MACOSX_DEPLOYMENT_TARGET")
         .env_remove("IPHONEOS_DEPLOYMENT_TARGET")
         .env_remove("WATCHOS_DEPLOYMENT_TARGET")
@@ -58,7 +57,6 @@ fn main() {
     run_in_tmpdir(|| {
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.crate_type("lib");
             rustc.emit("obj");
             rustc.input("foo.rs");
@@ -82,7 +80,6 @@ fn main() {
 
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.crate_type("dylib");
             rustc.input("foo.rs");
             rustc.output("libfoo.dylib");
@@ -108,7 +105,6 @@ fn main() {
     run_in_tmpdir(|| {
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.crate_type("bin");
             rustc.input("foo.rs");
             rustc.output("foo");
@@ -147,7 +143,6 @@ fn main() {
     run_in_tmpdir(|| {
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.incremental("incremental");
             rustc.crate_type("lib");
             rustc.emit("obj");

tests/run-make/apple-sdk-version/rmake.rs

@@ -24,8 +24,7 @@ fn has_sdk_version(file: &str, version: &str) {
 
 fn main() {
     // Fetch rustc's inferred deployment target.
-    let current_deployment_target =
-        rustc().target(target()).print("deployment-target").run().stdout_utf8();
+    let current_deployment_target = rustc().print("deployment-target").run().stdout_utf8();
     let current_deployment_target = current_deployment_target.split('=').last().unwrap().trim();
 
     // Fetch current SDK version via. xcrun.
@@ -45,15 +44,15 @@ fn main() {
     let current_sdk_version = current_sdk_version.trim();
 
     // Check the SDK version in the object file produced by the codegen backend.
-    rustc().target(target()).crate_type("lib").emit("obj").input("foo.rs").output("foo.o").run();
+    rustc().crate_type("lib").emit("obj").input("foo.rs").output("foo.o").run();
     // Set to 0, which means not set or "n/a".
     has_sdk_version("foo.o", "n/a");
 
     // Check the SDK version in the .rmeta file, as set in `create_object_file`.
     //
     // This is just to ensure that we don't set some odd version in `create_object_file`,
     // if the rmeta file is packed in a different way in the future, this can safely be removed.
-    rustc().target(target()).crate_type("rlib").input("foo.rs").output("libfoo.rlib").run();
+    rustc().crate_type("rlib").input("foo.rs").output("libfoo.rlib").run();
     // Extra .rmeta file (which is encoded as an object file).
     cmd("ar").arg("-x").arg("libfoo.rlib").arg("lib.rmeta").run();
     has_sdk_version("lib.rmeta", "n/a");
@@ -69,7 +68,6 @@ fn main() {
         // Test with clang
         let file_name = format!("foo_cc{file_ext}");
         rustc()
-            .target(target())
             .crate_type("bin")
             .arg("-Clinker-flavor=gcc")
             .input("foo.rs")
@@ -80,7 +78,6 @@ fn main() {
         // Test with ld64
         let file_name = format!("foo_ld{file_ext}");
         rustc()
-            .target(target())
             .crate_type("bin")
             .arg("-Clinker-flavor=ld")
             .input("foo.rs")

tests/run-make/artifact-incr-cache-no-obj/rmake.rs

@@ -1,3 +1,5 @@
+//@ needs-target-std
+//
 // emitting an object file is not necessary if user didn't ask for one
 //
 // This test is similar to run-make/artifact-incr-cache but it doesn't

tests/run-make/artifact-incr-cache/rmake.rs

@@ -1,3 +1,5 @@
+//@ needs-target-std
+//
 // rustc should be able to emit required files (asm, llvm-*, etc) during incremental
 // compilation on the first pass by running the code gen as well as on subsequent runs -
 // extracting them from the cache

tests/run-make/bin-emit-no-symbols/rmake.rs

@@ -1,3 +1,5 @@
+//@ needs-target-std
+//
 // When setting the crate type as a "bin" (in app.rs),
 // this could cause a bug where some symbols would not be
 // emitted in the object files. This has been fixed, and

tests/run-make/box-struct-no-segfault/rmake.rs

@@ -1,3 +1,5 @@
+//@ needs-target-std
+//
 // The crate "foo" tied to this test executes a very specific function,
 // which involves boxing an instance of the struct Foo. However,
 // this once caused a segmentation fault in cargo release builds due to an LLVM

tests/run-make/checksum-freshness/rmake.rs

@@ -1,3 +1,4 @@
+//@ needs-target-std
 use run_make_support::{rfs, rustc};
 
 fn main() {

tests/run-make/compiler-lookup-paths-2/rmake.rs

@@ -1,3 +1,5 @@
+//@ needs-target-std
+//
 // This test checks that extern crate declarations in Cargo without a corresponding declaration
 // in the manifest of a dependency are NOT allowed. The last rustc call does it anyways, which
 // should result in a compilation failure.

tests/run-make/compiler-lookup-paths/rmake.rs

@@ -1,3 +1,5 @@
+//@ needs-target-std
+//
 // Since #19941, rustc can accept specifications on its library search paths.
 // This test runs Rust programs with varied library dependencies, expecting them
 // to succeed or fail depending on the situation.

tests/run-make/const-trait-stable-toolchain/rmake.rs

@@ -1,3 +1,5 @@
+//@ needs-target-std
+//
 // Test output of const super trait errors in both stable and nightly.
 // We don't want to provide suggestions on stable that only make sense in nightly.
 

tests/run-make/crate-circular-deps-link/rmake.rs

@@ -1,3 +1,5 @@
+//@ needs-target-std
+//
 // Test that previously triggered a linker failure with root cause
 // similar to one found in the issue #69368.
 //

tests/run-make/cross-lang-lto-upstream-rlibs/rmake.rs

@@ -1,3 +1,5 @@
+//@ needs-target-std
+//
 // When using the flag -C linker-plugin-lto, static libraries could lose their upstream object
 // files during compilation. This bug was fixed in #53031, and this test compiles a staticlib
 // dependent on upstream, checking that the upstream object file still exists after no LTO and

tests/run-make/cross-lang-lto/rmake.rs

@@ -1,3 +1,5 @@
+//@ needs-target-std
+//
 // This test checks that the object files we generate are actually
 // LLVM bitcode files (as used by linker LTO plugins) when compiling with
 // -Clinker-plugin-lto.

tests/run-make/debugger-visualizer-dep-info/rmake.rs

@@ -1,3 +1,5 @@
+//@ needs-target-std
+//
 // This test checks that files referenced via #[debugger_visualizer] are
 // included in `--emit dep-info` output.
 // See https://github.com/rust-lang/rust/pull/111641

tests/run-make/dep-info/rmake.rs

@@ -1,3 +1,5 @@
+//@ needs-target-std
+//
 // This is a simple smoke test for rustc's `--emit dep-info` feature. It prints out
 // information about dependencies in a Makefile-compatible format, as a `.d` file.
 // Note that this test does not check that the `.d` file is Makefile-compatible.

tests/run-make/diagnostics-traits-from-duplicate-crates/rmake.rs

@@ -1,3 +1,5 @@
+//@ needs-target-std
+//
 // Non-regression test for issue #132920 where multiple versions of the same crate are present in
 // the dependency graph, and an unexpected error in a dependent crate caused an ICE in the
 // unsatisfied bounds diagnostics for traits present in multiple crate versions.

tests/run-make/doctests-merge/rmake.rs

@@ -1,3 +1,5 @@
+//@ ignore-cross-compile (needs to run doctests)
+
 use std::path::Path;
 
 use run_make_support::{cwd, diff, rustc, rustdoc};

tests/run-make/doctests-runtool/rmake.rs

@@ -1,3 +1,5 @@
+//@ ignore-cross-compile (needs to run host tool binary)
+
 // Tests behavior of rustdoc `--test-runtool`.
 
 use std::path::PathBuf;

tests/run-make/dump-mono-stats/rmake.rs

@@ -1,3 +1,5 @@
+//@ needs-target-std
+//
 // A flag named dump-mono-stats was added to the compiler in 2022, which
 // collects stats on instantiation of items and their associated costs.
 // This test checks that the output stat file exists, and that it contains

tests/run-make/duplicate-output-flavors/rmake.rs

@@ -1,3 +1,4 @@
+//@ needs-target-std
 use run_make_support::rustc;
 
 fn main() {

tests/run-make/embed-metadata/rmake.rs

@@ -1,3 +1,6 @@
+//@ needs-target-std
+//@ needs-crate-type: dylib
+
 // Tests the -Zembed-metadata compiler flag.
 // Tracking issue: https://github.com/rust-lang/rust/issues/139165