Update testing

.gitignore

@@ -1,10 +1,5 @@
 target
-examples/mps3-an536/target
-examples/mps3-an536/target-d32
-examples/mps3-an536-el2/target
-examples/mps3-an536-el2/target-d32
-examples/versatileab/target
-examples/versatileab/target-d32
+target-*
 Cargo.lock
 **/.DS_Store
 

aarch32-tests/tests/common/mod.rs

@@ -41,24 +41,37 @@ pub(crate) mod test_utils {
         bins
     }
 
-    pub fn run_bin(
+    pub fn run_bin<S1, S2>(
         dir: &Path,
         bin: &str,
         target: &str,
-        flags: &[&str],
-        rustflags: Option<&str>,
-    ) -> String {
+        flags: impl IntoIterator<Item = S1>,
+        rustflags: impl IntoIterator<Item = S2>,
+    ) -> String
+    where
+        S1: AsRef<std::ffi::OsStr>,
+        S2: AsRef<str>,
+    {
         let mut cmd = Command::new("cargo");
         cmd.current_dir(dir)
             .arg("run")
             .arg(format!("--target={target}"))
             .args(flags)
             .args(["--bin", bin]);
 
+        let mut rustflags_str = String::new();
+        for rustflag in rustflags {
+            rustflags_str.push_str(rustflag.as_ref());
+            rustflags_str.push_str(" ");
+        }
         // Some variants (e.g. fpu-d32) need a target-feature/-cpu; pass it only to
         // this cross build, never to the host test binary.
-        if let Some(rustflags) = rustflags {
-            cmd.env("RUSTFLAGS", rustflags);
+        if !rustflags_str.is_empty() {
+            cmd.env("RUSTFLAGS", rustflags_str);
+        }
+
+        if std::env::var("TEST_VERBOSE").is_ok() {
+            eprintln!("\nRunning {:?}", cmd);
         }
 
         let output = cmd.output().expect("failed to execute cargo run");

aarch32-tests/tests/tests.rs

@@ -15,42 +15,125 @@
 //! against the same `<bin>-<target>` snapshot as the plain build.
 //!
 //! Filter like any test binary: `cargo test -p aarch32-tests -- armv7a`.
+//!
+//! Set `TEST_VERBOSE=1` in your environment to see details of the commands executed.
 
 mod common;
 
 use common::test_utils;
 use libtest_mimic::{Arguments, Trial};
 
-/// A build variant of a target: a name suffix plus extra cargo flags. All
-/// variants of a target compare against the same snapshot.
+/// A build variant of a target: a name suffix plus extra cargo flags.
+///
+/// All variants within a group are compared against the same snapshot.
 struct Variant {
     label: &'static str,
-    extra_flags: &'static [&'static str],
+    flags: &'static [&'static str],
+    rustflags: &'static [&'static str],
 }
 
+/// Build with no features
 const PLAIN: Variant = Variant {
     label: "",
-    extra_flags: &[],
+    flags: &[],
+    rustflags: &[],
 };
+
+/// Build with the `svc-stack-interrupt` feature
 const SVC: Variant = Variant {
     label: "svc",
-    extra_flags: &["--features=svc-stack-interrupt"],
+    flags: &["--features=svc-stack-interrupt"],
+    rustflags: &[],
+};
+
+/// Build for a Cortex-R5
+const R5_CPU: Variant = Variant {
+    label: "r5-cpu",
+    flags: &[
+        // Tell the runtime we're using the FPU
+        "--features=eabi-fpu",
+    ],
+    rustflags: &[
+        // Optimise for Cortex-R5
+        "-Ctarget-cpu=cortex-r5",
+    ],
 };
-const FPU: Variant = Variant {
+
+/// Build for a Cortex-R5 no FPU
+const R5_CPU_NOFPU_FEAT: Variant = Variant {
+    label: "r5-cpu-nofpu",
+    flags: &[],
+    rustflags: &[
+        // Optimise for Cortex-R5
+        "-Ctarget-cpu=cortex-r5",
+        // Turn off the FPU that cortex-r5 implies
+        "-Ctarget-feature=-fpregs",
+    ],
+};
+
+/// Build for a Cortex-R5 no double precision
+const R5_CPU_NODP_FEAT: Variant = Variant {
+    label: "r5-cpu-nodp",
+    flags: &[
+        // Tell the runtime we're using the FPU
+        "--features=eabi-fpu",
+    ],
+    rustflags: &[
+        // Optimise for Cortex-R5
+        "-Ctarget-cpu=cortex-r5",
+        // Turn off the DP support that cortex-r5 implies
+        "-Ctarget-feature=-fp64",
+    ],
+};
+
+/// Build for a generic CPU with no double precision
+const NODP_FEAT: Variant = Variant {
+    label: "no-dp",
+    flags: &[],
+    rustflags: &[
+        // Turn off the DP support that is enabled by default
+        "-Ctarget-feature=-fp64",
+    ],
+};
+
+/// Build for a generic Arm CPU with 32 DP registers
+const D32_FEAT: Variant = Variant {
     label: "fpu-d32",
-    extra_flags: &[],
+    flags: &[
+        // Tell the aarch32-rt assembly to stack the high FPU registers,
+        "--features=fpu-d32",
+    ],
+    rustflags: &[
+        // Enable usage of all 32 double precision FPU registers
+        "-Ctarget-feature=+d32",
+    ],
 };
 
+/// Build for a Cortex-R52 with 32 DP registers
+const R52_CPU: Variant = Variant {
+    label: "r52-cpu",
+    flags: &[
+        // Tell the aarch32-rt assembly to stack the high FPU registers,
+        "--features=fpu-d32",
+    ],
+    rustflags: &[
+        // Optimise for Cortex-R52 (which also enables usage of all 32 DP registers)
+        "-Ctarget-cpu=cortex-r52",
+    ],
+};
+
+/// A group of programs to build and test
 struct Group {
+    name: &'static str,
     example: &'static str,
     targets: &'static [&'static str],
     flags: &'static [&'static str],
-    rustflags: Option<&'static str>,
     variants: &'static [Variant],
 }
 
 const MATRIX: &[Group] = &[
     Group {
+        name: "versatileab-legacy",
         example: "versatileab",
         targets: &[
             "armv4t-none-eabi",
@@ -62,61 +145,45 @@ const MATRIX: &[Group] = &[
             "thumbv6-none-eabi",
         ],
         flags: &["--release", "-Zbuild-std=core"],
-        rustflags: None,
         variants: &[PLAIN, SVC],
     },
     Group {
+        name: "versatileab-v7r",
         example: "versatileab",
-        targets: &[
-            "armv7r-none-eabi",
-            "thumbv7r-none-eabi",
-            "armv7r-none-eabihf",
-            "thumbv7r-none-eabihf",
-            "armv7a-none-eabi",
-            "thumbv7a-none-eabi",
-            "armv7a-none-eabihf",
-            "thumbv7a-none-eabihf",
-        ],
+        targets: &["armv7r-none-eabi", "thumbv7r-none-eabi"],
         flags: &["--release"],
-        rustflags: None,
-        variants: &[PLAIN, SVC],
+        variants: &[PLAIN, SVC, R5_CPU, R5_CPU_NOFPU_FEAT, R5_CPU_NODP_FEAT],
     },
     Group {
+        name: "versatileab-v7r-hf",
         example: "versatileab",
-        targets: &["armv7a-none-eabihf", "thumbv7a-none-eabihf"],
-        flags: &["--release", "--features=fpu-d32", "--target-dir=target-d32"],
-        rustflags: Some("-Ctarget-feature=+d32"),
-        variants: &[FPU],
+        targets: &["armv7r-none-eabihf", "thumbv7r-none-eabihf"],
+        flags: &["--release"],
+        variants: &[PLAIN, SVC, R5_CPU, R5_CPU_NODP_FEAT, NODP_FEAT],
     },
     Group {
-        example: "mps3-an536",
-        targets: &["armv8r-none-eabihf", "thumbv8r-none-eabihf"],
+        name: "versatileab-v7a",
+        example: "versatileab",
+        targets: &["armv7a-none-eabihf", "thumbv7a-none-eabihf"],
         flags: &["--release"],
-        rustflags: None,
-        variants: &[PLAIN, SVC],
+        variants: &[PLAIN, SVC, D32_FEAT],
     },
     Group {
+        name: "mps3-an536",
         example: "mps3-an536",
         targets: &["armv8r-none-eabihf", "thumbv8r-none-eabihf"],
-        flags: &["--release", "--features=fpu-d32", "--target-dir=target-d32"],
-        rustflags: Some("-Ctarget-cpu=cortex-r52"),
-        variants: &[FPU],
-    },
-    Group {
-        example: "mps3-an536-el2",
-        targets: &["armv8r-none-eabihf", "thumbv8r-none-eabihf"],
         flags: &["--release"],
-        rustflags: None,
-        variants: &[PLAIN],
+        variants: &[PLAIN, SVC, R52_CPU],
     },
     Group {
+        name: "mps3-an536-el2",
         example: "mps3-an536-el2",
         targets: &["armv8r-none-eabihf", "thumbv8r-none-eabihf"],
-        flags: &["--release", "--features=fpu-d32", "--target-dir=target-d32"],
-        rustflags: Some("-Ctarget-cpu=cortex-r52"),
-        variants: &[FPU],
+        flags: &["--release"],
+        variants: &[PLAIN, R52_CPU],
     },
     Group {
+        name: "xilinx-zynq-a9",
         example: "xilinx-zynq-a9",
         targets: &[
             "armv7a-none-eabi",
@@ -125,7 +192,6 @@ const MATRIX: &[Group] = &[
             "thumbv7a-none-eabihf",
         ],
         flags: &["--release"],
-        rustflags: None,
         variants: &[PLAIN],
     },
 ];
@@ -137,23 +203,26 @@ fn main() {
     for group in MATRIX {
         for &target in group.targets {
             for variant in group.variants {
-                let example = group.example;
-                let rustflags = group.rustflags;
-                let dir = test_utils::test_dir(&format!("examples/{example}"));
+                let dir = test_utils::test_dir(&format!("examples/{}", group.example));
                 for bin in test_utils::discover_bins(&dir) {
                     let target = target.to_string();
-                    let flags: Vec<&'static str> = group
-                        .flags
-                        .iter()
-                        .chain(variant.extra_flags)
-                        .copied()
-                        .collect();
-                    let mut name = format!("{}/{}/{}", group.example, target, bin);
+                    let mut flags: Vec<String> = Vec::new();
+                    for flag in group.flags {
+                        flags.push(flag.to_string());
+                    }
+                    for flag in variant.flags {
+                        flags.push(flag.to_string());
+                    }
+                    if !variant.rustflags.is_empty() {
+                        // nonstandard build gets special target dir
+                        flags.push(format!("--target-dir=target-{}", variant.label));
+                    }
+                    let mut name = format!("{}/{}/{}", group.name, target, bin);
                     if !variant.label.is_empty() {
                         name.push_str(&format!(" [{}]", variant.label));
                     }
                     tests.push(Trial::test(name, move || {
-                        run_target_bin(example, &bin, &target, &flags, rustflags);
+                        run_target_bin(group.example, &bin, &target, &flags, &variant.rustflags);
                         Ok(())
                     }));
                 }
@@ -164,7 +233,7 @@ fn main() {
     libtest_mimic::run(&args, tests).exit();
 }
 
-fn run_target_bin(example: &str, bin: &str, target: &str, flags: &[&str], rustflags: Option<&str>) {
+fn run_target_bin(example: &str, bin: &str, target: &str, flags: &[String], rustflags: &[&str]) {
     let dir = test_utils::test_dir(&format!("examples/{example}"));
 
     // Per-example folder: snapshots/<example>/<bin>-<target>.snap

examples/versatileab/.cargo/config.toml

@@ -5,14 +5,12 @@ runner = "qemu-system-arm -machine versatileab -cpu cortex-r5f -semihosting -nog
 runner = "qemu-system-arm -machine versatileab -cpu cortex-r5f -semihosting -nographic -audio none -kernel"
 
 [target.armv7r-none-eabi]
-# change '-mcpu=cortex-r5' to '-mcpu=cortex-r5f' if you use eabi-fpu feature, otherwise
-# qemu-system-arm will lock up
-runner = "qemu-system-arm -machine versatileab -cpu cortex-r5 -semihosting -nographic -audio none -kernel"
+# We test FPU support even on EABI targets
+runner = "qemu-system-arm -machine versatileab -cpu cortex-r5f -semihosting -nographic -audio none -kernel"
 
 [target.thumbv7r-none-eabi]
-# change '-mcpu=cortex-r5' to '-mcpu=cortex-r5f' if you use eabi-fpu feature, otherwise
-# qemu-system-arm will lock up
-runner = "qemu-system-arm -machine versatileab -cpu cortex-r5 -semihosting -nographic -audio none -kernel"
+# We test FPU support even on EABI targets
+runner = "qemu-system-arm -machine versatileab -cpu cortex-r5f -semihosting -nographic -audio none -kernel"
 
 [target.armv7a-none-eabihf]
 runner = "qemu-system-arm -machine versatileab -cpu cortex-a15 -semihosting -nographic -audio none -kernel"

justfile

@@ -9,7 +9,7 @@ verbose := if v == "1" { "--verbose" } else { "" }
 
 # The aarch32-tests harness invocation shared by every test-qemu recipe. Each
 # recipe appends a name filter; the build matrix lives in the harness itself.
-qemu_test := "cargo test -p aarch32-tests --test tests --"
+qemu_test := "cargo test -p aarch32-tests --test tests --  --test-threads=1"
 
 # Our default target. It does everything that you might want to do pre-checkin.
 check: build-all build-all-examples doc-all fmt-check clippy-all test
@@ -281,8 +281,7 @@ test-qemu-v7r:
 # Armv7-A (Versatile AB), incl. fpu-d32 on hf targets. Two filters because the
 # name must be scoped to versatileab (zynq is also v7a), which splits arm/thumb.
 test-qemu-v7a:
-	{{qemu_test}} versatileab/armv7a
-	{{qemu_test}} versatileab/thumbv7a
+	{{qemu_test}} versatileab-v7a
 
 # Armv7-A (Xilinx Zynq-A9)
 test-qemu-v7a-zynq: