From 9d387ed898dd2a153b322526777164e62b025d2d Mon Sep 17 00:00:00 2001 From: Ben Ranford <84072202+ben-ranford@users.noreply.github.com> Date: Sun, 13 Sep 2026 17:45:30 +1000 Subject: [PATCH 1/3] test(ci): add native hosted smoke checks --- .github/workflows/ci.yml | 34 ++++++++++ docs/README.md | 1 + docs/native-hosted-smokes.md | 17 +++++ .../rigor/generated/dependency-inventory.json | 11 +++ scripts/rigor/native-smoke/main.go | 68 +++++++++++++++++++ scripts/rigor/native-smoke/main_test.go | 20 ++++++ scripts/rigor/workflow-guard/main.go | 17 ++++- scripts/rigor/workflow-guard/main_test.go | 12 ++++ 8 files changed, 177 insertions(+), 3 deletions(-) create mode 100644 docs/native-hosted-smokes.md create mode 100644 scripts/rigor/native-smoke/main.go create mode 100644 scripts/rigor/native-smoke/main_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db1fac8..2313d8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -184,3 +184,37 @@ jobs: darwin/arm64 windows/amd64 TARGETS + + native-smoke-macos: + name: ci / native-smoke-macos + runs-on: macos-14 + timeout-minutes: 12 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version: ${{ env.GO_VERSION_CURRENT }} + cache: false + - name: Run native runtime and capability smoke tests + run: go run ./scripts/rigor/native-smoke + + native-smoke-windows: + name: ci / native-smoke-windows + runs-on: windows-2025 + timeout-minutes: 12 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version: ${{ env.GO_VERSION_CURRENT }} + cache: false + - name: Run native runtime and capability smoke tests + run: go run ./scripts/rigor/native-smoke diff --git a/docs/README.md b/docs/README.md index db71b70..fa79141 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,6 +11,7 @@ same semantic interface for people and automation. - [Understand compatibility guarantees](compatibility.md) - [Apply the security contract](security.md) - [Publish a release candidate](releasing.md) +- [Understand native hosted smoke checks](native-hosted-smokes.md) ## Maintainers diff --git a/docs/native-hosted-smokes.md b/docs/native-hosted-smokes.md new file mode 100644 index 0000000..09f0307 --- /dev/null +++ b/docs/native-hosted-smokes.md @@ -0,0 +1,17 @@ +# Native hosted smoke checks + +The CI workflow runs bounded root-module smoke tests on GitHub-hosted +`macos-14` and `windows-2025` runners. They execute startup, cancellation, +EOF, newline input, no-colour/non-TTY, and Unicode capability tests selected by +`scripts/rigor/native-smoke`. + +The helper consumes `go test -json` and fails if any named smoke test has zero +matching test events. It does not replace Linux minimum/current-Go, race, +security, cross-compile, or full verification jobs. + +Each native job has a 12-minute timeout. The pair therefore has a maximum +budget of 24 hosted runner minutes per workflow run; actual usage is visible in +the GitHub Actions run and should remain well below that ceiling. There are no +platform skips in this smoke set. Any future unsupported native behavior must +be skipped with a tracked issue and a documented rationale rather than treated +as a passing native check. diff --git a/scripts/rigor/generated/dependency-inventory.json b/scripts/rigor/generated/dependency-inventory.json index 117dc73..9fa3f00 100644 --- a/scripts/rigor/generated/dependency-inventory.json +++ b/scripts/rigor/generated/dependency-inventory.json @@ -549,6 +549,17 @@ "strings" ] }, + { + "importPath": "github.com/ben-ranford/stave/scripts/rigor/native-smoke", + "dir": "scripts/rigor/native-smoke", + "imports": [ + "bytes", + "encoding/json", + "fmt", + "os", + "os/exec" + ] + }, { "importPath": "github.com/ben-ranford/stave/secret", "dir": "secret", diff --git a/scripts/rigor/native-smoke/main.go b/scripts/rigor/native-smoke/main.go new file mode 100644 index 0000000..e8a3e13 --- /dev/null +++ b/scripts/rigor/native-smoke/main.go @@ -0,0 +1,68 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" +) + +var expected = map[string]bool{ + "TestRuntimeRestoresOnEOFAndClosesOnce": true, + "TestLineDriverEmitsCanonicalTextAndShutdown": true, + "TestLineDriverDrawUsesSafePlainWriter": true, + "TestLineDriverDoesNotInventUnsupportedTTYCapabilities": true, + "TestServeCancellationInterruptsBlockingReader": true, + "TestCapabilityEnumsRejectInvalidWireValues": true, + "TestDetectExplicitEnvironment": true, + "TestNonTTYMachineOutputIsNotReclassifiedAsPlain": true, + "TestNonTTYAccessibleOutputIsNotReclassifiedAsPlain": true, + "TestCrossPlatformTerminalColourDetection": true, +} + +const selectedTests = "^(TestRuntimeRestoresOnEOFAndClosesOnce|TestLineDriverEmitsCanonicalTextAndShutdown|TestLineDriverDrawUsesSafePlainWriter|TestLineDriverDoesNotInventUnsupportedTTYCapabilities|TestServeCancellationInterruptsBlockingReader|TestCapabilityEnumsRejectInvalidWireValues|TestDetectExplicitEnvironment|TestNonTTYMachineOutputIsNotReclassifiedAsPlain|TestNonTTYAccessibleOutputIsNotReclassifiedAsPlain|TestCrossPlatformTerminalColourDetection)$" + +type testEvent struct { + Action string + Test string +} + +func main() { + command := exec.Command("go", "test", "-json", "-count=1", "-run", selectedTests, "./runtime/human", "./runtime/agent", "./capability") + output, err := command.Output() + os.Stdout.Write(output) + if err != nil { + fmt.Fprintf(os.Stderr, "native smoke tests failed: %v\n", err) + os.Exit(1) + } + seen, err := selectedTestStarts(output) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + for name := range expected { + if !seen[name] { + fmt.Fprintf(os.Stderr, "native smoke selected zero instances of %s\n", name) + os.Exit(1) + } + } + fmt.Printf("native smoke selected and passed %d tests\n", len(expected)) +} + +func selectedTestStarts(output []byte) (map[string]bool, error) { + seen := make(map[string]bool, len(expected)) + for _, line := range bytes.Split(output, []byte{'\n'}) { + if len(line) == 0 { + continue + } + var event testEvent + if err := json.Unmarshal(line, &event); err != nil { + return nil, fmt.Errorf("decode go test event: %w", err) + } + if event.Action == "run" && expected[event.Test] { + seen[event.Test] = true + } + } + return seen, nil +} diff --git a/scripts/rigor/native-smoke/main_test.go b/scripts/rigor/native-smoke/main_test.go new file mode 100644 index 0000000..4481090 --- /dev/null +++ b/scripts/rigor/native-smoke/main_test.go @@ -0,0 +1,20 @@ +package main + +import "testing" + +func TestSelectedTestStartsCountsExpectedRootTests(t *testing.T) { + output := []byte(`{"Action":"run","Test":"TestRuntimeRestoresOnEOFAndClosesOnce"} +{"Action":"run","Test":"TestCrossPlatformTerminalColourDetection/linux_truecolor"} +{"Action":"run","Test":"TestCrossPlatformTerminalColourDetection"} +`) + seen, err := selectedTestStarts(output) + if err != nil { + t.Fatal(err) + } + if !seen["TestRuntimeRestoresOnEOFAndClosesOnce"] || !seen["TestCrossPlatformTerminalColourDetection"] { + t.Fatalf("expected root test events, got %#v", seen) + } + if seen["TestLineDriverEmitsCanonicalTextAndShutdown"] { + t.Fatalf("unexpected test event counted: %#v", seen) + } +} diff --git a/scripts/rigor/workflow-guard/main.go b/scripts/rigor/workflow-guard/main.go index 749c0b7..90cb260 100644 --- a/scripts/rigor/workflow-guard/main.go +++ b/scripts/rigor/workflow-guard/main.go @@ -80,15 +80,26 @@ func checkJob(path, jobID string, job *actionlint.Job) error { func checkRunner(path, jobID string, runner *actionlint.Runner) error { if runner == nil || runner.Group != nil || runner.LabelsExpr != nil || len(runner.Labels) != 1 { - return fmt.Errorf("%s job %q must use one literal ubuntu-24.04 runner label", path, jobID) + return fmt.Errorf("%s job %q must use one approved literal hosted runner label", path, jobID) } label := runner.Labels[0] - if label.Value != "ubuntu-24.04" || label.ContainsExpression() { - return fmt.Errorf("%s job %q must use one literal ubuntu-24.04 runner label", path, jobID) + if label.ContainsExpression() || !approvedRunner(path, jobID, label.Value) { + return fmt.Errorf("%s job %q must use one approved literal hosted runner label", path, jobID) } return nil } +func approvedRunner(path, jobID, label string) bool { + if label == "ubuntu-24.04" { + return true + } + if filepath.Base(path) != "ci.yml" { + return false + } + return (jobID == "native-smoke-macos" && label == "macos-14") || + (jobID == "native-smoke-windows" && label == "windows-2025") +} + func checkActionInputs(path, jobID string, action *actionlint.ExecAction) error { if action.Uses == nil { return fmt.Errorf("%s job %q has an action step without uses", path, jobID) diff --git a/scripts/rigor/workflow-guard/main_test.go b/scripts/rigor/workflow-guard/main_test.go index 6126edd..87f765c 100644 --- a/scripts/rigor/workflow-guard/main_test.go +++ b/scripts/rigor/workflow-guard/main_test.go @@ -32,6 +32,18 @@ jobs: } } +func TestCheckWorkflowAllowsNativeHostedSmokeJobsOnlyInCI(t *testing.T) { + for name, runner := range map[string]string{"native-smoke-macos": "macos-14", "native-smoke-windows": "windows-2025"} { + workflow := "on: push\njobs:\n " + name + ":\n runs-on: " + runner + "\n steps:\n - run: true\n" + if err := checkWorkflow(".github/workflows/ci.yml", []byte(workflow)); err != nil { + t.Fatalf("approved native runner rejected: %v", err) + } + if err := checkWorkflow(".github/workflows/release.yml", []byte(workflow)); err == nil { + t.Fatal("native runner allowed outside ci workflow") + } + } +} + func TestCheckWorkflowRejectsUnsafeTrustBoundaries(t *testing.T) { workflowPrefix := "on: push\njobs:\n" validJob := ` From 8ce22ea668feb52f2331a144a889f992b582f028 Mon Sep 17 00:00:00 2001 From: Ben Ranford <84072202+ben-ranford@users.noreply.github.com> Date: Sun, 13 Sep 2026 23:37:15 +1000 Subject: [PATCH 2/3] fix(ci): require passing native smoke results --- docs/native-hosted-smokes.md | 6 ++-- runtime/human/line_driver_test.go | 4 +-- .../rigor/generated/dependency-inventory.json | 3 +- scripts/rigor/native-smoke/main.go | 16 ++++++--- scripts/rigor/native-smoke/main_test.go | 36 +++++++++++++++++-- 5 files changed, 52 insertions(+), 13 deletions(-) diff --git a/docs/native-hosted-smokes.md b/docs/native-hosted-smokes.md index 09f0307..2e9ce29 100644 --- a/docs/native-hosted-smokes.md +++ b/docs/native-hosted-smokes.md @@ -2,11 +2,11 @@ The CI workflow runs bounded root-module smoke tests on GitHub-hosted `macos-14` and `windows-2025` runners. They execute startup, cancellation, -EOF, newline input, no-colour/non-TTY, and Unicode capability tests selected by +EOF, newline input, no-colour/non-TTY, and Unicode line-input tests selected by `scripts/rigor/native-smoke`. -The helper consumes `go test -json` and fails if any named smoke test has zero -matching test events. It does not replace Linux minimum/current-Go, race, +The helper consumes `go test -json` and requires a terminal passing result for every named +smoke test. A skipped or failed selected root or subtest fails accounting. It does not replace Linux minimum/current-Go, race, security, cross-compile, or full verification jobs. Each native job has a 12-minute timeout. The pair therefore has a maximum diff --git a/runtime/human/line_driver_test.go b/runtime/human/line_driver_test.go index 020fc38..b3dd40a 100644 --- a/runtime/human/line_driver_test.go +++ b/runtime/human/line_driver_test.go @@ -15,7 +15,7 @@ import ( func TestLineDriverEmitsCanonicalTextAndShutdown(t *testing.T) { var output bytes.Buffer - driver, err := NewLineDriver(LineDriverOptions{Input: strings.NewReader("alpha\nbeta\n"), Output: &output, Width: 40, Height: 10}) + driver, err := NewLineDriver(LineDriverOptions{Input: strings.NewReader("alpha\nbeta\n你好 café 👋\n"), Output: &output, Width: 40, Height: 10}) if err != nil { t.Fatal(err) } @@ -33,7 +33,7 @@ func TestLineDriverEmitsCanonicalTextAndShutdown(t *testing.T) { } got = append(got, ev) } - if len(got) != 3 || got[0].Payload.(event.TextPayload).Text != "alpha" || got[1].Payload.(event.TextPayload).Text != "beta" || got[2].Kind != event.Shutdown { + if len(got) != 4 || got[0].Payload.(event.TextPayload).Text != "alpha" || got[1].Payload.(event.TextPayload).Text != "beta" || got[2].Payload.(event.TextPayload).Text != "你好 café 👋" || got[3].Kind != event.Shutdown { t.Fatalf("unexpected line events: %#v", got) } } diff --git a/scripts/rigor/generated/dependency-inventory.json b/scripts/rigor/generated/dependency-inventory.json index 9fa3f00..49ee0f4 100644 --- a/scripts/rigor/generated/dependency-inventory.json +++ b/scripts/rigor/generated/dependency-inventory.json @@ -557,7 +557,8 @@ "encoding/json", "fmt", "os", - "os/exec" + "os/exec", + "strings" ] }, { diff --git a/scripts/rigor/native-smoke/main.go b/scripts/rigor/native-smoke/main.go index e8a3e13..0bfdc0d 100644 --- a/scripts/rigor/native-smoke/main.go +++ b/scripts/rigor/native-smoke/main.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "os/exec" + "strings" ) var expected = map[string]bool{ @@ -36,21 +37,21 @@ func main() { fmt.Fprintf(os.Stderr, "native smoke tests failed: %v\n", err) os.Exit(1) } - seen, err := selectedTestStarts(output) + seen, err := selectedTestPasses(output) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } for name := range expected { if !seen[name] { - fmt.Fprintf(os.Stderr, "native smoke selected zero instances of %s\n", name) + fmt.Fprintf(os.Stderr, "native smoke has no passing result for %s\n", name) os.Exit(1) } } fmt.Printf("native smoke selected and passed %d tests\n", len(expected)) } -func selectedTestStarts(output []byte) (map[string]bool, error) { +func selectedTestPasses(output []byte) (map[string]bool, error) { seen := make(map[string]bool, len(expected)) for _, line := range bytes.Split(output, []byte{'\n'}) { if len(line) == 0 { @@ -60,7 +61,14 @@ func selectedTestStarts(output []byte) (map[string]bool, error) { if err := json.Unmarshal(line, &event); err != nil { return nil, fmt.Errorf("decode go test event: %w", err) } - if event.Action == "run" && expected[event.Test] { + root, _, _ := strings.Cut(event.Test, "/") + if !expected[root] { + continue + } + if event.Action == "skip" || event.Action == "fail" { + return nil, fmt.Errorf("native smoke %s reported %s", event.Test, event.Action) + } + if event.Action == "pass" && expected[event.Test] { seen[event.Test] = true } } diff --git a/scripts/rigor/native-smoke/main_test.go b/scripts/rigor/native-smoke/main_test.go index 4481090..9cd1624 100644 --- a/scripts/rigor/native-smoke/main_test.go +++ b/scripts/rigor/native-smoke/main_test.go @@ -1,13 +1,18 @@ package main -import "testing" +import ( + "fmt" + "testing" +) -func TestSelectedTestStartsCountsExpectedRootTests(t *testing.T) { +func TestSelectedTestsRequirePassingRoots(t *testing.T) { output := []byte(`{"Action":"run","Test":"TestRuntimeRestoresOnEOFAndClosesOnce"} +{"Action":"pass","Test":"TestRuntimeRestoresOnEOFAndClosesOnce"} {"Action":"run","Test":"TestCrossPlatformTerminalColourDetection/linux_truecolor"} {"Action":"run","Test":"TestCrossPlatformTerminalColourDetection"} +{"Action":"pass","Test":"TestCrossPlatformTerminalColourDetection"} `) - seen, err := selectedTestStarts(output) + seen, err := selectedTestPasses(output) if err != nil { t.Fatal(err) } @@ -18,3 +23,28 @@ func TestSelectedTestStartsCountsExpectedRootTests(t *testing.T) { t.Fatalf("unexpected test event counted: %#v", seen) } } + +func TestSelectedTestsRejectSkippedOrFailedRootsAndChildren(t *testing.T) { + for _, action := range []string{"skip", "fail"} { + for _, suffix := range []string{"", "/native"} { + t.Run(action+suffix, func(t *testing.T) { + output := []byte(fmt.Sprintf(`{"Action":"run","Test":"TestRuntimeRestoresOnEOFAndClosesOnce"} +{"Action":%q,"Test":%q} +`, action, "TestRuntimeRestoresOnEOFAndClosesOnce"+suffix)) + if _, err := selectedTestPasses(output); err == nil { + t.Fatal("accepted skipped or failed selected test") + } + }) + } + } +} + +func TestSelectedTestsDoNotCountStartsAsPasses(t *testing.T) { + seen, err := selectedTestPasses([]byte(`{"Action":"run","Test":"TestRuntimeRestoresOnEOFAndClosesOnce"}`)) + if err != nil { + t.Fatal(err) + } + if len(seen) != 0 { + t.Fatal("counted incomplete test as passed") + } +} From 42ebce5affb5e0643c48bdda116083c42a2a57e8 Mon Sep 17 00:00:00 2001 From: Ben Ranford <84072202+ben-ranford@users.noreply.github.com> Date: Mon, 14 Sep 2026 23:03:35 +1000 Subject: [PATCH 3/3] fix(ci): resolve an absolute Go executable for native smoke --- .../rigor/generated/dependency-inventory.json | 1 + scripts/rigor/native-smoke/main.go | 22 +++++++++- scripts/rigor/native-smoke/main_test.go | 42 +++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/scripts/rigor/generated/dependency-inventory.json b/scripts/rigor/generated/dependency-inventory.json index 49ee0f4..aeb305e 100644 --- a/scripts/rigor/generated/dependency-inventory.json +++ b/scripts/rigor/generated/dependency-inventory.json @@ -558,6 +558,7 @@ "fmt", "os", "os/exec", + "path/filepath", "strings" ] }, diff --git a/scripts/rigor/native-smoke/main.go b/scripts/rigor/native-smoke/main.go index 0bfdc0d..ce3b1c0 100644 --- a/scripts/rigor/native-smoke/main.go +++ b/scripts/rigor/native-smoke/main.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "strings" ) @@ -30,7 +31,12 @@ type testEvent struct { } func main() { - command := exec.Command("go", "test", "-json", "-count=1", "-run", selectedTests, "./runtime/human", "./runtime/agent", "./capability") + executable, err := goExecutable() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + command := exec.Command(executable, "test", "-json", "-count=1", "-run", selectedTests, "./runtime/human", "./runtime/agent", "./capability") output, err := command.Output() os.Stdout.Write(output) if err != nil { @@ -51,6 +57,20 @@ func main() { fmt.Printf("native smoke selected and passed %d tests\n", len(expected)) } +// goExecutable preserves the operator-selected Go installation while +// resolving it before execution. A relative PATH entry is not a stable trust +// boundary for a hosted smoke runner, so only an absolute executable is used. +func goExecutable() (string, error) { + executable, err := exec.LookPath("go") + if err != nil { + return "", fmt.Errorf("locate Go executable: %w", err) + } + if !filepath.IsAbs(executable) { + return "", fmt.Errorf("resolved Go executable path %q must be absolute", executable) + } + return executable, nil +} + func selectedTestPasses(output []byte) (map[string]bool, error) { seen := make(map[string]bool, len(expected)) for _, line := range bytes.Split(output, []byte{'\n'}) { diff --git a/scripts/rigor/native-smoke/main_test.go b/scripts/rigor/native-smoke/main_test.go index 9cd1624..16a452a 100644 --- a/scripts/rigor/native-smoke/main_test.go +++ b/scripts/rigor/native-smoke/main_test.go @@ -2,6 +2,10 @@ package main import ( "fmt" + "os" + "path/filepath" + "runtime" + "strings" "testing" ) @@ -48,3 +52,41 @@ func TestSelectedTestsDoNotCountStartsAsPasses(t *testing.T) { t.Fatal("counted incomplete test as passed") } } + +func TestGoExecutableReportsMissingLookup(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + if _, err := goExecutable(); err == nil || !strings.Contains(err.Error(), "locate Go executable") { + t.Fatalf("missing Go executable error = %v", err) + } +} + +func TestGoExecutableRejectsRelativePathResolution(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("relative executable lookup is platform specific") + } + directory := t.TempDir() + bin := filepath.Join(directory, "bin") + if err := os.Mkdir(bin, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(bin, "go"), []byte("#!/bin/sh\n"), 0o755); err != nil { + t.Fatal(err) + } + workingDirectory, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(directory); err != nil { + t.Fatal(err) + } + defer func() { + if err := os.Chdir(workingDirectory); err != nil { + t.Error(err) + } + }() + t.Setenv("PATH", "bin") + t.Setenv("GODEBUG", "execerrdot=0") + if _, err := goExecutable(); err == nil || !strings.Contains(err.Error(), "must be absolute") { + t.Fatalf("relative Go path was not rejected: %v", err) + } +}