diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db1fac8..2313d8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -184,3 +184,37 @@ jobs: darwin/arm64 windows/amd64 TARGETS + + native-smoke-macos: + name: ci / native-smoke-macos + runs-on: macos-14 + timeout-minutes: 12 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version: ${{ env.GO_VERSION_CURRENT }} + cache: false + - name: Run native runtime and capability smoke tests + run: go run ./scripts/rigor/native-smoke + + native-smoke-windows: + name: ci / native-smoke-windows + runs-on: windows-2025 + timeout-minutes: 12 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version: ${{ env.GO_VERSION_CURRENT }} + cache: false + - name: Run native runtime and capability smoke tests + run: go run ./scripts/rigor/native-smoke diff --git a/docs/README.md b/docs/README.md index db71b70..fa79141 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,6 +11,7 @@ same semantic interface for people and automation. - [Understand compatibility guarantees](compatibility.md) - [Apply the security contract](security.md) - [Publish a release candidate](releasing.md) +- [Understand native hosted smoke checks](native-hosted-smokes.md) ## Maintainers diff --git a/docs/native-hosted-smokes.md b/docs/native-hosted-smokes.md new file mode 100644 index 0000000..2e9ce29 --- /dev/null +++ b/docs/native-hosted-smokes.md @@ -0,0 +1,17 @@ +# Native hosted smoke checks + +The CI workflow runs bounded root-module smoke tests on GitHub-hosted +`macos-14` and `windows-2025` runners. They execute startup, cancellation, +EOF, newline input, no-colour/non-TTY, and Unicode line-input tests selected by +`scripts/rigor/native-smoke`. + +The helper consumes `go test -json` and requires a terminal passing result for every named +smoke test. A skipped or failed selected root or subtest fails accounting. It does not replace Linux minimum/current-Go, race, +security, cross-compile, or full verification jobs. + +Each native job has a 12-minute timeout. The pair therefore has a maximum +budget of 24 hosted runner minutes per workflow run; actual usage is visible in +the GitHub Actions run and should remain well below that ceiling. There are no +platform skips in this smoke set. Any future unsupported native behavior must +be skipped with a tracked issue and a documented rationale rather than treated +as a passing native check. diff --git a/runtime/human/line_driver_test.go b/runtime/human/line_driver_test.go index 020fc38..b3dd40a 100644 --- a/runtime/human/line_driver_test.go +++ b/runtime/human/line_driver_test.go @@ -15,7 +15,7 @@ import ( func TestLineDriverEmitsCanonicalTextAndShutdown(t *testing.T) { var output bytes.Buffer - driver, err := NewLineDriver(LineDriverOptions{Input: strings.NewReader("alpha\nbeta\n"), Output: &output, Width: 40, Height: 10}) + driver, err := NewLineDriver(LineDriverOptions{Input: strings.NewReader("alpha\nbeta\n你好 café 👋\n"), Output: &output, Width: 40, Height: 10}) if err != nil { t.Fatal(err) } @@ -33,7 +33,7 @@ func TestLineDriverEmitsCanonicalTextAndShutdown(t *testing.T) { } got = append(got, ev) } - if len(got) != 3 || got[0].Payload.(event.TextPayload).Text != "alpha" || got[1].Payload.(event.TextPayload).Text != "beta" || got[2].Kind != event.Shutdown { + if len(got) != 4 || got[0].Payload.(event.TextPayload).Text != "alpha" || got[1].Payload.(event.TextPayload).Text != "beta" || got[2].Payload.(event.TextPayload).Text != "你好 café 👋" || got[3].Kind != event.Shutdown { t.Fatalf("unexpected line events: %#v", got) } } diff --git a/scripts/rigor/generated/dependency-inventory.json b/scripts/rigor/generated/dependency-inventory.json index 117dc73..aeb305e 100644 --- a/scripts/rigor/generated/dependency-inventory.json +++ b/scripts/rigor/generated/dependency-inventory.json @@ -549,6 +549,19 @@ "strings" ] }, + { + "importPath": "github.com/ben-ranford/stave/scripts/rigor/native-smoke", + "dir": "scripts/rigor/native-smoke", + "imports": [ + "bytes", + "encoding/json", + "fmt", + "os", + "os/exec", + "path/filepath", + "strings" + ] + }, { "importPath": "github.com/ben-ranford/stave/secret", "dir": "secret", diff --git a/scripts/rigor/native-smoke/main.go b/scripts/rigor/native-smoke/main.go new file mode 100644 index 0000000..ce3b1c0 --- /dev/null +++ b/scripts/rigor/native-smoke/main.go @@ -0,0 +1,96 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +var expected = map[string]bool{ + "TestRuntimeRestoresOnEOFAndClosesOnce": true, + "TestLineDriverEmitsCanonicalTextAndShutdown": true, + "TestLineDriverDrawUsesSafePlainWriter": true, + "TestLineDriverDoesNotInventUnsupportedTTYCapabilities": true, + "TestServeCancellationInterruptsBlockingReader": true, + "TestCapabilityEnumsRejectInvalidWireValues": true, + "TestDetectExplicitEnvironment": true, + "TestNonTTYMachineOutputIsNotReclassifiedAsPlain": true, + "TestNonTTYAccessibleOutputIsNotReclassifiedAsPlain": true, + "TestCrossPlatformTerminalColourDetection": true, +} + +const selectedTests = "^(TestRuntimeRestoresOnEOFAndClosesOnce|TestLineDriverEmitsCanonicalTextAndShutdown|TestLineDriverDrawUsesSafePlainWriter|TestLineDriverDoesNotInventUnsupportedTTYCapabilities|TestServeCancellationInterruptsBlockingReader|TestCapabilityEnumsRejectInvalidWireValues|TestDetectExplicitEnvironment|TestNonTTYMachineOutputIsNotReclassifiedAsPlain|TestNonTTYAccessibleOutputIsNotReclassifiedAsPlain|TestCrossPlatformTerminalColourDetection)$" + +type testEvent struct { + Action string + Test string +} + +func main() { + executable, err := goExecutable() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + command := exec.Command(executable, "test", "-json", "-count=1", "-run", selectedTests, "./runtime/human", "./runtime/agent", "./capability") + output, err := command.Output() + os.Stdout.Write(output) + if err != nil { + fmt.Fprintf(os.Stderr, "native smoke tests failed: %v\n", err) + os.Exit(1) + } + seen, err := selectedTestPasses(output) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + for name := range expected { + if !seen[name] { + fmt.Fprintf(os.Stderr, "native smoke has no passing result for %s\n", name) + os.Exit(1) + } + } + fmt.Printf("native smoke selected and passed %d tests\n", len(expected)) +} + +// goExecutable preserves the operator-selected Go installation while +// resolving it before execution. A relative PATH entry is not a stable trust +// boundary for a hosted smoke runner, so only an absolute executable is used. +func goExecutable() (string, error) { + executable, err := exec.LookPath("go") + if err != nil { + return "", fmt.Errorf("locate Go executable: %w", err) + } + if !filepath.IsAbs(executable) { + return "", fmt.Errorf("resolved Go executable path %q must be absolute", executable) + } + return executable, nil +} + +func selectedTestPasses(output []byte) (map[string]bool, error) { + seen := make(map[string]bool, len(expected)) + for _, line := range bytes.Split(output, []byte{'\n'}) { + if len(line) == 0 { + continue + } + var event testEvent + if err := json.Unmarshal(line, &event); err != nil { + return nil, fmt.Errorf("decode go test event: %w", err) + } + root, _, _ := strings.Cut(event.Test, "/") + if !expected[root] { + continue + } + if event.Action == "skip" || event.Action == "fail" { + return nil, fmt.Errorf("native smoke %s reported %s", event.Test, event.Action) + } + if event.Action == "pass" && expected[event.Test] { + seen[event.Test] = true + } + } + return seen, nil +} diff --git a/scripts/rigor/native-smoke/main_test.go b/scripts/rigor/native-smoke/main_test.go new file mode 100644 index 0000000..16a452a --- /dev/null +++ b/scripts/rigor/native-smoke/main_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestSelectedTestsRequirePassingRoots(t *testing.T) { + output := []byte(`{"Action":"run","Test":"TestRuntimeRestoresOnEOFAndClosesOnce"} +{"Action":"pass","Test":"TestRuntimeRestoresOnEOFAndClosesOnce"} +{"Action":"run","Test":"TestCrossPlatformTerminalColourDetection/linux_truecolor"} +{"Action":"run","Test":"TestCrossPlatformTerminalColourDetection"} +{"Action":"pass","Test":"TestCrossPlatformTerminalColourDetection"} +`) + seen, err := selectedTestPasses(output) + if err != nil { + t.Fatal(err) + } + if !seen["TestRuntimeRestoresOnEOFAndClosesOnce"] || !seen["TestCrossPlatformTerminalColourDetection"] { + t.Fatalf("expected root test events, got %#v", seen) + } + if seen["TestLineDriverEmitsCanonicalTextAndShutdown"] { + t.Fatalf("unexpected test event counted: %#v", seen) + } +} + +func TestSelectedTestsRejectSkippedOrFailedRootsAndChildren(t *testing.T) { + for _, action := range []string{"skip", "fail"} { + for _, suffix := range []string{"", "/native"} { + t.Run(action+suffix, func(t *testing.T) { + output := []byte(fmt.Sprintf(`{"Action":"run","Test":"TestRuntimeRestoresOnEOFAndClosesOnce"} +{"Action":%q,"Test":%q} +`, action, "TestRuntimeRestoresOnEOFAndClosesOnce"+suffix)) + if _, err := selectedTestPasses(output); err == nil { + t.Fatal("accepted skipped or failed selected test") + } + }) + } + } +} + +func TestSelectedTestsDoNotCountStartsAsPasses(t *testing.T) { + seen, err := selectedTestPasses([]byte(`{"Action":"run","Test":"TestRuntimeRestoresOnEOFAndClosesOnce"}`)) + if err != nil { + t.Fatal(err) + } + if len(seen) != 0 { + t.Fatal("counted incomplete test as passed") + } +} + +func TestGoExecutableReportsMissingLookup(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + if _, err := goExecutable(); err == nil || !strings.Contains(err.Error(), "locate Go executable") { + t.Fatalf("missing Go executable error = %v", err) + } +} + +func TestGoExecutableRejectsRelativePathResolution(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("relative executable lookup is platform specific") + } + directory := t.TempDir() + bin := filepath.Join(directory, "bin") + if err := os.Mkdir(bin, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(bin, "go"), []byte("#!/bin/sh\n"), 0o755); err != nil { + t.Fatal(err) + } + workingDirectory, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(directory); err != nil { + t.Fatal(err) + } + defer func() { + if err := os.Chdir(workingDirectory); err != nil { + t.Error(err) + } + }() + t.Setenv("PATH", "bin") + t.Setenv("GODEBUG", "execerrdot=0") + if _, err := goExecutable(); err == nil || !strings.Contains(err.Error(), "must be absolute") { + t.Fatalf("relative Go path was not rejected: %v", err) + } +} diff --git a/scripts/rigor/workflow-guard/main.go b/scripts/rigor/workflow-guard/main.go index 749c0b7..90cb260 100644 --- a/scripts/rigor/workflow-guard/main.go +++ b/scripts/rigor/workflow-guard/main.go @@ -80,15 +80,26 @@ func checkJob(path, jobID string, job *actionlint.Job) error { func checkRunner(path, jobID string, runner *actionlint.Runner) error { if runner == nil || runner.Group != nil || runner.LabelsExpr != nil || len(runner.Labels) != 1 { - return fmt.Errorf("%s job %q must use one literal ubuntu-24.04 runner label", path, jobID) + return fmt.Errorf("%s job %q must use one approved literal hosted runner label", path, jobID) } label := runner.Labels[0] - if label.Value != "ubuntu-24.04" || label.ContainsExpression() { - return fmt.Errorf("%s job %q must use one literal ubuntu-24.04 runner label", path, jobID) + if label.ContainsExpression() || !approvedRunner(path, jobID, label.Value) { + return fmt.Errorf("%s job %q must use one approved literal hosted runner label", path, jobID) } return nil } +func approvedRunner(path, jobID, label string) bool { + if label == "ubuntu-24.04" { + return true + } + if filepath.Base(path) != "ci.yml" { + return false + } + return (jobID == "native-smoke-macos" && label == "macos-14") || + (jobID == "native-smoke-windows" && label == "windows-2025") +} + func checkActionInputs(path, jobID string, action *actionlint.ExecAction) error { if action.Uses == nil { return fmt.Errorf("%s job %q has an action step without uses", path, jobID) diff --git a/scripts/rigor/workflow-guard/main_test.go b/scripts/rigor/workflow-guard/main_test.go index 6126edd..87f765c 100644 --- a/scripts/rigor/workflow-guard/main_test.go +++ b/scripts/rigor/workflow-guard/main_test.go @@ -32,6 +32,18 @@ jobs: } } +func TestCheckWorkflowAllowsNativeHostedSmokeJobsOnlyInCI(t *testing.T) { + for name, runner := range map[string]string{"native-smoke-macos": "macos-14", "native-smoke-windows": "windows-2025"} { + workflow := "on: push\njobs:\n " + name + ":\n runs-on: " + runner + "\n steps:\n - run: true\n" + if err := checkWorkflow(".github/workflows/ci.yml", []byte(workflow)); err != nil { + t.Fatalf("approved native runner rejected: %v", err) + } + if err := checkWorkflow(".github/workflows/release.yml", []byte(workflow)); err == nil { + t.Fatal("native runner allowed outside ci workflow") + } + } +} + func TestCheckWorkflowRejectsUnsafeTrustBoundaries(t *testing.T) { workflowPrefix := "on: push\njobs:\n" validJob := `