From 99e847734cbb42e169a6d20ee101ea602f2ad5d4 Mon Sep 17 00:00:00 2001 From: Eddie Knight Date: Fri, 14 Aug 2026 13:04:25 -0500 Subject: [PATCH 1/2] fix: Do not require start on checks that aren't expected to run Signed-off-by: Eddie Knight --- evaluationlog.cue | 7 +- test/schema_test.go | 4 ++ .../bad-evaluation-log-missing-start.yaml | 35 ++++++++++ .../good-evaluation-log-unstarted.yaml | 67 +++++++++++++++++++ 4 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 test/test-data/bad-evaluation-log-missing-start.yaml create mode 100644 test/test-data/good-evaluation-log-unstarted.yaml diff --git a/evaluationlog.cue b/evaluationlog.cue index 056e5704..609c1d00 100644 --- a/evaluationlog.cue +++ b/evaluationlog.cue @@ -48,7 +48,12 @@ package gemara // Steps-executed is the number of steps that were executed as part of the assessment. "steps-executed"?: int @go(StepsExecuted) // Start is the timestamp when the assessment began. - start: #Datetime + // Assessments that never executed have no start time to record. + start?: #Datetime + if result != "Not Run" && result != "Unknown" && result != "Not Applicable" { + start: #Datetime + } + // End is the timestamp when the assessment concluded. end?: #Datetime // Recommendation provides guidance on how to address a failed assessment. diff --git a/test/schema_test.go b/test/schema_test.go index 7a7be0f0..9a13ada6 100644 --- a/test/schema_test.go +++ b/test/schema_test.go @@ -107,6 +107,10 @@ func TestSchemaValidation(t *testing.T) { // EvaluationLog — positive {"valid PVTR baseline scan", "./test-data/pvtr-baseline-scan.yaml", "#EvaluationLog", false, ""}, + {"assessments that never ran omit start", "./test-data/good-evaluation-log-unstarted.yaml", "#EvaluationLog", false, ""}, + + // EvaluationLog — negative + {"executed assessment missing start", "./test-data/bad-evaluation-log-missing-start.yaml", "#EvaluationLog", true, ""}, // EnforcementLog — positive {"valid enforcement log", "./test-data/good-enforcement-log.yaml", "#EnforcementLog", false, ""}, diff --git a/test/test-data/bad-evaluation-log-missing-start.yaml b/test/test-data/bad-evaluation-log-missing-start.yaml new file mode 100644 index 00000000..bf784797 --- /dev/null +++ b/test/test-data/bad-evaluation-log-missing-start.yaml @@ -0,0 +1,35 @@ +metadata: + id: EVAL-MISSING-START + type: EvaluationLog + gemara-version: "1.1.0" + version: 1.0.0 + description: Evaluation log whose executed assessment omits its start time + author: + id: pvtr + name: PVTR + type: Software +result: Passed +target: + id: github-repo + name: GitHub Repository + type: Software +evaluations: +- name: access control + control: + reference-id: OSPS-B + entry-id: OSPS-AC-01 + result: Passed + message: Multi-factor authentication is required + assessment-logs: + # Passed assessments executed, so start is still required + - requirement: + entry-id: OSPS-AC-01.01 + description: Verify that multi-factor authentication is required. + result: Passed + message: Two-factor authentication is configured as required by the parent organization + applicability: + - Maturity Level 1 + steps: + - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/access_control.orgRequiresMFA + steps-executed: 1 + end: 2025-08-22T16:02:00.000003708Z diff --git a/test/test-data/good-evaluation-log-unstarted.yaml b/test/test-data/good-evaluation-log-unstarted.yaml new file mode 100644 index 00000000..227d835f --- /dev/null +++ b/test/test-data/good-evaluation-log-unstarted.yaml @@ -0,0 +1,67 @@ +metadata: + id: EVAL-UNSTARTED + type: EvaluationLog + gemara-version: "1.1.0" + version: 1.0.0 + description: Evaluation log containing assessments that never executed + author: + id: pvtr + name: PVTR + type: Software +result: Needs Review +target: + id: github-repo + name: GitHub Repository + type: Software +evaluations: +- name: access control + control: + reference-id: OSPS-B + entry-id: OSPS-AC-01 + result: Needs Review + message: Assessments did not execute + assessment-logs: + # Not Run: no start time is recorded because the procedure never began + - requirement: + entry-id: OSPS-AC-01.01 + description: Verify that multi-factor authentication is required. + result: Not Run + message: Halted before execution because a prior assessment failed + applicability: + - Maturity Level 1 + steps: + - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/access_control.orgRequiresMFA + steps-executed: 0 + # Unknown: the outcome could not be determined, so no timing is asserted + - requirement: + entry-id: OSPS-AC-01.02 + description: Verify that administrative access is restricted. + result: Unknown + message: Evaluator could not reach the target + applicability: + - Maturity Level 1 + steps: + - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/access_control.adminAccess + # Not Applicable: the procedure was skipped as out of scope + - requirement: + entry-id: OSPS-AC-01.03 + description: Verify that branch protection is enabled on release branches. + result: Not Applicable + message: Project has no release branches + applicability: + - Maturity Level 2 + steps: + - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/access_control.branchProtection + # Passed: an executed assessment still records its start time + - requirement: + entry-id: OSPS-AC-01.04 + description: Verify that the default branch requires review. + result: Passed + message: Reviews are required on the default branch + applicability: + - Maturity Level 1 + steps: + - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/access_control.reviewRequired + steps-executed: 1 + start: 2025-08-22T16:02:00.000000000Z + end: 2025-08-22T16:02:00.000003708Z From 11971300a5486bad624779bb9e09df9a67aa5735 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Fri, 14 Aug 2026 15:23:57 -0400 Subject: [PATCH 2/2] fix: separate start-time validation from Go codegen via hidden def CUE conditionals cause IncompleteKind, preventing Go struct generation. Move the "start required when executed" guard into #_AssessmentLogStrict (marked @go(-)) and apply it at the #ControlEvaluation level, keeping #AssessmentLog clean for codegen. Update compat test to handle hidden definition blocks: skip #_ defs in collectStableDefs and strip their structural noise in relaxForSubsume, the same treatment already applied for time.Format and list.Contains cross-context false positives. Assisted-by: Claude Code --- evaluationlog.cue | 15 ++++++-- test/compat_test.go | 93 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 104 insertions(+), 4 deletions(-) diff --git a/evaluationlog.cue b/evaluationlog.cue index 609c1d00..4fca36ec 100644 --- a/evaluationlog.cue +++ b/evaluationlog.cue @@ -27,6 +27,18 @@ package gemara "assessment-logs": [...{ requirement: "reference-id": (control."reference-id") }] + // Require start timestamp on assessments that actually executed + "assessment-logs": [#_AssessmentLogStrict, ...#_AssessmentLogStrict] +} + +// _AssessmentLogStrict layers the "start required unless unexecuted" rule on top of #AssessmentLog +#_AssessmentLogStrict: { + @go(-) +} & #AssessmentLog & { + result: #Result + if result != "Not Run" && result != "Unknown" && result != "Not Applicable" { + start: #Datetime + } } // AssessmentLog contains the results of executing a single assessment procedure for a control requirement. @@ -50,9 +62,6 @@ package gemara // Start is the timestamp when the assessment began. // Assessments that never executed have no start time to record. start?: #Datetime - if result != "Not Run" && result != "Unknown" && result != "Not Applicable" { - start: #Datetime - } // End is the timestamp when the assessment concluded. end?: #Datetime diff --git a/test/compat_test.go b/test/compat_test.go index 46e0d078..3fb33da6 100644 --- a/test/compat_test.go +++ b/test/compat_test.go @@ -161,6 +161,8 @@ func relaxForSubsume(content string) string { } result := strings.Join(lines, "\n") + result = stripHiddenDefBlocks(result) + result = strings.Replace(result, `#Datetime: time.Format("2006-01-02T15:04:05Z07:00")`, `#Datetime: string`, 1) @@ -175,6 +177,95 @@ func relaxForSubsume(content string) string { return result } +// stripHiddenDefBlocks removes hidden definition blocks (#_Foo: { ... }) and +// collapses references to them back to their underlying public type. Hidden +// definitions are validation-only wrappers (marked @go(-)) whose structural +// shape causes cross-context subsumption false positives — the same class of +// noise as time.Format and list.Contains. +func stripHiddenDefBlocks(content string) string { + hiddenDefs := findHiddenDefs(content) + + var out []string + lines := strings.Split(content, "\n") + i := 0 + for i < len(lines) { + trimmed := strings.TrimSpace(lines[i]) + + if strings.HasPrefix(trimmed, "#_") && strings.Contains(trimmed, ":") { + defName := strings.TrimSpace(strings.SplitN(trimmed, ":", 2)[0]) + if _, ok := hiddenDefs[defName]; ok { + for len(out) > 0 && strings.HasPrefix(strings.TrimSpace(out[len(out)-1]), "//") { + out = out[:len(out)-1] + } + depth := 0 + for i < len(lines) { + code := strings.TrimSpace(lines[i]) + if !strings.HasPrefix(code, "//") { + for _, ch := range code { + if ch == '{' { + depth++ + } else if ch == '}' { + depth-- + } + } + } + i++ + if depth <= 0 { + break + } + } + continue + } + } + + out = append(out, lines[i]) + i++ + } + result := strings.Join(out, "\n") + + for name, base := range hiddenDefs { + result = strings.ReplaceAll(result, name, base) + } + return result +} + +// findHiddenDefs scans for #_Foo: { @go(-) } & #Bar patterns and returns +// a map from hidden name to underlying public type. +func findHiddenDefs(content string) map[string]string { + defs := make(map[string]string) + lines := strings.Split(content, "\n") + for i, line := range lines { + trimmed := strings.TrimSpace(line) + if !strings.HasPrefix(trimmed, "#_") || !strings.Contains(trimmed, ":") { + continue + } + name := strings.TrimSpace(strings.SplitN(trimmed, ":", 2)[0]) + depth := 0 + for j := i; j < len(lines); j++ { + for _, ch := range lines[j] { + if ch == '{' { + depth++ + } else if ch == '}' { + depth-- + } + } + if idx := strings.Index(lines[j], "& #"); idx >= 0 { + rest := lines[j][idx+2:] + parts := strings.Fields(rest) + if len(parts) > 0 { + base := strings.TrimRight(parts[0], " &{") + defs[name] = base + break + } + } + if depth <= 0 && j > i { + break + } + } + } + return defs +} + func collectStableDefs(schemaDir string) ([]string, error) { var stableDefs []string @@ -197,7 +288,7 @@ func collectStableDefs(schemaDir string) ([]string, error) { } for _, line := range strings.Split(content, "\n") { line = strings.TrimSpace(line) - if strings.HasPrefix(line, "#") && strings.Contains(line, ":") { + if strings.HasPrefix(line, "#") && !strings.HasPrefix(line, "#_") && strings.Contains(line, ":") { def := strings.TrimSpace(strings.SplitN(line, ":", 2)[0]) stableDefs = append(stableDefs, def) }