diff --git a/docs/experiment-template.md b/docs/experiment-template.md index a4da8c4..14c3099 100644 --- a/docs/experiment-template.md +++ b/docs/experiment-template.md @@ -41,10 +41,30 @@ and says nothing about which licence the code is actually under or whether the result may be promoted anywhere. A borrowed directory in an experiment whose record declares nothing passes, because an absent field is never refused. +Add `Held-back` where the experiment is held back under +`docs/decisions/0010-a-flaw-in-shipped-software.md`, with the date of the report +the window is counted from, written as `YYYY-MM-DD`. It is not in the header +above for the reason `Measurement-Commit` is not, and here a value shipped +filled in would be worse than misleading: it would declare a hold on every +record copied from this file. The record stays in `asking` while it waits, the +listing prints that it is held back and when the clock started and nothing about +what it is about, and a value that is not a date is refused. How long the wait +is, what ends it and what the single extension costs are +`docs/decisions/0022-how-long-a-held-back-record-waits.md`, and `SECURITY.md` is +where that window is published for the project the report went to. + +What that refusal does not reach is the case the field exists for. A record +being held back that declares no `Held-back` is not refused and cannot be, since +an absent field is never a refusal. It sits in `asking` with a question that +says nothing and appears in the listing as ordinary unanswered work, which is +the misreport the field is written against and which only the person writing the +record can prevent. + The format is `docs/decisions/0008-the-experiment-record.md`, as added to by `docs/decisions/0015-an-experiment-declares-the-harness-it-needs.md`, by -`docs/decisions/0016-an-answer-names-the-commit-it-measured.md` and by -`docs/decisions/0019-code-under-another-licence.md`. This file is a +`docs/decisions/0016-an-answer-names-the-commit-it-measured.md`, by +`docs/decisions/0019-code-under-another-licence.md` and by +`docs/decisions/0022-how-long-a-held-back-record-waits.md`. This file is a convenience and those records are the authority. ## Question diff --git a/internal/check/check.go b/internal/check/check.go index 3c9304f..e0ccdb8 100644 --- a/internal/check/check.go +++ b/internal/check/check.go @@ -522,6 +522,7 @@ func walkExperiments(fsys fs.FS, root string, res *Result) error { res.Refusals = append(res.Refusals, refuseState(record, data)...) res.Refusals = append(res.Refusals, refuseHeaderDates(record, data)...) res.Refusals = append(res.Refusals, refuseMeasurementCommit(record, data)...) + res.Refusals = append(res.Refusals, refuseHeldBack(record, data)...) res.Refusals = append(res.Refusals, refuseDates(record, data, res.Now)...) res.Refusals = append(res.Refusals, refusePromotion(record, data)...) // The two rules here that read the directory as well as the record, diff --git a/internal/check/heldback.go b/internal/check/heldback.go new file mode 100644 index 0000000..75edd1a --- /dev/null +++ b/internal/check/heldback.go @@ -0,0 +1,86 @@ +package check + +import ( + "fmt" + "time" +) + +// FieldHeldBack is the date of the report a held-back record's window is +// counted from. Record 0022 adds it and record 0013 makes it optional, as it +// makes every field added after it. +// +// It is declared here rather than beside the fields record 0008 fixes, and +// that is the shape headerDateFields argues for at its own list: a field a +// later record adds arrives with the check that reads it, so a checker built +// before the field is unaware of it and a field with no check has nowhere to +// hide. +// +// What the field is for is not a property of this package and is written in +// record 0022 and in SECURITY.md. What this package does with it is read it, +// hold it to being a date, and give the listing something to print. +const FieldHeldBack = "Held-back" + +// RecordHeldBackIsNotADate refuses a held-back field carrying something that +// is not a date written the one way record 0008 names. +// +// The value is the start of a window somebody outside this board is entitled +// to plan against, and it is the only thing in the tree that says when that +// window started. A value nothing can read leaves the record saying it is held +// back and saying nothing about since when, which is the half of the +// disclosure that costs something: the fact of a hold with no date is what +// record 0022 rejected as waiting indefinitely, arriving through a typo +// instead of through a decision. +// +// A field written with nothing after the colon is refused here too, for the +// reason RecordHeaderDateIsNotADate gives. Record 0013 makes absence legal for +// every field and an empty declaration a different statement, and a declared +// date with no value claims there is a date rather than claiming there is +// none. +const RecordHeldBackIsNotADate = "record-held-back-is-not-a-date" + +// refuseHeldBack holds a held-back date to being a date. +// +// WHERE IT DOES NOT REACH, and this is the whole of the residual. +// +// An absent field is never refused. Record 0013 fixes that and record 0022 +// says so of this field in its own words: a record that is being held back and +// carries no field sits in asking with a question that says nothing, and +// nothing here can refuse it. What stands behind that half is the template, +// the review and whoever writes the record. +// +// A date this reads is not a date it believes. Nothing in a checkout says when +// a report arrived, so the value is a claim its author made and the only thing +// judged is the shape of that claim. A held-back date invented and well formed +// passes. +// +// Two shapes a reader might expect here are deliberately outside it. A date +// later than the time the run read is not refused, so a window that has not +// started yet passes, and a held-back field on a record that is not asking is +// not refused either, though record 0022 fixes the state a hold is recorded +// in. Both are visible in the listing rather than refused, and neither has a +// property in this tree. +// +// A record whose bytes do not parse as a record is not judged here, for the +// reason every other header rule gives: nothing can read a field out of a file +// that has no header. +func refuseHeldBack(path string, data []byte) []Refusal { + record, err := ParseRecord(data) + if err != nil { + return nil + } + + written, present := record.Field(FieldHeldBack) + if !present { + return nil + } + if _, err := time.Parse(DateFormat, written); err == nil { + return nil + } + + return []Refusal{{ + Property: RecordHeldBackIsNotADate, + Subject: path, + Detail: fmt.Sprintf("its %s is %q, and record 0022 counts the window from a date written as %s, for example %s", + FieldHeldBack, written, dateShape, dateExample), + }} +} diff --git a/internal/check/list.go b/internal/check/list.go index bf35025..d0be839 100644 --- a/internal/check/list.go +++ b/internal/check/list.go @@ -57,6 +57,25 @@ type Entry struct { // checker produces, and repeating that judgement here would put a verdict // in a report. NeedsHardware string + + // HeldBack is the Held-back field as it was written, and Holding says + // whether the record carried one at all. The two are separate for the + // reason the hardware column is: a record that declares nothing and a + // record that declares an empty value are different statements, and a + // single string collapses them. + HeldBack string + + // Holding says whether the record declared Held-back. + Holding bool + + // Started is HeldBack parsed as a date, and HeldBackDated says whether it + // was read from one. A value that is not a date is refused by the checker + // and printed here as it was written, so the listing shows what the record + // says rather than what it should have said. + Started time.Time + + // HeldBackDated says whether Started was read from a date. + HeldBackDated bool } // Waiting is how long an experiment has been asking, in whole days, and @@ -222,6 +241,15 @@ func readEntry(dir, slug string) Entry { entry.Dated = true } } + + if held, present := record.Field(FieldHeldBack); present { + entry.Holding = true + entry.HeldBack = held + if date, err := time.Parse(DateFormat, held); err == nil { + entry.Started = date + entry.HeldBackDated = true + } + } return entry } @@ -263,7 +291,7 @@ func (l Listing) Report(now time.Time) string { out += fmt.Sprintf("%d %s\n", len(l.Entries), plural(len(l.Entries), "experiment", "experiments")) out += fmt.Sprintf("the time this run read is %s\n", now.UTC().Format(time.RFC3339)) if len(l.Entries) == 0 { - return out + return out + l.holds() } rows := [][5]string{{"slug", "state", "question written", "waiting", "needs"}} @@ -294,5 +322,55 @@ func (l Listing) Report(now time.Time) string { } out += " " + strings.Join(line, " ") + "\n" } + return out + l.holds() +} + +// holds writes the held-back records, one dated line each, under a count. +// +// It is lines rather than a column, which is record 0022's own word for it and +// is also what the table can carry. A sixth column would sit on every listing +// to say nothing on almost all of them, and the comment at Report gives the +// reason: a listing wide enough to wrap stops being scannable, and scanning it +// is the whole point. +// +// The count is printed whatever it is, including zero. A reader who does not +// know the listing reports holds at all cannot tell a tree with none from a +// listing that never looked, and those are the two statements this line exists +// to separate. It is the same reason the count of experiments is printed above +// it. +// +// WHAT A LINE SAYS AND WHAT IT DOES NOT. It names the slug and the date the +// clock started, and nothing else. Record 0022 discloses the fact of a hold and +// when it began, and discloses what the experiment is about through neither, +// because a date and the fact of a hold give a reader nothing to act on and +// that is the whole point of holding the record back. +// +// Two things it reports rather than repairs. A value that is not a date is +// printed as it was written, because a line saying a record is held back since +// something unreadable is what sends somebody to the record, and +// refuseHeldBack is what refuses it. A hold on a record that is not asking is +// printed with the state beside it: record 0022 fixes a hold as a field beside +// asking, no property in this tree refuses one anywhere else, and a report that +// quietly dropped the state would hide the disagreement instead of the subject. +func (l Listing) holds() string { + var held []Entry + for _, entry := range l.Entries { + if entry.Holding { + held = append(held, entry) + } + } + + out := fmt.Sprintf("%d %s held back\n", len(held), plural(len(held), "record is", "records are")) + for _, entry := range held { + since := fmt.Sprintf("the clock started %s", entry.Started.Format(DateFormat)) + if !entry.HeldBackDated { + since = fmt.Sprintf("its %s is %q, which is not a date", FieldHeldBack, entry.HeldBack) + } + line := fmt.Sprintf(" %s is held back and %s", entry.Slug, since) + if entry.State != StateAsking { + line += fmt.Sprintf(", while its state is %s", entry.State) + } + out += line + "\n" + } return out } diff --git a/internal/check/list_test.go b/internal/check/list_test.go index 5e3bbb8..e406b33 100644 --- a/internal/check/list_test.go +++ b/internal/check/list_test.go @@ -3,6 +3,7 @@ package check import ( "os" "path/filepath" + "strings" "testing" ) @@ -185,3 +186,110 @@ func read(t *testing.T, name string) Listing { } return listing } + +// TestTheListingKeepsAHoldApartFromASilence is the half of this field that is +// easy to lose. Record 0013 makes a field added after it optional, so a record +// declaring nothing is the ordinary case and a record declaring a hold is the +// exception, and reading a missing field as an empty one would report every +// record in the tree as held back since nothing. +func TestTheListingKeepsAHoldApartFromASilence(t *testing.T) { + holding := map[string]bool{ + "still-asking": true, + "unreadable-hold": true, + "finished-but-still-held": true, + "no-hold": false, + } + + for _, entry := range read(t, "held-back-experiments").Entries { + want, named := holding[entry.Slug] + if !named { + t.Fatalf("%s is in the listing and this test does not name it", entry.Slug) + } + if entry.Holding != want { + t.Errorf("%s is held back %v, want %v", entry.Slug, entry.Holding, want) + } + } +} + +// TestAHoldIsDatedOnlyWhereTheRecordWroteADate keeps the parsed date apart from +// the value as it was written. The checker refuses a held-back field that is +// not a date, and the listing prints one, so a listing that silently read an +// unparseable value as the zero time would print the first day of year one as +// the moment the clock started. +func TestAHoldIsDatedOnlyWhereTheRecordWroteADate(t *testing.T) { + tests := []struct { + slug string + written string + dated bool + started string + }{ + {slug: "still-asking", written: "2026-06-01", dated: true, started: "2026-06-01"}, + {slug: "finished-but-still-held", written: "2026-04-05", dated: true, started: "2026-04-05"}, + {slug: "unreadable-hold", written: "some time in June", dated: false}, + {slug: "no-hold", written: "", dated: false}, + } + + entries := make(map[string]Entry) + for _, entry := range read(t, "held-back-experiments").Entries { + entries[entry.Slug] = entry + } + + for _, tc := range tests { + t.Run(tc.slug, func(t *testing.T) { + entry, listed := entries[tc.slug] + if !listed { + t.Fatalf("%s is not in the listing", tc.slug) + } + if entry.HeldBack != tc.written { + t.Errorf("its %s reads %q, want %q", FieldHeldBack, entry.HeldBack, tc.written) + } + if entry.HeldBackDated != tc.dated { + t.Fatalf("the hold is dated %v, want %v", entry.HeldBackDated, tc.dated) + } + if tc.dated && entry.Started.Format(DateFormat) != tc.started { + t.Errorf("the clock started %s, want %s", entry.Started.Format(DateFormat), tc.started) + } + }) + } +} + +// TestTheReportPrintsTheHoldsAndTheirDates compares the whole report against +// the one stored beside the tree, for the reason TestTheReportPrintsEveryColumn +// gives: an assertion that a slug appears somewhere passes on a report whose +// lines have silently swapped their dates. +func TestTheReportPrintsTheHoldsAndTheirDates(t *testing.T) { + name := "held-back-experiments" + got := read(t, name).Report(listingNow) + + data, err := os.ReadFile(filepath.Join(listingsDir, name, "expected-report")) + if err != nil { + t.Fatalf("cannot read the expected report: %v", err) + } + if got != string(data) { + t.Fatalf("the report is\n%s\nand the case expects\n%s", got, data) + } +} + +// TestTheReportCountsHoldsWhenThereAreNone is the line a reader needs to tell a +// tree with no hold from a listing that never looked for one. It is asserted on +// its own because it is the case almost every run of this verb produces, and a +// count printed only when it is not zero teaches nobody that it is printed. +func TestTheReportCountsHoldsWhenThereAreNone(t *testing.T) { + report := read(t, "several-experiments").Report(listingNow) + if !strings.Contains(report, "0 records are held back\n") { + t.Errorf("the report does not count the holds it did not find:\n%s", report) + } +} + +// TestAListingOfATreeWithNoExperimentsStillCountsTheHolds holds the same line +// on the path that returns before the table is built. A count that disappears +// with the table is a count a reader cannot rely on being there. +func TestAListingOfATreeWithNoExperimentsStillCountsTheHolds(t *testing.T) { + listing, err := List(filepath.Join(casesDir, "no-experiments-directory", "tree"), listingNow) + if err != nil { + t.Fatalf("the listing failed: %v", err) + } + if !strings.Contains(listing.Report(listingNow), "0 records are held back\n") { + t.Errorf("the report does not count the holds it did not find:\n%s", listing.Report(listingNow)) + } +} diff --git a/testdata/cases/record-held-back-with-a-date/expected b/testdata/cases/record-held-back-with-a-date/expected new file mode 100644 index 0000000..466fe70 --- /dev/null +++ b/testdata/cases/record-held-back-with-a-date/expected @@ -0,0 +1,4 @@ +directories 1 +records 1 +experiments present +decisions absent diff --git a/testdata/cases/record-held-back-with-a-date/expected-refusals b/testdata/cases/record-held-back-with-a-date/expected-refusals new file mode 100644 index 0000000..e69de29 diff --git a/testdata/cases/record-held-back-with-a-date/tree/experiments/one/EXPERIMENT.md b/testdata/cases/record-held-back-with-a-date/tree/experiments/one/EXPERIMENT.md new file mode 100644 index 0000000..23a5b41 --- /dev/null +++ b/testdata/cases/record-held-back-with-a-date/tree/experiments/one/EXPERIMENT.md @@ -0,0 +1,17 @@ +Slug: one +State: asking +Question-Written: 2026-01-01 +Held-back: 2026-02-14 + +## Question + +Whether a piece of software somebody runs behaves the way its documentation +says it does under one input it does not describe. The rest of this question is +held back under record 0010 and is not written here. + +## Method + +Reported to the project that ships it, and nothing about it is in this tree. + +## Answer + diff --git a/testdata/cases/record-held-back-with-a-value-that-is-not-a-date/expected b/testdata/cases/record-held-back-with-a-value-that-is-not-a-date/expected new file mode 100644 index 0000000..466fe70 --- /dev/null +++ b/testdata/cases/record-held-back-with-a-value-that-is-not-a-date/expected @@ -0,0 +1,4 @@ +directories 1 +records 1 +experiments present +decisions absent diff --git a/testdata/cases/record-held-back-with-a-value-that-is-not-a-date/expected-refusals b/testdata/cases/record-held-back-with-a-value-that-is-not-a-date/expected-refusals new file mode 100644 index 0000000..541fa0a --- /dev/null +++ b/testdata/cases/record-held-back-with-a-value-that-is-not-a-date/expected-refusals @@ -0,0 +1 @@ +record-held-back-is-not-a-date diff --git a/testdata/cases/record-held-back-with-a-value-that-is-not-a-date/near-neighbour b/testdata/cases/record-held-back-with-a-value-that-is-not-a-date/near-neighbour new file mode 100644 index 0000000..85baf29 --- /dev/null +++ b/testdata/cases/record-held-back-with-a-value-that-is-not-a-date/near-neighbour @@ -0,0 +1 @@ +record-held-back-with-a-date diff --git a/testdata/cases/record-held-back-with-a-value-that-is-not-a-date/tree/experiments/one/EXPERIMENT.md b/testdata/cases/record-held-back-with-a-value-that-is-not-a-date/tree/experiments/one/EXPERIMENT.md new file mode 100644 index 0000000..f00b13d --- /dev/null +++ b/testdata/cases/record-held-back-with-a-value-that-is-not-a-date/tree/experiments/one/EXPERIMENT.md @@ -0,0 +1,17 @@ +Slug: one +State: asking +Question-Written: 2026-01-01 +Held-back: reported in February + +## Question + +Whether a piece of software somebody runs behaves the way its documentation +says it does under one input it does not describe. The rest of this question is +held back under record 0010 and is not written here. + +## Method + +Reported to the project that ships it, and nothing about it is in this tree. + +## Answer + diff --git a/testdata/cases/record-held-back-with-nothing-after-the-colon/expected b/testdata/cases/record-held-back-with-nothing-after-the-colon/expected new file mode 100644 index 0000000..466fe70 --- /dev/null +++ b/testdata/cases/record-held-back-with-nothing-after-the-colon/expected @@ -0,0 +1,4 @@ +directories 1 +records 1 +experiments present +decisions absent diff --git a/testdata/cases/record-held-back-with-nothing-after-the-colon/expected-refusals b/testdata/cases/record-held-back-with-nothing-after-the-colon/expected-refusals new file mode 100644 index 0000000..541fa0a --- /dev/null +++ b/testdata/cases/record-held-back-with-nothing-after-the-colon/expected-refusals @@ -0,0 +1 @@ +record-held-back-is-not-a-date diff --git a/testdata/cases/record-held-back-with-nothing-after-the-colon/near-neighbour b/testdata/cases/record-held-back-with-nothing-after-the-colon/near-neighbour new file mode 100644 index 0000000..85baf29 --- /dev/null +++ b/testdata/cases/record-held-back-with-nothing-after-the-colon/near-neighbour @@ -0,0 +1 @@ +record-held-back-with-a-date diff --git a/testdata/cases/record-held-back-with-nothing-after-the-colon/tree/experiments/one/EXPERIMENT.md b/testdata/cases/record-held-back-with-nothing-after-the-colon/tree/experiments/one/EXPERIMENT.md new file mode 100644 index 0000000..1af28a5 --- /dev/null +++ b/testdata/cases/record-held-back-with-nothing-after-the-colon/tree/experiments/one/EXPERIMENT.md @@ -0,0 +1,17 @@ +Slug: one +State: asking +Question-Written: 2026-01-01 +Held-back: + +## Question + +Whether a piece of software somebody runs behaves the way its documentation +says it does under one input it does not describe. The rest of this question is +held back under record 0010 and is not written here. + +## Method + +Reported to the project that ships it, and nothing about it is in this tree. + +## Answer + diff --git a/testdata/listings/held-back-experiments/expected-report b/testdata/listings/held-back-experiments/expected-report new file mode 100644 index 0000000..616015a --- /dev/null +++ b/testdata/listings/held-back-experiments/expected-report @@ -0,0 +1,12 @@ +examined ../../testdata/listings/held-back-experiments/tree +4 experiments +the time this run read is 2026-09-15T12:00:00Z + slug state question written waiting needs + still-asking asking 2026-06-01 106 days not declared + unreadable-hold asking 2026-06-15 92 days not declared + no-hold asking 2026-07-01 76 days not declared + finished-but-still-held answered 2026-04-01 - not declared +3 records are held back + still-asking is held back and the clock started 2026-06-01 + unreadable-hold is held back and its Held-back is "some time in June", which is not a date + finished-but-still-held is held back and the clock started 2026-04-05, while its state is answered diff --git a/testdata/listings/held-back-experiments/tree/experiments/finished-but-still-held/EXPERIMENT.md b/testdata/listings/held-back-experiments/tree/experiments/finished-but-still-held/EXPERIMENT.md new file mode 100644 index 0000000..65da1c3 --- /dev/null +++ b/testdata/listings/held-back-experiments/tree/experiments/finished-but-still-held/EXPERIMENT.md @@ -0,0 +1,18 @@ +Slug: finished-but-still-held +State: answered +Question-Written: 2026-04-01 +Answer-Written: 2026-04-20 +Held-back: 2026-04-05 + +## Question + +Held back under record 0010. Nothing about it is written here. + +## Method + +Reported to the project that ships the software. + +## Answer + +The record was written once the project had said what it wanted said, and the +field saying when the clock started was left on it. diff --git a/testdata/listings/held-back-experiments/tree/experiments/no-hold/EXPERIMENT.md b/testdata/listings/held-back-experiments/tree/experiments/no-hold/EXPERIMENT.md new file mode 100644 index 0000000..94e6a2d --- /dev/null +++ b/testdata/listings/held-back-experiments/tree/experiments/no-hold/EXPERIMENT.md @@ -0,0 +1,14 @@ +Slug: no-hold +State: asking +Question-Written: 2026-07-01 + +## Question + +Does the listing leave a record alone when it declares no hold? + +## Method + +Point the listing at this tree. + +## Answer + diff --git a/testdata/listings/held-back-experiments/tree/experiments/still-asking/EXPERIMENT.md b/testdata/listings/held-back-experiments/tree/experiments/still-asking/EXPERIMENT.md new file mode 100644 index 0000000..5b1a482 --- /dev/null +++ b/testdata/listings/held-back-experiments/tree/experiments/still-asking/EXPERIMENT.md @@ -0,0 +1,15 @@ +Slug: still-asking +State: asking +Question-Written: 2026-06-01 +Held-back: 2026-06-01 + +## Question + +Held back under record 0010. Nothing about it is written here. + +## Method + +Reported to the project that ships the software. + +## Answer + diff --git a/testdata/listings/held-back-experiments/tree/experiments/unreadable-hold/EXPERIMENT.md b/testdata/listings/held-back-experiments/tree/experiments/unreadable-hold/EXPERIMENT.md new file mode 100644 index 0000000..888104f --- /dev/null +++ b/testdata/listings/held-back-experiments/tree/experiments/unreadable-hold/EXPERIMENT.md @@ -0,0 +1,15 @@ +Slug: unreadable-hold +State: asking +Question-Written: 2026-06-15 +Held-back: some time in June + +## Question + +Held back under record 0010. Nothing about it is written here. + +## Method + +Reported to the project that ships the software. + +## Answer + diff --git a/testdata/listings/several-experiments/expected-report b/testdata/listings/several-experiments/expected-report index abefd0e..aa659ca 100644 --- a/testdata/listings/several-experiments/expected-report +++ b/testdata/listings/several-experiments/expected-report @@ -8,3 +8,4 @@ the time this run read is 2026-09-15T12:00:00Z already-answered answered 2026-04-01 - not declared no-header unreadable unreadable - unreadable no-record-at-all no record no record - no record +0 records are held back