From 1888f04bd4747314c9758cc766d8b6ee26205936 Mon Sep 17 00:00:00 2001 From: avison9 Date: Mon, 21 Sep 2026 19:45:08 +0100 Subject: [PATCH] A column an exclude list matches is reported as matched, not as "not matched" Three messages named the list setting with a fixed "is not matched by": right for an include list, the opposite of what happened for an exclude list. "public.shipments.driver_phone ... is not matched by column.exclude.list" described a column the exclude list matched, which is why it was missing. Since v0.2.1 the diff rule never raises a pattern-excluded column, so the source-column-not-captured info line is the only message such a column gets, and it read backwards. leftOut(setting) words it per list mode, for the source inventory, the sink column read and the sink table read. Four expectations change, each by that clause; the corpus was edited first and the test failed before the code changed. RefuseRadar's tree, an include-list connector, prints the same 0 / 0 / 159. --- .../expected.txt | 2 +- .../expected.txt | 4 ++-- .../expected.txt | 4 ++-- corpus/iceberg-exclude-list/expected.txt | 2 +- internal/engine/engine.go | 19 ++++++++++++++++--- 5 files changed, 22 insertions(+), 9 deletions(-) diff --git a/corpus/diff-exclude-list-captures-by-default/expected.txt b/corpus/diff-exclude-list-captures-by-default/expected.txt index 37563e3..7adce47 100644 --- a/corpus/diff-exclude-list-captures-by-default/expected.txt +++ b/corpus/diff-exclude-list-captures-by-default/expected.txt @@ -1,4 +1,4 @@ info source-column-not-captured diff-exclude-list-captures-by-default/migrations/0033_report_validations.sql:7 - public.report_validations.assigned_by_user_id exists and is not matched by column.exclude.list in diff-exclude-list-captures-by-default/connector.json; nothing reads it yet + public.report_validations.assigned_by_user_id exists and is matched by column.exclude.list in diff-exclude-list-captures-by-default/connector.json; nothing reads it yet the day a sink asks for it is the day it is found missing 0 error(s), 0 warning(s), 1 info diff --git a/corpus/diff-exclude-list-excludes-new-column/expected.txt b/corpus/diff-exclude-list-excludes-new-column/expected.txt index 1aba132..fed1647 100644 --- a/corpus/diff-exclude-list-excludes-new-column/expected.txt +++ b/corpus/diff-exclude-list-excludes-new-column/expected.txt @@ -1,7 +1,7 @@ info source-column-not-captured diff-exclude-list-excludes-new-column/migrations/0033_report_validations.sql:7 - public.report_validations.assigned_by_user_id exists and is not matched by column.exclude.list in diff-exclude-list-excludes-new-column/connector.json; nothing reads it yet + public.report_validations.assigned_by_user_id exists and is matched by column.exclude.list in diff-exclude-list-excludes-new-column/connector.json; nothing reads it yet the day a sink asks for it is the day it is found missing info source-column-not-captured diff-exclude-list-excludes-new-column/migrations/0035_validation_response_distance.sql:5 - public.report_validations.response_distance_m exists and is not matched by column.exclude.list in diff-exclude-list-excludes-new-column/connector.json; nothing reads it yet + public.report_validations.response_distance_m exists and is matched by column.exclude.list in diff-exclude-list-excludes-new-column/connector.json; nothing reads it yet the day a sink asks for it is the day it is found missing 0 error(s), 0 warning(s), 2 info diff --git a/corpus/diff-exclude-list-pattern-already-matches/expected.txt b/corpus/diff-exclude-list-pattern-already-matches/expected.txt index 6814f2d..269387a 100644 --- a/corpus/diff-exclude-list-pattern-already-matches/expected.txt +++ b/corpus/diff-exclude-list-pattern-already-matches/expected.txt @@ -1,7 +1,7 @@ info source-column-not-captured diff-exclude-list-pattern-already-matches/migrations/0033_report_validations.sql:7 - public.report_validations.assigned_by_user_id exists and is not matched by column.exclude.list in diff-exclude-list-pattern-already-matches/connector.json; nothing reads it yet + public.report_validations.assigned_by_user_id exists and is matched by column.exclude.list in diff-exclude-list-pattern-already-matches/connector.json; nothing reads it yet the day a sink asks for it is the day it is found missing info source-column-not-captured diff-exclude-list-pattern-already-matches/migrations/0035_validation_response_distance.sql:5 - public.report_validations.response_distance_m exists and is not matched by column.exclude.list in diff-exclude-list-pattern-already-matches/connector.json; nothing reads it yet + public.report_validations.response_distance_m exists and is matched by column.exclude.list in diff-exclude-list-pattern-already-matches/connector.json; nothing reads it yet the day a sink asks for it is the day it is found missing 0 error(s), 0 warning(s), 2 info diff --git a/corpus/iceberg-exclude-list/expected.txt b/corpus/iceberg-exclude-list/expected.txt index bb2dcfd..76d2b36 100644 --- a/corpus/iceberg-exclude-list/expected.txt +++ b/corpus/iceberg-exclude-list/expected.txt @@ -1,5 +1,5 @@ error sink-column-not-captured iceberg-exclude-list/sink.iceberg/shipments.sql:6 - public.shipments.driver_phone is read by lake.cdc.shipments (is fed topic wh.public.shipments by iceberg-exclude-list/sink-connector.json) but is not matched by column.exclude.list in iceberg-exclude-list/connector.json + public.shipments.driver_phone is read by lake.cdc.shipments (is fed topic wh.public.shipments by iceberg-exclude-list/sink-connector.json) but is matched by column.exclude.list in iceberg-exclude-list/connector.json every row will carry the column's default, with no error anywhere fix: remove public.shipments.driver_phone from column.exclude.list, deploy the connector, then apply the sink schema; rows already written need a snapshot 1 error(s), 0 warning(s), 0 info diff --git a/internal/engine/engine.go b/internal/engine/engine.go index d4f6c16..8370e92 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -139,6 +139,19 @@ func Run(in *Input) []model.Finding { return fs } +// leftOut words why the connector does not carry a name, for the list +// mode: an include list that does not match it, or an exclude list that +// does. Until v0.2.2 every message said "is not matched by +// column.exclude.list" for a column the exclude list matched, the +// opposite of what happened, and since v0.2.1 that info line is the only +// message a pattern-excluded column gets. +func leftOut(setting string) string { + if excludes(setting) { + return "is matched by " + setting + } + return "is not matched by " + setting +} + // edit words the remedy for the contract's list mode: a column is added to // an include list and removed from an exclude list. func edit(setting, name string) string { @@ -186,7 +199,7 @@ func sinkTableNotCaptured(in *Input, reads []Read) []model.Finding { seen[r.Sink] = true fs = append(fs, model.Finding{ Rule: "sink-table-not-captured", Severity: model.Error, Pos: r.Sink.Pos, - Message: fmt.Sprintf("%s %s, but %s.%s is not matched by %s in %s\nnothing will ever arrive on that topic", r.Sink.Name, how(r), r.Table.Schema, r.Table.Name, in.Contract.TableListSetting(), in.Contract.Pos().File), + Message: fmt.Sprintf("%s %s, but %s.%s %s in %s\nnothing will ever arrive on that topic", r.Sink.Name, how(r), r.Table.Schema, r.Table.Name, leftOut(in.Contract.TableListSetting()), in.Contract.Pos().File), Fix: edit(in.Contract.TableListSetting(), r.Table.Schema+"."+r.Table.Name) + " and deploy the connector before the sink schema", }) } @@ -206,7 +219,7 @@ func sinkColumnNotCaptured(in *Input, reads []Read) []model.Finding { q := fmt.Sprintf("%s.%s.%s", r.Table.Schema, r.Table.Name, src.Name) fs = append(fs, model.Finding{ Rule: "sink-column-not-captured", Severity: model.Error, Pos: r.Column.Pos, - Message: fmt.Sprintf("%s is read by %s (%s) but is not matched by %s in %s\nevery row will carry the column's default, with no error anywhere", q, r.Sink.Name, how(r), in.Contract.ColumnListSetting(), in.Contract.Pos().File), + Message: fmt.Sprintf("%s is read by %s (%s) but %s in %s\nevery row will carry the column's default, with no error anywhere", q, r.Sink.Name, how(r), leftOut(in.Contract.ColumnListSetting()), in.Contract.Pos().File), Fix: edit(in.Contract.ColumnListSetting(), q) + ", deploy the connector, then apply the sink schema; rows already written need a snapshot", }) } @@ -248,7 +261,7 @@ func sourceColumnNotCaptured(in *Input, reads []Read, raised map[string]bool) [] } fs = append(fs, model.Finding{ Rule: "source-column-not-captured", Severity: model.Info, Pos: c.Pos, - Message: fmt.Sprintf("%s.%s exists and is not matched by %s in %s; nothing reads it yet\nthe day a sink asks for it is the day it is found missing", t.Qualified(), c.Name, in.Contract.ColumnListSetting(), in.Contract.Pos().File), + Message: fmt.Sprintf("%s.%s exists and %s in %s; nothing reads it yet\nthe day a sink asks for it is the day it is found missing", t.Qualified(), c.Name, leftOut(in.Contract.ColumnListSetting()), in.Contract.Pos().File), }) } }