From a61b9a1a46bc9ecc50cc211e790b464d71f8c2bc Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 30 Sep 2026 20:36:07 +0200 Subject: [PATCH] CAS: drop KeyStrictness, readers skip unknown ordinary keys Strict readers are a leftover of the codec rewrites. Every object is JSON with the `!` rule for decision-changing fields, so strictness has no job left, and it made any optional field on `cas_run`, fold seal, `cas_ref_ckpt`, `cas_ref_catalog` or `cas_gc_maintenance_state` a compatibility-version bump. Remove the enum, the traits column and the reader parameter; the seven sites that rejected unknown keys with their own throw go through `skipUnknown`. Per object, a test that an unknown ordinary key is skipped and one that an unknown `!` key fails. Reader-only change. Signed-off-by: Mikhail Filimonov --- .../cas/architecture/storage-layout.md | 48 ++++++++--------- .../Formats/CasBlobEnvelopeFormat.cpp | 4 +- .../Formats/CasBlobMetaFormat.cpp | 2 +- .../Formats/CasFoldSealFormat.cpp | 16 +++--- .../ContentAddressed/Formats/CasFormat.cpp | 45 ++++++++-------- .../ContentAddressed/Formats/CasFormat.h | 5 -- .../Formats/CasGcMaintenanceStateFormat.cpp | 2 +- .../Formats/CasGcOutcomesFormat.cpp | 2 +- .../Formats/CasGcStateFormat.cpp | 4 +- .../Formats/CasPartManifestFormat.cpp | 4 +- .../Formats/CasPoolMetaFormat.cpp | 2 +- .../Formats/CasRecordStreamFormat.cpp | 10 ++-- .../Formats/CasRecordStreamFormat.h | 2 +- .../Formats/CasRefCatalogFormat.cpp | 8 +-- .../Formats/CasRefCkptFormat.cpp | 9 ++-- .../Formats/CasRefCkptFormat.h | 10 ++-- .../Formats/CasRefLogFormat.cpp | 8 +-- .../Formats/CasRefSnapshotFormat.cpp | 4 +- .../Formats/CasServerRootFormats.cpp | 6 +-- .../Formats/CasTextFormat.cpp | 11 ++-- .../ContentAddressed/Formats/CasTextFormat.h | 13 ++--- .../ContentAddressed/Formats/README.md | 10 ++-- src/Disks/tests/cas_format_test_battery.h | 31 +++++++++++ .../tests/gtest_cas_fold_seal_format.cpp | 35 +++++++++++- .../gtest_cas_gc_maintenance_state_format.cpp | 16 ++++-- .../tests/gtest_cas_record_stream_format.cpp | 40 ++++++++++++++ src/Disks/tests/gtest_cas_ref_catalog.cpp | 37 ++++++------- src/Disks/tests/gtest_cas_ref_ckpt.cpp | 53 +++++++++---------- src/Disks/tests/gtest_cas_text_format.cpp | 33 +++++------- src/Disks/tests/gtest_cas_wire_vocab.cpp | 14 ++--- 30 files changed, 285 insertions(+), 199 deletions(-) diff --git a/docs/en/antalya/cas/architecture/storage-layout.md b/docs/en/antalya/cas/architecture/storage-layout.md index b136552acf37..871a502027cb 100644 --- a/docs/en/antalya/cas/architecture/storage-layout.md +++ b/docs/en/antalya/cas/architecture/storage-layout.md @@ -15,8 +15,9 @@ control-plane bodies are JSON Lines — one JSON object per line, sorted where t or a set of entries (`Formats/README.md`; see [Envelope format](#envelope-format) below for which parts are a single JSON object versus JSON Lines versus raw payload bytes). The format is deliberately this plain: any object can be fetched and read with ordinary line-oriented tools -while debugging, and a new field is additive — a tolerant reader skips it — so the format evolves -without a migration. +while debugging, and the format evolves without a migration. A new optional field is a plain key, +and an old reader skips it. A field that changes a reader's decisions is written with the `!` +prefix, and an old reader fails with `UNKNOWN_FORMAT_VERSION` instead of half-reading the object. ## Key table {#key-table} @@ -88,30 +89,27 @@ bytes. Condensed from the authoritative traits table in `CasFormat.cpp` (`TRAITS`, asserted complete by `gtest_cas_text_format.cpp`). -| Type string | Family | Key strictness | Compression | +| Type string | Family | Compression | |---|---|---|---| -| `cas_blob` | `PayloadHybrid` | tolerant | never (raw, fixed offset) | -| `cas_blob_meta` | `Control` | tolerant | never | -| `cas_pool_meta` | `Control` | tolerant | never | -| `cas_ref_log` | `Control` | tolerant | always (`.zst`) | -| `cas_ref_snap` | `Control` | tolerant | always (`.zst`) | -| `cas_ref_ckpt` | `Control` | strict | never | -| `cas_ref_catalog` | `Control` | strict | never | -| `cas_part_manifest` | `PayloadHybrid` | tolerant | always (`.zst`) | -| `cas_run` | `RecordStream` | strict | pinned raw | -| `cas_fold_seal` | `Control` | strict | pinned raw | -| `cas_gc_state` | `Control` | tolerant | never | -| `cas_gc_hb` | `Control` | tolerant | never | -| `cas_gc_outcomes` | `Control` | tolerant | always (`.zst`) | -| `cas_gc_maintenance_state` | `Control` | strict | never | -| `cas_owner` | `Control` | tolerant | never | -| `cas_epoch` | `Control` | tolerant | never | -| `cas_mount_lease` | `Control` | tolerant | never | - -"Strict" means unknown keys are rejected rather than skipped, used for objects where every field -decides a durability or cleanup decision (`cas_ref_ckpt`, `cas_ref_catalog`, `cas_fold_seal`, -`cas_run`, `cas_gc_maintenance_state`); a `!`-prefixed key is always critical regardless of the -kind's strictness. "Pinned raw" objects (`cas_run`, `cas_fold_seal`) need stable bytes across +| `cas_blob` | `PayloadHybrid` | never (raw, fixed offset) | +| `cas_blob_meta` | `Control` | never | +| `cas_pool_meta` | `Control` | never | +| `cas_ref_log` | `Control` | always (`.zst`) | +| `cas_ref_snap` | `Control` | always (`.zst`) | +| `cas_ref_ckpt` | `Control` | never | +| `cas_ref_catalog` | `Control` | never | +| `cas_part_manifest` | `PayloadHybrid` | always (`.zst`) | +| `cas_run` | `RecordStream` | pinned raw | +| `cas_fold_seal` | `Control` | pinned raw | +| `cas_gc_state` | `Control` | never | +| `cas_gc_hb` | `Control` | never | +| `cas_gc_outcomes` | `Control` | always (`.zst`) | +| `cas_gc_maintenance_state` | `Control` | never | +| `cas_owner` | `Control` | never | +| `cas_epoch` | `Control` | never | +| `cas_mount_lease` | `Control` | never | + +"Pinned raw" objects (`cas_run`, `cas_fold_seal`) need stable bytes across re-encodes for deterministic-artifact adoption, so their bytes are never recompressed once written. `cas_blob` and `cas_part_manifest` are the `PayloadHybrid` family: a text descriptor followed by a raw payload zone, rather than a single JSON body. diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobEnvelopeFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobEnvelopeFormat.cpp index 523912c519c4..40e4f96c8900 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobEnvelopeFormat.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobEnvelopeFormat.cpp @@ -235,7 +235,7 @@ String encodeEnvelopeHeader(EnvelopeHeader & header, uint32_t blob_header_len, EnvelopeHeader decodeEnvelopeHeader(std::string_view head_bytes, uint64_t /*object_size*/, ObjectKind expected_kind) { ReadBufferFromMemory in(head_bytes.data(), head_bytes.size()); - JsonObjectReader r(in, KeyStrictness::Tolerant, "blob envelope"); + JsonObjectReader r(in, "blob envelope"); EnvelopeHeader h; h.kind = ObjectKind::Blob; @@ -287,7 +287,7 @@ EnvelopeHeader decodeEnvelopeHeader(std::string_view head_bytes, uint64_t /*obje else if (key == EnvelopeWire::ref) h.intended_ref = r.readString(); else - r.skipUnknown(key); /// `!`-key -> UNKNOWN_FORMAT_VERSION; unknown plain key -> skipped (tolerant) + r.skipUnknown(key); /// `!`-key -> UNKNOWN_FORMAT_VERSION; unknown plain key -> skipped } if (!saw_type) throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS blob envelope: missing type"); diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobMetaFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobMetaFormat.cpp index 2b40584e0199..0b5d8c699189 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobMetaFormat.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobMetaFormat.cpp @@ -65,7 +65,7 @@ BlobMeta decodeBlobMeta(std::string_view bytes) expectHeaderLine(in, FormatId::BlobMeta); const String body = readLine(in, traitsFor(FormatId::BlobMeta).line_cap, "blob meta"); ReadBufferFromMemory body_in(body.data(), body.size()); - JsonObjectReader r(body_in, KeyStrictness::Tolerant, "blob meta"); + JsonObjectReader r(body_in, "blob meta"); // Start with the documented defaults. In particular, `version` stays at 1 because the header's // version is authoritative and is not copied into the body struct. diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFoldSealFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFoldSealFormat.cpp index b2c0bf1f9756..136509c41564 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFoldSealFormat.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFoldSealFormat.cpp @@ -380,13 +380,13 @@ CasFoldSeal decodeFoldSeal(std::string_view data, std::optional expect { const String meta = readLine(in, line_cap, "fold seal"); ReadBufferFromMemory m(meta.data(), meta.size()); - JsonObjectReader r(m, KeyStrictness::Strict, "fold seal"); + JsonObjectReader r(m, "fold seal"); String key; while (r.nextKey(key)) { if (key == FoldSealWire::generation) seal.generation = r.readU64String(); else if (key == FoldSealWire::parent_generation) seal.parent_generation = r.readU64String(); - else r.skipUnknown(key); /// Strict => any unknown key is CORRUPTED_DATA + else r.skipUnknown(key); } } @@ -400,7 +400,7 @@ CasFoldSeal decodeFoldSeal(std::string_view data, std::optional expect { readLineInto(in, row_line, line_cap, "fold seal"); ReadBufferFromMemory l(row_line.data(), row_line.size()); - row_reader.reset(l, KeyStrictness::Strict, "fold seal"); + row_reader.reset(l, "fold seal"); JsonObjectReader & r = row_reader; String key; if (!r.nextKey(key)) @@ -409,8 +409,8 @@ CasFoldSeal decodeFoldSeal(std::string_view data, std::optional expect if (key == "n") { const uint64_t n = r.readU64Number(); - if (r.nextKey(key)) - throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS fold seal: trailer has extra keys"); + while (r.nextKey(key)) + r.skipUnknown(key); if (!l.eof() || !in.eof()) throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS fold seal: bytes after trailer"); if (n != seen) @@ -458,7 +458,7 @@ CasFoldSeal decodeFoldSeal(std::string_view data, std::optional expect else if (key == FoldSealWire::retry_round) hold_next_retry_round = r.readU64String(); else if (key == FoldSealWire::remove_epoch) remove_txn_epoch = r.readU64String(); else if (key == FoldSealWire::remove_seq) remove_txn_sequence = r.readU64String(); - else throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS fold seal: unknown ref_life key '{}'", key); + else r.skipUnknown(key); } if (!life_id || *life_id == 0) @@ -539,7 +539,7 @@ CasFoldSeal decodeFoldSeal(std::string_view data, std::optional expect else if (key == FoldSealWire::checksum) checksum = r.readHex128(); else if (key == FoldSealWire::shard) shard = r.readU64Number(); else if (key == FoldSealWire::key_generation) generation = r.readU64String(); - else throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS fold seal: unknown run key '{}'", key); + else r.skipUnknown(key); } if (!run_key || !checksum || !shard || !generation) throw Exception(ErrorCodes::CORRUPTED_DATA, @@ -559,7 +559,7 @@ CasFoldSeal decodeFoldSeal(std::string_view data, std::optional expect else if (key == FoldSealWire::condemned_total) condemned_total = r.readU64Number(); else if (key == FoldSealWire::pending_total) pending_total = r.readU64Number(); else if (key == FoldSealWire::oldest_round) oldest_nonpending_condemn_round = r.readU64String(); - else throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS fold seal: unknown condemned key '{}'", key); + else r.skipUnknown(key); } if (!shard || !condemned_total || !pending_total || !oldest_nonpending_condemn_round) throw Exception(ErrorCodes::CORRUPTED_DATA, diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.cpp index 4ec24eeae61e..da8199257817 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.cpp @@ -100,39 +100,36 @@ constexpr uint64_t kMiB = 1024 * 1024; /// small raw singletons. constexpr FormatTraits TRAITS[] = { - {FormatId::Blob, "cas_blob", TextFamily::PayloadHybrid, KeyStrictness::Tolerant, CompressionPolicy::Never, 256, 256}, - {FormatId::BlobMeta, "cas_blob_meta", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, - {FormatId::PoolMeta, "cas_pool_meta", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, - {FormatId::RefLog, "cas_ref_log", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Always, 64 * kMiB, 64 * kMiB}, - {FormatId::RefSnapshot, "cas_ref_snap", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Always, 64 * kMiB, 64 * kMiB}, + {FormatId::Blob, "cas_blob", TextFamily::PayloadHybrid, CompressionPolicy::Never, 256, 256}, + {FormatId::BlobMeta, "cas_blob_meta", TextFamily::Control, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, + {FormatId::PoolMeta, "cas_pool_meta", TextFamily::Control, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, + {FormatId::RefLog, "cas_ref_log", TextFamily::Control, CompressionPolicy::Always, 64 * kMiB, 64 * kMiB}, + {FormatId::RefSnapshot, "cas_ref_snap", TextFamily::Control, CompressionPolicy::Always, 64 * kMiB, 64 * kMiB}, /// `cas_ref_ckpt` is a three-field mutable singleton read by a point GET on every recovery and on /// every cleanup decision, so its caps are deliberately TIGHT (64 KiB / 4 KiB rather than the /// megabyte scale its Control-family siblings use): nothing legitimate approaches them, and the cap - /// is the first thing that fires if a foreign object ever lands at the key. STRICT for the same - /// reason its decoder is -- every field changes what cleanup may delete, so nothing in it may be - /// skipped. Raw (`Never`): a small singleton, and `publishCkpt` re-encodes it on every attempt. - {FormatId::RefCkpt, "cas_ref_ckpt", TextFamily::Control, KeyStrictness::Strict, CompressionPolicy::Never, 64 * kKiB, 4 * kKiB}, + /// is the first thing that fires if a foreign object ever lands at the key. Raw (`Never`): a small + /// singleton, and `publishCkpt` re-encodes it on every attempt. + {FormatId::RefCkpt, "cas_ref_ckpt", TextFamily::Control, CompressionPolicy::Never, 64 * kKiB, 4 * kKiB}, /// `cas_ref_catalog` (INV-3): one object for the whole pool, token-CAS like `gc/state`, read on - /// every fold round and every recovery. STRICT for the same reason `cas_ref_ckpt` is -- every - /// field decides a namespace's lifecycle, so nothing in it may be skipped. Raw (`Never`): the - /// admission gate measures `encodeRefCatalog`'s own output directly, so a compressed size would - /// answer the wrong question. The object cap is the fold-seal's own 256 MiB (predicate (2) of the + /// every fold round and every recovery. Raw (`Never`): the admission gate measures + /// `encodeRefCatalog`'s own output directly, so a compressed size would answer the wrong question. The object cap is the fold-seal's own 256 MiB (predicate (2) of the /// additive admission check bounds it further via the entry count); the line cap is tight (4 KiB) /// because one entry's record is ordinarily small -- but not always small enough: a namespace or /// `server_root_id` near their own byte bounds, worst-case escaped, can push a single line past /// 4 KiB, and `encodeRefCatalog` REFUSES that entry (`LIMIT_EXCEEDED`, `CasRefCatalogFormat.cpp`'s /// `checkLineBytes`) rather than writing an object no reader could later decode. - {FormatId::RefCatalog, "cas_ref_catalog", TextFamily::Control, KeyStrictness::Strict, CompressionPolicy::Never, 256 * kMiB, 4 * kKiB}, - {FormatId::GcMaintenanceState, "cas_gc_maintenance_state", TextFamily::Control, KeyStrictness::Strict, CompressionPolicy::Never, 512 * kKiB, 512 * kKiB}, - {FormatId::PartManifest, "cas_part_manifest", TextFamily::PayloadHybrid, KeyStrictness::Tolerant, CompressionPolicy::Always, 256 * kMiB, 64 * kKiB}, - {FormatId::RunFile, "cas_run", TextFamily::RecordStream, KeyStrictness::Strict, CompressionPolicy::PinnedRaw, 0, 4 * kKiB}, - {FormatId::FoldSeal, "cas_fold_seal", TextFamily::Control, KeyStrictness::Strict, CompressionPolicy::PinnedRaw, 256 * kMiB, 64 * kKiB}, - {FormatId::GcState, "cas_gc_state", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, - {FormatId::GcHeartbeat, "cas_gc_hb", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, - {FormatId::GcOutcomes, "cas_gc_outcomes", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Always, 256 * kMiB, 64 * kKiB}, - {FormatId::Owner, "cas_owner", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, - {FormatId::ServerEpoch, "cas_epoch", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, - {FormatId::MountLease, "cas_mount_lease", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, + {FormatId::RefCatalog, "cas_ref_catalog", TextFamily::Control, CompressionPolicy::Never, 256 * kMiB, 4 * kKiB}, + {FormatId::GcMaintenanceState, "cas_gc_maintenance_state", TextFamily::Control, CompressionPolicy::Never, 512 * kKiB, 512 * kKiB}, + {FormatId::PartManifest, "cas_part_manifest", TextFamily::PayloadHybrid, CompressionPolicy::Always, 256 * kMiB, 64 * kKiB}, + {FormatId::RunFile, "cas_run", TextFamily::RecordStream, CompressionPolicy::PinnedRaw, 0, 4 * kKiB}, + {FormatId::FoldSeal, "cas_fold_seal", TextFamily::Control, CompressionPolicy::PinnedRaw, 256 * kMiB, 64 * kKiB}, + {FormatId::GcState, "cas_gc_state", TextFamily::Control, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, + {FormatId::GcHeartbeat, "cas_gc_hb", TextFamily::Control, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, + {FormatId::GcOutcomes, "cas_gc_outcomes", TextFamily::Control, CompressionPolicy::Always, 256 * kMiB, 64 * kKiB}, + {FormatId::Owner, "cas_owner", TextFamily::Control, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, + {FormatId::ServerEpoch, "cas_epoch", TextFamily::Control, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, + {FormatId::MountLease, "cas_mount_lease", TextFamily::Control, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, }; } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.h index bfcb8cf652fb..849a5ff06d2d 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.h @@ -98,10 +98,6 @@ std::span changePoints(FormatId id); /// or a descriptor followed by raw payload bytes. enum class TextFamily : uint8_t { Control = 1, RecordStream = 2, PayloadHybrid = 3 }; -/// Whether a decoder skips unknown ordinary keys or rejects them. Critical keys prefixed with `!` -/// are rejected by all families because they signal a required extension. -enum class KeyStrictness : uint8_t { Tolerant = 1, Strict = 2 }; - /// Deterministic storage policy. `Always` uses whole-object zstd and a `.zst` key suffix; `Never` /// remains raw; `PinnedRaw` is raw because byte adoption compares the serialized bytes. enum class CompressionPolicy : uint8_t { Never = 1, Always = 2, PinnedRaw = 3 }; @@ -113,7 +109,6 @@ struct FormatTraits FormatId id; std::string_view type; /// header-line "type" value TextFamily family; - KeyStrictness strictness; CompressionPolicy compression; uint64_t object_cap; /// max DECOMPRESSED object bytes; 0 = uncapped (streamed) uint64_t line_cap; /// max bytes of one text line diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcMaintenanceStateFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcMaintenanceStateFormat.cpp index cc13021b020d..03c64aa58cfb 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcMaintenanceStateFormat.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcMaintenanceStateFormat.cpp @@ -43,7 +43,7 @@ GcMaintenanceState decodeGcMaintenanceState(std::string_view data) expectHeaderLine(in, FormatId::GcMaintenanceState); const String body = readLine(in, traitsFor(FormatId::GcMaintenanceState).line_cap, "cas_gc_maintenance_state"); ReadBufferFromMemory body_in(body.data(), body.size()); - JsonObjectReader reader(body_in, KeyStrictness::Strict, "cas_gc_maintenance_state"); + JsonObjectReader reader(body_in, "cas_gc_maintenance_state"); GcMaintenanceState result; bool has_cursor = false; diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcOutcomesFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcOutcomesFormat.cpp index 2f9aa0e141d2..a4f92401b651 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcOutcomesFormat.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcOutcomesFormat.cpp @@ -79,7 +79,7 @@ OutcomeLog decodeOutcomeLog(std::string_view data) { readLineInto(in, row_line, line_cap, "outcome log"); ReadBufferFromMemory line_in(row_line.data(), row_line.size()); - row_reader.reset(line_in, KeyStrictness::Tolerant, "outcome log"); + row_reader.reset(line_in, "outcome log"); JsonObjectReader & r = row_reader; String key; diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcStateFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcStateFormat.cpp index 5cc268a4d9c3..7d064e757f2b 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcStateFormat.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcStateFormat.cpp @@ -60,7 +60,7 @@ GcState decodeGcState(std::string_view data) expectHeaderLine(in, FormatId::GcState); const String body = readLine(in, traitsFor(FormatId::GcState).line_cap, "gc/state"); ReadBufferFromMemory body_in(body.data(), body.size()); - JsonObjectReader r(body_in, KeyStrictness::Tolerant, "gc/state"); + JsonObjectReader r(body_in, "gc/state"); GcState state; bool saw_gcs = false; @@ -118,7 +118,7 @@ GcHeartbeat decodeGcHeartbeat(std::string_view data) expectHeaderLine(in, FormatId::GcHeartbeat); const String body = readLine(in, traitsFor(FormatId::GcHeartbeat).line_cap, "gc heartbeat"); ReadBufferFromMemory body_in(body.data(), body.size()); - JsonObjectReader r(body_in, KeyStrictness::Tolerant, "gc heartbeat"); + JsonObjectReader r(body_in, "gc heartbeat"); GcHeartbeat hb; bool saw_by = false; diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPartManifestFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPartManifestFormat.cpp index e0722add9bdc..7c5109c0e4b7 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPartManifestFormat.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPartManifestFormat.cpp @@ -148,7 +148,7 @@ PartManifest decodePartManifest(std::string_view data) { const String meta = readLine(in, line_cap, "cas_part_manifest"); ReadBufferFromMemory mm(meta.data(), meta.size()); - JsonObjectReader r(mm, KeyStrictness::Tolerant, "cas_part_manifest"); + JsonObjectReader r(mm, "cas_part_manifest"); ManifestRefFields fields; std::optional ns; std::optional pd; @@ -185,7 +185,7 @@ PartManifest decodePartManifest(std::string_view data) { readLineInto(in, row_line, line_cap, "cas_part_manifest"); ReadBufferFromMemory l(row_line.data(), row_line.size()); - row_reader.reset(l, KeyStrictness::Tolerant, "cas_part_manifest"); + row_reader.reset(l, "cas_part_manifest"); JsonObjectReader & r = row_reader; String key; if (!r.nextKey(key)) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPoolMetaFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPoolMetaFormat.cpp index 80f9572c4547..c024e8b87a3c 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPoolMetaFormat.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPoolMetaFormat.cpp @@ -110,7 +110,7 @@ PoolMeta decodePoolMeta(std::string_view data) const String body = readLine(in, traitsFor(FormatId::PoolMeta).line_cap, "pool meta"); ReadBufferFromMemory body_in(body.data(), body.size()); - JsonObjectReader r(body_in, KeyStrictness::Tolerant, "pool meta"); + JsonObjectReader r(body_in, "pool meta"); PoolMeta pm; bool saw_pid = false; diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRecordStreamFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRecordStreamFormat.cpp index 78dedf30a4b2..1399139b3809 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRecordStreamFormat.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRecordStreamFormat.cpp @@ -140,7 +140,7 @@ void expectRunHeaderLine(ReadBuffer & in, std::string_view expected_kind) const FormatTraits & t = traitsFor(FormatId::RunFile); const String line = readLine(in, t.line_cap, t.type); ReadBufferFromMemory buf(line.data(), line.size()); - JsonObjectReader r(buf, KeyStrictness::Tolerant, t.type); + JsonObjectReader r(buf, t.type); String key; if (!r.nextKey(key) || key != "type") @@ -250,7 +250,7 @@ bool SourceEdgeRunReader::next(SourceEdgeRecord & rec) ReadBufferFromMemory line_in(scratch.data(), scratch.size()); /// Re-point the reader rather than building one per row: a fresh reader re-allocates its /// seen-key store and value scratch every row, and this loop runs once per record. - reader.reset(line_in, KeyStrictness::Strict, "cas_run"); + reader.reset(line_in, "cas_run"); JsonObjectReader & r = reader; String key; @@ -260,8 +260,8 @@ bool SourceEdgeRunReader::next(SourceEdgeRecord & rec) if (key == "n") { const uint64_t n = r.readU64Number(); - if (r.nextKey(key)) - throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: trailer has extra keys"); + while (r.nextKey(key)) + r.skipUnknown(key); if (!line_in.eof()) throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: junk after trailer object"); /// The trailer must be the last line of the object; hashing must be at EOF (this also drains and @@ -295,7 +295,7 @@ bool SourceEdgeRunReader::next(SourceEdgeRecord & rec) else if (key == RunWire::size) { out.size = r.readU64Number(); have_size = true; } else if (key == RunWire::condemn_round) { out.condemn_round = r.readU64String(); have_condemn_round = true; } else if (key == RunWire::confirmed) { out.marker_confirmed = r.readBool(); have_confirmed = true; } - else r.skipUnknown(key); /// Strict => any unknown key is CORRUPTED_DATA + else r.skipUnknown(key); } while (r.nextKey(key)); if (!have_ref || !have_src || !have_mark) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRecordStreamFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRecordStreamFormat.h index 5741a61fc9b7..404c6fa2f189 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRecordStreamFormat.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRecordStreamFormat.h @@ -55,7 +55,7 @@ inline RunMarker runMarkerFromByte(char byte, std::string_view what) /// the `RecordStream` family /// (`FormatId::RunFile`): unbounded-cardinality sorted records, `object_cap = 0` (NEVER materialized /// whole — streamed one line at a time over a `ReadBuffer`), `line_cap = 4 KiB`, `PinnedRaw` (no -/// compression) + `Strict` (byte-deterministic for `putDeterministicArtifact` adoption). +/// compression; byte-deterministic for `putDeterministicArtifact` adoption). /// /// This file is backend-free: it accepts caller-owned `ReadBuffer`/`WriteBuffer` objects and reaches /// no backend or GC machinery -- `PersistedEtag` is a value type with no live backend behind diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCatalogFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCatalogFormat.cpp index 6cf44651508e..d527fe1ce458 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCatalogFormat.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCatalogFormat.cpp @@ -187,7 +187,7 @@ RefCatalog decodeRefCatalog(std::string_view data) { readLineInto(in, row_line, line_cap, "ref catalog"); ReadBufferFromMemory l(row_line.data(), row_line.size()); - row_reader.reset(l, KeyStrictness::Strict, "ref catalog"); + row_reader.reset(l, "ref catalog"); JsonObjectReader & r = row_reader; String key; if (!r.nextKey(key)) @@ -196,8 +196,8 @@ RefCatalog decodeRefCatalog(std::string_view data) if (key == "n") { const uint64_t n = r.readU64Number(); - if (r.nextKey(key)) - throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS ref catalog: trailer has extra keys"); + while (r.nextKey(key)) + r.skipUnknown(key); if (!l.eof() || !in.eof()) throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS ref catalog: bytes after trailer"); if (n != seen) @@ -227,7 +227,7 @@ RefCatalog decodeRefCatalog(std::string_view data) else if (key == RefCatalogWire::creator_epoch) cwe = r.readU64String(); else if (key == RefCatalogWire::creator_fence) cfg = r.readU64String(); else if (key == RefCatalogWire::remove_round) removal_started_round = r.readU64String(); - else throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS ref catalog: unknown entry key '{}'", key); + else r.skipUnknown(key); } if (!l.eof()) throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS ref catalog: junk after record"); diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCkptFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCkptFormat.cpp index 4034f5c244c1..720266337bc3 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCkptFormat.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCkptFormat.cpp @@ -132,12 +132,9 @@ RefCkpt decodeRefCkpt(std::string_view data) expectHeaderLine(in, FormatId::RefCkpt); const String body = readLine(in, traitsFor(FormatId::RefCkpt).line_cap, "cas_ref_ckpt"); ReadBufferFromMemory body_in(body.data(), body.size()); - /// STRICT: an unknown ordinary key is `CORRUPTED_DATA` and a `!`-prefixed one is - /// `UNKNOWN_FORMAT_VERSION`. `_ckpt` is a control object whose every field changes what cleanup is - /// allowed to delete, so a reader that silently ignored a key it did not understand would be - /// deciding deletions from a body it only partially read. Duplicate keys are rejected by - /// `JsonObjectReader` itself. - JsonObjectReader r(body_in, KeyStrictness::Strict, "cas_ref_ckpt"); + /// A field that changes what cleanup may delete is written `!`-prefixed, so an old reader fails + /// instead of half-reading. + JsonObjectReader r(body_in, "cas_ref_ckpt"); RefCkpt ckpt; std::optional snapshot_epoch; diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCkptFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCkptFormat.h index d0331248db6f..97fd9aa139be 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCkptFormat.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCkptFormat.h @@ -105,11 +105,11 @@ struct RefCkpt /// fails. That assertion is the tripwire for this shortcut, not an incidental check of the key shape. String encodeRefCkpt(const RefCkpt & ckpt); -/// Decode a complete `cas_ref_ckpt` text object. STRICT (`KeyStrictness::Strict`): an unknown ordinary -/// key, a duplicate key, a truncated object (a missing body line, or half of an optional -/// id pair), or trailing bytes all raise `CORRUPTED_DATA` -- never a partially-populated struct. This -/// object gates destructive cleanup and names recovery's base, so "decoded something" must mean -/// "decoded exactly what a writer of this format wrote". +/// Decode a complete `cas_ref_ckpt` text object. A duplicate key, a truncated object (a missing body +/// line, or half of an optional id pair), or trailing bytes raise `CORRUPTED_DATA` -- never a +/// partially-populated struct. An unknown `!`-prefixed key raises `UNKNOWN_FORMAT_VERSION`; an unknown +/// ordinary key is skipped. This object gates destructive cleanup and names recovery's base, so +/// "decoded something" must mean "decoded everything the writer made decision-relevant". RefCkpt decodeRefCkpt(std::string_view data); /// The shared field-level validity rule, applied on both encode and decode: every PRESENT field is a diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefLogFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefLogFormat.cpp index 8c9d010ee678..17fe47894881 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefLogFormat.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefLogFormat.cpp @@ -126,7 +126,7 @@ struct BindingFields /// `!`-prefixed: `prev_epoch_seal` is INV-2 chain evidence, not cosmetic metadata -- a decoder that /// doesn't understand it must refuse the object rather than silently drop the chain link while /// otherwise passing the structural grammar (`JsonObjectReader::skipUnknown` rejects any unrecognized -/// `!`-key with `UNKNOWN_FORMAT_VERSION`, tolerant or not). +/// `!`-key with `UNKNOWN_FORMAT_VERSION`). void writeLogMeta(CasJsonWriter & out, const String & ns, const RefTxnId & txn_id, const std::optional & prev_epoch_seal) { bool first = true; @@ -308,7 +308,7 @@ RefLogTxn decodeRefLogTxn(std::string_view data, const String & expected_ns, con { const String line = readLine(in, line_cap, "cas_ref_log"); ReadBufferFromMemory m(line.data(), line.size()); - JsonObjectReader r(m, KeyStrictness::Tolerant, "cas_ref_log"); + JsonObjectReader r(m, "cas_ref_log"); bool saw_ns = false; bool saw_txn_epoch = false; bool saw_txn_seq = false; @@ -373,7 +373,7 @@ RefLogTxn decodeRefLogTxn(std::string_view data, const String & expected_ns, con { readLineInto(in, row_line, line_cap, "cas_ref_log"); ReadBufferFromMemory l(row_line.data(), row_line.size()); - row_reader.reset(l, KeyStrictness::Tolerant, "cas_ref_log"); + row_reader.reset(l, "cas_ref_log"); JsonObjectReader & r = row_reader; String key; if (!r.nextKey(key)) @@ -455,7 +455,7 @@ std::optional peekRefLogMeta(const String & sealed_bytes) readLine(in, line_cap, "cas_ref_log"); /// header line -- skipped, the version is not judged here const String meta = readLine(in, line_cap, "cas_ref_log"); ReadBufferFromMemory m(meta.data(), meta.size()); - JsonObjectReader r(m, KeyStrictness::Tolerant, "cas_ref_log"); + JsonObjectReader r(m, "cas_ref_log"); RefLogMetaPeek peek; bool saw_ns = false; bool saw_epoch = false; diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefSnapshotFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefSnapshotFormat.cpp index 8d8b9e6f8244..0b886466e85c 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefSnapshotFormat.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefSnapshotFormat.cpp @@ -149,7 +149,7 @@ RefTableSnapshot decodeRefTableSnapshot( { const String line = readLine(in, line_cap, "cas_ref_snap"); ReadBufferFromMemory meta_buf(line.data(), line.size()); - JsonObjectReader r(meta_buf, KeyStrictness::Tolerant, "cas_ref_snap"); + JsonObjectReader r(meta_buf, "cas_ref_snap"); bool saw_ns = false; bool saw_snapshot_epoch = false; bool saw_snapshot_seq = false; @@ -189,7 +189,7 @@ RefTableSnapshot decodeRefTableSnapshot( { readLineInto(in, row_line, line_cap, "cas_ref_snap"); ReadBufferFromMemory l(row_line.data(), row_line.size()); - row_reader.reset(l, KeyStrictness::Tolerant, "cas_ref_snap"); + row_reader.reset(l, "cas_ref_snap"); JsonObjectReader & r = row_reader; String key; if (!r.nextKey(key)) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.cpp index a01a62ed30f6..f430d2e4d612 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.cpp @@ -71,7 +71,7 @@ OwnerObject decodeOwner(std::string_view data) expectHeaderLine(in, FormatId::Owner); const String body = readBodyLine(in, FormatId::Owner, "owner"); ReadBufferFromMemory body_in(body.data(), body.size()); - JsonObjectReader r(body_in, KeyStrictness::Tolerant, "owner"); + JsonObjectReader r(body_in, "owner"); OwnerObject o; bool saw = false; @@ -114,7 +114,7 @@ ServerEpoch decodeServerEpoch(std::string_view data) expectHeaderLine(in, FormatId::ServerEpoch); const String body = readBodyLine(in, FormatId::ServerEpoch, "server-epoch"); ReadBufferFromMemory body_in(body.data(), body.size()); - JsonObjectReader r(body_in, KeyStrictness::Tolerant, "server-epoch"); + JsonObjectReader r(body_in, "server-epoch"); ServerEpoch e; bool saw = false; @@ -162,7 +162,7 @@ MountLease decodeMountLease(std::string_view data) expectHeaderLine(in, FormatId::MountLease); const String body = readBodyLine(in, FormatId::MountLease, "mount-lease"); ReadBufferFromMemory body_in(body.data(), body.size()); - JsonObjectReader r(body_in, KeyStrictness::Tolerant, "mount-lease"); + JsonObjectReader r(body_in, "mount-lease"); MountLease m; bool saw_su = false; diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasTextFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasTextFormat.cpp index aefa25c871c6..6aa6eb1a0099 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasTextFormat.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasTextFormat.cpp @@ -166,16 +166,15 @@ auto JsonObjectReader::guarded(F && f) } } -JsonObjectReader::JsonObjectReader(ReadBuffer & in_, KeyStrictness strictness_, std::string_view what_) - : in(&in_), strictness(strictness_), what(what_) +JsonObjectReader::JsonObjectReader(ReadBuffer & in_, std::string_view what_) + : in(&in_), what(what_) { guarded([&] { assertChar('{', *in); }); } -void JsonObjectReader::reset(ReadBuffer & in_, KeyStrictness strictness_, std::string_view what_) +void JsonObjectReader::reset(ReadBuffer & in_, std::string_view what_) { in = &in_; - strictness = strictness_; what = what_; /// `clear` on both keeps their buffers: that is the whole point of reusing the reader. seen_keys.clear(); @@ -320,8 +319,6 @@ void JsonObjectReader::skipUnknown(const String & key) if (!key.empty() && key[0] == '!') throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, "CAS {}: critical key '{}' is not understood by this build", what, key); - if (strictness == KeyStrictness::Strict) - throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: unknown key '{}' in a strict format", what, key); skipJSONField(*in, key, jsonReadSettings()); }); } @@ -399,7 +396,7 @@ namespace TextHeader parseHeaderObject(std::string_view line, std::string_view what) { ReadBufferFromMemory buf(line.data(), line.size()); - JsonObjectReader r(buf, KeyStrictness::Tolerant, what); + JsonObjectReader r(buf, what); String key; if (!r.nextKey(key) || key != "type") throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: header line must start with \"type\"", what); diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasTextFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasTextFormat.h index 8bbbf58a4efd..141893433227 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasTextFormat.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasTextFormat.h @@ -226,15 +226,14 @@ constexpr bool isLowercaseHexChar(char c) /// /// The reader borrows the input buffer and records the object name for exception messages. It /// enforces unique keys and translates the several low-level parser exceptions into the CAS -/// `CORRUPTED_DATA` contract. Unknown keys follow the supplied evolution policy: ordinary keys -/// may be skipped in tolerant objects, while `!`-prefixed keys always fail with -/// `UNKNOWN_FORMAT_VERSION`. +/// `CORRUPTED_DATA` contract. An unknown ordinary key is skipped; an unknown `!`-prefixed key fails +/// with `UNKNOWN_FORMAT_VERSION`. class JsonObjectReader { public: /// Consumes the opening `{`; throws `CORRUPTED_DATA` when the object does not start there. - JsonObjectReader(ReadBuffer & in_, KeyStrictness strictness_, std::string_view what_); + JsonObjectReader(ReadBuffer & in_, std::string_view what_); /// An unbound reader, for a decoder that wants one reader outside its row loop and re-points it /// per row. `reset` must be called before any read; nothing else is valid on it. @@ -247,7 +246,7 @@ class JsonObjectReader /// instructions executed inside the decoder. Reusing one reader amortises that away. The /// object-level state -- the key set and the position in the object -- is reset in full, so a /// reused reader accepts and rejects exactly what a fresh one would. - void reset(ReadBuffer & in_, KeyStrictness strictness_, std::string_view what_); + void reset(ReadBuffer & in_, std::string_view what_); /// Advances to the next key; false when the closing '}' was consumed. The caller must /// consume the value (one read* / skipUnknown) before the next call. Duplicate keys are /// rejected with `CORRUPTED_DATA`. @@ -267,8 +266,7 @@ class JsonObjectReader /// Reads the bare JSON literals `true` and `false`. bool readBool(); /// Applies the evolution rule for an unrecognized key: `!`-prefixed keys produce - /// `UNKNOWN_FORMAT_VERSION`; strict objects produce `CORRUPTED_DATA`; tolerant objects skip - /// the value. + /// `UNKNOWN_FORMAT_VERSION`; any other unknown key has its value skipped. void skipUnknown(const String & key); private: @@ -282,7 +280,6 @@ class JsonObjectReader std::string_view readStringIntoScratch(); ReadBuffer * in = nullptr; - KeyStrictness strictness = KeyStrictness::Strict; String what; std::vector seen_keys; String scratch; diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/README.md b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/README.md index 50fe520b649e..f56b3d0c9028 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/README.md +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/README.md @@ -37,7 +37,7 @@ trailer, followed by a banner-framed raw payload zone for inline file bytes. ## Codec table -Authoritative per-format traits (type string, family, strictness, compression policy, caps) live +Authoritative per-format traits (type string, family, compression policy, caps) live in `CasFormat.cpp` (`TRAITS`), asserted complete by `gtest_cas_text_format.cpp`. Key naming follows a deliberate split between metadata written once per object and fields repeated @@ -70,13 +70,15 @@ use the corresponding `old_*` and `new_*` key bundles. - `v` (header line) is the ONLY version field; reader gate: `v > G_BUILD` → `UNKNOWN_FORMAT_VERSION`, checked before the body. -- Additive change = new tolerant key, no `v` bump; on MUTABLE objects the field is best-effort +- Additive change = new ordinary key, no `v` bump (every reader skips unknown ordinary keys); on + MUTABLE objects the field is best-effort until the pool floor rises (an old writer's fresh re-encode drops it). - Breaking change = `v` bump + `changePoints` + write-down-to-floor; the floor raise is what fences old builds out (mount gates: `min_reader_generation` forward, pool-meta `v` backward). -- Deterministic formats (`cas_fold_seal`, `cas_run`): strict keys, pinned raw, and the adoption +- Deterministic formats (`cas_fold_seal`, `cas_run`): pinned raw, and the adoption pin — on a `putDeterministicArtifact` conflict, re-encode at the `v` of the EXISTING object. -- A key prefixed `!` is critical: a reader that does not understand it fails closed. +- A key prefixed `!` is critical: a reader that does not understand it fails with + `UNKNOWN_FORMAT_VERSION`. A field that changes a reader's decisions is written with `!`. - Padding zones (blob header pad, manifest banners) are deterministic and verified — no unaccounted bytes in any object. - `openObject` policy asymmetry: a compressed body under a raw-compression policy is rejected diff --git a/src/Disks/tests/cas_format_test_battery.h b/src/Disks/tests/cas_format_test_battery.h index efed9ce44a74..3add5bbeda59 100644 --- a/src/Disks/tests/cas_format_test_battery.h +++ b/src/Disks/tests/cas_format_test_battery.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -61,6 +62,36 @@ void expectCode(int code, F && f, const String & context) } } +namespace cas_battery_detail +{ +/// Adds `"":1` as the last key of the `line`-th (0-based) line of `text`. +inline String withExtraKeyInLine(const String & text, size_t line, std::string_view key) +{ + size_t begin = 0; + for (size_t i = 0; i < line; ++i) + { + begin = text.find('\n', begin); + EXPECT_NE(begin, String::npos) << "no line " << line; + if (begin == String::npos) + return text; + ++begin; + } + const size_t end = text.find('\n', begin); + const size_t close = text.rfind('}', end == String::npos ? text.size() : end); + EXPECT_TRUE(close != String::npos && close >= begin) << "no object on line " << line; + if (close == String::npos || close < begin) + return text; + String out = text; + out.insert(close, fmt::format("{}\"{}\":1", text[close - 1] == '{' ? "" : ",", key)); + return out; +} + +inline size_t lineCount(const String & text) +{ + return static_cast(std::count(text.begin(), text.end(), '\n')); +} +} + namespace DB::Cas::tests { inline std::set & batteryCoveredIds() diff --git a/src/Disks/tests/gtest_cas_fold_seal_format.cpp b/src/Disks/tests/gtest_cas_fold_seal_format.cpp index ba8c6f131be9..8ffaa3a05629 100644 --- a/src/Disks/tests/gtest_cas_fold_seal_format.cpp +++ b/src/Disks/tests/gtest_cas_fold_seal_format.cpp @@ -8,7 +8,7 @@ using namespace DB::Cas; -namespace DB::ErrorCodes { extern const int CORRUPTED_DATA; extern const int LOGICAL_ERROR; } +namespace DB::ErrorCodes { extern const int CORRUPTED_DATA; extern const int LOGICAL_ERROR; extern const int UNKNOWN_FORMAT_VERSION; } namespace { @@ -217,6 +217,39 @@ TEST(CASFoldSealFormat, RejectsUnexpectedGeneration) EXPECT_EQ(decodeFoldSeal(encoded).generation, 5); } +namespace +{ +CasFoldSeal sealWithEveryRowKind() +{ + CasFoldSeal seal; + seal.generation = 5; + seal.parent_generation = 4; + seal.ref_lives[UInt128{1}].coverage = RefCoverage{.classification = CoverageClass::Folded, .last_folded_ref_id = RefTxnId{7, 11}}; + seal.blob_target_runs.push_back(RunRef{.key = "r0", .checksum = UInt128(0x0f), .shard = 0, .key_generation = 5}); + seal.condemned_summary[0] = CondemnedSummary{.condemned_total = 3, .pending_total = 1, .oldest_nonpending_condemn_round = 4}; + return seal; +} +} + +/// Every line of the object (meta, each row kind, trailer) skips an unknown ordinary key. +TEST(CASFoldSealFormat, UnknownOrdinaryKeyIsSkippedOnEveryLine) +{ + const CasFoldSeal seal = sealWithEveryRowKind(); + const String encoded = encodeFoldSeal(seal); + ASSERT_EQ(cas_battery_detail::lineCount(encoded), 6u); + for (size_t line = 1; line < 6; ++line) + EXPECT_EQ(decodeFoldSeal(cas_battery_detail::withExtraKeyInLine(encoded, line, "zz")), seal) << "line " << line; +} + +/// Every line of the object fails with the version code on an unknown `!`-prefixed key. +TEST(CASFoldSealFormat, UnknownCriticalKeyIsUnknownFormatVersionOnEveryLine) +{ + const String encoded = encodeFoldSeal(sealWithEveryRowKind()); + for (size_t line = 1; line < 6; ++line) + cas_battery_detail::expectCode(DB::ErrorCodes::UNKNOWN_FORMAT_VERSION, + [&] { decodeFoldSeal(cas_battery_detail::withExtraKeyInLine(encoded, line, "!zz")); }, fmt::format("line {}", line)); +} + TEST(CASFoldSeal, EncodingIsByteDeterministic) { const CasFoldSeal in = sampleFoldSeal(); diff --git a/src/Disks/tests/gtest_cas_gc_maintenance_state_format.cpp b/src/Disks/tests/gtest_cas_gc_maintenance_state_format.cpp index 309d20f8de89..e1d989481497 100644 --- a/src/Disks/tests/gtest_cas_gc_maintenance_state_format.cpp +++ b/src/Disks/tests/gtest_cas_gc_maintenance_state_format.cpp @@ -46,7 +46,6 @@ TEST(CASGCMaintenanceStateFormat, RegistryLayoutAndCanonicalCodec) const FormatTraits & traits = traitsFor(FormatId::GcMaintenanceState); EXPECT_EQ(traits.type, "cas_gc_maintenance_state"); EXPECT_EQ(traits.family, TextFamily::Control); - EXPECT_EQ(traits.strictness, KeyStrictness::Strict); EXPECT_EQ(traits.compression, CompressionPolicy::Never); EXPECT_EQ(traits.object_cap, 512 * 1024); EXPECT_EQ(traits.line_cap, 512 * 1024); @@ -65,6 +64,19 @@ TEST(CASGCMaintenanceStateFormat, RegistryLayoutAndCanonicalCodec) EXPECT_EQ(decodeGcMaintenanceState(encodeGcMaintenanceState(state)), state); } +TEST(CASGCMaintenanceStateFormat, SkipsUnknownOrdinaryKey) +{ + const String text = "{\"type\":\"cas_gc_maintenance_state\",\"v\":1}\n{\"janitor_cursor\":\"a\",\"extra\":1}\n"; + EXPECT_EQ(decodeGcMaintenanceState(text), (GcMaintenanceState{.janitor_cursor = "a"})); +} + +TEST(CASGCMaintenanceStateFormat, UnknownCriticalKeyIsUnknownFormatVersion) +{ + const String text = "{\"type\":\"cas_gc_maintenance_state\",\"v\":1}\n{\"janitor_cursor\":\"a\",\"!extra\":1}\n"; + DB::Cas::tests::expectThrowsCode(DB::ErrorCodes::UNKNOWN_FORMAT_VERSION, + [&] { (void)decodeGcMaintenanceState(text); }); +} + TEST(CASGCMaintenanceStateFormat, RejectsMalformedAndBoundsCursor) { const auto bad = [](std::string_view body) @@ -75,8 +87,6 @@ TEST(CASGCMaintenanceStateFormat, RejectsMalformedAndBoundsCursor) [&] { (void)decodeGcMaintenanceState(bad("{}\n")); }); DB::Cas::tests::expectThrowsCode(DB::ErrorCodes::CORRUPTED_DATA, [&] { (void)decodeGcMaintenanceState(bad("{\"janitor_cursor\":\"a\",\"janitor_cursor\":\"b\"}\n")); }); - DB::Cas::tests::expectThrowsCode(DB::ErrorCodes::CORRUPTED_DATA, - [&] { (void)decodeGcMaintenanceState(bad("{\"janitor_cursor\":\"a\",\"extra\":1}\n")); }); DB::Cas::tests::expectThrowsCode(DB::ErrorCodes::CORRUPTED_DATA, [&] { (void)decodeGcMaintenanceState(bad("{\"janitor_cursor\":\"a\"}\nx")); }); diff --git a/src/Disks/tests/gtest_cas_record_stream_format.cpp b/src/Disks/tests/gtest_cas_record_stream_format.cpp index e796b394ca8b..84ed94d2610c 100644 --- a/src/Disks/tests/gtest_cas_record_stream_format.cpp +++ b/src/Disks/tests/gtest_cas_record_stream_format.cpp @@ -357,6 +357,46 @@ TEST(CASRecordStream, TruncationAtLineBoundaryFailsClosed) EXPECT_THROW(decodeRun(bytes.substr(0, trailer)), DB::Exception); } +namespace +{ +std::vector sampleRunRecords() +{ + return {edge(chRef(1), 10), condemned(chRef(2), PersistedEtag{"etag", "e-1"}, 4242, 7, true), zero(chRef(3))}; +} +} + +/// The header, each row and the trailer skip an unknown ordinary key. +TEST(CASRecordStream, UnknownOrdinaryKeyIsSkippedOnEveryLine) +{ + const String bytes = encodeRun(sampleRunRecords()); + const std::vector expected = decodeRun(bytes); + const size_t lines = cas_battery_detail::lineCount(bytes); + ASSERT_EQ(lines, 5u); + for (size_t line = 0; line < lines; ++line) + { + const std::vector back = decodeRun(cas_battery_detail::withExtraKeyInLine(bytes, line, "zz")); + ASSERT_EQ(back.size(), expected.size()) << "line " << line; + for (size_t i = 0; i < back.size(); ++i) + { + EXPECT_EQ(back[i].ref, expected[i].ref) << "line " << line; + EXPECT_EQ(back[i].source_id, expected[i].source_id) << "line " << line; + EXPECT_EQ(back[i].marker, expected[i].marker) << "line " << line; + EXPECT_EQ(back[i].size, expected[i].size) << "line " << line; + EXPECT_EQ(back[i].condemn_round, expected[i].condemn_round) << "line " << line; + } + } +} + +/// The header, each row and the trailer fail with the version code on an unknown `!`-prefixed key. +TEST(CASRecordStream, UnknownCriticalKeyIsUnknownFormatVersionOnEveryLine) +{ + const String bytes = encodeRun(sampleRunRecords()); + const size_t lines = cas_battery_detail::lineCount(bytes); + for (size_t line = 0; line < lines; ++line) + cas_battery_detail::expectCode(DB::ErrorCodes::UNKNOWN_FORMAT_VERSION, + [&] { decodeRun(cas_battery_detail::withExtraKeyInLine(bytes, line, "!zz")); }, fmt::format("line {}", line)); +} + TEST(CASRecordStream, HeaderGates) { /// Wrong type. diff --git a/src/Disks/tests/gtest_cas_ref_catalog.cpp b/src/Disks/tests/gtest_cas_ref_catalog.cpp index 8d4c364cb68f..449a683a10a9 100644 --- a/src/Disks/tests/gtest_cas_ref_catalog.cpp +++ b/src/Disks/tests/gtest_cas_ref_catalog.cpp @@ -63,6 +63,7 @@ namespace DB::ErrorCodes extern const int CORRUPTED_DATA; extern const int LOGICAL_ERROR; extern const int LIMIT_EXCEEDED; + extern const int UNKNOWN_FORMAT_VERSION; extern const int NETWORK_ERROR; extern const int BAD_ARGUMENTS; extern const int S3_ERROR; @@ -618,20 +619,22 @@ TEST(CASRefCatalogFormat, DecodeRejectsUnknownState) DB::Cas::tests::expectThrowsCode(DB::ErrorCodes::CORRUPTED_DATA, [&] { decodeRefCatalog(bad); }); } -TEST(CASRefCatalogFormat, DecodeRejectsUnknownEntryKey) +TEST(CASRefCatalogFormat, DecodeSkipsUnknownOrdinaryKeyOnEntryAndTrailer) { - const String bad = rawCatalog( - {R"({"kind":"entry","ns":"a","state":"live","life":"00000000000000000000000000000001","unknown":"x"})"}); - try - { - (void)decodeRefCatalog(bad); - FAIL() << "expected CORRUPTED_DATA"; - } - catch (const DB::Exception & e) - { - EXPECT_EQ(e.code(), DB::ErrorCodes::CORRUPTED_DATA); - EXPECT_NE(e.message().find("unknown entry key"), String::npos) << e.message(); - } + const String inc = "00000000000000000000000000000001"; + const String good = rawCatalog({rawEntryLine("a", "live", inc)}); + const RefCatalog expected = decodeRefCatalog(good); + ASSERT_EQ(expected.entries.size(), 1u); + for (size_t line = 1; line < cas_battery_detail::lineCount(good); ++line) + EXPECT_EQ(decodeRefCatalog(cas_battery_detail::withExtraKeyInLine(good, line, "unknown")), expected) << "line " << line; +} + +TEST(CASRefCatalogFormat, DecodeRejectsUnknownCriticalKeyOnEntryAndTrailer) +{ + const String good = rawCatalog({rawEntryLine("a", "live", "00000000000000000000000000000001")}); + for (size_t line = 1; line < cas_battery_detail::lineCount(good); ++line) + DB::Cas::tests::expectThrowsCode(DB::ErrorCodes::UNKNOWN_FORMAT_VERSION, + [&] { (void)decodeRefCatalog(cas_battery_detail::withExtraKeyInLine(good, line, "!unknown")); }); } TEST(CASRefCatalogFormat, DecodeRejectsEmptyNamespace) @@ -669,15 +672,13 @@ TEST(CASRefCatalogFormatDeathTest, NsStateToWordRaisesLogicalErrorOnImpossibleVa /// ---------- registry row / raw-storage tripwire ---------- -/// The registry row is part of the contract, mirroring `gtest_cas_ref_ckpt.cpp`'s -/// `RegistryRowIsControlStrictWithTightCaps`: Control/Strict decides how the decoder treats unknown -/// keys, and the caps are the first thing that fires if a foreign object ever lands at the key. -TEST(CASRefCatalogFormat, RegistryRowIsControlStrictWithRawStorage) +/// The registry row is part of the contract: the caps are the first thing that fires if a foreign +/// object ever lands at the key. +TEST(CASRefCatalogFormat, RegistryRowIsControlWithRawStorage) { const FormatTraits & traits = traitsFor(FormatId::RefCatalog); EXPECT_EQ(traits.type, "cas_ref_catalog"); EXPECT_EQ(traits.family, TextFamily::Control); - EXPECT_EQ(traits.strictness, KeyStrictness::Strict); EXPECT_EQ(traits.object_cap, 256u * 1024u * 1024u); EXPECT_EQ(traits.line_cap, 4u * 1024u); EXPECT_EQ(traitsForType("cas_ref_catalog"), &traits); diff --git a/src/Disks/tests/gtest_cas_ref_ckpt.cpp b/src/Disks/tests/gtest_cas_ref_ckpt.cpp index f3f7e08efbd7..ddbf9f33c26f 100644 --- a/src/Disks/tests/gtest_cas_ref_ckpt.cpp +++ b/src/Disks/tests/gtest_cas_ref_ckpt.cpp @@ -340,39 +340,36 @@ TEST(CASRefCheckpoint, CodecRejectsIncoherentCommittedFrontierAndSealEpochs) expectThrowsCode(DB::ErrorCodes::CORRUPTED_DATA, [&] { decodeRefCkpt(malformed); }); } -/// STRICT means an unknown key is corruption, not something to skip. A `_ckpt` decides deletions, so a -/// reader that ignored a field it did not understand would be authorizing them from a body it only -/// partly read. -TEST(CASRefCheckpoint, RejectsAnUnknownKey) +TEST(CASRefCheckpoint, SkipsAnUnknownOrdinaryKey) { - const String good = encodeRefCkpt(RefCkpt{.life_epoch = std::optional{1}, .committed_through = ID_1_1, .checkpoint_snapshot_id = ID_1_1, - .last_epoch_seal = std::nullopt}); - String with_unknown = good; + const RefCkpt ckpt{.life_epoch = std::optional{1}, .committed_through = ID_1_1, .checkpoint_snapshot_id = ID_1_1, + .last_epoch_seal = std::nullopt}; + String with_unknown = encodeRefCkpt(ckpt); with_unknown.replace(with_unknown.rfind('}'), 1, R"(,"zz":"1"})"); - expectThrowsCode(DB::ErrorCodes::CORRUPTED_DATA, [&] { decodeRefCkpt(with_unknown); }); + EXPECT_EQ(decodeRefCkpt(with_unknown), ckpt); +} - /// A `!`-prefixed key is a REQUIRED extension and reports the version, not corruption -- the - /// distinction is what lets an operator tell "this build is too old" from "this object is broken". - String with_critical = good; +/// A `!`-prefixed key is a REQUIRED extension and reports the version, not corruption -- the +/// distinction is what lets an operator tell "this build is too old" from "this object is broken". +TEST(CASRefCheckpoint, UnknownCriticalKeyIsUnknownFormatVersion) +{ + String with_critical = encodeRefCkpt(RefCkpt{.life_epoch = std::optional{1}, .committed_through = ID_1_1, + .checkpoint_snapshot_id = ID_1_1, .last_epoch_seal = std::nullopt}); with_critical.replace(with_critical.rfind('}'), 1, R"(,"!zz":"1"})"); expectThrowsCode(DB::ErrorCodes::UNKNOWN_FORMAT_VERSION, [&] { decodeRefCkpt(with_critical); }); } -/// Replacing the abbreviated key is a format cut, not an alias. Treating it as an optional partial -/// pair would make an old writer's checkpoint appear to have no committed frontier. -TEST(CASRefCheckpoint, RejectsOldCommittedEpochKeyRatherThanAliasingIt) +/// The abbreviated key is a removed spelling, not an alias: it is skipped like any unknown key and +/// never read as `committed_epoch`. +TEST(CASRefCheckpoint, OldCommittedEpochKeyIsSkippedNotAliased) { - /// The values are chosen so ALIASING would be harmless: the spliced `"cte":"9"` re-assigns the - /// epoch the object already carries, leaving a valid checkpoint. A reader that honoured the old - /// spelling would therefore DECODE, and this test fails; only the strict unknown-key rejection - /// makes it throw. Values under which aliasing corrupts the object would let the invariant - /// checker throw the same code and hide the alias. - String with_old_key = encodeRefCkpt(RefCkpt{.life_epoch = std::optional{9}, - .committed_through = RefTxnId{9, 1}, - .checkpoint_snapshot_id = RefTxnId{9, 1}, - .last_epoch_seal = std::nullopt}); - with_old_key.replace(with_old_key.rfind('}'), 1, R"(,"cte":"9"})"); - expectThrowsCode(DB::ErrorCodes::CORRUPTED_DATA, [&] { decodeRefCkpt(with_old_key); }); + /// The spliced value differs from the real `committed_epoch`, so an alias would change the + /// decoded frontier (or trip the invariant checker) and the equality below would fail. + const RefCkpt ckpt{.life_epoch = std::optional{9}, .committed_through = RefTxnId{9, 1}, + .checkpoint_snapshot_id = RefTxnId{9, 1}, .last_epoch_seal = std::nullopt}; + String with_old_key = encodeRefCkpt(ckpt); + with_old_key.replace(with_old_key.rfind('}'), 1, R"(,"cte":"5"})"); + EXPECT_EQ(decodeRefCkpt(with_old_key), ckpt); } /// A duplicate key has no single meaning, so it can never be resolved by a reader's preference. @@ -447,14 +444,12 @@ TEST(CASRefCheckpoint, RejectsInvalidFieldsOnEncodeAndOnDecode) [&] { decodeRefCkpt(prefix + R"({"life_epoch":"7","snapshot_epoch":"1","snapshot_seq":"0"})" + "\n"); }); } -/// The registry row is part of the contract: Control/Strict decides how the decoder treats unknown -/// keys, and the caps are the first thing that fires if a foreign object ever lands at the key. -TEST(CASRefCheckpoint, RegistryRowIsControlStrictWithTightCaps) +/// The caps are the first thing that fires if a foreign object ever lands at the key. +TEST(CASRefCheckpoint, RegistryRowIsControlWithTightCaps) { const FormatTraits & traits = traitsFor(FormatId::RefCkpt); EXPECT_EQ(traits.type, "cas_ref_ckpt"); EXPECT_EQ(traits.family, TextFamily::Control); - EXPECT_EQ(traits.strictness, KeyStrictness::Strict); EXPECT_EQ(traits.object_cap, 64u * 1024u); EXPECT_EQ(traits.line_cap, 4u * 1024u); EXPECT_EQ(traitsForType("cas_ref_ckpt"), &traits); diff --git a/src/Disks/tests/gtest_cas_text_format.cpp b/src/Disks/tests/gtest_cas_text_format.cpp index c300e522b739..119dc801a146 100644 --- a/src/Disks/tests/gtest_cas_text_format.cpp +++ b/src/Disks/tests/gtest_cas_text_format.cpp @@ -84,11 +84,9 @@ TEST(CASFormatTraits, CompleteUniqueAndGated) /// CASFormatTraitsDeathTest below proves the abort positively in those builds instead. EXPECT_THROW(traitsFor(FormatId::Roster), DB::Exception); #endif - /// Deterministic formats are pinned raw + strict; spot-check the two. + /// Deterministic formats are pinned raw; spot-check the two. EXPECT_EQ(traitsFor(FormatId::RunFile).compression, CompressionPolicy::PinnedRaw); - EXPECT_EQ(traitsFor(FormatId::RunFile).strictness, KeyStrictness::Strict); EXPECT_EQ(traitsFor(FormatId::FoldSeal).compression, CompressionPolicy::PinnedRaw); - EXPECT_EQ(traitsFor(FormatId::FoldSeal).strictness, KeyStrictness::Strict); /// .zst key suffix is exactly the Always set (can-grow-large types). EXPECT_EQ(storedSuffix(FormatId::RefSnapshot), ".zst"); EXPECT_EQ(storedSuffix(FormatId::RefLog), ".zst"); @@ -128,7 +126,7 @@ TEST(CASJsonVocab, WriteAndReadBack) EXPECT_EQ(rendered.substr(0, 45), R"({"tag":"000102030405060708090a0b0c0d0e0f","se)"); DB::ReadBufferFromMemory in(rendered.data(), rendered.size()); - JsonObjectReader r(in, KeyStrictness::Strict, "test"); + JsonObjectReader r(in, "test"); String key; ASSERT_TRUE(r.nextKey(key)); EXPECT_EQ(key, "tag"); EXPECT_EQ(r.readHex128(), hexToU128("000102030405060708090a0b0c0d0e0f")); @@ -153,7 +151,7 @@ TEST(CASJsonVocab, WordArrayFieldAndReaderRejectInvalidValues) const auto read = [](std::string_view text) { DB::ReadBufferFromMemory in(text.data(), text.size()); - JsonObjectReader r(in, KeyStrictness::Tolerant, "test"); + JsonObjectReader r(in, "test"); String key; EXPECT_TRUE(r.nextKey(key)); return r.readStringArray(); @@ -165,50 +163,45 @@ TEST(CASJsonVocab, WordArrayFieldAndReaderRejectInvalidValues) TEST(CASJsonVocab, FailClosedRules) { - auto reader = [](std::string_view text, KeyStrictness s, auto && consume) + auto reader = [](std::string_view text, auto && consume) { DB::ReadBufferFromMemory in(text.data(), text.size()); - JsonObjectReader r(in, s, "test"); + JsonObjectReader r(in, "test"); consume(r); }; /// duplicate key - expectCode(DB::ErrorCodes::CORRUPTED_DATA, [&] { reader(R"({"a":1,"a":2})", KeyStrictness::Tolerant, [](auto & r) + expectCode(DB::ErrorCodes::CORRUPTED_DATA, [&] { reader(R"({"a":1,"a":2})", [](auto & r) { String k; while (r.nextKey(k)) r.readU64Number(); }); }); - /// unknown key: Tolerant skips (nested value), Strict rejects - reader(R"({"zz":{"deep":[1,2]},"n":5})", KeyStrictness::Tolerant, [](auto & r) + /// unknown ordinary key: skipped, including a nested value + reader(R"({"zz":{"deep":[1,2]},"n":5})", [](auto & r) { String k; ASSERT_TRUE(r.nextKey(k)); r.skipUnknown(k); ASSERT_TRUE(r.nextKey(k)); EXPECT_EQ(r.readU64Number(), 5u); EXPECT_FALSE(r.nextKey(k)); }); - expectCode(DB::ErrorCodes::CORRUPTED_DATA, [&] { reader(R"({"zz":1})", KeyStrictness::Strict, [](auto & r) - { - String k; - ASSERT_TRUE(r.nextKey(k)); r.skipUnknown(k); - }); }); - /// critical key fails closed regardless of strictness - expectCode(DB::ErrorCodes::UNKNOWN_FORMAT_VERSION, [&] { reader(R"({"!x":1})", KeyStrictness::Tolerant, [](auto & r) + /// critical key fails closed + expectCode(DB::ErrorCodes::UNKNOWN_FORMAT_VERSION, [&] { reader(R"({"!x":1})", [](auto & r) { String k; ASSERT_TRUE(r.nextKey(k)); r.skipUnknown(k); }); }); /// whitespace is not canonical - expectCode(DB::ErrorCodes::CORRUPTED_DATA, [&] { reader(R"({ "a":1})", KeyStrictness::Tolerant, [](auto & r) + expectCode(DB::ErrorCodes::CORRUPTED_DATA, [&] { reader(R"({ "a":1})", [](auto & r) { String k; r.nextKey(k); }); }); /// bad hex width / junk in u64 string - expectCode(DB::ErrorCodes::CORRUPTED_DATA, [&] { reader(R"({"digest":"0102"})", KeyStrictness::Tolerant, [](auto & r) + expectCode(DB::ErrorCodes::CORRUPTED_DATA, [&] { reader(R"({"digest":"0102"})", [](auto & r) { String k; r.nextKey(k); r.readHex128(); }); }); - expectCode(DB::ErrorCodes::CORRUPTED_DATA, [&] { reader(R"({"u64_string_field":"12x"})", KeyStrictness::Tolerant, [](auto & r) + expectCode(DB::ErrorCodes::CORRUPTED_DATA, [&] { reader(R"({"u64_string_field":"12x"})", [](auto & r) { String k; r.nextKey(k); r.readU64String(); diff --git a/src/Disks/tests/gtest_cas_wire_vocab.cpp b/src/Disks/tests/gtest_cas_wire_vocab.cpp index 97bef46f5e73..aa4976c3bd74 100644 --- a/src/Disks/tests/gtest_cas_wire_vocab.cpp +++ b/src/Disks/tests/gtest_cas_wire_vocab.cpp @@ -95,7 +95,7 @@ TEST(CASWireVocab, SiblingFieldsWriteAndReadBack) R"({"token_type":"etag","token":"etag-abc\"x","algo":"ch128","digest":"00112233445566778899aabbccddeeff"})"); DB::ReadBufferFromMemory in(rendered.data(), rendered.size()); - JsonObjectReader r(in, KeyStrictness::Tolerant, "t"); + JsonObjectReader r(in, "t"); String key; String tv; String ha; @@ -130,7 +130,7 @@ TEST(CASWireVocab, MatchAndBuildRoundTripsABlobRef) using namespace DB::Cas; const String rendered = R"({"algo":"ch128","digest":"00112233445566778899aabbccddeeff"})"; DB::ReadBufferFromMemory in(rendered.data(), rendered.size()); - JsonObjectReader r(in, KeyStrictness::Tolerant, "t"); + JsonObjectReader r(in, "t"); BlobRefFields fields; String key; while (r.nextKey(key)) @@ -173,7 +173,7 @@ TEST(CASWireVocab, MatchManifestRefFieldsAndBuildRefRoundTripInAnyKeyOrder) /// would fail to parse this literal. const String rendered = R"({"ord":3,"epoch":"7","build":"9"})"; DB::ReadBufferFromMemory in(rendered.data(), rendered.size()); - JsonObjectReader r(in, KeyStrictness::Tolerant, "t"); + JsonObjectReader r(in, "t"); ManifestRefFields fields; String key; while (r.nextKey(key)) @@ -198,7 +198,7 @@ TEST(CASWireVocab, MatchTokenFieldsConsumesSemanticKeysAndLeavesUnrelatedKeyUnma using namespace DB::Cas; const String rendered = R"({"token_type":"etag","token":"abc","zz":1})"; DB::ReadBufferFromMemory in(rendered.data(), rendered.size()); - JsonObjectReader r(in, KeyStrictness::Tolerant, "t"); + JsonObjectReader r(in, "t"); TokenFields fields; String key; bool saw_unmatched = false; @@ -220,7 +220,7 @@ TEST(CASWireVocab, TokenFieldsBuildsInAnyKeyOrderAndRequiresBothFields) { const String rendered = R"({"token":"abc","token_type":"etag"})"; DB::ReadBufferFromMemory in(rendered.data(), rendered.size()); - JsonObjectReader r(in, KeyStrictness::Tolerant, "t"); + JsonObjectReader r(in, "t"); TokenFields fields; String key; while (r.nextKey(key)) @@ -242,7 +242,7 @@ TEST(CASWireVocab, OldManifestEpochKeyDoesNotAliasTheSemanticKey) { const String rendered = R"({"me":"1","build":"2","ord":3})"; DB::ReadBufferFromMemory in(rendered.data(), rendered.size()); - JsonObjectReader r(in, KeyStrictness::Tolerant, "t"); + JsonObjectReader r(in, "t"); ManifestRefFields fields; String key; while (r.nextKey(key)) @@ -281,7 +281,7 @@ TEST(CASPersistedEtag, RoundTripsThroughEveryFormatAndNeverBecomesAnIncarnation) closeObject(out, first); const String rendered = std::move(out).take(); DB::ReadBufferFromMemory in(rendered.data(), rendered.size()); - JsonObjectReader r(in, KeyStrictness::Strict, "t"); + JsonObjectReader r(in, "t"); TokenFields fields; String key; while (r.nextKey(key))