diff --git a/config/frac_version.go b/config/frac_version.go index 1ed8f27a..ef9ee4bd 100644 --- a/config/frac_version.go +++ b/config/frac_version.go @@ -24,6 +24,9 @@ const ( // BinaryDataV5 - token blocks have zone maps (eng letters presense, max token length) BinaryDataV5 + + // BinaryDataV6 - the ID count is no longer stored in the offsets section. + BinaryDataV6 ) -const CurrentFracVersion = BinaryDataV5 +const CurrentFracVersion = BinaryDataV6 diff --git a/frac/sealed/block_offsets.go b/frac/sealed/block_offsets.go index d644a0f7..1961cef8 100644 --- a/frac/sealed/block_offsets.go +++ b/frac/sealed/block_offsets.go @@ -3,6 +3,8 @@ package sealed import ( "encoding/binary" "errors" + + "github.com/ozontech/seq-db/config" ) type BlockOffsets struct { @@ -12,12 +14,6 @@ type BlockOffsets struct { func (b *BlockOffsets) Pack(buf []byte) []byte { buf = binary.LittleEndian.AppendUint32(buf, uint32(len(b.Offsets))) - // NOTE(dkharms): Previously we stored here amount of documents ids. - // - // I've created a task which will require fraction binary version bumping - // to get rid of this: https://github.com/ozontech/seq-db/issues/409 - buf = binary.LittleEndian.AppendUint32(buf, 0) - var prev uint64 for _, pos := range b.Offsets { buf = binary.AppendVarint(buf, int64(pos-prev)) @@ -26,7 +22,7 @@ func (b *BlockOffsets) Pack(buf []byte) []byte { return buf } -func (b *BlockOffsets) Unpack(data []byte) error { +func (b *BlockOffsets) Unpack(data []byte, fracVer config.BinaryDataVersion) error { if len(data) < 4 { return errors.New("blocks offset decoding error: truncated header (missing offsets count)") } @@ -34,13 +30,13 @@ func (b *BlockOffsets) Unpack(data []byte) error { idsBlocksCount := binary.LittleEndian.Uint32(data) data = data[4:] - if len(data) < 4 { - return errors.New("blocks offset decoding error: truncated header (missing IDsTotal)") - } + if fracVer < config.BinaryDataV6 { + if len(data) < 4 { + return errors.New("blocks offset decoding error: truncated header (missing IDsTotal)") + } - // NOTE(dkharms): Previously we stored here amount of documents ids. - _ = binary.LittleEndian.Uint32(data) - data = data[4:] + data = data[4:] + } offset := uint64(0) b.Offsets = make([]uint64, 0, idsBlocksCount) diff --git a/frac/sealed/block_offsets_test.go b/frac/sealed/block_offsets_test.go new file mode 100644 index 00000000..6ecf3ae1 --- /dev/null +++ b/frac/sealed/block_offsets_test.go @@ -0,0 +1,95 @@ +package sealed + +import ( + "encoding/binary" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/ozontech/seq-db/config" +) + +func TestBlockOffsetsPack(t *testing.T) { + t.Parallel() + + offsets := []uint64{10, 25, 40} + block := BlockOffsets{Offsets: offsets} + + packed := block.Pack(nil) + + require.Equal(t, packBlockOffsetsForTest(offsets, config.BinaryDataV6), packed) +} + +func TestBlockOffsetsUnpack(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + data []byte + fracVer config.BinaryDataVersion + want []uint64 + wantErr string + }{ + { + name: "current_format", + data: packBlockOffsetsForTest([]uint64{10, 25, 40}, config.BinaryDataV6), + fracVer: config.BinaryDataV6, + want: []uint64{10, 25, 40}, + }, + { + name: "legacy_format", + data: packBlockOffsetsForTest([]uint64{10, 25, 40}, config.BinaryDataV5), + fracVer: config.BinaryDataV5, + want: []uint64{10, 25, 40}, + }, + { + name: "current_empty", + data: packBlockOffsetsForTest(nil, config.BinaryDataV6), + fracVer: config.BinaryDataV6, + want: []uint64{}, + }, + { + name: "legacy_header_truncated", + data: binary.LittleEndian.AppendUint32(nil, 0), + fracVer: config.BinaryDataV5, + wantErr: "missing IDsTotal", + }, + { + name: "offsets_count_mismatch", + data: binary.LittleEndian.AppendUint32(nil, 1), + fracVer: config.BinaryDataV6, + wantErr: "offset count mismatch", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var block BlockOffsets + err := block.Unpack(tt.data, tt.fracVer) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + return + } + + require.NoError(t, err) + require.Equal(t, tt.want, block.Offsets) + }) + } +} + +func packBlockOffsetsForTest(offsets []uint64, fracVer config.BinaryDataVersion) []byte { + data := binary.LittleEndian.AppendUint32(nil, uint32(len(offsets))) + if fracVer < config.BinaryDataV6 { + data = binary.LittleEndian.AppendUint32(data, 42) + } + + var prev uint64 + for _, offset := range offsets { + data = binary.AppendVarint(data, int64(offset-prev)) + prev = offset + } + + return data +} diff --git a/frac/sealed_loader.go b/frac/sealed_loader.go index 10d95a2d..07d9ae24 100644 --- a/frac/sealed_loader.go +++ b/frac/sealed_loader.go @@ -86,7 +86,7 @@ func (l *LegacyLoader) loadIDs(info *common.Info) (seqids.Table, []uint64, error } var offsets sealed.BlockOffsets - if err := offsets.Unpack(data); err != nil { + if err := offsets.Unpack(data, info.BinaryDataVer); err != nil { return seqids.Table{}, nil, err } @@ -179,7 +179,7 @@ func (l *Loader) Load(blocksData *sealed.BlocksData, info *common.Info, readers blockOffsets sealed.BlockOffsets ) - blockOffsets, err = l.loadBlocksOffsets(readers.Offsets) + blockOffsets, err = l.loadBlocksOffsets(readers.Offsets, info.BinaryDataVer) if err != nil { logger.Fatal("load offsets error", zap.Error(err)) } @@ -207,7 +207,10 @@ func (l *Loader) Load(blocksData *sealed.BlocksData, info *common.Info, readers } // loadBlocksOffsets reads block 0 from the .offsets file. -func (l *Loader) loadBlocksOffsets(r storage.IndexReader) (sealed.BlockOffsets, error) { +func (l *Loader) loadBlocksOffsets( + r storage.IndexReader, + fracVer config.BinaryDataVersion, +) (sealed.BlockOffsets, error) { data, _, err := r.ReadIndexBlock(0, l.buf) l.buf = data @@ -216,7 +219,7 @@ func (l *Loader) loadBlocksOffsets(r storage.IndexReader) (sealed.BlockOffsets, } var b sealed.BlockOffsets - if err := b.Unpack(data); err != nil { + if err := b.Unpack(data, fracVer); err != nil { return sealed.BlockOffsets{}, err }