refactor: remove ID count from offsets

config/frac_version.go

@@ -24,6 +24,9 @@ const (
 
 	// BinaryDataV5 - token blocks have zone maps (eng letters presense, max token length)
 	BinaryDataV5
+
+	// BinaryDataV6 - the ID count is no longer stored in the offsets section.
+	BinaryDataV6
 )
 
-const CurrentFracVersion = BinaryDataV5
+const CurrentFracVersion = BinaryDataV6

frac/sealed/block_offsets.go

@@ -3,6 +3,8 @@ package sealed
 import (
 	"encoding/binary"
 	"errors"
+
+	"github.com/ozontech/seq-db/config"
 )
 
 type BlockOffsets struct {
@@ -12,12 +14,6 @@ type BlockOffsets struct {
 func (b *BlockOffsets) Pack(buf []byte) []byte {
 	buf = binary.LittleEndian.AppendUint32(buf, uint32(len(b.Offsets)))
 
-	// NOTE(dkharms): Previously we stored here amount of documents ids.
-	//
-	// I've created a task which will require fraction binary version bumping
-	// to get rid of this: https://github.com/ozontech/seq-db/issues/409
-	buf = binary.LittleEndian.AppendUint32(buf, 0)
-
 	var prev uint64
 	for _, pos := range b.Offsets {
 		buf = binary.AppendVarint(buf, int64(pos-prev))
@@ -26,21 +22,21 @@ func (b *BlockOffsets) Pack(buf []byte) []byte {
 	return buf
 }
 
-func (b *BlockOffsets) Unpack(data []byte) error {
+func (b *BlockOffsets) Unpack(data []byte, fracVer config.BinaryDataVersion) error {
 	if len(data) < 4 {
 		return errors.New("blocks offset decoding error: truncated header (missing offsets count)")
 	}
 
 	idsBlocksCount := binary.LittleEndian.Uint32(data)
 	data = data[4:]
 
-	if len(data) < 4 {
-		return errors.New("blocks offset decoding error: truncated header (missing IDsTotal)")
-	}
+	if fracVer < config.BinaryDataV6 {
+		if len(data) < 4 {
+			return errors.New("blocks offset decoding error: truncated header (missing IDsTotal)")
+		}
 
-	// NOTE(dkharms): Previously we stored here amount of documents ids.
-	_ = binary.LittleEndian.Uint32(data)
-	data = data[4:]
+		data = data[4:]
+	}
 
 	offset := uint64(0)
 	b.Offsets = make([]uint64, 0, idsBlocksCount)

frac/sealed/block_offsets_test.go

@@ -0,0 +1,95 @@
+package sealed
+
+import (
+	"encoding/binary"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ozontech/seq-db/config"
+)
+
+func TestBlockOffsetsPack(t *testing.T) {
+	t.Parallel()
+
+	offsets := []uint64{10, 25, 40}
+	block := BlockOffsets{Offsets: offsets}
+
+	packed := block.Pack(nil)
+
+	require.Equal(t, packBlockOffsetsForTest(offsets, config.BinaryDataV6), packed)
+}
+
+func TestBlockOffsetsUnpack(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name    string
+		data    []byte
+		fracVer config.BinaryDataVersion
+		want    []uint64
+		wantErr string
+	}{
+		{
+			name:    "current_format",
+			data:    packBlockOffsetsForTest([]uint64{10, 25, 40}, config.BinaryDataV6),
+			fracVer: config.BinaryDataV6,
+			want:    []uint64{10, 25, 40},
+		},
+		{
+			name:    "legacy_format",
+			data:    packBlockOffsetsForTest([]uint64{10, 25, 40}, config.BinaryDataV5),
+			fracVer: config.BinaryDataV5,
+			want:    []uint64{10, 25, 40},
+		},
+		{
+			name:    "current_empty",
+			data:    packBlockOffsetsForTest(nil, config.BinaryDataV6),
+			fracVer: config.BinaryDataV6,
+			want:    []uint64{},
+		},
+		{
+			name:    "legacy_header_truncated",
+			data:    binary.LittleEndian.AppendUint32(nil, 0),
+			fracVer: config.BinaryDataV5,
+			wantErr: "missing IDsTotal",
+		},
+		{
+			name:    "offsets_count_mismatch",
+			data:    binary.LittleEndian.AppendUint32(nil, 1),
+			fracVer: config.BinaryDataV6,
+			wantErr: "offset count mismatch",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			var block BlockOffsets
+			err := block.Unpack(tt.data, tt.fracVer)
+			if tt.wantErr != "" {
+				require.ErrorContains(t, err, tt.wantErr)
+				return
+			}
+
+			require.NoError(t, err)
+			require.Equal(t, tt.want, block.Offsets)
+		})
+	}
+}
+
+func packBlockOffsetsForTest(offsets []uint64, fracVer config.BinaryDataVersion) []byte {
+	data := binary.LittleEndian.AppendUint32(nil, uint32(len(offsets)))
+	if fracVer < config.BinaryDataV6 {
+		data = binary.LittleEndian.AppendUint32(data, 42)
+	}
+
+	var prev uint64
+	for _, offset := range offsets {
+		data = binary.AppendVarint(data, int64(offset-prev))
+		prev = offset
+	}
+
+	return data
+}

frac/sealed_loader.go

@@ -86,7 +86,7 @@ func (l *LegacyLoader) loadIDs(info *common.Info) (seqids.Table, []uint64, error
 	}
 
 	var offsets sealed.BlockOffsets
-	if err := offsets.Unpack(data); err != nil {
+	if err := offsets.Unpack(data, info.BinaryDataVer); err != nil {
 		return seqids.Table{}, nil, err
 	}
 
@@ -179,7 +179,7 @@ func (l *Loader) Load(blocksData *sealed.BlocksData, info *common.Info, readers
 		blockOffsets sealed.BlockOffsets
 	)
 
-	blockOffsets, err = l.loadBlocksOffsets(readers.Offsets)
+	blockOffsets, err = l.loadBlocksOffsets(readers.Offsets, info.BinaryDataVer)
 	if err != nil {
 		logger.Fatal("load offsets error", zap.Error(err))
 	}
@@ -207,7 +207,10 @@ func (l *Loader) Load(blocksData *sealed.BlocksData, info *common.Info, readers
 }
 
 // loadBlocksOffsets reads block 0 from the .offsets file.
-func (l *Loader) loadBlocksOffsets(r storage.IndexReader) (sealed.BlockOffsets, error) {
+func (l *Loader) loadBlocksOffsets(
+	r storage.IndexReader,
+	fracVer config.BinaryDataVersion,
+) (sealed.BlockOffsets, error) {
 	data, _, err := r.ReadIndexBlock(0, l.buf)
 	l.buf = data
 
@@ -216,7 +219,7 @@ func (l *Loader) loadBlocksOffsets(r storage.IndexReader) (sealed.BlockOffsets,
 	}
 
 	var b sealed.BlockOffsets
-	if err := b.Unpack(data); err != nil {
+	if err := b.Unpack(data, fracVer); err != nil {
 		return sealed.BlockOffsets{}, err
 	}