Skip to content

feat: add token frequencies #438

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cheb0 merged 7 commits into from
Jul 28, 2026
Merged

feat: add token frequencies #438

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ef4601d
token frequency (configurable)
cheb0 Jun 11, 2026
57042bd
fix merge
cheb0 Jul 27, 2026
6c7835e
fix test - reuse buffer
cheb0 Jul 27, 2026
0555aef
reassign slices
cheb0 Jul 27, 2026
eba0386
specify token threshold in percentage
cheb0 Jul 27, 2026
1a607b6
rename, default values, validation
cheb0 Jul 28, 2026
6863629
docs
cheb0 Jul 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 10 additions & 9 deletions cmd/seq-db/seq-db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,15 +265,16 @@ func startStore(
CacheCleanupDelay: 0,
MinSealFracSize: uint64(cfg.Storage.FracSize) * consts.DefaultMinSealPercent / 100,
SealParams: common.SealParams{
IDsZstdLevel: cfg.Compression.SealedZstdCompressionLevel,
LIDsZstdLevel: cfg.Compression.SealedZstdCompressionLevel,
LIDBlockSize: cfg.Sealing.Lids.BlockSize,
TokenBlockSize: int(cfg.Sealing.Tokens.BlockSize),
TokenListZstdLevel: cfg.Compression.SealedZstdCompressionLevel,
DocsPositionsZstdLevel: cfg.Compression.SealedZstdCompressionLevel,
TokenTableZstdLevel: cfg.Compression.SealedZstdCompressionLevel,
DocBlocksZstdLevel: cfg.Compression.DocBlockZstdCompressionLevel,
DocBlockSize: int(cfg.DocsSorting.DocBlockSize),
IDsZstdLevel: cfg.Compression.SealedZstdCompressionLevel,
LIDsZstdLevel: cfg.Compression.SealedZstdCompressionLevel,
LIDBlockSize: cfg.Sealing.Lids.BlockSize,
TokenBlockSize: int(cfg.Sealing.Tokens.BlockSize),
TokenListZstdLevel: cfg.Compression.SealedZstdCompressionLevel,
DocsPositionsZstdLevel: cfg.Compression.SealedZstdCompressionLevel,
TokenTableZstdLevel: cfg.Compression.SealedZstdCompressionLevel,
DocBlocksZstdLevel: cfg.Compression.DocBlockZstdCompressionLevel,
DocBlockSize: int(cfg.DocsSorting.DocBlockSize),
TokenFreqThresholdPercentage: cfg.Sealing.Tokens.FreqThresholdPercentage,
},
Fraction: frac.Config{
Search: frac.SearchConfig{
Expand Down
4 changes: 4 additions & 0 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,10 @@ type Config struct {
Tokens struct {
// BlockSize sets max token block size in bytes.
BlockSize Bytes `config:"block_size" default:"16KiB"`
// FreqThresholdPercentage specifies the minimum posting-list length as a percentage
// of the fraction's document count. For example, with 1_000_000 docs and FreqThresholdPercentage=1,
// frequency is stored for tokens that appear in at least 10_000 documents.
FreqThresholdPercentage float64 `config:"freq_threshold_percentage" default:"0.005"`
} `config:"tokens"`

Lids struct {
Expand Down
2 changes: 1 addition & 1 deletion config/frac_version.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ const (
// BinaryDataV4 - delta bitpack encoded MIDs and LIDs
BinaryDataV4

// BinaryDataV5 - token blocks have zone maps (eng letters presense, max token length)
// BinaryDataV5 - token blocks have zone maps (eng letters presense) and doc frequencies for heavy tokens
BinaryDataV5
)

Expand Down
2 changes: 2 additions & 0 deletions config/validation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ func (c *Config) storeValidations() []validateFn {
greaterThan("sealing.lids.block_size", 0, c.Sealing.Lids.BlockSize),
lessOrEqThan("sealing.lids.block_size", 65536, c.Sealing.Lids.BlockSize),
greaterThan("sealing.tokens.block_size", 0, c.Sealing.Tokens.BlockSize),
greaterOrEqThan("sealing.tokens.freq_threshold_percentage", 0.0, c.Sealing.Tokens.FreqThresholdPercentage),
lessOrEqThan("sealing.tokens.freq_threshold_percentage", 100.0, c.Sealing.Tokens.FreqThresholdPercentage),
inRange("offloading.queue_size_percent", 0, 100, c.Offloading.QueueSizePercent),

greaterThan("experimental.max_regex_tokens_check", -1, c.Experimental.MaxRegexTokensCheck),
Expand Down
45 changes: 45 additions & 0 deletions config/validation_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,51 @@ limits:
sealing:
tokens:
block_size: -1B
`),
expectErr: true,
},
{
name: "Valid sealing.tokens.freq_threshold_percentage",
cfg: createCfgFile(t, base+`
sealing:
tokens:
freq_threshold_percentage: 0.005
`),
expectErr: false,
},
{
name: "Valid large sealing.tokens.freq_threshold_percentage",
cfg: createCfgFile(t, base+`
sealing:
tokens:
freq_threshold_percentage: 99.5
`),
expectErr: false,
},
{
name: "Valid max value for sealing.tokens.freq_threshold_percentage",
cfg: createCfgFile(t, base+`
sealing:
tokens:
freq_threshold_percentage: 100.0
`),
expectErr: false,
},
{
name: "Invalid negative sealing.tokens.freq_threshold_percentage",
cfg: createCfgFile(t, base+`
sealing:
tokens:
freq_threshold_percentage: -0.005
`),
expectErr: true,
},
{
name: "Invalid large sealing.tokens.freq_threshold_percentage",
cfg: createCfgFile(t, base+`
sealing:
tokens:
freq_threshold_percentage: 100.03
`),
expectErr: true,
},
Expand Down
5 changes: 3 additions & 2 deletions consts/consts.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,9 @@ const (

DefaultMaxTokenSize = 72

DefaultBulkRequestsLimit = 32
DefaultSearchRequestsLimit = 32
DefaultBulkRequestsLimit = 32
DefaultSearchRequestsLimit = 32
DefaultTokenFreqThresholdPercentage = 0.005

BulkMaxTries = 3

Expand Down
7 changes: 4 additions & 3 deletions docs/en/02-configuration.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,10 @@ Settings for fraction sealing.

### Tokens

| Field | Type | Default | Description |
|-------|------|---------|-------------|
| `sealing.tokens.block_size` | Bytes | `16KiB` | Max token block size in bytes |
| Field | Type | Default | Description |
|--------------------------------------------|---------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------|
| `sealing.tokens.block_size` | Bytes | `16KiB` | Max token block size in bytes |
| `sealing.tokens.freq_threshold_percentage` | float64 | `0.005` | The minimum posting-list length as a percentage of the fraction's document count a token should have to have a doc frequency stored in fraction |

### Lids

Expand Down
7 changes: 4 additions & 3 deletions docs/ru/02-configuration.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,10 @@ id: configuration

### Tokens

| Параметр | Тип | Значение по умолчанию | Описание |
|----------|-----|----------------------|-----------|
| `sealing.tokens.block_size` | Bytes | `16KiB` | Максимальный размер блока токенов в байтах |
| Параметр | Тип | Значение по умолчанию | Описание |
|--------------------------------------------|---------|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `sealing.tokens.block_size` | Bytes | `16KiB` | Максимальный размер блока токенов в байтах |
| `sealing.tokens.freq_threshold_percentage` | float64 | `0.005` | Минимальный размер списка лидов, по достижению которого для токена сохраняется количество документов, в которых встречается данный токен. Указывается в процентах от количества документов во фаркции. |

### Lids

Expand Down
7 changes: 4 additions & 3 deletions frac/common/seal_params.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ type SealParams struct {
TokenTableZstdLevel int
DocBlocksZstdLevel int // DocBlocksZstdLevel is the zstd compress level of each document block.

LIDBlockSize int
TokenBlockSize int
DocBlockSize int // DocBlockSize is decompressed payload size of document block.
LIDBlockSize int
TokenBlockSize int
TokenFreqThresholdPercentage float64
DocBlockSize int // DocBlockSize is decompressed payload size of document block.
}
2 changes: 1 addition & 1 deletion frac/remote.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ func (f *Remote) createDataProvider(ctx context.Context) (*sealedDataProvider, e
blocksOffsets: f.blocksData.BlocksOffsets,
lidsTable: f.blocksData.LIDsTable,
lidsLoader: lids.NewLoader(f.info.BinaryDataVer, lidReader, f.indexCache.LIDs),
tokenBlockLoader: token.NewBlockLoader(f.BaseFileName, tokenReader, f.indexCache.Tokens),
tokenBlockLoader: token.NewBlockLoader(f.BaseFileName, f.Info().BinaryDataVer, tokenReader, f.indexCache.Tokens),
tokenTableLoader: token.NewTableLoader(f.BaseFileName, f.Info().BinaryDataVer, f.IsLegacy, tokenReader, f.indexCache.TokenTable),

idsTable: &f.blocksData.IDsTable,
Expand Down
2 changes: 1 addition & 1 deletion frac/sealed.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -515,7 +515,7 @@ func (f *Sealed) createDataProvider(ctx context.Context) *sealedDataProvider {
blocksOffsets: f.blocksData.BlocksOffsets,
lidsTable: f.blocksData.LIDsTable,
lidsLoader: lids.NewLoader(f.info.BinaryDataVer, lidReader, f.indexCache.LIDs),
tokenBlockLoader: token.NewBlockLoader(f.BaseFileName, tokenReader, f.indexCache.Tokens),
tokenBlockLoader: token.NewBlockLoader(f.BaseFileName, f.Info().BinaryDataVer, tokenReader, f.indexCache.Tokens),
tokenTableLoader: token.NewTableLoader(f.BaseFileName, f.Info().BinaryDataVer, f.IsLegacy, tokenReader, f.indexCache.TokenTable),

idsTable: &f.blocksData.IDsTable,
Expand Down
Loading
Loading