feat: add roaring bitmaps in posing lists

cmd/seq-db/seq-db.go

@@ -275,6 +275,7 @@
 				DocBlocksZstdLevel:           cfg.Compression.DocBlockZstdCompressionLevel,
 				DocBlockSize:                 int(cfg.DocsSorting.DocBlockSize),
 				TokenFreqThresholdPercentage: cfg.Sealing.Tokens.FreqThresholdPercentage,
+				LIDsBitmapThreshold:    cfg.Sealing.Lids.BitmapThreshold,
 			},
 			Fraction: frac.Config{
 				Search: frac.SearchConfig{

config/config.go

@@ -84,6 +84,10 @@ type Config struct {
 		Lids struct {
 			// BlockSize sets max lids (postings) saved per LIDs block.
 			BlockSize int `config:"block_size" default:"65536"`
+			// BitmapThreshold specifies minimum number of LIDs in the lid list
+			// which are serialized as bitmap. LIDs lists with more elements use bitmap encoding,
+			// while smaller lists use delta encoding.
+			BitmapThreshold int `config:"bitmap_threshold" default:"65536"`
 		} `config:"lids"`
 	} `config:"sealing"`
 

config/frac_version.go

@@ -24,6 +24,9 @@ const (
 
 	// BinaryDataV5 - token blocks have zone maps (eng letters presense) and doc frequencies for heavy tokens
 	BinaryDataV5
+
+	// BinaryDataV6 - bitmap for sufficiently large LID lists, mixed LIDs block format
+	BinaryDataV6
 )
 
-const CurrentFracVersion = BinaryDataV5
+const CurrentFracVersion = BinaryDataV6

frac/common/seal_params.go

@@ -13,6 +13,7 @@
 	DocBlocksZstdLevel     int // DocBlocksZstdLevel is the zstd compress level of each document block.
 
 	LIDBlockSize                 int
+	LIDsBitmapThreshold 		 int // LIDsBitmapThreshold is the minimum number of LIDs in the lid list to serialize as bitmap.
 	TokenBlockSize               int
 	TokenFreqThresholdPercentage float64
 	DocBlockSize                 int // DocBlockSize is decompressed payload size of document block.

frac/fraction_test.go

@@ -105,6 +105,7 @@ func (s *FractionTestSuite) SetupTestCommon() {
 		LIDBlockSize:           512,
 		TokenBlockSize:         128,
 		DocBlockSize:           128 * int(units.KiB),
+		LIDsBitmapThreshold:    25,
 	}
 
 	var err error
@@ -1415,6 +1416,18 @@ func (s *FractionTestSuite) TestSearchLargeFrac() {
 			fromTime: fromTime,
 			toTime:   toTime,
 		},
+		{
+			name: "complex AND+OR",
+			query: "(service:gateway OR service:proxy OR service:scheduler) AND " +
+				"(message:request OR message:failed) AND (level:1 OR level:2 OR level:3)",
+			filter: func(doc *testDoc) bool {
+				return (doc.service == gateway || doc.service == proxy || doc.service == "scheduler") &&
+					(strings.Contains(doc.message, "request") || strings.Contains(doc.message, "failed")) &&
+					(doc.level >= 1 && doc.level <= 3)
+			},
+			fromTime: fromTime,
+			toTime:   toTime,
+		},
 		{
 			name:  "service:gateway AND NOT (message:request OR message:timed OR level:[0 to 3])",
 			query: "service:gateway AND NOT (message:request OR message:timed OR level:[0 to 3])",

frac/processor/search.go

@@ -175,10 +175,10 @@ func IndexSearch(
 	return qpr, nil
 }
 
-func batcher(evalTree node.Node, buf []node.LID) func(need int) []node.LID {
+func batcher(evalTree node.Node, buf []node.LID, desc bool) func(need int) []node.LID {
 	if batchNode, ok := tryConvertToBatchedTree(evalTree); ok {
 		return func(need int) []node.LID {
-			buf = batchNode.NextBatch().LIDs(buf[:0])
+			buf = batchNode.NextBatch(need).CopyLIDs(desc, buf[:0])
 			if len(buf) > need {
 				buf = buf[:need]
 			}
@@ -227,7 +227,7 @@ func iterateEvalTree(
 	mids := buffers.mids
 	rids := buffers.rids
 
-	batchedEvalTree := batcher(evalTree, buffers.lids)
+	batchedEvalTree := batcher(evalTree, buffers.lids, params.Order.IsDesc())
 
 	timerEval := sw.Timer("eval_tree_next")
 	timerMID := sw.Timer("get_mid")
@@ -355,9 +355,9 @@ func sampler(n uint32) func(in []node.LID) []node.LID {
 func tryConvertToBatchedTree(evalTree node.Node) (node.BatchedNode, bool) {
 	switch it := evalTree.(type) {
 	case *lids.IteratorDesc:
-		return it, true
+		return lids.NewBatchedIteratorDesc(it), true
 	case *lids.IteratorAsc:
-		return it, true
+		return lids.NewBatchedIteratorAsc(it), true
 	default:
 		return nil, false
 	}