From c01426eb6da2221c438d5ae7c6610f9f3ecb55cc Mon Sep 17 00:00:00 2001 From: Shaojie Li Date: Mon, 1 Jun 2026 02:28:25 +0000 Subject: [PATCH] perf(parquet): SIMD bit-unpack for DeltaBpDecoder dense and sparse visitor paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds on jaylisde/velox#3 (PR A) and jaylisde/velox#4 (PR B) to close the remaining DELTA-vs-PLAIN/DICT scan gap on TPC-H Q12. 1. Inline SIMD bit-unpack kernel in DeltaBpDecoder::decodeLongs. When the read aligns at a miniblock start and consumes a whole miniblock, dispatch on bit_width 0..32 to a compile-time specialized kernel: - bw 0: arithmetic-sequence fast path (no bit-extract). - bw 1..16: 4-value/iter, single unaligned 64-bit load (4*16 = 64 bits fit in one u64 window). - bw 17..32: 2-value/iter, __uint128_t funnel-shift (lowers to two u64 loads + SHRD on x86_64). The trailing u64 read is safe because intra-page overshoots fall into the next miniblock and the last miniblock has PageReader::kPageReadPadding (8) trailing bytes guaranteed. bit_widths 33..64 fall through to the per-row scalar inner loop, which is unchanged. 2. New readWithVisitorSparseBuffered path for the !Visitor::dense + !hasNulls + deterministic filter + NoHook + integral DataType case (the hot path on Q12 because filter chaining produces sparse row sets after l_shipmode is applied). Decodes kBatch=1024 physical values into a stack buffer via decodeLongs (which now uses the SIMD kernel above), then walks the visitor's sparse rows array using rows[k] - batchPhysStart as buffer index. The existing per-row visitor.process is preserved — only the decode side is batched. n is capped to the visitor's residual physical span so the decoder never advances past what the visitor will consume. 3. readWithVisitorDenseBatched kBatch raised from 256 to 1024 to amortize the chunk-loop overhead now that decodeLongs is much faster per call. 4. Add velox/dwio/parquet/tests/reader/DeltaBpDecoderTest.cpp: - Hand-rolled DeltaEncoder for byte-stream control in tests. - 11 boundary tests: bit_widths 0/8/10/16/24/32, multi-block in a single readValues, mid-miniblock split across two readValues calls, negative minDelta, bit_width > 32 fallback, narrowing to int32_t. - 32 parameterized roundtrip tests covering bit_widths 1..32, each forcing the encoder to pick exactly that width by saturating one residual to (1< && + std::is_integral_v) { + readWithVisitorSparseBuffered(visitor); + return; + } int32_t toSkip; bool atEnd = false; const bool allowNulls = hasNulls && visitor.allowNulls(); @@ -116,7 +125,7 @@ class DeltaBpDecoder { constexpr bool kHasFilter = !std:: is_same_v; - constexpr int32_t kBatch = 256; + constexpr int32_t kBatch = 1024; const int32_t total = visitor.numRows(); DataType* output = visitor.rawValues(total); int32_t* filterHits = kHasFilter ? visitor.outputRows(total) : nullptr; @@ -141,8 +150,53 @@ class DeltaBpDecoder { visitor.setNumValues(numValues); } + // Sparse + deterministic-filter + NoHook + integral DataType fast + // path. DELTA's delta-chain forces every physical row to be decoded + // even when the row set is sparse, so we batch the decode side into + // a stack buffer and run visitor.process() scalar over the buffer + // (buffer index instead of readLong() per row). + template + void readWithVisitorSparseBuffered(Visitor& visitor) { + using DataType = typename Visitor::DataType; + constexpr int32_t kBatch = 1024; + DataType buf[kBatch]; + + const auto* rows = visitor.rows(); + const int32_t numRows = visitor.numRows(); + int32_t rowIdx = 0; + int32_t currentPhys = (numRows > 0) ? rows[0] : 0; + bool atEnd = false; + while (!atEnd && rowIdx < numRows) { + const int32_t remaining = static_cast(totalValuesRemaining_); + if (remaining <= 0) { + return; + } + // Cap n to the visitor's residual physical span; decoding past it + // would leave the decoder at the wrong physical position and + // misalign the next readWithVisitor call. + const int32_t lastRow = rows[numRows - 1]; + const int32_t maxSpan = lastRow - currentPhys + 1; + const int32_t n = std::min({kBatch, remaining, maxSpan}); + decodeLongs(buf, n); + const int32_t batchPhysStart = currentPhys; + const int32_t batchPhysEnd = currentPhys + n; // exclusive + currentPhys = batchPhysEnd; + while (rowIdx < numRows && rows[rowIdx] < batchPhysEnd) { + const int32_t i = rows[rowIdx] - batchPhysStart; + visitor.process(buf[i], atEnd); + ++rowIdx; + if (atEnd) { + return; + } + } + } + } + // Inlined readLong() with state hoisted to locals; advances a running - // bitOffset to avoid a multiply per row. DataType narrowing covers int32. + // bitOffset to avoid a multiply per row. Whole bit-aligned miniblocks + // dispatch to decodeMiniBlockSimd(); the scalar tail handles partial + // miniblocks, bit_width > 32, and the page's first (header) value. + // DataType narrowing covers int32. template void decodeLongs(DataType* out, int32_t n) { const char* bufStart = bufferStart_; @@ -157,21 +211,113 @@ class DeltaBpDecoder { int32_t i = 0; while (i < n) { if (miniBlockRemaining == 0) { + // Refill the miniblock. Two distinct cases: + // 1. The very first value of the page lives in the page + // header (lastValue_), not in a packed miniblock. Emit + // it and initialize the first block if there is one. + // 2. Subsequent miniblocks: advance to the next miniblock + // (or new block) but DO NOT consume any value. Leaves + // miniBlockRemaining = valsPerMiniBlk so the SIMD fast + // path below can swallow the whole miniblock. + if (!firstBlockInitialized_) { + bufferStart_ = bufStart; + valuesRemainingCurrentMiniBlock_ = 0; + totalValuesRemaining_ = totalRemaining; + lastValue_ = lastValue; + int64_t v = readLong(); + out[i] = static_cast(v); + bufStart = bufferStart_; + miniBlockRemaining = valuesRemainingCurrentMiniBlock_; + totalRemaining = totalValuesRemaining_; + lastValue = lastValue_; + minDelta = minDelta_; + deltaBitWidth = deltaBitWidth_; + bitOffset = (valsPerMiniBlk - miniBlockRemaining) * deltaBitWidth; + ++i; + continue; + } bufferStart_ = bufStart; valuesRemainingCurrentMiniBlock_ = 0; totalValuesRemaining_ = totalRemaining; lastValue_ = lastValue; - int64_t v = readLong(); - out[i] = static_cast(v); + advanceMiniBlock(); bufStart = bufferStart_; miniBlockRemaining = valuesRemainingCurrentMiniBlock_; - totalRemaining = totalValuesRemaining_; - lastValue = lastValue_; minDelta = minDelta_; deltaBitWidth = deltaBitWidth_; - bitOffset = (valsPerMiniBlk - miniBlockRemaining) * deltaBitWidth; - ++i; - continue; + bitOffset = 0; + } + + // Whole-miniblock SIMD fast path. Eligibility: + // - We are at the miniblock start (bit-aligned, full count + // remaining). + // - The whole miniblock fits within the requested chunk and + // the remaining page values. + // - bit_width <= 32: an inline kernel materialized for each + // compile-time bit width handles the unpack. Widths 33..64 + // fall through to the per-row scalar inner loop. Width 0 + // is a constant-stride arithmetic sequence (no bit-extract + // needed). + if (miniBlockRemaining == valsPerMiniBlk && + static_cast(n - i) >= valsPerMiniBlk && + totalRemaining >= valsPerMiniBlk && deltaBitWidth <= 32) { + const int32_t mbValues = static_cast(valsPerMiniBlk); + DataType* dst = out + i; + bool dispatched = true; + switch (deltaBitWidth) { + case 0: + decodeMiniBlockConstantDelta(mbValues, minDelta, lastValue, dst); + break; +#define VELOX_DELTA_BP_DISPATCH(bw) \ + case bw: \ + decodeMiniBlockSimdImpl( \ + bufStart, mbValues, minDelta, lastValue, dst); \ + break; + VELOX_DELTA_BP_DISPATCH(1) + VELOX_DELTA_BP_DISPATCH(2) + VELOX_DELTA_BP_DISPATCH(3) + VELOX_DELTA_BP_DISPATCH(4) + VELOX_DELTA_BP_DISPATCH(5) + VELOX_DELTA_BP_DISPATCH(6) + VELOX_DELTA_BP_DISPATCH(7) + VELOX_DELTA_BP_DISPATCH(8) + VELOX_DELTA_BP_DISPATCH(9) + VELOX_DELTA_BP_DISPATCH(10) + VELOX_DELTA_BP_DISPATCH(11) + VELOX_DELTA_BP_DISPATCH(12) + VELOX_DELTA_BP_DISPATCH(13) + VELOX_DELTA_BP_DISPATCH(14) + VELOX_DELTA_BP_DISPATCH(15) + VELOX_DELTA_BP_DISPATCH(16) + VELOX_DELTA_BP_DISPATCH(17) + VELOX_DELTA_BP_DISPATCH(18) + VELOX_DELTA_BP_DISPATCH(19) + VELOX_DELTA_BP_DISPATCH(20) + VELOX_DELTA_BP_DISPATCH(21) + VELOX_DELTA_BP_DISPATCH(22) + VELOX_DELTA_BP_DISPATCH(23) + VELOX_DELTA_BP_DISPATCH(24) + VELOX_DELTA_BP_DISPATCH(25) + VELOX_DELTA_BP_DISPATCH(26) + VELOX_DELTA_BP_DISPATCH(27) + VELOX_DELTA_BP_DISPATCH(28) + VELOX_DELTA_BP_DISPATCH(29) + VELOX_DELTA_BP_DISPATCH(30) + VELOX_DELTA_BP_DISPATCH(31) + VELOX_DELTA_BP_DISPATCH(32) +#undef VELOX_DELTA_BP_DISPATCH + default: + dispatched = false; + } + if (dispatched) { + bufStart += bits::nbytes(deltaBitWidth * valsPerMiniBlk); + const uint64_t consumed = valsPerMiniBlk; + miniBlockRemaining = 0; + totalRemaining -= consumed; + i += static_cast(consumed); + bitOffset = 0; + continue; + } } uint64_t value = 0; @@ -202,6 +348,90 @@ class DeltaBpDecoder { lastValue_ = lastValue; } + /// Decode one whole miniblock with prefix-sum fused into the + /// bit-extract loop. All arithmetic is unsigned mod-2^64 (Parquet + /// spec). Safety: the unpack reads up to 7 bytes past the last + /// miniblock byte; PageReader::readBytes guarantees + /// kPageReadPadding (8) trailing bytes, and intra-page overshoots + /// fall into the next miniblock — both valid memory. + template + FOLLY_ALWAYS_INLINE void decodeMiniBlockSimdImpl( + const char* src, + int32_t numValues, + int64_t minDelta, + int64_t& lastValue, + DataType* out) { + static_assert(bitWidth >= 1 && bitWidth <= 32); + constexpr uint64_t mask = + (bitWidth == 32) ? 0xFFFFFFFFULL : ((1ULL << bitWidth) - 1); + const uint8_t* p = reinterpret_cast(src); + uint64_t cumulative = static_cast(lastValue); + const uint64_t step = static_cast(minDelta); + if constexpr (bitWidth <= 16) { + // Process 4 values per iteration. A single unaligned 64-bit + // load holds at least 4 values for any bitWidth in [1..16] + // (4*bw <= 64). All 4 values come from one load + one shift. + for (int32_t i = 0; i < numValues; i += 4) { + const int32_t bitPos = i * bitWidth; + const int32_t byteOff = bitPos >> 3; + const int32_t bitInByte = bitPos & 7; + const uint64_t word = + *reinterpret_cast(p + byteOff) >> bitInByte; + cumulative += step + (word & mask); + out[i + 0] = static_cast(cumulative); + cumulative += step + ((word >> bitWidth) & mask); + out[i + 1] = static_cast(cumulative); + cumulative += step + ((word >> (2 * bitWidth)) & mask); + out[i + 2] = static_cast(cumulative); + cumulative += step + ((word >> (3 * bitWidth)) & mask); + out[i + 3] = static_cast(cumulative); + } + } else { + // bitWidth in [17..32]: 2 values per iteration. After shifting + // by bitInByte (0..7), 2*bw + bitInByte can reach 71 bits, past + // a u64 window. Use __uint128_t for the unaligned load — on + // x86_64 this lowers to two u64 loads and a SHRD-style funnel + // shift, with no branch on bitInByte. The trailing u64 read + // (byteOff + 8) is always safe: intra-page it falls into the + // next miniblock; at page end the kPageReadPadding (8) bytes + // cover it. + for (int32_t i = 0; i < numValues; i += 2) { + const int32_t bitPos = i * bitWidth; + const int32_t byteOff = bitPos >> 3; + const int32_t bitInByte = bitPos & 7; + const __uint128_t window = + static_cast<__uint128_t>( + *reinterpret_cast(p + byteOff)) | + (static_cast<__uint128_t>( + *reinterpret_cast(p + byteOff + 8)) + << 64); + const uint64_t word = static_cast(window >> bitInByte); + cumulative += step + (word & mask); + out[i + 0] = static_cast(cumulative); + cumulative += step + ((word >> bitWidth) & mask); + out[i + 1] = static_cast(cumulative); + } + } + lastValue = static_cast(cumulative); + } + + // Specialization for bit_width == 0: every value is constant-stride + // (lastValue + (i+1)*minDelta). + template + FOLLY_ALWAYS_INLINE void decodeMiniBlockConstantDelta( + int32_t numValues, + int64_t minDelta, + int64_t& lastValue, + DataType* out) { + uint64_t cumulative = static_cast(lastValue); + const uint64_t step = static_cast(minDelta); + for (int32_t i = 0; i < numValues; ++i) { + cumulative += step; + out[i] = static_cast(cumulative); + } + lastValue = static_cast(cumulative); + } + bool getVlqInt(uint64_t& v); bool getZigZagVlqInt(int64_t& v); @@ -212,6 +442,22 @@ class DeltaBpDecoder { void initMiniBlock(int32_t bitWidth); + // Advances to the next miniblock without consuming a value. Mirrors + // the inner branch of readLong()'s `valuesRemainingCurrentMiniBlock_ + // == 0` case (post-firstBlockInitialized_) but stops short of + // decoding the first value, so callers that decode a whole miniblock + // in one shot (decodeLongs SIMD path) can pick it up cleanly. + void advanceMiniBlock() { + VELOX_DCHECK(firstBlockInitialized_); + VELOX_DCHECK_EQ(valuesRemainingCurrentMiniBlock_, 0); + ++miniBlockIdx_; + if (miniBlockIdx_ < miniBlocksPerBlock_) { + initMiniBlock(deltaBitWidths_[miniBlockIdx_]); + } else { + initBlock(); + } + } + FOLLY_ALWAYS_INLINE int64_t readLong() { int64_t value = 0; if (valuesRemainingCurrentMiniBlock_ == 0) { diff --git a/velox/dwio/parquet/tests/reader/CMakeLists.txt b/velox/dwio/parquet/tests/reader/CMakeLists.txt index 48c96c24daa..3da96dc384d 100644 --- a/velox/dwio/parquet/tests/reader/CMakeLists.txt +++ b/velox/dwio/parquet/tests/reader/CMakeLists.txt @@ -93,6 +93,19 @@ target_link_libraries( ${TEST_LINK_LIBS} ) +add_executable(velox_dwio_parquet_delta_bp_decoder_test DeltaBpDecoderTest.cpp) +add_test( + NAME velox_dwio_parquet_delta_bp_decoder_test + COMMAND velox_dwio_parquet_delta_bp_decoder_test + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) +target_link_libraries( + velox_dwio_parquet_delta_bp_decoder_test + velox_dwio_native_parquet_reader + velox_link_libs + ${TEST_LINK_LIBS} +) + if(VELOX_ENABLE_BENCHMARKS) add_executable(velox_dwio_parquet_structure_decoder_benchmark NestedStructureDecoderBenchmark.cpp) target_link_libraries( diff --git a/velox/dwio/parquet/tests/reader/DeltaBpDecoderTest.cpp b/velox/dwio/parquet/tests/reader/DeltaBpDecoderTest.cpp new file mode 100644 index 00000000000..73f5bf76376 --- /dev/null +++ b/velox/dwio/parquet/tests/reader/DeltaBpDecoderTest.cpp @@ -0,0 +1,357 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "velox/dwio/parquet/reader/DeltaBpDecoder.h" + +#include +#include +#include + +namespace facebook::velox::parquet::test { +namespace { + +// Tiny DELTA_BINARY_PACKED encoder used to feed DeltaBpDecoder with byte +// streams whose contents we control exactly. Mirrors the reference encoder +// in the Parquet spec; not optimized. +class DeltaEncoder { + public: + // Encodes 'values' with one block of 'valuesPerBlock' values divided into + // 'miniBlocksPerBlock' miniblocks. The encoder picks one bit width per + // miniblock equal to the number of bits required to represent the largest + // (delta - minDelta) within that miniblock. The first value of the page + // is stored in the header. The result includes 'kPageReadPadding' trailing + // zero bytes so that DeltaBpDecoder's readLong() loadBits never reads past + // valid memory. + std::vector encode( + const std::vector& values, + int valuesPerBlock, + int miniBlocksPerBlock) { + VELOX_CHECK_GT(valuesPerBlock, 0); + VELOX_CHECK_EQ(valuesPerBlock % 128, 0); + VELOX_CHECK_GT(miniBlocksPerBlock, 0); + const int valuesPerMiniBlock = valuesPerBlock / miniBlocksPerBlock; + VELOX_CHECK_EQ(valuesPerMiniBlock % 32, 0); + + out_.clear(); + writeVlq(static_cast(valuesPerBlock)); + writeVlq(static_cast(miniBlocksPerBlock)); + writeVlq(static_cast(values.size())); + if (values.empty()) { + writeZigZag(0); + out_.insert(out_.end(), 8, 0); // padding + return out_; + } + writeZigZag(values.front()); + + int64_t prev = values.front(); + size_t pos = 1; + while (pos < values.size()) { + const int blockSize = std::min(valuesPerBlock, values.size() - pos); + + std::vector deltas(valuesPerBlock, 0); + int64_t minDelta = std::numeric_limits::max(); + for (int i = 0; i < blockSize; ++i) { + deltas[i] = values[pos + i] - prev; + prev = values[pos + i]; + minDelta = std::min(minDelta, deltas[i]); + } + // Per spec, missing trailing miniblocks may use any width; we use 0. + // Compute per-miniblock bit width over the residual (delta - minDelta). + std::vector widths(miniBlocksPerBlock, 0); + for (int m = 0; m < miniBlocksPerBlock; ++m) { + const int begin = m * valuesPerMiniBlock; + const int end = std::min(begin + valuesPerMiniBlock, blockSize); + if (end <= begin) { + continue; + } + uint64_t maxResidual = 0; + for (int i = begin; i < end; ++i) { + maxResidual = std::max(maxResidual, deltas[i] - minDelta); + } + widths[m] = bitsRequired(maxResidual); + } + + writeZigZag(minDelta); + for (int m = 0; m < miniBlocksPerBlock; ++m) { + out_.push_back(widths[m]); + } + + // Pack each miniblock: write valuesPerMiniBlock residuals at the + // miniblock's bit width even when the block has < valuesPerBlock real + // values (trailing slots use zero residual, which is fine — the + // decoder advances totalValuesRemaining_ separately and will not emit + // them). + for (int m = 0; m < miniBlocksPerBlock; ++m) { + const int begin = m * valuesPerMiniBlock; + const int end = std::min(begin + valuesPerMiniBlock, blockSize); + std::vector residuals(valuesPerMiniBlock, 0); + for (int i = begin; i < end; ++i) { + residuals[i - begin] = static_cast(deltas[i] - minDelta); + } + writeBitPacked(residuals, widths[m]); + } + + pos += blockSize; + } + out_.insert(out_.end(), 8, 0); // padding + return out_; + } + + private: + static uint8_t bitsRequired(uint64_t v) { + if (v == 0) { + return 0; + } + uint8_t b = 0; + while (v > 0) { + ++b; + v >>= 1; + } + return b; + } + + void writeVlq(uint64_t v) { + while (v >= 0x80) { + out_.push_back(static_cast(v | 0x80)); + v >>= 7; + } + out_.push_back(static_cast(v)); + } + + void writeZigZag(int64_t v) { + auto u = static_cast((v << 1) ^ (v >> 63)); + writeVlq(u); + } + + void writeBitPacked(const std::vector& vals, uint8_t bitWidth) { + if (bitWidth == 0) { + return; + } + uint64_t buffer = 0; + int filled = 0; + for (uint64_t v : vals) { + buffer |= (v & ((1ULL << bitWidth) - 1)) << filled; + filled += bitWidth; + while (filled >= 8) { + out_.push_back(static_cast(buffer & 0xff)); + buffer >>= 8; + filled -= 8; + } + } + if (filled > 0) { + out_.push_back(static_cast(buffer & 0xff)); + } + } + + std::vector out_; +}; + +// Decodes an entire page through the same path the SIMD-eligible production +// caller uses: readValues. Returns the produced values. +std::vector decodeAll( + const std::vector& bytes, + size_t numValues) { + DeltaBpDecoder decoder(reinterpret_cast(bytes.data())); + std::vector out(numValues); + decoder.readValues(out.data(), static_cast(numValues)); + return out; +} + +} // namespace + +// Encodes 'numValues' values whose first value is 'first' and whose +// successive deltas are bounded by 'maxDelta' (so residuals fit in +// ceil(log2(maxDelta+1)) bits). +std::vector +makeAscendingValues(int numValues, int64_t first, int64_t maxDelta) { + std::vector v; + v.reserve(numValues); + v.push_back(first); + for (int i = 1; i < numValues; ++i) { + const int64_t pseudo = static_cast((i + 7) * 1'000'003LL); + v.push_back(v.back() + (pseudo % (maxDelta + 1))); + } + return v; +} + +void runRoundtrip( + const std::vector& values, + int valuesPerBlock, + int miniBlocksPerBlock) { + DeltaEncoder enc; + const auto bytes = enc.encode(values, valuesPerBlock, miniBlocksPerBlock); + const auto decoded = decodeAll(bytes, values.size()); + EXPECT_EQ(decoded, values); +} + +// Roundtrip test: encode then decode 32 sequential values whose deltas +// require exactly 10 bits. Targets the unpack9to15 PDEP path. +TEST(DeltaBpDecoderTest, bitWidth10SingleMiniBlock) { + const auto values = makeAscendingValues(32, 1'000'000, 1023); + runRoundtrip(values, /*valuesPerBlock=*/128, /*miniBlocksPerBlock=*/4); +} + +// bit width 8 — targets unpack8 (8→32 cvt path). +TEST(DeltaBpDecoderTest, bitWidth8SingleMiniBlock) { + const auto values = makeAscendingValues(32, 0, 255); + runRoundtrip(values, 128, 4); +} + +// bit width 16 — targets unpack16 (16→32 cvt path). +TEST(DeltaBpDecoderTest, bitWidth16SingleMiniBlock) { + const auto values = makeAscendingValues(32, 0, 65'535); + runRoundtrip(values, 128, 4); +} + +// bit width 24 — targets unpack22to31 PDEP path. +TEST(DeltaBpDecoderTest, bitWidth24SingleMiniBlock) { + const auto values = makeAscendingValues(32, 0, (1 << 24) - 1); + runRoundtrip(values, 128, 4); +} + +// bit width 32 — targets unpack32 (memcpy path). +TEST(DeltaBpDecoderTest, bitWidth32SingleMiniBlock) { + const auto values = makeAscendingValues(32, 0, (1LL << 32) - 1); + runRoundtrip(values, 128, 4); +} + +// bit width 0 — every delta equals minDelta. Decoded values form a +// constant-stride arithmetic sequence; the bit-unpack body produces zero. +TEST(DeltaBpDecoderTest, bitWidth0ConstantDelta) { + std::vector values{42}; + for (int i = 1; i < 128; ++i) { + values.push_back(values.back() + 7); + } + runRoundtrip(values, 128, 4); +} + +// Multi-block decode in a single readValues call. With valuesPerBlock=128 +// and 384 total values, the decoder crosses two block boundaries +// (initBlock invoked twice during the call). Every miniblock and every +// block boundary must be handled without losing state. +TEST(DeltaBpDecoderTest, multiBlockSingleCall) { + const auto values = makeAscendingValues(384, 0, 1023); + runRoundtrip(values, 128, 4); +} + +// Decode the page in two calls, with the split landing mid-miniblock. The +// SIMD fast path is only safe at miniblock-start (bitOffset == 0); the +// second call must fall back to the scalar inner loop for the partial +// miniblock and then resume the SIMD path on the next miniblock. +TEST(DeltaBpDecoderTest, splitInsideMiniBlock) { + const auto values = makeAscendingValues(128, 1'000, 1023); + DeltaEncoder enc; + const auto bytes = enc.encode(values, 128, 4); + DeltaBpDecoder decoder(reinterpret_cast(bytes.data())); + std::vector out(128); + decoder.readValues(out.data(), 17); // mid-miniblock split + decoder.readValues(out.data() + 17, 128 - 17); + EXPECT_EQ(out, values); +} + +// Negative minDelta — checks that the unsigned arithmetic on the +// hot-path correctly wraps when minDelta is sign-extended. +TEST(DeltaBpDecoderTest, negativeMinDelta) { + std::vector values{1'000'000}; + // Deltas oscillate around -100, residuals (delta - minDelta) ∈ [0,200]. + for (int i = 1; i < 64; ++i) { + values.push_back(values.back() - 100 + ((i * 17) % 201)); + } + runRoundtrip(values, 128, 4); +} + +// bit width > 32 — must take the scalar fallback because +// dwio::common::unpack only handles up to 32 bits. +TEST(DeltaBpDecoderTest, bitWidthAbove32Fallback) { + std::vector values{0}; + for (int i = 1; i < 32; ++i) { + values.push_back(values.back() + (1LL << 33) + (i * 7)); + } + runRoundtrip(values, 128, 4); +} + +// int32_t output type — exercises the narrowing static_cast in +// readValues / decodeLongs. +TEST(DeltaBpDecoderTest, narrowingToInt32) { + std::vector values64; + values64.reserve(32); + values64.push_back(100); + for (int i = 1; i < 32; ++i) { + values64.push_back(values64.back() + ((i * 11) & 0x3ff)); + } + + DeltaEncoder enc; + const auto bytes = enc.encode(values64, 128, 4); + DeltaBpDecoder decoder(reinterpret_cast(bytes.data())); + std::vector out(32); + decoder.readValues(out.data(), 32); + for (size_t i = 0; i < values64.size(); ++i) { + EXPECT_EQ(out[i], static_cast(values64[i])); + } +} + +// Parameterized roundtrip across every supported SIMD-dispatched bit +// width (1..32). Each width drives one miniblock of 32 values whose +// residuals saturate the requested bit width — guaranteeing the +// encoder picks exactly that width — and verifies decoded output +// matches input byte-for-byte. Exercises: +// - bit_width <= 16 (4-value/iter unaligned 64-bit load path) +// - bit_width 17..32 (2-value/iter __uint128_t funnel-shift path) +// - bit-offset misalignment within u64/u128 windows +class DeltaBpDecoderBitWidthTest : public ::testing::TestWithParam {}; + +TEST_P(DeltaBpDecoderBitWidthTest, roundtrip32Values) { + const int bitWidth = GetParam(); + // Saturate residual range for this bit width: max residual is + // (1 << bitWidth) - 1. Use minDelta = 0 so encoded residual == + // delta, forcing the encoder to pick exactly 'bitWidth'. + const int64_t maxDelta = + (bitWidth == 32) ? 0xFFFFFFFFLL : ((1LL << bitWidth) - 1); + std::vector values; + values.reserve(32); + // Start far from zero so int64 narrowing still works for bw=32. + values.push_back(1'000'000); + for (int i = 1; i < 32; ++i) { + int64_t delta; + if (i == 1) { + // Ensure the encoder sees at least one value at the maximum, + // so bitsRequired() picks 'bitWidth' — not a smaller width. + delta = maxDelta; + } else { + const int64_t pseudo = static_cast((i + 7) * 1'000'003LL); + delta = pseudo % (maxDelta + 1); + } + values.push_back(values.back() + delta); + } + + DeltaEncoder enc; + const auto bytes = + enc.encode(values, /*valuesPerBlock=*/128, /*miniBlocksPerBlock=*/4); + DeltaBpDecoder decoder(reinterpret_cast(bytes.data())); + std::vector out(values.size()); + decoder.readValues(out.data(), static_cast(out.size())); + EXPECT_EQ(out, values) << "bit_width=" << bitWidth; +} + +INSTANTIATE_TEST_SUITE_P( + AllBitWidths, + DeltaBpDecoderBitWidthTest, + ::testing::Range(1, 33), + [](const ::testing::TestParamInfo& info) { + return "bw" + std::to_string(info.param); + }); + +} // namespace facebook::velox::parquet::test