From a16dcc1aca01349cb664a99d23d54604bbef59c2 Mon Sep 17 00:00:00 2001 From: aabduvakhobov Date: Mon, 4 Aug 2025 14:55:52 +0200 Subject: [PATCH 01/15] Add MacaqueV model type's implementation --- .../modelardb_compression/src/compression.rs | 126 ++++++++-------- .../src/models/{gorilla.rs => macaque_v.rs} | 137 +++++++++++++----- .../modelardb_compression/src/models/mod.rs | 24 ++- crates/modelardb_compression/src/types.rs | 4 +- 4 files changed, 182 insertions(+), 109 deletions(-) rename crates/modelardb_compression/src/models/{gorilla.rs => macaque_v.rs} (80%) diff --git a/crates/modelardb_compression/src/compression.rs b/crates/modelardb_compression/src/compression.rs index d90b25343..6e4afb883 100644 --- a/crates/modelardb_compression/src/compression.rs +++ b/crates/modelardb_compression/src/compression.rs @@ -24,8 +24,8 @@ use arrow::record_batch::RecordBatch; use modelardb_types::types::{ErrorBound, TimestampArray, ValueArray}; use crate::error::{ModelarDbCompressionError, Result}; -use crate::models::gorilla::Gorilla; -use crate::models::{self, GORILLA_ID, timestamps}; +use crate::models::macaque_v::MacaqueV; +use crate::models::{self, timestamps, MACAQUE_V_ID}; use crate::types::{CompressedSegmentBatchBuilder, CompressedSegmentBuilder, ModelBuilder}; /// Maximum number of residuals that can be stored as part of a compressed segment. The number of @@ -238,13 +238,13 @@ fn compress_and_store_residuals_in_a_separate_segment( // Compute metadata and compress the values stored in this segment without residuals. let uncompressed_values = &uncompressed_values.values()[start_index..=end_index]; - let mut gorilla = Gorilla::new(error_bound); + let mut gorilla = MacaqueV::new(error_bound); gorilla.compress_values(uncompressed_values); let (values, min_value, max_value) = gorilla.model(); compressed_segment_batch_builder.append_compressed_segment( - GORILLA_ID, + MACAQUE_V_ID, start_time, end_time, ×tamps, @@ -269,7 +269,7 @@ mod tests { use modelardb_types::schemas::COMPRESSED_SCHEMA; use modelardb_types::types::{TimestampBuilder, ValueBuilder}; - use crate::{MODEL_TYPE_NAMES, models}; + use crate::{models, MODEL_TYPE_NAMES}; const TAG_VALUE: &str = "tag"; const ADD_NOISE_RANGE: Option> = Some(1.0..1.05); @@ -470,7 +470,7 @@ mod tests { false, ValuesStructure::largest_random_without_overflow(), ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - &[models::GORILLA_ID], + &[models::MACAQUE_V_ID], ); } @@ -480,7 +480,7 @@ mod tests { false, ValuesStructure::largest_random_without_overflow(), ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - &[models::GORILLA_ID], + &[models::MACAQUE_V_ID], ); } @@ -490,7 +490,7 @@ mod tests { true, ValuesStructure::largest_random_without_overflow(), ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - &[models::GORILLA_ID], + &[models::MACAQUE_V_ID], ); } @@ -500,7 +500,7 @@ mod tests { true, ValuesStructure::largest_random_without_overflow(), ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - &[models::GORILLA_ID], + &[models::MACAQUE_V_ID], ); } @@ -534,89 +534,89 @@ mod tests { } #[test] - fn test_try_compress_regular_random_linear_constant_time_series_within_absolute_error_bound_zero() - { + fn test_try_compress_regular_random_linear_constant_time_series_within_absolute_error_bound_zero( + ) { generate_compress_and_assert_known_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), false, - &[models::GORILLA_ID, models::SWING_ID, models::PMC_MEAN_ID], - &[models::GORILLA_ID, models::SWING_ID, models::PMC_MEAN_ID], + &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], + &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], ); } #[test] - fn test_try_compress_regular_random_linear_constant_time_series_within_relative_error_bound_zero() - { + fn test_try_compress_regular_random_linear_constant_time_series_within_relative_error_bound_zero( + ) { generate_compress_and_assert_known_time_series( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), false, - &[models::GORILLA_ID, models::SWING_ID, models::PMC_MEAN_ID], - &[models::GORILLA_ID, models::SWING_ID, models::PMC_MEAN_ID], + &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], + &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], ); } #[test] - fn test_try_compress_irregular_random_linear_constant_time_series_within_absolute_error_bound_zero() - { + fn test_try_compress_irregular_random_linear_constant_time_series_within_absolute_error_bound_zero( + ) { generate_compress_and_assert_known_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), true, - &[models::GORILLA_ID, models::SWING_ID, models::PMC_MEAN_ID], - &[models::GORILLA_ID, models::SWING_ID, models::PMC_MEAN_ID], + &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], + &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], ); } #[test] - fn test_try_compress_irregular_random_linear_constant_time_series_within_relative_error_bound_zero() - { + fn test_try_compress_irregular_random_linear_constant_time_series_within_relative_error_bound_zero( + ) { generate_compress_and_assert_known_time_series( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), true, - &[models::GORILLA_ID, models::SWING_ID, models::PMC_MEAN_ID], - &[models::GORILLA_ID, models::SWING_ID, models::PMC_MEAN_ID], + &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], + &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], ); } #[test] - fn test_try_compress_regular_constant_linear_random_time_series_within_absolute_error_bound_zero() - { + fn test_try_compress_regular_constant_linear_random_time_series_within_absolute_error_bound_zero( + ) { generate_compress_and_assert_known_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), false, - &[models::PMC_MEAN_ID, models::SWING_ID, models::GORILLA_ID], + &[models::PMC_MEAN_ID, models::SWING_ID, models::MACAQUE_V_ID], &[models::PMC_MEAN_ID, models::SWING_ID], ); } #[test] - fn test_try_compress_regular_constant_linear_random_time_series_within_relative_error_bound_zero() - { + fn test_try_compress_regular_constant_linear_random_time_series_within_relative_error_bound_zero( + ) { generate_compress_and_assert_known_time_series( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), false, - &[models::PMC_MEAN_ID, models::SWING_ID, models::GORILLA_ID], + &[models::PMC_MEAN_ID, models::SWING_ID, models::MACAQUE_V_ID], &[models::PMC_MEAN_ID, models::SWING_ID], ); } #[test] - fn test_try_compress_irregular_constant_linear_random_time_series_within_absolute_error_bound_zero() - { + fn test_try_compress_irregular_constant_linear_random_time_series_within_absolute_error_bound_zero( + ) { generate_compress_and_assert_known_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), true, - &[models::PMC_MEAN_ID, models::SWING_ID, models::GORILLA_ID], + &[models::PMC_MEAN_ID, models::SWING_ID, models::MACAQUE_V_ID], &[models::PMC_MEAN_ID, models::SWING_ID], ); } #[test] - fn test_try_compress_irregular_constant_linear_random_time_series_within_relative_error_bound_zero() - { + fn test_try_compress_irregular_constant_linear_random_time_series_within_relative_error_bound_zero( + ) { generate_compress_and_assert_known_time_series( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), true, - &[models::PMC_MEAN_ID, models::SWING_ID, models::GORILLA_ID], + &[models::PMC_MEAN_ID, models::SWING_ID, models::MACAQUE_V_ID], &[models::PMC_MEAN_ID, models::SWING_ID], ); } @@ -649,7 +649,7 @@ mod tests { [uncompressed_timestamps_start_index..uncompressed_timestamps_end_index], ValuesStructure::Linear(None), ), - models::GORILLA_ID => data_generation::generate_values( + models::MACAQUE_V_ID => data_generation::generate_values( &uncompressed_timestamps.values() [uncompressed_timestamps_start_index..uncompressed_timestamps_end_index], ValuesStructure::largest_random_without_overflow(), @@ -707,8 +707,8 @@ mod tests { } #[test] - fn test_try_compress_regular_synthetic_time_series_without_noise_within_absolute_error_bound_zero() - { + fn test_try_compress_regular_synthetic_time_series_without_noise_within_absolute_error_bound_zero( + ) { generate_compress_and_assert_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), false, @@ -717,8 +717,8 @@ mod tests { } #[test] - fn test_try_compress_regular_synthetic_time_series_without_noise_within_relative_error_bound_zero() - { + fn test_try_compress_regular_synthetic_time_series_without_noise_within_relative_error_bound_zero( + ) { generate_compress_and_assert_time_series( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), false, @@ -727,8 +727,8 @@ mod tests { } #[test] - fn test_try_compress_regular_synthetic_time_series_without_noise_within_absolute_error_bound_five() - { + fn test_try_compress_regular_synthetic_time_series_without_noise_within_absolute_error_bound_five( + ) { generate_compress_and_assert_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_FIVE).unwrap(), false, @@ -737,8 +737,8 @@ mod tests { } #[test] - fn test_try_compress_regular_synthetic_time_series_without_noise_within_relative_error_bound_five() - { + fn test_try_compress_regular_synthetic_time_series_without_noise_within_relative_error_bound_five( + ) { generate_compress_and_assert_time_series( ErrorBound::try_new_relative(ERROR_BOUND_FIVE).unwrap(), false, @@ -787,8 +787,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_synthetic_time_series_without_noise_within_absolute_error_bound_zero() - { + fn test_try_compress_irregular_synthetic_time_series_without_noise_within_absolute_error_bound_zero( + ) { generate_compress_and_assert_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), true, @@ -797,8 +797,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_synthetic_time_series_without_noise_within_relative_error_bound_zero() - { + fn test_try_compress_irregular_synthetic_time_series_without_noise_within_relative_error_bound_zero( + ) { generate_compress_and_assert_time_series( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), true, @@ -807,8 +807,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_synthetic_time_series_without_noise_within_absolute_error_bound_five() - { + fn test_try_compress_irregular_synthetic_time_series_without_noise_within_absolute_error_bound_five( + ) { generate_compress_and_assert_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_FIVE).unwrap(), true, @@ -817,8 +817,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_synthetic_time_series_without_noise_within_relative_error_bound_five() - { + fn test_try_compress_irregular_synthetic_time_series_without_noise_within_relative_error_bound_five( + ) { generate_compress_and_assert_time_series( ErrorBound::try_new_relative(ERROR_BOUND_FIVE).unwrap(), true, @@ -827,8 +827,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_synthetic_time_series_with_noise_within_absolute_error_bound_zero() - { + fn test_try_compress_irregular_synthetic_time_series_with_noise_within_absolute_error_bound_zero( + ) { generate_compress_and_assert_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), true, @@ -837,8 +837,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_synthetic_time_series_with_noise_within_relative_error_bound_zero() - { + fn test_try_compress_irregular_synthetic_time_series_with_noise_within_relative_error_bound_zero( + ) { generate_compress_and_assert_time_series( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), true, @@ -847,8 +847,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_synthetic_time_series_with_noise_within_absolute_error_bound_five() - { + fn test_try_compress_irregular_synthetic_time_series_with_noise_within_absolute_error_bound_five( + ) { generate_compress_and_assert_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_FIVE).unwrap(), true, @@ -857,8 +857,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_synthetic_time_series_with_noise_within_relative_error_bound_five() - { + fn test_try_compress_irregular_synthetic_time_series_with_noise_within_relative_error_bound_five( + ) { generate_compress_and_assert_time_series( ErrorBound::try_new_relative(ERROR_BOUND_FIVE).unwrap(), true, @@ -1002,7 +1002,7 @@ mod tests { ); assert_eq!(1, compressed_record_batch.num_rows()); - assert_eq!(GORILLA_ID, model_type_ids.value(0)); + assert_eq!(MACAQUE_V_ID, model_type_ids.value(0)); assert_eq!(100, start_times.value(0)); assert_eq!(500, end_times.value(0)); assert_eq!(1, timestamps.value(0).len()); diff --git a/crates/modelardb_compression/src/models/gorilla.rs b/crates/modelardb_compression/src/models/macaque_v.rs similarity index 80% rename from crates/modelardb_compression/src/models/gorilla.rs rename to crates/modelardb_compression/src/models/macaque_v.rs index 3582ac129..7524b3e1d 100644 --- a/crates/modelardb_compression/src/models/gorilla.rs +++ b/crates/modelardb_compression/src/models/macaque_v.rs @@ -13,24 +13,32 @@ * limitations under the License. */ -//! Implementation of the Gorilla model type which uses the lossless compression method for -//! floating-point values proposed for the time series management system Gorilla in the [Gorilla -//! paper]. The compression method described in the paper has been extended with support for lossy -//! compression by replacing values with the previous value if possible within the error bound. As -//! this compression method compresses the values of a time series segment using XOR and a variable -//! length binary encoding, aggregates are computed by iterating over all values in the segment. +//! Implementation of MacaqueV model type which extends the lossless +//! compression method for floating-point values proposed for the time series +//! management system Gorilla in the [Gorilla paper] by 1) adding support for +//! error-bounded lossy compression, and 2) optimizing flag bits for better +//! compression of real-life sensor data. MacaqueV adds support for lossy +//! compression by 1) replacing values with the previous value if possible +//! within the error bound, and 2) rewriting the least mantissa bits of a value +//! within the error bound so that Gorilla uses fewer bits for encoding. +//! MacaqueV optimizes Gorilla's flag bits by swapping the flag bits 0 and 10. +//! This modification proved to be effective when Gorilla is used alongside +//! PMC-Mean and Swing for multi-model compression. As this compression method +//! compresses the values of a time series segment using XOR and a +//! variable length binary encoding, aggregates are computed by iterating +//! over all values in the segment. //! //! [Gorilla paper]: https://www.vldb.org/pvldb/vol8/p1816-teller.pdf use modelardb_types::types::{Timestamp, Value, ValueBuilder}; use crate::models; -use crate::models::ErrorBound; use crate::models::bits::{BitReader, BitVecBuilder}; +use crate::models::ErrorBound; /// The state the Gorilla model type needs while compressing the values of a /// time series segment. -pub struct Gorilla { +pub struct MacaqueV { /// Maximum relative error for the value of each data point. error_bound: ErrorBound, /// Min value compressed and added to `compressed_values`. @@ -51,7 +59,7 @@ pub struct Gorilla { length: usize, } -impl Gorilla { +impl MacaqueV { pub fn new(error_bound: ErrorBound) -> Self { Self { error_bound, @@ -92,12 +100,18 @@ impl Gorilla { /// Compress `value` using XOR and a variable length binary encoding and then store it. fn compress_value_xor_last_value(&mut self, value: Value) { - // The best case for Gorilla is storing duplicate values. - let value = if models::is_value_within_error_bound(self.error_bound, value, self.last_value) - { - self.last_value - } else { + // Rewrite the value bits using the error bound + let value = if models::is_lossless_compression(self.error_bound) { value + } else { + // If compression is lossy we try to rewrite value first. + // The best case for Gorilla is storing duplicate values. + if models::is_value_within_error_bound(self.error_bound, value, self.last_value) { + self.last_value + } else { + // we only do binary rewriting if value rewriting is not possible + self.rewrite_value_with_log_method(value) + } }; let value_as_integer = value.to_bits(); @@ -105,19 +119,19 @@ impl Gorilla { let value_xor_last_value = value_as_integer ^ last_value_as_integer; if value_xor_last_value == 0 { - // Store each repeated value as a single zero bit. + // Store each repeated value as a one and zero bit. + self.compressed_values.append_a_one_bit(); self.compressed_values.append_a_zero_bit(); } else { // Store each new value as its leading zero bits (if necessary) and // meaningful bits (all bits from the first to the last one bit). let leading_zero_bits = value_xor_last_value.leading_zeros() as u8; let trailing_zero_bits = value_xor_last_value.trailing_zeros() as u8; - self.compressed_values.append_a_one_bit(); if leading_zero_bits >= self.last_leading_zero_bits && trailing_zero_bits >= self.last_trailing_zero_bits { - // Store only the meaningful bits. + // Store only the meaningful bits after a flag bit zero. self.compressed_values.append_a_zero_bit(); let meaningful_bits = models::VALUE_SIZE_IN_BITS - self.last_leading_zero_bits @@ -130,6 +144,7 @@ impl Gorilla { // Store the leading zero bits before the meaningful bits using // 5 and 6 bits respectively as described in the Gorilla paper. self.compressed_values.append_a_one_bit(); + self.compressed_values.append_a_one_bit(); self.compressed_values .append_bits(leading_zero_bits as u64, 5); @@ -150,6 +165,32 @@ impl Gorilla { self.update_min_max_and_last_value(value); } + /// MacaqueV's value rewrite method. + fn rewrite_value_with_log_method(&self, value: Value) -> Value { + if value == 0.0 || value.is_infinite() || value.is_nan() { + return value; + } + let value_as_u32 = value.to_bits(); + let abs_error_bound = + models::maximum_allowed_deviation(self.error_bound, value as f64) as f32; + let exponent = get_exponent(value); + let factorized_epsilon = abs_error_bound / 2f32.powi(exponent); + // Rewriting value using by 23 - ⌈log2 factorized_epsilon⌉ bits + // never exceeds the error bound. However, one more bit can be + // rewritten when the majority of the least mantissa bits are 0, + // thus we use 23 - ⌊log2 factorized_epsilon⌋ and then + // perform extra check if the error bound is not exceeded. + let mut rewrite_position = 23 - factorized_epsilon.log2().abs().floor() as i32; + // Extra check to ensure if the error is within the error_bound + let mut rewritten_value = f32::from_bits(rewrite_bits_by_n(value_as_u32, rewrite_position)); + // If the error bound exceeded, value is rewritten with one les bit i.e., 23 − ⌈log2 factorized_epsilon⌉. + if !models::is_value_within_error_bound(self.error_bound, value, rewritten_value) { + rewrite_position -= 1; + rewritten_value = f32::from_bits(rewrite_bits_by_n(value.to_bits(), rewrite_position)); + } + rewritten_value + } + /// Update the current minimum, maximum, and last value based on `value`. fn update_min_max_and_last_value(&mut self, value: Value) { self.min_value = Value::min(self.min_value, value); @@ -198,8 +239,15 @@ pub fn sum(length: usize, values: &[u8], maybe_model_last_value: Option) leading_zeros = bits.read_bits(5) as u8; let meaningful_bits = bits.read_bits(6) as u8; trailing_zeros = models::VALUE_SIZE_IN_BITS - meaningful_bits - leading_zeros; + // Decompress the value by reading its meaningful bits, restoring + // its trailing zeroes through shifting, and reversing the XOR. + let meaningful_bits = models::VALUE_SIZE_IN_BITS - leading_zeros - trailing_zeros; + let mut value = bits.read_bits(meaningful_bits) as u32; + value <<= trailing_zeros; + value ^= last_value; + last_value = value; } - + } else { // Decompress the value by reading its meaningful bits, restoring // its trailing zeroes through shifting, and reversing the XOR. let meaningful_bits = models::VALUE_SIZE_IN_BITS - leading_zeros - trailing_zeros; @@ -249,8 +297,15 @@ pub fn grid( leading_zeros = bits.read_bits(5) as u8; let meaningful_bits = bits.read_bits(6) as u8; trailing_zeros = models::VALUE_SIZE_IN_BITS - meaningful_bits - leading_zeros; + // Decompress the value by reading its meaningful bits, restoring + // its trailing zeroes through shifting, and reversing the XOR. + let meaningful_bits = models::VALUE_SIZE_IN_BITS - leading_zeros - trailing_zeros; + let mut value = bits.read_bits(meaningful_bits) as u32; + value <<= trailing_zeros; + value ^= last_value; + last_value = value; } - + } else { // Decompress the value by reading its meaningful bits, restoring // its trailing zeroes through shifting, and reversing the XOR. let meaningful_bits = models::VALUE_SIZE_IN_BITS - leading_zeros - trailing_zeros; @@ -263,6 +318,18 @@ pub fn grid( } } +fn get_exponent(value: f32) -> i32 { + let n_bits: u32 = value.to_bits(); + let exponent_ = ((n_bits >> 23) & 0xff) as i32; + let exponent = exponent_ - 127; + exponent +} + +fn rewrite_bits_by_n(bits_to_rewrite: u32, erase_by_n: i32) -> u32 { + let mask = u32::MAX << erase_by_n; + bits_to_rewrite & mask +} + #[cfg(test)] mod tests { use super::*; @@ -277,20 +344,20 @@ mod tests { #[test] fn test_empty_sequence_with_absolute_error_bound_zero() { let error_bound = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - assert!(Gorilla::new(error_bound).model().0.is_empty()); + assert!(MacaqueV::new(error_bound).model().0.is_empty()); } #[test] fn test_empty_sequence_with_relative_error_bound_zero() { let error_bound = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); - assert!(Gorilla::new(error_bound).model().0.is_empty()); + assert!(MacaqueV::new(error_bound).model().0.is_empty()); } proptest! { #[test] fn test_append_single_value_with_absolute_error_bound_zero(value in ProptestValue::ANY) { let error_bound = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); model_type.compress_values(&[value]); @@ -302,7 +369,7 @@ mod tests { #[test] fn test_append_single_value_with_relative_error_bound_zero(value in ProptestValue::ANY) { let error_bound = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); model_type.compress_values(&[value]); @@ -314,7 +381,7 @@ mod tests { #[test] fn test_append_repeated_values_with_absolute_error_bound_zero(value in ProptestValue::ANY) { let error_bound = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); model_type.compress_values(&[value, value]); @@ -326,7 +393,7 @@ mod tests { #[test] fn test_append_repeated_values_with_relative_error_bound_zero(value in ProptestValue::ANY) { let error_bound = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); model_type.compress_values(&[value, value]); @@ -339,7 +406,7 @@ mod tests { #[test] fn test_append_different_values_with_leading_zero_bits_with_absolute_error_bound_zero() { let error_bound = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); model_type.compress_values(&[37.0, 73.0]); @@ -351,7 +418,7 @@ mod tests { #[test] fn test_append_different_values_with_leading_zero_bits_with_relative_error_bound_zero() { let error_bound = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); model_type.compress_values(&[37.0, 73.0]); @@ -363,7 +430,7 @@ mod tests { #[test] fn test_append_different_values_without_leading_zero_bits_with_absolute_error_bound_zero() { let error_bound = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); model_type.compress_values(&[37.0, 71.0, 73.0]); @@ -375,7 +442,7 @@ mod tests { #[test] fn test_append_different_values_without_leading_zero_bits_with_relative_error_bound_zero() { let error_bound = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); model_type.compress_values(&[37.0, 71.0, 73.0]); @@ -398,7 +465,7 @@ mod tests { } fn test_append_values_within_error_bound(error_bound: ErrorBound) { - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); model_type.compress_values(&[10.0]); let before_last_value = model_type.last_value; @@ -519,11 +586,9 @@ mod tests { let values_array = value_builder.finish(); assert!(values.len() == timestamps.len() && values.len() == values_array.len()); - assert!( - timestamps - .windows(2) - .all(|window| window[1] - window[0] == 1) - ); + assert!(timestamps + .windows(2) + .all(|window| window[1] - window[0] == 1)); assert!(slice_of_value_equal(values_array.values(), values)); } @@ -582,7 +647,7 @@ mod tests { values: &[Value], maybe_model_last_value: Option, ) -> Vec { - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); if let Some(model_last_value) = maybe_model_last_value { model_type.compress_values_without_first(values, model_last_value); } else { diff --git a/crates/modelardb_compression/src/models/mod.rs b/crates/modelardb_compression/src/models/mod.rs index 4fbcbb6fa..2eab3718b 100644 --- a/crates/modelardb_compression/src/models/mod.rs +++ b/crates/modelardb_compression/src/models/mod.rs @@ -18,7 +18,7 @@ //! contains general functionality used by the model types. pub mod bits; -pub mod gorilla; +pub mod macaque_v; pub mod pmc_mean; pub mod swing; pub mod timestamps; @@ -35,13 +35,13 @@ use crate::types::CompressedSegmentBuilder; /// to the ids must be reflected in all statements matching on them. pub const PMC_MEAN_ID: i8 = 0; pub const SWING_ID: i8 = 1; -pub const GORILLA_ID: i8 = 2; +pub const MACAQUE_V_ID: i8 = 2; /// Number of implemented model types. It is usize instead of u8 as it is used as an array length. pub const MODEL_TYPE_COUNT: usize = 3; /// Mapping of model type ids to names. -pub const MODEL_TYPE_NAMES: [&str; MODEL_TYPE_COUNT] = ["pmc_mean", "swing", "gorilla"]; +pub const MODEL_TYPE_NAMES: [&str; MODEL_TYPE_COUNT] = ["pmc_mean", "swing", "macaque_v"]; /// Size of [`Value`] in bytes. pub(super) const VALUE_SIZE_IN_BYTES: u8 = mem::size_of::() as u8; @@ -87,6 +87,14 @@ pub fn maximum_allowed_deviation(error_bound: ErrorBound, value: f64) -> f64 { } } +/// Returns true is compression is lossless i.e., `error_bound` is 0. +pub fn is_lossless_compression(error_bound: ErrorBound) -> bool { + match error_bound { + ErrorBound::Absolute(error_bound) => return error_bound == 0.0, + ErrorBound::Relative(error_bound) => return error_bound == 0.0, + } +} + /// Returns true if `v1` and `v2` are equivalent or both values are NAN. fn equal_or_nan(v1: f64, v2: f64) -> bool { v1 == v2 || (v1.is_nan() && v2.is_nan()) @@ -160,9 +168,9 @@ pub fn sum( ), ) } - GORILLA_ID => ( + MACAQUE_V_ID => ( f32::NAN, // A segment with values compressed by Gorilla never has residuals. - gorilla::sum(model_length, values, None), + macaque_v::sum(model_length, values, None), ), _ => panic!("Unknown model type."), }; @@ -171,7 +179,7 @@ pub fn sum( if residuals.is_empty() { model_sum } else { - let residuals_sum = gorilla::sum( + let residuals_sum = macaque_v::sum( residuals_length, &residuals[..residuals.len() - 1], Some(model_last_value), @@ -228,7 +236,7 @@ pub fn grid( value_builder, ) } - GORILLA_ID => gorilla::grid(values, model_timestamps, value_builder, None), + MACAQUE_V_ID => macaque_v::grid(values, model_timestamps, value_builder, None), _ => panic!("Unknown model type."), } @@ -236,7 +244,7 @@ pub fn grid( if !residuals.is_empty() { let model_last_value = value_builder.values_slice()[value_builder.len() - 1]; - gorilla::grid( + macaque_v::grid( &residuals[..residuals.len() - 1], residuals_timestamps, value_builder, diff --git a/crates/modelardb_compression/src/types.rs b/crates/modelardb_compression/src/types.rs index 32cbe9780..61b70705f 100644 --- a/crates/modelardb_compression/src/types.rs +++ b/crates/modelardb_compression/src/types.rs @@ -27,7 +27,7 @@ use modelardb_types::types::{ ErrorBound, Timestamp, TimestampArray, TimestampBuilder, Value, ValueArray, ValueBuilder, }; -use crate::models::gorilla::Gorilla; +use crate::models::macaque_v::MacaqueV; use crate::models::pmc_mean::PMCMean; use crate::models::swing::Swing; use crate::models::{PMC_MEAN_ID, SWING_ID}; @@ -271,7 +271,7 @@ impl CompressedSegmentBuilder { error_bound: ErrorBound, uncompressed_residuals: &[Value], ) -> (Vec, Value, Value) { - let mut gorilla = Gorilla::new(error_bound); + let mut gorilla = MacaqueV::new(error_bound); gorilla.compress_values_without_first(uncompressed_residuals, self.model_last_value); gorilla.model() } From e96a275b8e7b5c888fea43bf97ebffc5b2e1b31a Mon Sep 17 00:00:00 2001 From: aabduvakhobov Date: Mon, 4 Aug 2025 15:47:38 +0200 Subject: [PATCH 02/15] Improve comments and remove redundant code --- .../src/models/macaque_v.rs | 34 ++++++++++--------- .../modelardb_compression/src/models/mod.rs | 4 +-- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/crates/modelardb_compression/src/models/macaque_v.rs b/crates/modelardb_compression/src/models/macaque_v.rs index 7524b3e1d..3dfd132fe 100644 --- a/crates/modelardb_compression/src/models/macaque_v.rs +++ b/crates/modelardb_compression/src/models/macaque_v.rs @@ -13,20 +13,21 @@ * limitations under the License. */ -//! Implementation of MacaqueV model type which extends the lossless +//! Implementation of MacaqueV model type which extends the lossless //! compression method for floating-point values proposed for the time series //! management system Gorilla in the [Gorilla paper] by 1) adding support for -//! error-bounded lossy compression, and 2) optimizing flag bits for better -//! compression of real-life sensor data. MacaqueV adds support for lossy -//! compression by 1) replacing values with the previous value if possible -//! within the error bound, and 2) rewriting the least mantissa bits of a value -//! within the error bound so that Gorilla uses fewer bits for encoding. -//! MacaqueV optimizes Gorilla's flag bits by swapping the flag bits 0 and 10. -//! This modification proved to be effective when Gorilla is used alongside -//! PMC-Mean and Swing for multi-model compression. As this compression method -//! compresses the values of a time series segment using XOR and a -//! variable length binary encoding, aggregates are computed by iterating -//! over all values in the segment. +//! error-bounded lossy compression, and 2) optimizing flag bits for better +//! compression of real-life sensor data. MacaqueV adds support for lossy +//! compression by 1) replacing a value with the previous value if possible +//! within the error bound, or 2) rewriting the least mantissa bits of the value +//! within the error bound so that Gorilla uses fewer bits for encoding. +//! MacaqueV optimizes Gorilla's flag bits by swapping the flag bits 0 and 10. +//! This modification proved to be effective when Gorilla is used alongside +//! PMC-Mean and Swing for multi-model compression. As this compression method +//! uses Gorilla that compresses the values of a time series segment using +//! XOR and a variable length binary encoding, aggregates are computed by +//! iterating over all values in the segment. A paper describing MacaqueV +//! was submitted to ICDE 2026. //! //! [Gorilla paper]: https://www.vldb.org/pvldb/vol8/p1816-teller.pdf @@ -318,15 +319,16 @@ pub fn grid( } } +// Extract the unbiased exponent from single precision float. fn get_exponent(value: f32) -> i32 { let n_bits: u32 = value.to_bits(); let exponent_ = ((n_bits >> 23) & 0xff) as i32; - let exponent = exponent_ - 127; - exponent + exponent_ - 127 } -fn rewrite_bits_by_n(bits_to_rewrite: u32, erase_by_n: i32) -> u32 { - let mask = u32::MAX << erase_by_n; +// Left shift unsigned 32-bit integer by a given number of places. +fn rewrite_bits_by_n(bits_to_rewrite: u32, left_shift_by: i32) -> u32 { + let mask = u32::MAX << left_shift_by; bits_to_rewrite & mask } diff --git a/crates/modelardb_compression/src/models/mod.rs b/crates/modelardb_compression/src/models/mod.rs index 2eab3718b..d5e2e39b4 100644 --- a/crates/modelardb_compression/src/models/mod.rs +++ b/crates/modelardb_compression/src/models/mod.rs @@ -90,8 +90,8 @@ pub fn maximum_allowed_deviation(error_bound: ErrorBound, value: f64) -> f64 { /// Returns true is compression is lossless i.e., `error_bound` is 0. pub fn is_lossless_compression(error_bound: ErrorBound) -> bool { match error_bound { - ErrorBound::Absolute(error_bound) => return error_bound == 0.0, - ErrorBound::Relative(error_bound) => return error_bound == 0.0, + ErrorBound::Absolute(error_bound) => error_bound == 0.0, + ErrorBound::Relative(error_bound) => error_bound == 0.0, } } From f0bdccdbfafb2b4604c522dd3e0fd4a021595277 Mon Sep 17 00:00:00 2001 From: aabduvakhobov Date: Mon, 4 Aug 2025 18:50:43 +0200 Subject: [PATCH 03/15] Ensure all comments for Gorilla are consistent --- .../modelardb_compression/src/compression.rs | 6 +-- .../src/models/macaque_v.rs | 41 +++++++++---------- crates/modelardb_compression/src/types.rs | 6 +-- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/crates/modelardb_compression/src/compression.rs b/crates/modelardb_compression/src/compression.rs index 6e4afb883..1f61409bb 100644 --- a/crates/modelardb_compression/src/compression.rs +++ b/crates/modelardb_compression/src/compression.rs @@ -238,10 +238,10 @@ fn compress_and_store_residuals_in_a_separate_segment( // Compute metadata and compress the values stored in this segment without residuals. let uncompressed_values = &uncompressed_values.values()[start_index..=end_index]; - let mut gorilla = MacaqueV::new(error_bound); - gorilla.compress_values(uncompressed_values); + let mut macaque_v = MacaqueV::new(error_bound); + macaque_v.compress_values(uncompressed_values); - let (values, min_value, max_value) = gorilla.model(); + let (values, min_value, max_value) = macaque_v.model(); compressed_segment_batch_builder.append_compressed_segment( MACAQUE_V_ID, diff --git a/crates/modelardb_compression/src/models/macaque_v.rs b/crates/modelardb_compression/src/models/macaque_v.rs index 3dfd132fe..3c2c7f814 100644 --- a/crates/modelardb_compression/src/models/macaque_v.rs +++ b/crates/modelardb_compression/src/models/macaque_v.rs @@ -37,7 +37,7 @@ use crate::models; use crate::models::bits::{BitReader, BitVecBuilder}; use crate::models::ErrorBound; -/// The state the Gorilla model type needs while compressing the values of a +/// The state the MacaqueV model type needs while compressing the values of a /// time series segment. pub struct MacaqueV { /// Maximum relative error for the value of each data point. @@ -74,7 +74,7 @@ impl MacaqueV { } } - /// Store the first value in full if this instance of [`Gorilla`] is empty and then compress the + /// Store the first value in full if this instance of [`MacaqueV`] is empty and then compress the /// remaining `values` using XOR and a variable length binary encoding before storing them. pub fn compress_values(&mut self, values: &[Value]) { for value in values { @@ -106,7 +106,7 @@ impl MacaqueV { value } else { // If compression is lossy we try to rewrite value first. - // The best case for Gorilla is storing duplicate values. + // The best case for MacaqueV is storing duplicate values. if models::is_value_within_error_bound(self.error_bound, value, self.last_value) { self.last_value } else { @@ -212,13 +212,12 @@ impl MacaqueV { } /// Compute the sum of the values for a time series segment whose values are compressed using -/// Gorilla's compression method for floating-point values. If `maybe_model_last_value` is provided, -/// it is assumed the first value in `values` is compressed against it instead of being stored in -/// full, i.e., uncompressed. +/// MacaqueV. If `maybe_model_last_value` is provided, it is assumed the first value in `values` +/// is compressed against it instead of being stored in full, i.e., uncompressed. pub fn sum(length: usize, values: &[u8], maybe_model_last_value: Option) -> Value { - // This function replicates code from gorilla::grid() as it isn't necessary to store the + // This function replicates code from macaque_v::grid() as it isn't necessary to store the // timestamps and values in arrays for a sum. So any changes to the decompression must be - // mirrored in gorilla::grid(). + // mirrored in macaque_v::grid(). let mut bits = BitReader::try_new(values).unwrap(); let mut leading_zeros = u8::MAX; let mut trailing_zeros: u8 = 0; @@ -263,8 +262,8 @@ pub fn sum(length: usize, values: &[u8], maybe_model_last_value: Option) } /// Decompress all the values in `values` for the `timestamps` without matching values in -/// `value_builder`. The values in `values` are compressed using Gorilla's compression method for -/// floating-point values. `values` are appended to `value_builder`. If `maybe_model_last_value` +/// `value_builder`. The values in `values` are compressed using MacaqueV. +/// `values` are appended to `value_builder`. If `maybe_model_last_value` /// is provided, it is assumed the first value in `values` is compressed against it instead of being /// stored in full, i.e., uncompressed. pub fn grid( @@ -273,7 +272,7 @@ pub fn grid( value_builder: &mut ValueBuilder, maybe_model_last_value: Option, ) { - // Changes to the decompression must be mirrored in gorilla::sum(). + // Changes to the decompression must be mirrored in macaque_v::sum(). // unwrap() is safe as values is from a segment and thus cannot be empty. let mut bits = BitReader::try_new(values).unwrap(); let mut leading_zeros = u8::MAX; @@ -342,7 +341,7 @@ mod tests { use crate::models; - // Tests for Gorilla. + // Tests for MacaqueV. #[test] fn test_empty_sequence_with_absolute_error_bound_zero() { let error_bound = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); @@ -494,7 +493,7 @@ mod tests { fn test_sum_with_absolute_error_bound_zero(values in collection::vec(ProptestValue::ANY, 0..50)) { prop_assume!(!values.is_empty()); let expected_sum = values.iter().sum::(); - let compressed_values = compress_values_using_gorilla( + let compressed_values = compress_values_using_macaque_v( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), &values, None); let sum = sum(values.len(), &compressed_values, None); @@ -505,7 +504,7 @@ mod tests { fn test_sum_with_relative_error_bound_zero(values in collection::vec(ProptestValue::ANY, 0..50)) { prop_assume!(!values.is_empty()); let expected_sum = values.iter().sum::(); - let compressed_values = compress_values_using_gorilla( + let compressed_values = compress_values_using_macaque_v( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), &values, None); let sum = sum(values.len(), &compressed_values, None); @@ -515,7 +514,7 @@ mod tests { #[test] fn test_sum_model_single_value_with_absolute_error_bound_zero() { - let compressed_values = compress_values_using_gorilla( + let compressed_values = compress_values_using_macaque_v( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), &[37.0], None, @@ -526,7 +525,7 @@ mod tests { #[test] fn test_sum_model_single_value_with_relative_error_bound_zero() { - let compressed_values = compress_values_using_gorilla( + let compressed_values = compress_values_using_macaque_v( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), &[37.0], None, @@ -538,7 +537,7 @@ mod tests { #[test] fn test_sum_residuals_single_value_with_absolute_error_bound_zero() { let maybe_model_last_value = Some(37.0); - let compressed_values = compress_values_using_gorilla( + let compressed_values = compress_values_using_macaque_v( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), &[37.0], maybe_model_last_value, @@ -550,7 +549,7 @@ mod tests { #[test] fn test_sum_residuals_single_value_with_relative_error_bound_zero() { let maybe_model_last_value = Some(37.0); - let compressed_values = compress_values_using_gorilla( + let compressed_values = compress_values_using_macaque_v( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), &[37.0], maybe_model_last_value, @@ -579,7 +578,7 @@ mod tests { } fn assert_grid_with_error_bound(error_bound: ErrorBound, values: &[Value]) { - let compressed_values = compress_values_using_gorilla(error_bound, values, None); + let compressed_values = compress_values_using_macaque_v(error_bound, values, None); let timestamps: Vec = (1..=values.len() as i64).step_by(1).collect(); let mut value_builder = ValueBuilder::with_capacity(values.len()); @@ -628,7 +627,7 @@ mod tests { fn assert_grid_single(error_bound: ErrorBound, maybe_model_last_value: Option) { let compressed_values = - compress_values_using_gorilla(error_bound, &[37.0], maybe_model_last_value); + compress_values_using_macaque_v(error_bound, &[37.0], maybe_model_last_value); let mut value_builder = ValueBuilder::new(); grid( @@ -644,7 +643,7 @@ mod tests { assert_eq!(values.value(0), 37.0); } - fn compress_values_using_gorilla( + fn compress_values_using_macaque_v( error_bound: ErrorBound, values: &[Value], maybe_model_last_value: Option, diff --git a/crates/modelardb_compression/src/types.rs b/crates/modelardb_compression/src/types.rs index 61b70705f..f12403818 100644 --- a/crates/modelardb_compression/src/types.rs +++ b/crates/modelardb_compression/src/types.rs @@ -271,9 +271,9 @@ impl CompressedSegmentBuilder { error_bound: ErrorBound, uncompressed_residuals: &[Value], ) -> (Vec, Value, Value) { - let mut gorilla = MacaqueV::new(error_bound); - gorilla.compress_values_without_first(uncompressed_residuals, self.model_last_value); - gorilla.model() + let mut macaque_v = MacaqueV::new(error_bound); + macaque_v.compress_values_without_first(uncompressed_residuals, self.model_last_value); + macaque_v.model() } /// Encode the information required for a [`PMCMean`] model where the `residuals_min_value` From a89722734ec944c2c8069864a59917bb920e3f60 Mon Sep 17 00:00:00 2001 From: aabduvakhobov Date: Mon, 4 Aug 2025 18:55:27 +0200 Subject: [PATCH 04/15] Ensure formatting is consistent --- .../modelardb_compression/src/models/macaque_v.rs | 14 +++++++------- crates/modelardb_compression/src/types.rs | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/modelardb_compression/src/models/macaque_v.rs b/crates/modelardb_compression/src/models/macaque_v.rs index 3c2c7f814..3e2f77c9c 100644 --- a/crates/modelardb_compression/src/models/macaque_v.rs +++ b/crates/modelardb_compression/src/models/macaque_v.rs @@ -26,7 +26,7 @@ //! PMC-Mean and Swing for multi-model compression. As this compression method //! uses Gorilla that compresses the values of a time series segment using //! XOR and a variable length binary encoding, aggregates are computed by -//! iterating over all values in the segment. A paper describing MacaqueV +//! iterating over all values in the segment. A paper describing MacaqueV //! was submitted to ICDE 2026. //! //! [Gorilla paper]: https://www.vldb.org/pvldb/vol8/p1816-teller.pdf @@ -166,7 +166,7 @@ impl MacaqueV { self.update_min_max_and_last_value(value); } - /// MacaqueV's value rewrite method. + /// MacaqueV's value rewrite method. fn rewrite_value_with_log_method(&self, value: Value) -> Value { if value == 0.0 || value.is_infinite() || value.is_nan() { return value; @@ -176,10 +176,10 @@ impl MacaqueV { models::maximum_allowed_deviation(self.error_bound, value as f64) as f32; let exponent = get_exponent(value); let factorized_epsilon = abs_error_bound / 2f32.powi(exponent); - // Rewriting value using by 23 - ⌈log2 factorized_epsilon⌉ bits - // never exceeds the error bound. However, one more bit can be - // rewritten when the majority of the least mantissa bits are 0, - // thus we use 23 - ⌊log2 factorized_epsilon⌋ and then + // Rewriting value using by 23 - ⌈log2 factorized_epsilon⌉ bits + // never exceeds the error bound. However, one more bit can be + // rewritten when the majority of the least mantissa bits are 0, + // thus we use 23 - ⌊log2 factorized_epsilon⌋ and then // perform extra check if the error bound is not exceeded. let mut rewrite_position = 23 - factorized_epsilon.log2().abs().floor() as i32; // Extra check to ensure if the error is within the error_bound @@ -262,7 +262,7 @@ pub fn sum(length: usize, values: &[u8], maybe_model_last_value: Option) } /// Decompress all the values in `values` for the `timestamps` without matching values in -/// `value_builder`. The values in `values` are compressed using MacaqueV. +/// `value_builder`. The values in `values` are compressed using MacaqueV. /// `values` are appended to `value_builder`. If `maybe_model_last_value` /// is provided, it is assumed the first value in `values` is compressed against it instead of being /// stored in full, i.e., uncompressed. diff --git a/crates/modelardb_compression/src/types.rs b/crates/modelardb_compression/src/types.rs index f12403818..ba629efe2 100644 --- a/crates/modelardb_compression/src/types.rs +++ b/crates/modelardb_compression/src/types.rs @@ -19,7 +19,7 @@ use std::sync::Arc; use std::{debug_assert, iter}; use arrow::array::{ - ArrayBuilder, ArrayRef, BinaryBuilder, Float32Builder, Int8Builder, Int16Array, StringArray, + ArrayBuilder, ArrayRef, BinaryBuilder, Float32Builder, Int16Array, Int8Builder, StringArray, }; use arrow::datatypes::Schema; use arrow::record_batch::RecordBatch; @@ -30,8 +30,8 @@ use modelardb_types::types::{ use crate::models::macaque_v::MacaqueV; use crate::models::pmc_mean::PMCMean; use crate::models::swing::Swing; +use crate::models::{timestamps, VALUE_SIZE_IN_BYTES}; use crate::models::{PMC_MEAN_ID, SWING_ID}; -use crate::models::{VALUE_SIZE_IN_BYTES, timestamps}; /// A model being built from an uncompressed segment using the potentially lossy model types in /// [`models`]. Each of the potentially lossy model types is used to fit models to the data points, From 5e52697f047eed35977460217a9f9f59de5526c5 Mon Sep 17 00:00:00 2001 From: aabduvakhobov Date: Wed, 6 Aug 2025 15:35:41 +0200 Subject: [PATCH 05/15] Add missing punctuations in comments --- .../src/models/macaque_v.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/crates/modelardb_compression/src/models/macaque_v.rs b/crates/modelardb_compression/src/models/macaque_v.rs index 3e2f77c9c..83cec3cbb 100644 --- a/crates/modelardb_compression/src/models/macaque_v.rs +++ b/crates/modelardb_compression/src/models/macaque_v.rs @@ -18,16 +18,15 @@ //! management system Gorilla in the [Gorilla paper] by 1) adding support for //! error-bounded lossy compression, and 2) optimizing flag bits for better //! compression of real-life sensor data. MacaqueV adds support for lossy -//! compression by 1) replacing a value with the previous value if possible +//! compression by 1) rewriting the current value with the previous one if possible //! within the error bound, or 2) rewriting the least mantissa bits of the value -//! within the error bound so that Gorilla uses fewer bits for encoding. +//! to zero within the error bound so that Gorilla uses fewer bits for encoding. //! MacaqueV optimizes Gorilla's flag bits by swapping the flag bits 0 and 10. //! This modification proved to be effective when Gorilla is used alongside //! PMC-Mean and Swing for multi-model compression. As this compression method //! uses Gorilla that compresses the values of a time series segment using //! XOR and a variable length binary encoding, aggregates are computed by -//! iterating over all values in the segment. A paper describing MacaqueV -//! was submitted to ICDE 2026. +//! iterating over all values in the segment. //! //! [Gorilla paper]: https://www.vldb.org/pvldb/vol8/p1816-teller.pdf @@ -101,16 +100,16 @@ impl MacaqueV { /// Compress `value` using XOR and a variable length binary encoding and then store it. fn compress_value_xor_last_value(&mut self, value: Value) { - // Rewrite the value bits using the error bound let value = if models::is_lossless_compression(self.error_bound) { value } else { - // If compression is lossy we try to rewrite value first. - // The best case for MacaqueV is storing duplicate values. + // The best case for MacaqueV is rewriting the current value + // with the previous one. if models::is_value_within_error_bound(self.error_bound, value, self.last_value) { self.last_value } else { - // we only do binary rewriting if value rewriting is not possible + // If value rewriting is not possible, the least mantissa bits of the + // value are rewritten to zero. self.rewrite_value_with_log_method(value) } }; @@ -182,7 +181,7 @@ impl MacaqueV { // thus we use 23 - ⌊log2 factorized_epsilon⌋ and then // perform extra check if the error bound is not exceeded. let mut rewrite_position = 23 - factorized_epsilon.log2().abs().floor() as i32; - // Extra check to ensure if the error is within the error_bound + // Extra check to ensure if the error is within the error_bound. let mut rewritten_value = f32::from_bits(rewrite_bits_by_n(value_as_u32, rewrite_position)); // If the error bound exceeded, value is rewritten with one les bit i.e., 23 − ⌈log2 factorized_epsilon⌉. if !models::is_value_within_error_bound(self.error_bound, value, rewritten_value) { From 309c3edcedf74f0afd77bdbeaa9b99540b9d7fee Mon Sep 17 00:00:00 2001 From: aabduvakhobov Date: Wed, 6 Aug 2025 15:50:36 +0200 Subject: [PATCH 06/15] Ensure bit rewriting method checks for a negative zero --- crates/modelardb_compression/src/models/macaque_v.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/modelardb_compression/src/models/macaque_v.rs b/crates/modelardb_compression/src/models/macaque_v.rs index 83cec3cbb..1ef840a02 100644 --- a/crates/modelardb_compression/src/models/macaque_v.rs +++ b/crates/modelardb_compression/src/models/macaque_v.rs @@ -167,7 +167,7 @@ impl MacaqueV { /// MacaqueV's value rewrite method. fn rewrite_value_with_log_method(&self, value: Value) -> Value { - if value == 0.0 || value.is_infinite() || value.is_nan() { + if value.abs() == 0.0 || value.is_infinite() || value.is_nan() { return value; } let value_as_u32 = value.to_bits(); From 6516d7aa58e81f2f17d2d90331acf9799f56114b Mon Sep 17 00:00:00 2001 From: aabduvakhobov Date: Thu, 7 Aug 2025 11:13:35 +0200 Subject: [PATCH 07/15] Run rustfmt with --edition 2024 --- .../modelardb_compression/src/compression.rs | 84 +++++++++---------- .../src/models/macaque_v.rs | 12 +-- crates/modelardb_compression/src/types.rs | 4 +- 3 files changed, 51 insertions(+), 49 deletions(-) diff --git a/crates/modelardb_compression/src/compression.rs b/crates/modelardb_compression/src/compression.rs index 1f61409bb..455daf0c2 100644 --- a/crates/modelardb_compression/src/compression.rs +++ b/crates/modelardb_compression/src/compression.rs @@ -25,7 +25,7 @@ use modelardb_types::types::{ErrorBound, TimestampArray, ValueArray}; use crate::error::{ModelarDbCompressionError, Result}; use crate::models::macaque_v::MacaqueV; -use crate::models::{self, timestamps, MACAQUE_V_ID}; +use crate::models::{self, MACAQUE_V_ID, timestamps}; use crate::types::{CompressedSegmentBatchBuilder, CompressedSegmentBuilder, ModelBuilder}; /// Maximum number of residuals that can be stored as part of a compressed segment. The number of @@ -269,7 +269,7 @@ mod tests { use modelardb_types::schemas::COMPRESSED_SCHEMA; use modelardb_types::types::{TimestampBuilder, ValueBuilder}; - use crate::{models, MODEL_TYPE_NAMES}; + use crate::{MODEL_TYPE_NAMES, models}; const TAG_VALUE: &str = "tag"; const ADD_NOISE_RANGE: Option> = Some(1.0..1.05); @@ -534,8 +534,8 @@ mod tests { } #[test] - fn test_try_compress_regular_random_linear_constant_time_series_within_absolute_error_bound_zero( - ) { + fn test_try_compress_regular_random_linear_constant_time_series_within_absolute_error_bound_zero() + { generate_compress_and_assert_known_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), false, @@ -545,8 +545,8 @@ mod tests { } #[test] - fn test_try_compress_regular_random_linear_constant_time_series_within_relative_error_bound_zero( - ) { + fn test_try_compress_regular_random_linear_constant_time_series_within_relative_error_bound_zero() + { generate_compress_and_assert_known_time_series( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), false, @@ -556,8 +556,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_random_linear_constant_time_series_within_absolute_error_bound_zero( - ) { + fn test_try_compress_irregular_random_linear_constant_time_series_within_absolute_error_bound_zero() + { generate_compress_and_assert_known_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), true, @@ -567,8 +567,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_random_linear_constant_time_series_within_relative_error_bound_zero( - ) { + fn test_try_compress_irregular_random_linear_constant_time_series_within_relative_error_bound_zero() + { generate_compress_and_assert_known_time_series( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), true, @@ -578,8 +578,8 @@ mod tests { } #[test] - fn test_try_compress_regular_constant_linear_random_time_series_within_absolute_error_bound_zero( - ) { + fn test_try_compress_regular_constant_linear_random_time_series_within_absolute_error_bound_zero() + { generate_compress_and_assert_known_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), false, @@ -589,8 +589,8 @@ mod tests { } #[test] - fn test_try_compress_regular_constant_linear_random_time_series_within_relative_error_bound_zero( - ) { + fn test_try_compress_regular_constant_linear_random_time_series_within_relative_error_bound_zero() + { generate_compress_and_assert_known_time_series( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), false, @@ -600,8 +600,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_constant_linear_random_time_series_within_absolute_error_bound_zero( - ) { + fn test_try_compress_irregular_constant_linear_random_time_series_within_absolute_error_bound_zero() + { generate_compress_and_assert_known_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), true, @@ -611,8 +611,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_constant_linear_random_time_series_within_relative_error_bound_zero( - ) { + fn test_try_compress_irregular_constant_linear_random_time_series_within_relative_error_bound_zero() + { generate_compress_and_assert_known_time_series( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), true, @@ -707,8 +707,8 @@ mod tests { } #[test] - fn test_try_compress_regular_synthetic_time_series_without_noise_within_absolute_error_bound_zero( - ) { + fn test_try_compress_regular_synthetic_time_series_without_noise_within_absolute_error_bound_zero() + { generate_compress_and_assert_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), false, @@ -717,8 +717,8 @@ mod tests { } #[test] - fn test_try_compress_regular_synthetic_time_series_without_noise_within_relative_error_bound_zero( - ) { + fn test_try_compress_regular_synthetic_time_series_without_noise_within_relative_error_bound_zero() + { generate_compress_and_assert_time_series( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), false, @@ -727,8 +727,8 @@ mod tests { } #[test] - fn test_try_compress_regular_synthetic_time_series_without_noise_within_absolute_error_bound_five( - ) { + fn test_try_compress_regular_synthetic_time_series_without_noise_within_absolute_error_bound_five() + { generate_compress_and_assert_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_FIVE).unwrap(), false, @@ -737,8 +737,8 @@ mod tests { } #[test] - fn test_try_compress_regular_synthetic_time_series_without_noise_within_relative_error_bound_five( - ) { + fn test_try_compress_regular_synthetic_time_series_without_noise_within_relative_error_bound_five() + { generate_compress_and_assert_time_series( ErrorBound::try_new_relative(ERROR_BOUND_FIVE).unwrap(), false, @@ -787,8 +787,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_synthetic_time_series_without_noise_within_absolute_error_bound_zero( - ) { + fn test_try_compress_irregular_synthetic_time_series_without_noise_within_absolute_error_bound_zero() + { generate_compress_and_assert_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), true, @@ -797,8 +797,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_synthetic_time_series_without_noise_within_relative_error_bound_zero( - ) { + fn test_try_compress_irregular_synthetic_time_series_without_noise_within_relative_error_bound_zero() + { generate_compress_and_assert_time_series( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), true, @@ -807,8 +807,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_synthetic_time_series_without_noise_within_absolute_error_bound_five( - ) { + fn test_try_compress_irregular_synthetic_time_series_without_noise_within_absolute_error_bound_five() + { generate_compress_and_assert_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_FIVE).unwrap(), true, @@ -817,8 +817,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_synthetic_time_series_without_noise_within_relative_error_bound_five( - ) { + fn test_try_compress_irregular_synthetic_time_series_without_noise_within_relative_error_bound_five() + { generate_compress_and_assert_time_series( ErrorBound::try_new_relative(ERROR_BOUND_FIVE).unwrap(), true, @@ -827,8 +827,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_synthetic_time_series_with_noise_within_absolute_error_bound_zero( - ) { + fn test_try_compress_irregular_synthetic_time_series_with_noise_within_absolute_error_bound_zero() + { generate_compress_and_assert_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), true, @@ -837,8 +837,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_synthetic_time_series_with_noise_within_relative_error_bound_zero( - ) { + fn test_try_compress_irregular_synthetic_time_series_with_noise_within_relative_error_bound_zero() + { generate_compress_and_assert_time_series( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), true, @@ -847,8 +847,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_synthetic_time_series_with_noise_within_absolute_error_bound_five( - ) { + fn test_try_compress_irregular_synthetic_time_series_with_noise_within_absolute_error_bound_five() + { generate_compress_and_assert_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_FIVE).unwrap(), true, @@ -857,8 +857,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_synthetic_time_series_with_noise_within_relative_error_bound_five( - ) { + fn test_try_compress_irregular_synthetic_time_series_with_noise_within_relative_error_bound_five() + { generate_compress_and_assert_time_series( ErrorBound::try_new_relative(ERROR_BOUND_FIVE).unwrap(), true, diff --git a/crates/modelardb_compression/src/models/macaque_v.rs b/crates/modelardb_compression/src/models/macaque_v.rs index 1ef840a02..233332c6e 100644 --- a/crates/modelardb_compression/src/models/macaque_v.rs +++ b/crates/modelardb_compression/src/models/macaque_v.rs @@ -33,8 +33,8 @@ use modelardb_types::types::{Timestamp, Value, ValueBuilder}; use crate::models; -use crate::models::bits::{BitReader, BitVecBuilder}; use crate::models::ErrorBound; +use crate::models::bits::{BitReader, BitVecBuilder}; /// The state the MacaqueV model type needs while compressing the values of a /// time series segment. @@ -108,7 +108,7 @@ impl MacaqueV { if models::is_value_within_error_bound(self.error_bound, value, self.last_value) { self.last_value } else { - // If value rewriting is not possible, the least mantissa bits of the + // If value rewriting is not possible, the least mantissa bits of the // value are rewritten to zero. self.rewrite_value_with_log_method(value) } @@ -586,9 +586,11 @@ mod tests { let values_array = value_builder.finish(); assert!(values.len() == timestamps.len() && values.len() == values_array.len()); - assert!(timestamps - .windows(2) - .all(|window| window[1] - window[0] == 1)); + assert!( + timestamps + .windows(2) + .all(|window| window[1] - window[0] == 1) + ); assert!(slice_of_value_equal(values_array.values(), values)); } diff --git a/crates/modelardb_compression/src/types.rs b/crates/modelardb_compression/src/types.rs index ba629efe2..f12403818 100644 --- a/crates/modelardb_compression/src/types.rs +++ b/crates/modelardb_compression/src/types.rs @@ -19,7 +19,7 @@ use std::sync::Arc; use std::{debug_assert, iter}; use arrow::array::{ - ArrayBuilder, ArrayRef, BinaryBuilder, Float32Builder, Int16Array, Int8Builder, StringArray, + ArrayBuilder, ArrayRef, BinaryBuilder, Float32Builder, Int8Builder, Int16Array, StringArray, }; use arrow::datatypes::Schema; use arrow::record_batch::RecordBatch; @@ -30,8 +30,8 @@ use modelardb_types::types::{ use crate::models::macaque_v::MacaqueV; use crate::models::pmc_mean::PMCMean; use crate::models::swing::Swing; -use crate::models::{timestamps, VALUE_SIZE_IN_BYTES}; use crate::models::{PMC_MEAN_ID, SWING_ID}; +use crate::models::{VALUE_SIZE_IN_BYTES, timestamps}; /// A model being built from an uncompressed segment using the potentially lossy model types in /// [`models`]. Each of the potentially lossy model types is used to fit models to the data points, From 77e5eb92a5a929accc8f42d55e83df3e88f405b1 Mon Sep 17 00:00:00 2001 From: aabduvakhobov Date: Thu, 7 Aug 2025 12:51:05 +0200 Subject: [PATCH 08/15] Ensure comments are consistent as per CGodiksen's feedback --- .../src/models/macaque_v.rs | 62 +++++++++---------- .../modelardb_compression/src/models/mod.rs | 2 +- crates/modelardb_compression/src/types.rs | 8 +-- 3 files changed, 34 insertions(+), 38 deletions(-) diff --git a/crates/modelardb_compression/src/models/macaque_v.rs b/crates/modelardb_compression/src/models/macaque_v.rs index 233332c6e..9eeca9b84 100644 --- a/crates/modelardb_compression/src/models/macaque_v.rs +++ b/crates/modelardb_compression/src/models/macaque_v.rs @@ -13,28 +13,25 @@ * limitations under the License. */ -//! Implementation of MacaqueV model type which extends the lossless -//! compression method for floating-point values proposed for the time series -//! management system Gorilla in the [Gorilla paper] by 1) adding support for -//! error-bounded lossy compression, and 2) optimizing flag bits for better -//! compression of real-life sensor data. MacaqueV adds support for lossy -//! compression by 1) rewriting the current value with the previous one if possible -//! within the error bound, or 2) rewriting the least mantissa bits of the value -//! to zero within the error bound so that Gorilla uses fewer bits for encoding. -//! MacaqueV optimizes Gorilla's flag bits by swapping the flag bits 0 and 10. -//! This modification proved to be effective when Gorilla is used alongside -//! PMC-Mean and Swing for multi-model compression. As this compression method -//! uses Gorilla that compresses the values of a time series segment using -//! XOR and a variable length binary encoding, aggregates are computed by -//! iterating over all values in the segment. +//! Implementation of the MacaqueV model type which extends the lossless compression method for +//! floating-point values proposed for the time series management system Gorilla in the [Gorilla +//! paper] by 1) adding support for error-bounded lossy compression, and 2) optimizing flag bits +//! for better compression of real-life sensor data. MacaqueV adds support for lossy compression by +//! 1) rewriting the current value with the previous one if possible within the error bound, or +//! 2) rewriting the least mantissa bits of the value to zero within the error bound so that Gorilla +//! uses fewer bits for encoding. MacaqueV optimizes Gorilla's flag bits by swapping the flag bits +//! 0 and 10. This modification proved to be effective when Gorilla is used alongside PMC-Mean +//! and Swing for multi-model compression. As this compression method uses Gorilla that compresses +//! the values of a time series segment using XOR and a variable length binary encoding, aggregates +//! are computed by iterating over all values in the segment. //! //! [Gorilla paper]: https://www.vldb.org/pvldb/vol8/p1816-teller.pdf use modelardb_types::types::{Timestamp, Value, ValueBuilder}; use crate::models; -use crate::models::ErrorBound; use crate::models::bits::{BitReader, BitVecBuilder}; +use crate::models::ErrorBound; /// The state the MacaqueV model type needs while compressing the values of a /// time series segment. @@ -103,13 +100,12 @@ impl MacaqueV { let value = if models::is_lossless_compression(self.error_bound) { value } else { - // The best case for MacaqueV is rewriting the current value - // with the previous one. + // The best case for MacaqueV is rewriting the current value with the previous one. if models::is_value_within_error_bound(self.error_bound, value, self.last_value) { self.last_value } else { - // If value rewriting is not possible, the least mantissa bits of the - // value are rewritten to zero. + // If rewriting the value is not possible, the least mantissa bits of the value + // are rewritten to zero. self.rewrite_value_with_log_method(value) } }; @@ -131,7 +127,7 @@ impl MacaqueV { if leading_zero_bits >= self.last_leading_zero_bits && trailing_zero_bits >= self.last_trailing_zero_bits { - // Store only the meaningful bits after a flag bit zero. + // Store only the meaningful bits after a flag zero bit. self.compressed_values.append_a_zero_bit(); let meaningful_bits = models::VALUE_SIZE_IN_BITS - self.last_leading_zero_bits @@ -165,29 +161,31 @@ impl MacaqueV { self.update_min_max_and_last_value(value); } - /// MacaqueV's value rewrite method. + /// Rewrite least mantissa bits of the `value` to zero within the `error_bound`. fn rewrite_value_with_log_method(&self, value: Value) -> Value { if value.abs() == 0.0 || value.is_infinite() || value.is_nan() { return value; } + let value_as_u32 = value.to_bits(); let abs_error_bound = models::maximum_allowed_deviation(self.error_bound, value as f64) as f32; let exponent = get_exponent(value); let factorized_epsilon = abs_error_bound / 2f32.powi(exponent); - // Rewriting value using by 23 - ⌈log2 factorized_epsilon⌉ bits - // never exceeds the error bound. However, one more bit can be - // rewritten when the majority of the least mantissa bits are 0, - // thus we use 23 - ⌊log2 factorized_epsilon⌋ and then - // perform extra check if the error bound is not exceeded. + // Rewriting value using 23 - ⌈log2 factorized_epsilon⌉ bits never exceeds the error bound. + // However, one more bit can be rewritten when the majority of the least mantissa bits + // are 0. Thus we use 23 - ⌊log2 factorized_epsilon⌋ and then perform an extra check + // to ensure if the error bound is not exceeded. let mut rewrite_position = 23 - factorized_epsilon.log2().abs().floor() as i32; - // Extra check to ensure if the error is within the error_bound. let mut rewritten_value = f32::from_bits(rewrite_bits_by_n(value_as_u32, rewrite_position)); - // If the error bound exceeded, value is rewritten with one les bit i.e., 23 − ⌈log2 factorized_epsilon⌉. + + // If the error bound is exceeded, value is rewritten with one less bit i.e., + // using 23 − ⌈log2 factorized_epsilon⌉. if !models::is_value_within_error_bound(self.error_bound, value, rewritten_value) { rewrite_position -= 1; rewritten_value = f32::from_bits(rewrite_bits_by_n(value.to_bits(), rewrite_position)); } + rewritten_value } @@ -586,11 +584,9 @@ mod tests { let values_array = value_builder.finish(); assert!(values.len() == timestamps.len() && values.len() == values_array.len()); - assert!( - timestamps - .windows(2) - .all(|window| window[1] - window[0] == 1) - ); + assert!(timestamps + .windows(2) + .all(|window| window[1] - window[0] == 1)); assert!(slice_of_value_equal(values_array.values(), values)); } diff --git a/crates/modelardb_compression/src/models/mod.rs b/crates/modelardb_compression/src/models/mod.rs index d5e2e39b4..2c0f0f46a 100644 --- a/crates/modelardb_compression/src/models/mod.rs +++ b/crates/modelardb_compression/src/models/mod.rs @@ -87,7 +87,7 @@ pub fn maximum_allowed_deviation(error_bound: ErrorBound, value: f64) -> f64 { } } -/// Returns true is compression is lossless i.e., `error_bound` is 0. +/// Returns true if compression is lossless i.e., `error_bound` value is 0. pub fn is_lossless_compression(error_bound: ErrorBound) -> bool { match error_bound { ErrorBound::Absolute(error_bound) => error_bound == 0.0, diff --git a/crates/modelardb_compression/src/types.rs b/crates/modelardb_compression/src/types.rs index f12403818..00a3c6033 100644 --- a/crates/modelardb_compression/src/types.rs +++ b/crates/modelardb_compression/src/types.rs @@ -215,7 +215,7 @@ impl CompressedSegmentBuilder { &uncompressed_timestamps.values()[self.start_index..=residuals_end_index], ); - // Compress residual values using Gorilla if any exists. + // Compress residual values using MacaqueV if any exists. let residuals = if self.end_index < residuals_end_index { let residuals_start_index = self.end_index + 1; @@ -265,7 +265,7 @@ impl CompressedSegmentBuilder { ) } - /// Compress `uncompressed_residuals` within `error_bound` using [`Gorilla`]. + /// Compress `uncompressed_residuals` within `error_bound` using [`MacaqueV`]. fn compress_residuals( &self, error_bound: ErrorBound, @@ -423,7 +423,7 @@ pub(crate) struct CompressedSegmentBatchBuilder { /// the values of each compressed segment in the batch within an error /// bound. values: BinaryBuilder, - /// Values between this and the next segment, compressed using [`Gorilla`], + /// Values between this and the next segment, compressed using [`MacaqueV`], /// that the models could not represent efficiently within the error bound /// and which are too few for a new segment due to the amount of metadata. residuals: BinaryBuilder, @@ -808,7 +808,7 @@ mod tests { let model_end_index = model.end_index; // Create a segment that represents its values using a model of the expected type and its - // residuals using Gorilla, and then assert that the expected encoding is used for it. + // residuals using MacaqueV, and then assert that the expected encoding is used for it. let residuals_end_index = uncompressed_timestamps.len() - 1; let mut compressed_schema_fields = COMPRESSED_SCHEMA.0.fields.clone().to_vec(); From 8a2e39cf85efd0838ea845583c6a66311debcbe0 Mon Sep 17 00:00:00 2001 From: aabduvakhobov Date: Thu, 7 Aug 2025 14:01:53 +0200 Subject: [PATCH 09/15] Add comments for MacaqueTS --- .../src/models/timestamps.rs | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/crates/modelardb_compression/src/models/timestamps.rs b/crates/modelardb_compression/src/models/timestamps.rs index 4ae41c5ab..c445563ba 100644 --- a/crates/modelardb_compression/src/models/timestamps.rs +++ b/crates/modelardb_compression/src/models/timestamps.rs @@ -13,22 +13,26 @@ * limitations under the License. */ -//! Implementation of lossless compression for timestamps. Optimized compression -//! methods are used depending on the number of data points in a compressed -//! segment and if its timestamps have been sampled at a regular sampling -//! interval. +//! Implementation of MacaqueTS, a lossless compression for timestamps. MacaqueTS uses +//! optimized compression methods depending on the number of data points in a compressed segment +//! and if its timestamps have been sampled at a regular sampling interval. +//! +//! If a segment only contains one data point, its timestamp is stored as both the segment's +//! `start_time` and `end_time`, and if a segment only contains two data points, +//! the timestamps are stored as the segment's `start_time` and `end_time`, respectively. +//! If a segment contains more than two data points, the first and last timestamps +//! are stored as the segment's `start_time` and `end_time`, respectively, while its +//! residual timestamps are compressed using one of two methods. If the data points +//! in the segment have been collected at a regular sampling interval, the residual timestamps +//! are compressed as the segment's length with the prefix zero bits stripped. +//! If none of the above apply, an extended version of the compression method proposed +//! for timestamps for the time series management system Gorilla in the [Gorilla paper] +//! is used as a fallback. MacaqueTS extends Gorilla's timestamp compression method by +//! using zero as the first delta instead of computing it explicitly. This avoids the need +//! to encode the first delta as a special case. MacaqueTS also extends the flag ranges +//! used by Gorilla for integer bit-packing to support encoding finer granularity timestamps +//! with a sampling interval of lower than one second. //! -//! If a segment only contains one data point its timestamp is stored as both -//! the segment's `start_time` and `end_time`, and if a segment only contains -//! two data points the timestamps are stored as the segment's `start_time` and -//! `end_time`, respectively. If a segment contains more than two data points, -//! the first and last timestamps are stored as the segment's `start_time` and -//! `end_time`, respectively, while its residual timestamps are compressed using -//! one of two methods. If the data points in the segment have been collected at -//! a regular sampling interval, the residual timestamps are compressed as the -//! segment's length with the prefix zero bits stripped. If none of the above -//! apply, the compression method proposed for timestamps for the time series -//! management system Gorilla in the [Gorilla paper] is used as a fallback. //! //! [Gorilla paper]: https://www.vldb.org/pvldb/vol8/p1816-teller.pdf @@ -150,8 +154,9 @@ fn compress_irregular_residual_timestamps(uncompressed_timestamps: &[Timestamp]) /// Decompress all of a segment's timestamps which are compressed as /// `start_time` for segments of length one, `start_time` and `end_time` for /// segments of length two, the segment's length for regular time series, or -/// using Gorilla's compression method for timestamps for irregular time series. -/// The decompressed timestamps are appended to `timestamp_builder`. +/// using extended version of Gorilla's compression method for timestamps +/// for irregular time series. The decompressed timestamps are appended +/// to `timestamp_builder`. pub fn decompress_all_timestamps( start_time: Timestamp, end_time: Timestamp, @@ -215,9 +220,8 @@ fn decompress_all_regular_timestamps( } /// Decompress all of a segment's timestamps, which for this segment are sampled -/// at an irregular sampling interval, and thus compressed using Gorilla's -/// compression method for timestamps. The decompressed timestamps are appended -/// to `timestamp_builder`. +/// at an irregular sampling interval, and thus compressed using MacaqueTS. +/// The decompressed timestamps are appended to `timestamp_builder`. fn decompress_all_irregular_timestamps( start_time: Timestamp, end_time: Timestamp, @@ -227,7 +231,7 @@ fn decompress_all_irregular_timestamps( // Add the first timestamp stored as `start_time` in the segment. timestamp_builder.append_value(start_time); - // Remove the one bit used as a flag to specify that Gorilla is used. + // Remove the one bit used as a flag to specify that MacaqueTS is used. let mut bits = BitReader::try_new(residual_timestamps).unwrap(); bits.read_bit(); From f13f052ef52b826231f3aee0c6d1211acf57293a Mon Sep 17 00:00:00 2001 From: aabduvakhobov Date: Thu, 7 Aug 2025 15:29:13 +0200 Subject: [PATCH 10/15] Remove listing in MacaqueV documentation --- crates/modelardb_compression/src/models/macaque_v.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/modelardb_compression/src/models/macaque_v.rs b/crates/modelardb_compression/src/models/macaque_v.rs index 9eeca9b84..1b3adcaac 100644 --- a/crates/modelardb_compression/src/models/macaque_v.rs +++ b/crates/modelardb_compression/src/models/macaque_v.rs @@ -15,12 +15,12 @@ //! Implementation of the MacaqueV model type which extends the lossless compression method for //! floating-point values proposed for the time series management system Gorilla in the [Gorilla -//! paper] by 1) adding support for error-bounded lossy compression, and 2) optimizing flag bits +//! paper] by adding support for error-bounded lossy compression, and optimizing flag bits //! for better compression of real-life sensor data. MacaqueV adds support for lossy compression by -//! 1) rewriting the current value with the previous one if possible within the error bound, or -//! 2) rewriting the least mantissa bits of the value to zero within the error bound so that Gorilla +//! rewriting the current value with the previous one if possible within the error bound, or +//! rewriting the least mantissa bits of the value to zero within the error bound so that Gorilla //! uses fewer bits for encoding. MacaqueV optimizes Gorilla's flag bits by swapping the flag bits -//! 0 and 10. This modification proved to be effective when Gorilla is used alongside PMC-Mean +//! 0 and 10. This modification proved to be effective when Gorilla is used alongside PMC-Mean //! and Swing for multi-model compression. As this compression method uses Gorilla that compresses //! the values of a time series segment using XOR and a variable length binary encoding, aggregates //! are computed by iterating over all values in the segment. From f46e3b6c743de68e8acfe5848563352d509f26f5 Mon Sep 17 00:00:00 2001 From: aabduvakhobov Date: Sat, 16 Aug 2025 15:29:11 +0200 Subject: [PATCH 11/15] Fix SKJ's comments --- .../src/models/macaque_v.rs | 54 ++++++++++--------- .../src/models/timestamps.rs | 37 +++++++------ 2 files changed, 50 insertions(+), 41 deletions(-) diff --git a/crates/modelardb_compression/src/models/macaque_v.rs b/crates/modelardb_compression/src/models/macaque_v.rs index 1b3adcaac..0e27a300a 100644 --- a/crates/modelardb_compression/src/models/macaque_v.rs +++ b/crates/modelardb_compression/src/models/macaque_v.rs @@ -13,16 +13,16 @@ * limitations under the License. */ -//! Implementation of the MacaqueV model type which extends the lossless compression method for -//! floating-point values proposed for the time series management system Gorilla in the [Gorilla +//! Implementation of the MacaqueV model type which extends the lossless compression method for +//! floating-point values proposed for the time series management system Gorilla in the [Gorilla //! paper] by adding support for error-bounded lossy compression, and optimizing flag bits //! for better compression of real-life sensor data. MacaqueV adds support for lossy compression by //! rewriting the current value with the previous one if possible within the error bound, or //! rewriting the least mantissa bits of the value to zero within the error bound so that Gorilla //! uses fewer bits for encoding. MacaqueV optimizes Gorilla's flag bits by swapping the flag bits -//! 0 and 10. This modification proved to be effective when Gorilla is used alongside PMC-Mean -//! and Swing for multi-model compression. As this compression method uses Gorilla that compresses -//! the values of a time series segment using XOR and a variable length binary encoding, aggregates +//! 0 and 10. The experiments showed this modification's effectiveness when Gorilla is used alongside +//! PMC-Mean and Swing for multi-model compression. As MacaqueV uses Gorilla that compresses the +//! values of a time series segment using XOR and a variable length binary encoding, aggregates //! are computed by iterating over all values in the segment. //! //! [Gorilla paper]: https://www.vldb.org/pvldb/vol8/p1816-teller.pdf @@ -30,8 +30,8 @@ use modelardb_types::types::{Timestamp, Value, ValueBuilder}; use crate::models; -use crate::models::bits::{BitReader, BitVecBuilder}; use crate::models::ErrorBound; +use crate::models::bits::{BitReader, BitVecBuilder}; /// The state the MacaqueV model type needs while compressing the values of a /// time series segment. @@ -104,9 +104,9 @@ impl MacaqueV { if models::is_value_within_error_bound(self.error_bound, value, self.last_value) { self.last_value } else { - // If rewriting the value is not possible, the least mantissa bits of the value - // are rewritten to zero. - self.rewrite_value_with_log_method(value) + // When the value rewriting is not possible within the error bound, + // the least mantissa bits of the value are rewritten to zero within the error bound. + self.rewrite_least_mantissa_bits(value) } }; @@ -127,7 +127,7 @@ impl MacaqueV { if leading_zero_bits >= self.last_leading_zero_bits && trailing_zero_bits >= self.last_trailing_zero_bits { - // Store only the meaningful bits after a flag zero bit. + // Store a flag zero bit and the meaningful bits. self.compressed_values.append_a_zero_bit(); let meaningful_bits = models::VALUE_SIZE_IN_BITS - self.last_leading_zero_bits @@ -161,8 +161,9 @@ impl MacaqueV { self.update_min_max_and_last_value(value); } - /// Rewrite least mantissa bits of the `value` to zero within the `error_bound`. - fn rewrite_value_with_log_method(&self, value: Value) -> Value { + /// Rewrite the highest number of mantissa bits possible for `value` within the `error_bound` + /// starting from the least significant bits. + fn rewrite_least_mantissa_bits(&self, value: Value) -> Value { if value.abs() == 0.0 || value.is_infinite() || value.is_nan() { return value; } @@ -172,10 +173,13 @@ impl MacaqueV { models::maximum_allowed_deviation(self.error_bound, value as f64) as f32; let exponent = get_exponent(value); let factorized_epsilon = abs_error_bound / 2f32.powi(exponent); - // Rewriting value using 23 - ⌈log2 factorized_epsilon⌉ bits never exceeds the error bound. - // However, one more bit can be rewritten when the majority of the least mantissa bits - // are 0. Thus we use 23 - ⌊log2 factorized_epsilon⌋ and then perform an extra check - // to ensure if the error bound is not exceeded. + // Rewriting the last 23 - ⌈log2 factorized_epsilon⌉ mantissa bits + // never exceeds the error bound. However, in that case, one more bit + // can be rewritten if the majority of the least mantissa bits are 0. + // Thus, we rewrite bits using 23 - ⌊log2 factorized_epsilon⌋ and + // perform an extra check to ensure the error bound is not exceeded. + // If the error bound is exceeded, we rewrite 23 - ⌈log2 factorized_epsilon⌉ + // least mantissa bits. let mut rewrite_position = 23 - factorized_epsilon.log2().abs().floor() as i32; let mut rewritten_value = f32::from_bits(rewrite_bits_by_n(value_as_u32, rewrite_position)); @@ -318,13 +322,13 @@ pub fn grid( // Extract the unbiased exponent from single precision float. fn get_exponent(value: f32) -> i32 { let n_bits: u32 = value.to_bits(); - let exponent_ = ((n_bits >> 23) & 0xff) as i32; - exponent_ - 127 + let biased_exponent = ((n_bits >> 23) & 0xff) as i32; + biased_exponent - 127 } -// Left shift unsigned 32-bit integer by a given number of places. -fn rewrite_bits_by_n(bits_to_rewrite: u32, left_shift_by: i32) -> u32 { - let mask = u32::MAX << left_shift_by; +// Left shift `bits_to_rewrite` by `positions_to_shift`. +fn rewrite_bits_by_n(bits_to_rewrite: u32, positions_to_shift: i32) -> u32 { + let mask = u32::MAX << positions_to_shift; bits_to_rewrite & mask } @@ -584,9 +588,11 @@ mod tests { let values_array = value_builder.finish(); assert!(values.len() == timestamps.len() && values.len() == values_array.len()); - assert!(timestamps - .windows(2) - .all(|window| window[1] - window[0] == 1)); + assert!( + timestamps + .windows(2) + .all(|window| window[1] - window[0] == 1) + ); assert!(slice_of_value_equal(values_array.values(), values)); } diff --git a/crates/modelardb_compression/src/models/timestamps.rs b/crates/modelardb_compression/src/models/timestamps.rs index c445563ba..4f8e1a47b 100644 --- a/crates/modelardb_compression/src/models/timestamps.rs +++ b/crates/modelardb_compression/src/models/timestamps.rs @@ -13,25 +13,28 @@ * limitations under the License. */ -//! Implementation of MacaqueTS, a lossless compression for timestamps. MacaqueTS uses +//! Implementation of MacaqueTS, a lossless compression for timestamps. MacaqueTS uses //! optimized compression methods depending on the number of data points in a compressed segment //! and if its timestamps have been sampled at a regular sampling interval. //! -//! If a segment only contains one data point, its timestamp is stored as both the segment's -//! `start_time` and `end_time`, and if a segment only contains two data points, -//! the timestamps are stored as the segment's `start_time` and `end_time`, respectively. -//! If a segment contains more than two data points, the first and last timestamps -//! are stored as the segment's `start_time` and `end_time`, respectively, while its -//! residual timestamps are compressed using one of two methods. If the data points -//! in the segment have been collected at a regular sampling interval, the residual timestamps -//! are compressed as the segment's length with the prefix zero bits stripped. -//! If none of the above apply, an extended version of the compression method proposed -//! for timestamps for the time series management system Gorilla in the [Gorilla paper] -//! is used as a fallback. MacaqueTS extends Gorilla's timestamp compression method by -//! using zero as the first delta instead of computing it explicitly. This avoids the need -//! to encode the first delta as a special case. MacaqueTS also extends the flag ranges -//! used by Gorilla for integer bit-packing to support encoding finer granularity timestamps -//! with a sampling interval of lower than one second. +//! - (1) If a segment only contains one data point, its timestamp is stored as both the segment's +//! `start_time` and `end_time`. +//! - (2) If a segment only contains two data points, the timestamps are stored as the segment's +//! `start_time` and `end_time`, respectively. +//! - (3) If a segment contains more than two data points, the first and last timestamps +//! are stored as the segment's `start_time` and `end_time`, respectively, while its +//! residual timestamps are compressed using one of two methods. +//! - (1) If the data points in the segment have been collected at a regular sampling interval, +//! the residual timestamps are compressed as the segment's length with the +//! prefix zero bits stripped. +//! - (2) If none of the above apply, an extended version of the compression method proposed +//! for timestamps for the time series management system Gorilla in the [Gorilla paper] +//! is used as a fallback. +//! +//! MacaqueTS extends Gorilla's timestamp compression method by using zero as the first delta +//! instead of computing it explicitly. MacaqueTS also extends the flag ranges used by Gorilla +//! for integer bit-packing to support encoding finer granularity timestamps with a sampling interval +//! of lower than one second. //! //! //! [Gorilla paper]: https://www.vldb.org/pvldb/vol8/p1816-teller.pdf @@ -220,7 +223,7 @@ fn decompress_all_regular_timestamps( } /// Decompress all of a segment's timestamps, which for this segment are sampled -/// at an irregular sampling interval, and thus compressed using MacaqueTS. +/// at an irregular sampling interval, and thus compressed using MacaqueTS. /// The decompressed timestamps are appended to `timestamp_builder`. fn decompress_all_irregular_timestamps( start_time: Timestamp, From 0711a360ec57de679e0ebedff07fc20311cb90f6 Mon Sep 17 00:00:00 2001 From: aabduvakhobov Date: Sat, 16 Aug 2025 18:33:30 +0200 Subject: [PATCH 12/15] Run tests again --- crates/modelardb_compression/src/models/macaque_v.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/modelardb_compression/src/models/macaque_v.rs b/crates/modelardb_compression/src/models/macaque_v.rs index 0e27a300a..09c62d308 100644 --- a/crates/modelardb_compression/src/models/macaque_v.rs +++ b/crates/modelardb_compression/src/models/macaque_v.rs @@ -164,7 +164,7 @@ impl MacaqueV { /// Rewrite the highest number of mantissa bits possible for `value` within the `error_bound` /// starting from the least significant bits. fn rewrite_least_mantissa_bits(&self, value: Value) -> Value { - if value.abs() == 0.0 || value.is_infinite() || value.is_nan() { + if value.abs() == 0.0 || value.is_nan() || value.is_infinite() { return value; } From da6c8c3eeb322300cbbb8720c67dcad746ecada4 Mon Sep 17 00:00:00 2001 From: aabduvakhobov Date: Thu, 28 Aug 2025 16:28:23 +0200 Subject: [PATCH 13/15] Describe MacaqueTS using bullet list --- .../src/models/macaque_v.rs | 8 +++---- .../src/models/timestamps.rs | 23 +++++++++---------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/crates/modelardb_compression/src/models/macaque_v.rs b/crates/modelardb_compression/src/models/macaque_v.rs index 09c62d308..8007406f3 100644 --- a/crates/modelardb_compression/src/models/macaque_v.rs +++ b/crates/modelardb_compression/src/models/macaque_v.rs @@ -175,11 +175,11 @@ impl MacaqueV { let factorized_epsilon = abs_error_bound / 2f32.powi(exponent); // Rewriting the last 23 - ⌈log2 factorized_epsilon⌉ mantissa bits // never exceeds the error bound. However, in that case, one more bit - // can be rewritten if the majority of the least mantissa bits are 0. - // Thus, we rewrite bits using 23 - ⌊log2 factorized_epsilon⌋ and + // can be rewritten if the majority of the least significant mantissa bits + // are 0. Thus, we rewrite bits using 23 - ⌊log2 factorized_epsilon⌋ and // perform an extra check to ensure the error bound is not exceeded. // If the error bound is exceeded, we rewrite 23 - ⌈log2 factorized_epsilon⌉ - // least mantissa bits. + // least significant mantissa bits. let mut rewrite_position = 23 - factorized_epsilon.log2().abs().floor() as i32; let mut rewritten_value = f32::from_bits(rewrite_bits_by_n(value_as_u32, rewrite_position)); @@ -319,7 +319,7 @@ pub fn grid( } } -// Extract the unbiased exponent from single precision float. +// Extract the unbiased exponent from `value`. fn get_exponent(value: f32) -> i32 { let n_bits: u32 = value.to_bits(); let biased_exponent = ((n_bits >> 23) & 0xff) as i32; diff --git a/crates/modelardb_compression/src/models/timestamps.rs b/crates/modelardb_compression/src/models/timestamps.rs index 4f8e1a47b..dd8f295bd 100644 --- a/crates/modelardb_compression/src/models/timestamps.rs +++ b/crates/modelardb_compression/src/models/timestamps.rs @@ -15,21 +15,20 @@ //! Implementation of MacaqueTS, a lossless compression for timestamps. MacaqueTS uses //! optimized compression methods depending on the number of data points in a compressed segment -//! and if its timestamps have been sampled at a regular sampling interval. -//! -//! - (1) If a segment only contains one data point, its timestamp is stored as both the segment's +//! and if its timestamps have been sampled at a regular sampling interval: +//! * If a segment only contains one data point, its timestamp is stored as both the segment's //! `start_time` and `end_time`. -//! - (2) If a segment only contains two data points, the timestamps are stored as the segment's +//! * If a segment only contains two data points, the timestamps are stored as the segment's //! `start_time` and `end_time`, respectively. -//! - (3) If a segment contains more than two data points, the first and last timestamps +//! * If a segment contains more than two data points, the first and last timestamps //! are stored as the segment's `start_time` and `end_time`, respectively, while its -//! residual timestamps are compressed using one of two methods. -//! - (1) If the data points in the segment have been collected at a regular sampling interval, -//! the residual timestamps are compressed as the segment's length with the -//! prefix zero bits stripped. -//! - (2) If none of the above apply, an extended version of the compression method proposed -//! for timestamps for the time series management system Gorilla in the [Gorilla paper] -//! is used as a fallback. +//! residual timestamps are compressed using one of two methods: +//! * If the data points in the segment have been collected at a regular sampling interval, +//! the residual timestamps are compressed as the segment's length with the +//! prefix zero bits stripped. +//! * If none of the above apply, an extended version of the compression method proposed +//! for timestamps for the time series management system Gorilla in the [Gorilla paper] +//! is used as a fallback. //! //! MacaqueTS extends Gorilla's timestamp compression method by using zero as the first delta //! instead of computing it explicitly. MacaqueTS also extends the flag ranges used by Gorilla From ca04016dc4765ae4e538408295cb8ba2e42638a1 Mon Sep 17 00:00:00 2001 From: aabduvakhobov Date: Thu, 28 Aug 2025 16:40:19 +0200 Subject: [PATCH 14/15] Ensure the least significant mantissa bits is everywhere --- crates/modelardb_compression/src/models/macaque_v.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/modelardb_compression/src/models/macaque_v.rs b/crates/modelardb_compression/src/models/macaque_v.rs index 8007406f3..57b85b992 100644 --- a/crates/modelardb_compression/src/models/macaque_v.rs +++ b/crates/modelardb_compression/src/models/macaque_v.rs @@ -18,7 +18,7 @@ //! paper] by adding support for error-bounded lossy compression, and optimizing flag bits //! for better compression of real-life sensor data. MacaqueV adds support for lossy compression by //! rewriting the current value with the previous one if possible within the error bound, or -//! rewriting the least mantissa bits of the value to zero within the error bound so that Gorilla +//! rewriting the least significant mantissa bits of the value to zero within the error bound so that Gorilla //! uses fewer bits for encoding. MacaqueV optimizes Gorilla's flag bits by swapping the flag bits //! 0 and 10. The experiments showed this modification's effectiveness when Gorilla is used alongside //! PMC-Mean and Swing for multi-model compression. As MacaqueV uses Gorilla that compresses the @@ -105,7 +105,8 @@ impl MacaqueV { self.last_value } else { // When the value rewriting is not possible within the error bound, - // the least mantissa bits of the value are rewritten to zero within the error bound. + // the least significant mantissa bits of the value are rewritten to zero + // within the error bound. self.rewrite_least_mantissa_bits(value) } }; From 1126cbec0a5ed3c43fb4801320e8c75cd9ecb798 Mon Sep 17 00:00:00 2001 From: aabduvakhobov Date: Wed, 3 Sep 2025 09:59:27 +0200 Subject: [PATCH 15/15] Clarify MacaqueV doc comment --- .../modelardb_compression/src/models/macaque_v.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/crates/modelardb_compression/src/models/macaque_v.rs b/crates/modelardb_compression/src/models/macaque_v.rs index 57b85b992..3c259ce28 100644 --- a/crates/modelardb_compression/src/models/macaque_v.rs +++ b/crates/modelardb_compression/src/models/macaque_v.rs @@ -18,12 +18,13 @@ //! paper] by adding support for error-bounded lossy compression, and optimizing flag bits //! for better compression of real-life sensor data. MacaqueV adds support for lossy compression by //! rewriting the current value with the previous one if possible within the error bound, or -//! rewriting the least significant mantissa bits of the value to zero within the error bound so that Gorilla -//! uses fewer bits for encoding. MacaqueV optimizes Gorilla's flag bits by swapping the flag bits -//! 0 and 10. The experiments showed this modification's effectiveness when Gorilla is used alongside -//! PMC-Mean and Swing for multi-model compression. As MacaqueV uses Gorilla that compresses the -//! values of a time series segment using XOR and a variable length binary encoding, aggregates -//! are computed by iterating over all values in the segment. +//! rewriting the least significant mantissa bits of the value to zero within the error bound so +//! that Gorilla uses fewer bits for encoding. MacaqueV optimizes Gorilla's flag bits by swapping the +//! flag bits 0 and 10. Experiments showed that this modification is very effective when Gorilla is used +//! after PMC-Mean and Swing i.e., for compressing residuals. This is because neighboring residuals +//! have rarely the same value and they are mostly similar to each other. As MacaqueV uses Gorilla +//! that compresses the values of a time series segment using XOR and a variable length binary encoding, +//! aggregates are computed by iterating over all values in the segment. //! //! [Gorilla paper]: https://www.vldb.org/pvldb/vol8/p1816-teller.pdf @@ -105,7 +106,7 @@ impl MacaqueV { self.last_value } else { // When the value rewriting is not possible within the error bound, - // the least significant mantissa bits of the value are rewritten to zero + // the least significant mantissa bits of the value are rewritten to zero // within the error bound. self.rewrite_least_mantissa_bits(value) }