diff --git a/crates/modelardb_compression/src/compression.rs b/crates/modelardb_compression/src/compression.rs index d90b25343..455daf0c2 100644 --- a/crates/modelardb_compression/src/compression.rs +++ b/crates/modelardb_compression/src/compression.rs @@ -24,8 +24,8 @@ use arrow::record_batch::RecordBatch; use modelardb_types::types::{ErrorBound, TimestampArray, ValueArray}; use crate::error::{ModelarDbCompressionError, Result}; -use crate::models::gorilla::Gorilla; -use crate::models::{self, GORILLA_ID, timestamps}; +use crate::models::macaque_v::MacaqueV; +use crate::models::{self, MACAQUE_V_ID, timestamps}; use crate::types::{CompressedSegmentBatchBuilder, CompressedSegmentBuilder, ModelBuilder}; /// Maximum number of residuals that can be stored as part of a compressed segment. The number of @@ -238,13 +238,13 @@ fn compress_and_store_residuals_in_a_separate_segment( // Compute metadata and compress the values stored in this segment without residuals. let uncompressed_values = &uncompressed_values.values()[start_index..=end_index]; - let mut gorilla = Gorilla::new(error_bound); - gorilla.compress_values(uncompressed_values); + let mut macaque_v = MacaqueV::new(error_bound); + macaque_v.compress_values(uncompressed_values); - let (values, min_value, max_value) = gorilla.model(); + let (values, min_value, max_value) = macaque_v.model(); compressed_segment_batch_builder.append_compressed_segment( - GORILLA_ID, + MACAQUE_V_ID, start_time, end_time, ×tamps, @@ -470,7 +470,7 @@ mod tests { false, ValuesStructure::largest_random_without_overflow(), ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - &[models::GORILLA_ID], + &[models::MACAQUE_V_ID], ); } @@ -480,7 +480,7 @@ mod tests { false, ValuesStructure::largest_random_without_overflow(), ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - &[models::GORILLA_ID], + &[models::MACAQUE_V_ID], ); } @@ -490,7 +490,7 @@ mod tests { true, ValuesStructure::largest_random_without_overflow(), ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - &[models::GORILLA_ID], + &[models::MACAQUE_V_ID], ); } @@ -500,7 +500,7 @@ mod tests { true, ValuesStructure::largest_random_without_overflow(), ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - &[models::GORILLA_ID], + &[models::MACAQUE_V_ID], ); } @@ -539,8 +539,8 @@ mod tests { generate_compress_and_assert_known_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), false, - &[models::GORILLA_ID, models::SWING_ID, models::PMC_MEAN_ID], - &[models::GORILLA_ID, models::SWING_ID, models::PMC_MEAN_ID], + &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], + &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], ); } @@ -550,8 +550,8 @@ mod tests { generate_compress_and_assert_known_time_series( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), false, - &[models::GORILLA_ID, models::SWING_ID, models::PMC_MEAN_ID], - &[models::GORILLA_ID, models::SWING_ID, models::PMC_MEAN_ID], + &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], + &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], ); } @@ -561,8 +561,8 @@ mod tests { generate_compress_and_assert_known_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), true, - &[models::GORILLA_ID, models::SWING_ID, models::PMC_MEAN_ID], - &[models::GORILLA_ID, models::SWING_ID, models::PMC_MEAN_ID], + &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], + &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], ); } @@ -572,8 +572,8 @@ mod tests { generate_compress_and_assert_known_time_series( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), true, - &[models::GORILLA_ID, models::SWING_ID, models::PMC_MEAN_ID], - &[models::GORILLA_ID, models::SWING_ID, models::PMC_MEAN_ID], + &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], + &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], ); } @@ -583,7 +583,7 @@ mod tests { generate_compress_and_assert_known_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), false, - &[models::PMC_MEAN_ID, models::SWING_ID, models::GORILLA_ID], + &[models::PMC_MEAN_ID, models::SWING_ID, models::MACAQUE_V_ID], &[models::PMC_MEAN_ID, models::SWING_ID], ); } @@ -594,7 +594,7 @@ mod tests { generate_compress_and_assert_known_time_series( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), false, - &[models::PMC_MEAN_ID, models::SWING_ID, models::GORILLA_ID], + &[models::PMC_MEAN_ID, models::SWING_ID, models::MACAQUE_V_ID], &[models::PMC_MEAN_ID, models::SWING_ID], ); } @@ -605,7 +605,7 @@ mod tests { generate_compress_and_assert_known_time_series( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), true, - &[models::PMC_MEAN_ID, models::SWING_ID, models::GORILLA_ID], + &[models::PMC_MEAN_ID, models::SWING_ID, models::MACAQUE_V_ID], &[models::PMC_MEAN_ID, models::SWING_ID], ); } @@ -616,7 +616,7 @@ mod tests { generate_compress_and_assert_known_time_series( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), true, - &[models::PMC_MEAN_ID, models::SWING_ID, models::GORILLA_ID], + &[models::PMC_MEAN_ID, models::SWING_ID, models::MACAQUE_V_ID], &[models::PMC_MEAN_ID, models::SWING_ID], ); } @@ -649,7 +649,7 @@ mod tests { [uncompressed_timestamps_start_index..uncompressed_timestamps_end_index], ValuesStructure::Linear(None), ), - models::GORILLA_ID => data_generation::generate_values( + models::MACAQUE_V_ID => data_generation::generate_values( &uncompressed_timestamps.values() [uncompressed_timestamps_start_index..uncompressed_timestamps_end_index], ValuesStructure::largest_random_without_overflow(), @@ -1002,7 +1002,7 @@ mod tests { ); assert_eq!(1, compressed_record_batch.num_rows()); - assert_eq!(GORILLA_ID, model_type_ids.value(0)); + assert_eq!(MACAQUE_V_ID, model_type_ids.value(0)); assert_eq!(100, start_times.value(0)); assert_eq!(500, end_times.value(0)); assert_eq!(1, timestamps.value(0).len()); diff --git a/crates/modelardb_compression/src/models/gorilla.rs b/crates/modelardb_compression/src/models/macaque_v.rs similarity index 75% rename from crates/modelardb_compression/src/models/gorilla.rs rename to crates/modelardb_compression/src/models/macaque_v.rs index 3582ac129..3c259ce28 100644 --- a/crates/modelardb_compression/src/models/gorilla.rs +++ b/crates/modelardb_compression/src/models/macaque_v.rs @@ -13,12 +13,18 @@ * limitations under the License. */ -//! Implementation of the Gorilla model type which uses the lossless compression method for +//! Implementation of the MacaqueV model type which extends the lossless compression method for //! floating-point values proposed for the time series management system Gorilla in the [Gorilla -//! paper]. The compression method described in the paper has been extended with support for lossy -//! compression by replacing values with the previous value if possible within the error bound. As -//! this compression method compresses the values of a time series segment using XOR and a variable -//! length binary encoding, aggregates are computed by iterating over all values in the segment. +//! paper] by adding support for error-bounded lossy compression, and optimizing flag bits +//! for better compression of real-life sensor data. MacaqueV adds support for lossy compression by +//! rewriting the current value with the previous one if possible within the error bound, or +//! rewriting the least significant mantissa bits of the value to zero within the error bound so +//! that Gorilla uses fewer bits for encoding. MacaqueV optimizes Gorilla's flag bits by swapping the +//! flag bits 0 and 10. Experiments showed that this modification is very effective when Gorilla is used +//! after PMC-Mean and Swing i.e., for compressing residuals. This is because neighboring residuals +//! have rarely the same value and they are mostly similar to each other. As MacaqueV uses Gorilla +//! that compresses the values of a time series segment using XOR and a variable length binary encoding, +//! aggregates are computed by iterating over all values in the segment. //! //! [Gorilla paper]: https://www.vldb.org/pvldb/vol8/p1816-teller.pdf @@ -28,9 +34,9 @@ use crate::models; use crate::models::ErrorBound; use crate::models::bits::{BitReader, BitVecBuilder}; -/// The state the Gorilla model type needs while compressing the values of a +/// The state the MacaqueV model type needs while compressing the values of a /// time series segment. -pub struct Gorilla { +pub struct MacaqueV { /// Maximum relative error for the value of each data point. error_bound: ErrorBound, /// Min value compressed and added to `compressed_values`. @@ -51,7 +57,7 @@ pub struct Gorilla { length: usize, } -impl Gorilla { +impl MacaqueV { pub fn new(error_bound: ErrorBound) -> Self { Self { error_bound, @@ -65,7 +71,7 @@ impl Gorilla { } } - /// Store the first value in full if this instance of [`Gorilla`] is empty and then compress the + /// Store the first value in full if this instance of [`MacaqueV`] is empty and then compress the /// remaining `values` using XOR and a variable length binary encoding before storing them. pub fn compress_values(&mut self, values: &[Value]) { for value in values { @@ -92,12 +98,18 @@ impl Gorilla { /// Compress `value` using XOR and a variable length binary encoding and then store it. fn compress_value_xor_last_value(&mut self, value: Value) { - // The best case for Gorilla is storing duplicate values. - let value = if models::is_value_within_error_bound(self.error_bound, value, self.last_value) - { - self.last_value - } else { + let value = if models::is_lossless_compression(self.error_bound) { value + } else { + // The best case for MacaqueV is rewriting the current value with the previous one. + if models::is_value_within_error_bound(self.error_bound, value, self.last_value) { + self.last_value + } else { + // When the value rewriting is not possible within the error bound, + // the least significant mantissa bits of the value are rewritten to zero + // within the error bound. + self.rewrite_least_mantissa_bits(value) + } }; let value_as_integer = value.to_bits(); @@ -105,19 +117,19 @@ impl Gorilla { let value_xor_last_value = value_as_integer ^ last_value_as_integer; if value_xor_last_value == 0 { - // Store each repeated value as a single zero bit. + // Store each repeated value as a one and zero bit. + self.compressed_values.append_a_one_bit(); self.compressed_values.append_a_zero_bit(); } else { // Store each new value as its leading zero bits (if necessary) and // meaningful bits (all bits from the first to the last one bit). let leading_zero_bits = value_xor_last_value.leading_zeros() as u8; let trailing_zero_bits = value_xor_last_value.trailing_zeros() as u8; - self.compressed_values.append_a_one_bit(); if leading_zero_bits >= self.last_leading_zero_bits && trailing_zero_bits >= self.last_trailing_zero_bits { - // Store only the meaningful bits. + // Store a flag zero bit and the meaningful bits. self.compressed_values.append_a_zero_bit(); let meaningful_bits = models::VALUE_SIZE_IN_BITS - self.last_leading_zero_bits @@ -130,6 +142,7 @@ impl Gorilla { // Store the leading zero bits before the meaningful bits using // 5 and 6 bits respectively as described in the Gorilla paper. self.compressed_values.append_a_one_bit(); + self.compressed_values.append_a_one_bit(); self.compressed_values .append_bits(leading_zero_bits as u64, 5); @@ -150,6 +163,38 @@ impl Gorilla { self.update_min_max_and_last_value(value); } + /// Rewrite the highest number of mantissa bits possible for `value` within the `error_bound` + /// starting from the least significant bits. + fn rewrite_least_mantissa_bits(&self, value: Value) -> Value { + if value.abs() == 0.0 || value.is_nan() || value.is_infinite() { + return value; + } + + let value_as_u32 = value.to_bits(); + let abs_error_bound = + models::maximum_allowed_deviation(self.error_bound, value as f64) as f32; + let exponent = get_exponent(value); + let factorized_epsilon = abs_error_bound / 2f32.powi(exponent); + // Rewriting the last 23 - ⌈log2 factorized_epsilon⌉ mantissa bits + // never exceeds the error bound. However, in that case, one more bit + // can be rewritten if the majority of the least significant mantissa bits + // are 0. Thus, we rewrite bits using 23 - ⌊log2 factorized_epsilon⌋ and + // perform an extra check to ensure the error bound is not exceeded. + // If the error bound is exceeded, we rewrite 23 - ⌈log2 factorized_epsilon⌉ + // least significant mantissa bits. + let mut rewrite_position = 23 - factorized_epsilon.log2().abs().floor() as i32; + let mut rewritten_value = f32::from_bits(rewrite_bits_by_n(value_as_u32, rewrite_position)); + + // If the error bound is exceeded, value is rewritten with one less bit i.e., + // using 23 − ⌈log2 factorized_epsilon⌉. + if !models::is_value_within_error_bound(self.error_bound, value, rewritten_value) { + rewrite_position -= 1; + rewritten_value = f32::from_bits(rewrite_bits_by_n(value.to_bits(), rewrite_position)); + } + + rewritten_value + } + /// Update the current minimum, maximum, and last value based on `value`. fn update_min_max_and_last_value(&mut self, value: Value) { self.min_value = Value::min(self.min_value, value); @@ -170,13 +215,12 @@ impl Gorilla { } /// Compute the sum of the values for a time series segment whose values are compressed using -/// Gorilla's compression method for floating-point values. If `maybe_model_last_value` is provided, -/// it is assumed the first value in `values` is compressed against it instead of being stored in -/// full, i.e., uncompressed. +/// MacaqueV. If `maybe_model_last_value` is provided, it is assumed the first value in `values` +/// is compressed against it instead of being stored in full, i.e., uncompressed. pub fn sum(length: usize, values: &[u8], maybe_model_last_value: Option) -> Value { - // This function replicates code from gorilla::grid() as it isn't necessary to store the + // This function replicates code from macaque_v::grid() as it isn't necessary to store the // timestamps and values in arrays for a sum. So any changes to the decompression must be - // mirrored in gorilla::grid(). + // mirrored in macaque_v::grid(). let mut bits = BitReader::try_new(values).unwrap(); let mut leading_zeros = u8::MAX; let mut trailing_zeros: u8 = 0; @@ -198,8 +242,15 @@ pub fn sum(length: usize, values: &[u8], maybe_model_last_value: Option) leading_zeros = bits.read_bits(5) as u8; let meaningful_bits = bits.read_bits(6) as u8; trailing_zeros = models::VALUE_SIZE_IN_BITS - meaningful_bits - leading_zeros; + // Decompress the value by reading its meaningful bits, restoring + // its trailing zeroes through shifting, and reversing the XOR. + let meaningful_bits = models::VALUE_SIZE_IN_BITS - leading_zeros - trailing_zeros; + let mut value = bits.read_bits(meaningful_bits) as u32; + value <<= trailing_zeros; + value ^= last_value; + last_value = value; } - + } else { // Decompress the value by reading its meaningful bits, restoring // its trailing zeroes through shifting, and reversing the XOR. let meaningful_bits = models::VALUE_SIZE_IN_BITS - leading_zeros - trailing_zeros; @@ -214,8 +265,8 @@ pub fn sum(length: usize, values: &[u8], maybe_model_last_value: Option) } /// Decompress all the values in `values` for the `timestamps` without matching values in -/// `value_builder`. The values in `values` are compressed using Gorilla's compression method for -/// floating-point values. `values` are appended to `value_builder`. If `maybe_model_last_value` +/// `value_builder`. The values in `values` are compressed using MacaqueV. +/// `values` are appended to `value_builder`. If `maybe_model_last_value` /// is provided, it is assumed the first value in `values` is compressed against it instead of being /// stored in full, i.e., uncompressed. pub fn grid( @@ -224,7 +275,7 @@ pub fn grid( value_builder: &mut ValueBuilder, maybe_model_last_value: Option, ) { - // Changes to the decompression must be mirrored in gorilla::sum(). + // Changes to the decompression must be mirrored in macaque_v::sum(). // unwrap() is safe as values is from a segment and thus cannot be empty. let mut bits = BitReader::try_new(values).unwrap(); let mut leading_zeros = u8::MAX; @@ -249,8 +300,15 @@ pub fn grid( leading_zeros = bits.read_bits(5) as u8; let meaningful_bits = bits.read_bits(6) as u8; trailing_zeros = models::VALUE_SIZE_IN_BITS - meaningful_bits - leading_zeros; + // Decompress the value by reading its meaningful bits, restoring + // its trailing zeroes through shifting, and reversing the XOR. + let meaningful_bits = models::VALUE_SIZE_IN_BITS - leading_zeros - trailing_zeros; + let mut value = bits.read_bits(meaningful_bits) as u32; + value <<= trailing_zeros; + value ^= last_value; + last_value = value; } - + } else { // Decompress the value by reading its meaningful bits, restoring // its trailing zeroes through shifting, and reversing the XOR. let meaningful_bits = models::VALUE_SIZE_IN_BITS - leading_zeros - trailing_zeros; @@ -263,6 +321,19 @@ pub fn grid( } } +// Extract the unbiased exponent from `value`. +fn get_exponent(value: f32) -> i32 { + let n_bits: u32 = value.to_bits(); + let biased_exponent = ((n_bits >> 23) & 0xff) as i32; + biased_exponent - 127 +} + +// Left shift `bits_to_rewrite` by `positions_to_shift`. +fn rewrite_bits_by_n(bits_to_rewrite: u32, positions_to_shift: i32) -> u32 { + let mask = u32::MAX << positions_to_shift; + bits_to_rewrite & mask +} + #[cfg(test)] mod tests { use super::*; @@ -273,24 +344,24 @@ mod tests { use crate::models; - // Tests for Gorilla. + // Tests for MacaqueV. #[test] fn test_empty_sequence_with_absolute_error_bound_zero() { let error_bound = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - assert!(Gorilla::new(error_bound).model().0.is_empty()); + assert!(MacaqueV::new(error_bound).model().0.is_empty()); } #[test] fn test_empty_sequence_with_relative_error_bound_zero() { let error_bound = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); - assert!(Gorilla::new(error_bound).model().0.is_empty()); + assert!(MacaqueV::new(error_bound).model().0.is_empty()); } proptest! { #[test] fn test_append_single_value_with_absolute_error_bound_zero(value in ProptestValue::ANY) { let error_bound = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); model_type.compress_values(&[value]); @@ -302,7 +373,7 @@ mod tests { #[test] fn test_append_single_value_with_relative_error_bound_zero(value in ProptestValue::ANY) { let error_bound = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); model_type.compress_values(&[value]); @@ -314,7 +385,7 @@ mod tests { #[test] fn test_append_repeated_values_with_absolute_error_bound_zero(value in ProptestValue::ANY) { let error_bound = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); model_type.compress_values(&[value, value]); @@ -326,7 +397,7 @@ mod tests { #[test] fn test_append_repeated_values_with_relative_error_bound_zero(value in ProptestValue::ANY) { let error_bound = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); model_type.compress_values(&[value, value]); @@ -339,7 +410,7 @@ mod tests { #[test] fn test_append_different_values_with_leading_zero_bits_with_absolute_error_bound_zero() { let error_bound = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); model_type.compress_values(&[37.0, 73.0]); @@ -351,7 +422,7 @@ mod tests { #[test] fn test_append_different_values_with_leading_zero_bits_with_relative_error_bound_zero() { let error_bound = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); model_type.compress_values(&[37.0, 73.0]); @@ -363,7 +434,7 @@ mod tests { #[test] fn test_append_different_values_without_leading_zero_bits_with_absolute_error_bound_zero() { let error_bound = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); model_type.compress_values(&[37.0, 71.0, 73.0]); @@ -375,7 +446,7 @@ mod tests { #[test] fn test_append_different_values_without_leading_zero_bits_with_relative_error_bound_zero() { let error_bound = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); model_type.compress_values(&[37.0, 71.0, 73.0]); @@ -398,7 +469,7 @@ mod tests { } fn test_append_values_within_error_bound(error_bound: ErrorBound) { - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); model_type.compress_values(&[10.0]); let before_last_value = model_type.last_value; @@ -425,7 +496,7 @@ mod tests { fn test_sum_with_absolute_error_bound_zero(values in collection::vec(ProptestValue::ANY, 0..50)) { prop_assume!(!values.is_empty()); let expected_sum = values.iter().sum::(); - let compressed_values = compress_values_using_gorilla( + let compressed_values = compress_values_using_macaque_v( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), &values, None); let sum = sum(values.len(), &compressed_values, None); @@ -436,7 +507,7 @@ mod tests { fn test_sum_with_relative_error_bound_zero(values in collection::vec(ProptestValue::ANY, 0..50)) { prop_assume!(!values.is_empty()); let expected_sum = values.iter().sum::(); - let compressed_values = compress_values_using_gorilla( + let compressed_values = compress_values_using_macaque_v( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), &values, None); let sum = sum(values.len(), &compressed_values, None); @@ -446,7 +517,7 @@ mod tests { #[test] fn test_sum_model_single_value_with_absolute_error_bound_zero() { - let compressed_values = compress_values_using_gorilla( + let compressed_values = compress_values_using_macaque_v( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), &[37.0], None, @@ -457,7 +528,7 @@ mod tests { #[test] fn test_sum_model_single_value_with_relative_error_bound_zero() { - let compressed_values = compress_values_using_gorilla( + let compressed_values = compress_values_using_macaque_v( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), &[37.0], None, @@ -469,7 +540,7 @@ mod tests { #[test] fn test_sum_residuals_single_value_with_absolute_error_bound_zero() { let maybe_model_last_value = Some(37.0); - let compressed_values = compress_values_using_gorilla( + let compressed_values = compress_values_using_macaque_v( ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), &[37.0], maybe_model_last_value, @@ -481,7 +552,7 @@ mod tests { #[test] fn test_sum_residuals_single_value_with_relative_error_bound_zero() { let maybe_model_last_value = Some(37.0); - let compressed_values = compress_values_using_gorilla( + let compressed_values = compress_values_using_macaque_v( ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), &[37.0], maybe_model_last_value, @@ -510,7 +581,7 @@ mod tests { } fn assert_grid_with_error_bound(error_bound: ErrorBound, values: &[Value]) { - let compressed_values = compress_values_using_gorilla(error_bound, values, None); + let compressed_values = compress_values_using_macaque_v(error_bound, values, None); let timestamps: Vec = (1..=values.len() as i64).step_by(1).collect(); let mut value_builder = ValueBuilder::with_capacity(values.len()); @@ -561,7 +632,7 @@ mod tests { fn assert_grid_single(error_bound: ErrorBound, maybe_model_last_value: Option) { let compressed_values = - compress_values_using_gorilla(error_bound, &[37.0], maybe_model_last_value); + compress_values_using_macaque_v(error_bound, &[37.0], maybe_model_last_value); let mut value_builder = ValueBuilder::new(); grid( @@ -577,12 +648,12 @@ mod tests { assert_eq!(values.value(0), 37.0); } - fn compress_values_using_gorilla( + fn compress_values_using_macaque_v( error_bound: ErrorBound, values: &[Value], maybe_model_last_value: Option, ) -> Vec { - let mut model_type = Gorilla::new(error_bound); + let mut model_type = MacaqueV::new(error_bound); if let Some(model_last_value) = maybe_model_last_value { model_type.compress_values_without_first(values, model_last_value); } else { diff --git a/crates/modelardb_compression/src/models/mod.rs b/crates/modelardb_compression/src/models/mod.rs index 4fbcbb6fa..2c0f0f46a 100644 --- a/crates/modelardb_compression/src/models/mod.rs +++ b/crates/modelardb_compression/src/models/mod.rs @@ -18,7 +18,7 @@ //! contains general functionality used by the model types. pub mod bits; -pub mod gorilla; +pub mod macaque_v; pub mod pmc_mean; pub mod swing; pub mod timestamps; @@ -35,13 +35,13 @@ use crate::types::CompressedSegmentBuilder; /// to the ids must be reflected in all statements matching on them. pub const PMC_MEAN_ID: i8 = 0; pub const SWING_ID: i8 = 1; -pub const GORILLA_ID: i8 = 2; +pub const MACAQUE_V_ID: i8 = 2; /// Number of implemented model types. It is usize instead of u8 as it is used as an array length. pub const MODEL_TYPE_COUNT: usize = 3; /// Mapping of model type ids to names. -pub const MODEL_TYPE_NAMES: [&str; MODEL_TYPE_COUNT] = ["pmc_mean", "swing", "gorilla"]; +pub const MODEL_TYPE_NAMES: [&str; MODEL_TYPE_COUNT] = ["pmc_mean", "swing", "macaque_v"]; /// Size of [`Value`] in bytes. pub(super) const VALUE_SIZE_IN_BYTES: u8 = mem::size_of::() as u8; @@ -87,6 +87,14 @@ pub fn maximum_allowed_deviation(error_bound: ErrorBound, value: f64) -> f64 { } } +/// Returns true if compression is lossless i.e., `error_bound` value is 0. +pub fn is_lossless_compression(error_bound: ErrorBound) -> bool { + match error_bound { + ErrorBound::Absolute(error_bound) => error_bound == 0.0, + ErrorBound::Relative(error_bound) => error_bound == 0.0, + } +} + /// Returns true if `v1` and `v2` are equivalent or both values are NAN. fn equal_or_nan(v1: f64, v2: f64) -> bool { v1 == v2 || (v1.is_nan() && v2.is_nan()) @@ -160,9 +168,9 @@ pub fn sum( ), ) } - GORILLA_ID => ( + MACAQUE_V_ID => ( f32::NAN, // A segment with values compressed by Gorilla never has residuals. - gorilla::sum(model_length, values, None), + macaque_v::sum(model_length, values, None), ), _ => panic!("Unknown model type."), }; @@ -171,7 +179,7 @@ pub fn sum( if residuals.is_empty() { model_sum } else { - let residuals_sum = gorilla::sum( + let residuals_sum = macaque_v::sum( residuals_length, &residuals[..residuals.len() - 1], Some(model_last_value), @@ -228,7 +236,7 @@ pub fn grid( value_builder, ) } - GORILLA_ID => gorilla::grid(values, model_timestamps, value_builder, None), + MACAQUE_V_ID => macaque_v::grid(values, model_timestamps, value_builder, None), _ => panic!("Unknown model type."), } @@ -236,7 +244,7 @@ pub fn grid( if !residuals.is_empty() { let model_last_value = value_builder.values_slice()[value_builder.len() - 1]; - gorilla::grid( + macaque_v::grid( &residuals[..residuals.len() - 1], residuals_timestamps, value_builder, diff --git a/crates/modelardb_compression/src/models/timestamps.rs b/crates/modelardb_compression/src/models/timestamps.rs index 4ae41c5ab..dd8f295bd 100644 --- a/crates/modelardb_compression/src/models/timestamps.rs +++ b/crates/modelardb_compression/src/models/timestamps.rs @@ -13,22 +13,28 @@ * limitations under the License. */ -//! Implementation of lossless compression for timestamps. Optimized compression -//! methods are used depending on the number of data points in a compressed -//! segment and if its timestamps have been sampled at a regular sampling -//! interval. +//! Implementation of MacaqueTS, a lossless compression for timestamps. MacaqueTS uses +//! optimized compression methods depending on the number of data points in a compressed segment +//! and if its timestamps have been sampled at a regular sampling interval: +//! * If a segment only contains one data point, its timestamp is stored as both the segment's +//! `start_time` and `end_time`. +//! * If a segment only contains two data points, the timestamps are stored as the segment's +//! `start_time` and `end_time`, respectively. +//! * If a segment contains more than two data points, the first and last timestamps +//! are stored as the segment's `start_time` and `end_time`, respectively, while its +//! residual timestamps are compressed using one of two methods: +//! * If the data points in the segment have been collected at a regular sampling interval, +//! the residual timestamps are compressed as the segment's length with the +//! prefix zero bits stripped. +//! * If none of the above apply, an extended version of the compression method proposed +//! for timestamps for the time series management system Gorilla in the [Gorilla paper] +//! is used as a fallback. +//! +//! MacaqueTS extends Gorilla's timestamp compression method by using zero as the first delta +//! instead of computing it explicitly. MacaqueTS also extends the flag ranges used by Gorilla +//! for integer bit-packing to support encoding finer granularity timestamps with a sampling interval +//! of lower than one second. //! -//! If a segment only contains one data point its timestamp is stored as both -//! the segment's `start_time` and `end_time`, and if a segment only contains -//! two data points the timestamps are stored as the segment's `start_time` and -//! `end_time`, respectively. If a segment contains more than two data points, -//! the first and last timestamps are stored as the segment's `start_time` and -//! `end_time`, respectively, while its residual timestamps are compressed using -//! one of two methods. If the data points in the segment have been collected at -//! a regular sampling interval, the residual timestamps are compressed as the -//! segment's length with the prefix zero bits stripped. If none of the above -//! apply, the compression method proposed for timestamps for the time series -//! management system Gorilla in the [Gorilla paper] is used as a fallback. //! //! [Gorilla paper]: https://www.vldb.org/pvldb/vol8/p1816-teller.pdf @@ -150,8 +156,9 @@ fn compress_irregular_residual_timestamps(uncompressed_timestamps: &[Timestamp]) /// Decompress all of a segment's timestamps which are compressed as /// `start_time` for segments of length one, `start_time` and `end_time` for /// segments of length two, the segment's length for regular time series, or -/// using Gorilla's compression method for timestamps for irregular time series. -/// The decompressed timestamps are appended to `timestamp_builder`. +/// using extended version of Gorilla's compression method for timestamps +/// for irregular time series. The decompressed timestamps are appended +/// to `timestamp_builder`. pub fn decompress_all_timestamps( start_time: Timestamp, end_time: Timestamp, @@ -215,9 +222,8 @@ fn decompress_all_regular_timestamps( } /// Decompress all of a segment's timestamps, which for this segment are sampled -/// at an irregular sampling interval, and thus compressed using Gorilla's -/// compression method for timestamps. The decompressed timestamps are appended -/// to `timestamp_builder`. +/// at an irregular sampling interval, and thus compressed using MacaqueTS. +/// The decompressed timestamps are appended to `timestamp_builder`. fn decompress_all_irregular_timestamps( start_time: Timestamp, end_time: Timestamp, @@ -227,7 +233,7 @@ fn decompress_all_irregular_timestamps( // Add the first timestamp stored as `start_time` in the segment. timestamp_builder.append_value(start_time); - // Remove the one bit used as a flag to specify that Gorilla is used. + // Remove the one bit used as a flag to specify that MacaqueTS is used. let mut bits = BitReader::try_new(residual_timestamps).unwrap(); bits.read_bit(); diff --git a/crates/modelardb_compression/src/types.rs b/crates/modelardb_compression/src/types.rs index 32cbe9780..00a3c6033 100644 --- a/crates/modelardb_compression/src/types.rs +++ b/crates/modelardb_compression/src/types.rs @@ -27,7 +27,7 @@ use modelardb_types::types::{ ErrorBound, Timestamp, TimestampArray, TimestampBuilder, Value, ValueArray, ValueBuilder, }; -use crate::models::gorilla::Gorilla; +use crate::models::macaque_v::MacaqueV; use crate::models::pmc_mean::PMCMean; use crate::models::swing::Swing; use crate::models::{PMC_MEAN_ID, SWING_ID}; @@ -215,7 +215,7 @@ impl CompressedSegmentBuilder { &uncompressed_timestamps.values()[self.start_index..=residuals_end_index], ); - // Compress residual values using Gorilla if any exists. + // Compress residual values using MacaqueV if any exists. let residuals = if self.end_index < residuals_end_index { let residuals_start_index = self.end_index + 1; @@ -265,15 +265,15 @@ impl CompressedSegmentBuilder { ) } - /// Compress `uncompressed_residuals` within `error_bound` using [`Gorilla`]. + /// Compress `uncompressed_residuals` within `error_bound` using [`MacaqueV`]. fn compress_residuals( &self, error_bound: ErrorBound, uncompressed_residuals: &[Value], ) -> (Vec, Value, Value) { - let mut gorilla = Gorilla::new(error_bound); - gorilla.compress_values_without_first(uncompressed_residuals, self.model_last_value); - gorilla.model() + let mut macaque_v = MacaqueV::new(error_bound); + macaque_v.compress_values_without_first(uncompressed_residuals, self.model_last_value); + macaque_v.model() } /// Encode the information required for a [`PMCMean`] model where the `residuals_min_value` @@ -423,7 +423,7 @@ pub(crate) struct CompressedSegmentBatchBuilder { /// the values of each compressed segment in the batch within an error /// bound. values: BinaryBuilder, - /// Values between this and the next segment, compressed using [`Gorilla`], + /// Values between this and the next segment, compressed using [`MacaqueV`], /// that the models could not represent efficiently within the error bound /// and which are too few for a new segment due to the amount of metadata. residuals: BinaryBuilder, @@ -808,7 +808,7 @@ mod tests { let model_end_index = model.end_index; // Create a segment that represents its values using a model of the expected type and its - // residuals using Gorilla, and then assert that the expected encoding is used for it. + // residuals using MacaqueV, and then assert that the expected encoding is used for it. let residuals_end_index = uncompressed_timestamps.len() - 1; let mut compressed_schema_fields = COMPRESSED_SCHEMA.0.fields.clone().to_vec();