diff --git a/crates/modelardb_compression/src/compression.rs b/crates/modelardb_compression/src/compression.rs index 455daf0c2..f2bcd83da 100644 --- a/crates/modelardb_compression/src/compression.rs +++ b/crates/modelardb_compression/src/compression.rs @@ -264,8 +264,8 @@ mod tests { use arrow::array::{ArrayBuilder, BinaryArray, Float32Array, Int8Array}; use arrow::datatypes::{DataType, Field}; + use modelardb_test::ERROR_BOUND_FIVE; use modelardb_test::data_generation::{self, ValuesStructure}; - use modelardb_test::{ERROR_BOUND_FIVE, ERROR_BOUND_ZERO}; use modelardb_types::schemas::COMPRESSED_SCHEMA; use modelardb_types::types::{TimestampBuilder, ValueBuilder}; @@ -277,11 +277,11 @@ mod tests { // Tests for try_compress(). #[test] - fn test_try_compress_empty_time_series_within_absolute_error_bound_zero() { + fn test_try_compress_empty_time_series_within_lossless_error_bound() { let compressed_record_batch = try_compress( &TimestampBuilder::new().finish(), &ValueBuilder::new().finish(), - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, compressed_schema(), vec![TAG_VALUE.to_owned()], 0, @@ -291,55 +291,21 @@ mod tests { } #[test] - fn test_try_compress_empty_time_series_within_relative_error_bound_zero() { - let compressed_record_batch = try_compress( - &TimestampBuilder::new().finish(), - &ValueBuilder::new().finish(), - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - compressed_schema(), - vec![TAG_VALUE.to_owned()], - 0, - ) - .unwrap(); - assert_eq!(0, compressed_record_batch.num_rows()); - } - - #[test] - fn test_try_compress_regular_constant_time_series_within_absolute_error_bound_zero() { - generate_compress_and_assert_known_segment( - false, - ValuesStructure::Constant(None), - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - &[models::PMC_MEAN_ID], - ); - } - - #[test] - fn test_try_compress_regular_constant_time_series_within_relative_error_bound_zero() { + fn test_try_compress_regular_constant_time_series_within_lossless_error_bound() { generate_compress_and_assert_known_segment( false, ValuesStructure::Constant(None), - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, &[models::PMC_MEAN_ID], ); } #[test] - fn test_try_compress_irregular_constant_time_series_within_absolute_error_bound_zero() { + fn test_try_compress_irregular_constant_time_series_within_lossless_error_bound() { generate_compress_and_assert_known_segment( true, ValuesStructure::Constant(None), - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - &[models::PMC_MEAN_ID], - ); - } - - #[test] - fn test_try_compress_irregular_constant_time_series_within_relative_error_bound_zero() { - generate_compress_and_assert_known_segment( - true, - ValuesStructure::Constant(None), - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, &[models::PMC_MEAN_ID], ); } @@ -385,41 +351,21 @@ mod tests { } #[test] - fn test_try_compress_regular_linear_time_series_within_absolute_error_bound_zero() { + fn test_try_compress_regular_linear_time_series_within_lossless_error_bound() { generate_compress_and_assert_known_segment( false, ValuesStructure::Linear(None), - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - &[models::SWING_ID], - ); - } - - #[test] - fn test_try_compress_regular_linear_time_series_within_relative_error_bound_zero() { - generate_compress_and_assert_known_segment( - false, - ValuesStructure::Linear(None), - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - &[models::SWING_ID], - ); - } - - #[test] - fn test_try_compress_irregular_linear_time_series_within_absolute_error_bound_zero() { - generate_compress_and_assert_known_segment( - true, - ValuesStructure::Linear(None), - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, &[models::SWING_ID], ); } #[test] - fn test_try_compress_irregular_linear_time_series_within_relative_error_bound_zero() { + fn test_try_compress_irregular_linear_time_series_within_lossless_error_bound() { generate_compress_and_assert_known_segment( true, ValuesStructure::Linear(None), - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, &[models::SWING_ID], ); } @@ -465,41 +411,21 @@ mod tests { } #[test] - fn test_try_compress_regular_random_time_series_within_absolute_error_bound_zero() { - generate_compress_and_assert_known_segment( - false, - ValuesStructure::largest_random_without_overflow(), - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - &[models::MACAQUE_V_ID], - ); - } - - #[test] - fn test_try_compress_regular_random_time_series_within_relative_error_bound_zero() { + fn test_try_compress_regular_random_time_series_within_lossless_error_bound() { generate_compress_and_assert_known_segment( false, ValuesStructure::largest_random_without_overflow(), - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, &[models::MACAQUE_V_ID], ); } #[test] - fn test_try_compress_irregular_random_time_series_within_absolute_error_bound_zero() { + fn test_try_compress_irregular_random_time_series_within_lossless_error_bound() { generate_compress_and_assert_known_segment( true, ValuesStructure::largest_random_without_overflow(), - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - &[models::MACAQUE_V_ID], - ); - } - - #[test] - fn test_try_compress_irregular_random_time_series_within_relative_error_bound_zero() { - generate_compress_and_assert_known_segment( - true, - ValuesStructure::largest_random_without_overflow(), - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, &[models::MACAQUE_V_ID], ); } @@ -534,10 +460,9 @@ mod tests { } #[test] - fn test_try_compress_regular_random_linear_constant_time_series_within_absolute_error_bound_zero() - { + fn test_try_compress_regular_random_linear_constant_time_series_within_lossless_error_bound() { generate_compress_and_assert_known_time_series( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, false, &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], @@ -545,32 +470,10 @@ mod tests { } #[test] - fn test_try_compress_regular_random_linear_constant_time_series_within_relative_error_bound_zero() - { - generate_compress_and_assert_known_time_series( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - false, - &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], - &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], - ); - } - - #[test] - fn test_try_compress_irregular_random_linear_constant_time_series_within_absolute_error_bound_zero() - { - generate_compress_and_assert_known_time_series( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - true, - &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], - &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], - ); - } - - #[test] - fn test_try_compress_irregular_random_linear_constant_time_series_within_relative_error_bound_zero() - { + fn test_try_compress_irregular_random_linear_constant_time_series_within_lossless_error_bound() + { generate_compress_and_assert_known_time_series( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, true, &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], &[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID], @@ -578,21 +481,9 @@ mod tests { } #[test] - fn test_try_compress_regular_constant_linear_random_time_series_within_absolute_error_bound_zero() - { - generate_compress_and_assert_known_time_series( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - false, - &[models::PMC_MEAN_ID, models::SWING_ID, models::MACAQUE_V_ID], - &[models::PMC_MEAN_ID, models::SWING_ID], - ); - } - - #[test] - fn test_try_compress_regular_constant_linear_random_time_series_within_relative_error_bound_zero() - { + fn test_try_compress_regular_constant_linear_random_time_series_within_lossless_error_bound() { generate_compress_and_assert_known_time_series( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, false, &[models::PMC_MEAN_ID, models::SWING_ID, models::MACAQUE_V_ID], &[models::PMC_MEAN_ID, models::SWING_ID], @@ -600,21 +491,10 @@ mod tests { } #[test] - fn test_try_compress_irregular_constant_linear_random_time_series_within_absolute_error_bound_zero() - { - generate_compress_and_assert_known_time_series( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - true, - &[models::PMC_MEAN_ID, models::SWING_ID, models::MACAQUE_V_ID], - &[models::PMC_MEAN_ID, models::SWING_ID], - ); - } - - #[test] - fn test_try_compress_irregular_constant_linear_random_time_series_within_relative_error_bound_zero() - { + fn test_try_compress_irregular_constant_linear_random_time_series_within_lossless_error_bound() + { generate_compress_and_assert_known_time_series( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, true, &[models::PMC_MEAN_ID, models::SWING_ID, models::MACAQUE_V_ID], &[models::PMC_MEAN_ID, models::SWING_ID], @@ -707,23 +587,8 @@ mod tests { } #[test] - fn test_try_compress_regular_synthetic_time_series_without_noise_within_absolute_error_bound_zero() - { - generate_compress_and_assert_time_series( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - false, - None, - ) - } - - #[test] - fn test_try_compress_regular_synthetic_time_series_without_noise_within_relative_error_bound_zero() - { - generate_compress_and_assert_time_series( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - false, - None, - ) + fn test_try_compress_regular_synthetic_time_series_without_noise_within_lossless_error_bound() { + generate_compress_and_assert_time_series(ErrorBound::Lossless, false, None) } #[test] @@ -747,23 +612,8 @@ mod tests { } #[test] - fn test_try_compress_regular_synthetic_time_series_with_noise_within_absolute_error_bound_zero() - { - generate_compress_and_assert_time_series( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - false, - ADD_NOISE_RANGE, - ) - } - - #[test] - fn test_try_compress_regular_synthetic_time_series_with_noise_within_relative_error_bound_zero() - { - generate_compress_and_assert_time_series( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - false, - ADD_NOISE_RANGE, - ) + fn test_try_compress_regular_synthetic_time_series_with_noise_within_lossless_error_bound() { + generate_compress_and_assert_time_series(ErrorBound::Lossless, false, ADD_NOISE_RANGE) } #[test] @@ -787,23 +637,9 @@ mod tests { } #[test] - fn test_try_compress_irregular_synthetic_time_series_without_noise_within_absolute_error_bound_zero() - { - generate_compress_and_assert_time_series( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - true, - None, - ) - } - - #[test] - fn test_try_compress_irregular_synthetic_time_series_without_noise_within_relative_error_bound_zero() - { - generate_compress_and_assert_time_series( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - true, - None, - ) + fn test_try_compress_irregular_synthetic_time_series_without_noise_within_lossless_error_bound() + { + generate_compress_and_assert_time_series(ErrorBound::Lossless, true, None) } #[test] @@ -827,23 +663,8 @@ mod tests { } #[test] - fn test_try_compress_irregular_synthetic_time_series_with_noise_within_absolute_error_bound_zero() - { - generate_compress_and_assert_time_series( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - true, - ADD_NOISE_RANGE, - ) - } - - #[test] - fn test_try_compress_irregular_synthetic_time_series_with_noise_within_relative_error_bound_zero() - { - generate_compress_and_assert_time_series( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - true, - ADD_NOISE_RANGE, - ) + fn test_try_compress_irregular_synthetic_time_series_with_noise_within_lossless_error_bound() { + generate_compress_and_assert_time_series(ErrorBound::Lossless, true, ADD_NOISE_RANGE) } #[test] @@ -967,7 +788,6 @@ mod tests { // Tests for compress_and_store_residuals_in_a_separate_segment(). #[test] fn test_compress_and_store_residuals_in_a_separate_segment() { - let error_bound = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); let uncompressed_timestamps = TimestampArray::from_iter_values((100..=500).step_by(100)); let uncompressed_values = ValueArray::from(vec![73.0, 37.0, 37.0, 37.0, 73.0]); @@ -979,7 +799,7 @@ mod tests { ); compress_and_store_residuals_in_a_separate_segment( - error_bound, + ErrorBound::Lossless, 0, uncompressed_timestamps.len() - 1, &uncompressed_timestamps, diff --git a/crates/modelardb_compression/src/models/macaque_v.rs b/crates/modelardb_compression/src/models/macaque_v.rs index 3c259ce28..1d22fe5e3 100644 --- a/crates/modelardb_compression/src/models/macaque_v.rs +++ b/crates/modelardb_compression/src/models/macaque_v.rs @@ -98,7 +98,7 @@ impl MacaqueV { /// Compress `value` using XOR and a variable length binary encoding and then store it. fn compress_value_xor_last_value(&mut self, value: Value) { - let value = if models::is_lossless_compression(self.error_bound) { + let value = if self.error_bound == ErrorBound::Lossless { value } else { // The best case for MacaqueV is rewriting the current value with the previous one. @@ -338,7 +338,7 @@ fn rewrite_bits_by_n(bits_to_rewrite: u32, positions_to_shift: i32) -> u32 { mod tests { use super::*; - use modelardb_test::{ERROR_BOUND_TEN, ERROR_BOUND_ZERO}; + use modelardb_test::ERROR_BOUND_TEN; use proptest::num::f32 as ProptestValue; use proptest::{bool, collection, prop_assert, prop_assert_eq, prop_assume, proptest}; @@ -346,34 +346,14 @@ mod tests { // Tests for MacaqueV. #[test] - fn test_empty_sequence_with_absolute_error_bound_zero() { - let error_bound = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - assert!(MacaqueV::new(error_bound).model().0.is_empty()); - } - - #[test] - fn test_empty_sequence_with_relative_error_bound_zero() { - let error_bound = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); - assert!(MacaqueV::new(error_bound).model().0.is_empty()); + fn test_empty_sequence_with_lossless_error_bound() { + assert!(MacaqueV::new(ErrorBound::Lossless).model().0.is_empty()); } proptest! { #[test] - fn test_append_single_value_with_absolute_error_bound_zero(value in ProptestValue::ANY) { - let error_bound = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = MacaqueV::new(error_bound); - - model_type.compress_values(&[value]); - - prop_assert!(models::equal_or_nan(value as f64, model_type.last_value as f64)); - prop_assert_eq!(model_type.last_leading_zero_bits, u8::MAX); - prop_assert_eq!(model_type.last_trailing_zero_bits, 0); - } - - #[test] - fn test_append_single_value_with_relative_error_bound_zero(value in ProptestValue::ANY) { - let error_bound = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = MacaqueV::new(error_bound); + fn test_append_single_value_with_lossless_error_bound(value in ProptestValue::ANY) { + let mut model_type = MacaqueV::new(ErrorBound::Lossless); model_type.compress_values(&[value]); @@ -383,21 +363,8 @@ mod tests { } #[test] - fn test_append_repeated_values_with_absolute_error_bound_zero(value in ProptestValue::ANY) { - let error_bound = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = MacaqueV::new(error_bound); - - model_type.compress_values(&[value, value]); - - prop_assert!(models::equal_or_nan(value as f64, model_type.last_value as f64)); - prop_assert_eq!(model_type.last_leading_zero_bits, u8::MAX); - prop_assert_eq!(model_type.last_trailing_zero_bits, 0); - } - - #[test] - fn test_append_repeated_values_with_relative_error_bound_zero(value in ProptestValue::ANY) { - let error_bound = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = MacaqueV::new(error_bound); + fn test_append_repeated_values_with_lossless_error_bound(value in ProptestValue::ANY) { + let mut model_type = MacaqueV::new(ErrorBound::Lossless); model_type.compress_values(&[value, value]); @@ -408,21 +375,8 @@ mod tests { } #[test] - fn test_append_different_values_with_leading_zero_bits_with_absolute_error_bound_zero() { - let error_bound = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = MacaqueV::new(error_bound); - - model_type.compress_values(&[37.0, 73.0]); - - assert!(models::equal_or_nan(73.0, model_type.last_value as f64)); - assert_eq!(model_type.last_leading_zero_bits, 8); - assert_eq!(model_type.last_trailing_zero_bits, 17); - } - - #[test] - fn test_append_different_values_with_leading_zero_bits_with_relative_error_bound_zero() { - let error_bound = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = MacaqueV::new(error_bound); + fn test_append_different_values_with_leading_zero_bits_with_lossless_error_bound() { + let mut model_type = MacaqueV::new(ErrorBound::Lossless); model_type.compress_values(&[37.0, 73.0]); @@ -432,21 +386,8 @@ mod tests { } #[test] - fn test_append_different_values_without_leading_zero_bits_with_absolute_error_bound_zero() { - let error_bound = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = MacaqueV::new(error_bound); - - model_type.compress_values(&[37.0, 71.0, 73.0]); - - assert!(models::equal_or_nan(73.0, model_type.last_value as f64)); - assert_eq!(model_type.last_leading_zero_bits, 8); - assert_eq!(model_type.last_trailing_zero_bits, 17); - } - - #[test] - fn test_append_different_values_without_leading_zero_bits_with_relative_error_bound_zero() { - let error_bound = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = MacaqueV::new(error_bound); + fn test_append_different_values_without_leading_zero_bits_with_lossless_error_bound() { + let mut model_type = MacaqueV::new(ErrorBound::Lossless); model_type.compress_values(&[37.0, 71.0, 73.0]); @@ -493,70 +434,30 @@ mod tests { // Tests for sum(). proptest! { #[test] - fn test_sum_with_absolute_error_bound_zero(values in collection::vec(ProptestValue::ANY, 0..50)) { + fn test_sum_with_lossless_error_bound(values in collection::vec(ProptestValue::ANY, 0..50)) { prop_assume!(!values.is_empty()); let expected_sum = values.iter().sum::(); let compressed_values = compress_values_using_macaque_v( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, &values, None); let sum = sum(values.len(), &compressed_values, None); prop_assert!(models::equal_or_nan(expected_sum as f64, sum as f64)); } - - #[test] - fn test_sum_with_relative_error_bound_zero(values in collection::vec(ProptestValue::ANY, 0..50)) { - prop_assume!(!values.is_empty()); - let expected_sum = values.iter().sum::(); - let compressed_values = compress_values_using_macaque_v( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - &values, None); - let sum = sum(values.len(), &compressed_values, None); - prop_assert!(models::equal_or_nan(expected_sum as f64, sum as f64)); - } - } - - #[test] - fn test_sum_model_single_value_with_absolute_error_bound_zero() { - let compressed_values = compress_values_using_macaque_v( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - &[37.0], - None, - ); - let sum = sum(1, &compressed_values, None); - assert_eq!(sum, 37.0); } #[test] - fn test_sum_model_single_value_with_relative_error_bound_zero() { - let compressed_values = compress_values_using_macaque_v( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - &[37.0], - None, - ); + fn test_sum_model_single_value_with_lossless_error_bound() { + let compressed_values = + compress_values_using_macaque_v(ErrorBound::Lossless, &[37.0], None); let sum = sum(1, &compressed_values, None); assert_eq!(sum, 37.0); } #[test] - fn test_sum_residuals_single_value_with_absolute_error_bound_zero() { + fn test_sum_residuals_single_value_with_lossless_error_bound() { let maybe_model_last_value = Some(37.0); - let compressed_values = compress_values_using_macaque_v( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - &[37.0], - maybe_model_last_value, - ); - let sum = sum(1, &compressed_values, maybe_model_last_value); - assert_eq!(sum, 37.0); - } - - #[test] - fn test_sum_residuals_single_value_with_relative_error_bound_zero() { - let maybe_model_last_value = Some(37.0); - let compressed_values = compress_values_using_macaque_v( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - &[37.0], - maybe_model_last_value, - ); + let compressed_values = + compress_values_using_macaque_v(ErrorBound::Lossless, &[37.0], maybe_model_last_value); let sum = sum(1, &compressed_values, maybe_model_last_value); assert_eq!(sum, 37.0); } @@ -564,18 +465,10 @@ mod tests { // Tests for grid(). proptest! { #[test] - fn test_grid_with_absolute_error_bound_zero(values in collection::vec(ProptestValue::ANY, 0..50)) { + fn test_grid_with_lossless_error_bound(values in collection::vec(ProptestValue::ANY, 0..50)) { prop_assume!(!values.is_empty()); assert_grid_with_error_bound( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - &values); - } - - #[test] - fn test_grid_with_relative_error_bound_zero(values in collection::vec(ProptestValue::ANY, 0..50)) { - prop_assume!(!values.is_empty()); - assert_grid_with_error_bound( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, &values); } } @@ -599,35 +492,13 @@ mod tests { } #[test] - fn test_grid_model_single_value_with_absolute_error_bound_zero() { - assert_grid_single( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - None, - ); - } - - #[test] - fn test_grid_model_single_value_with_relative_error_bound_zero() { - assert_grid_single( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - None, - ); + fn test_grid_model_single_value_with_lossless_error_bound() { + assert_grid_single(ErrorBound::Lossless, None); } #[test] - fn test_grid_residuals_single_value_with_absolute_error_bound_zero() { - assert_grid_single( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - Some(37.0), - ); - } - - #[test] - fn test_grid_residuals_single_value_with_relative_error_bound_zero() { - assert_grid_single( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - Some(37.0), - ); + fn test_grid_residuals_single_value_with_lossless_error_bound() { + assert_grid_single(ErrorBound::Lossless, Some(37.0)); } fn assert_grid_single(error_bound: ErrorBound, maybe_model_last_value: Option) { diff --git a/crates/modelardb_compression/src/models/mod.rs b/crates/modelardb_compression/src/models/mod.rs index 2c0f0f46a..c6b2a306b 100644 --- a/crates/modelardb_compression/src/models/mod.rs +++ b/crates/modelardb_compression/src/models/mod.rs @@ -75,6 +75,7 @@ pub fn is_value_within_error_bound( (result * 100.0) <= error_bound } } + ErrorBound::Lossless => equal_or_nan(real_value as f64, approximate_value as f64), } } @@ -84,14 +85,7 @@ pub fn maximum_allowed_deviation(error_bound: ErrorBound, value: f64) -> f64 { match error_bound { ErrorBound::Absolute(error_bound) => error_bound as f64 * 0.99, ErrorBound::Relative(error_bound) => f64::abs(value * (error_bound as f64 / 100.1)), - } -} - -/// Returns true if compression is lossless i.e., `error_bound` value is 0. -pub fn is_lossless_compression(error_bound: ErrorBound) -> bool { - match error_bound { - ErrorBound::Absolute(error_bound) => error_bound == 0.0, - ErrorBound::Relative(error_bound) => error_bound == 0.0, + ErrorBound::Lossless => 0.0, } } @@ -292,7 +286,6 @@ mod tests { use modelardb_test::{ ERROR_BOUND_ABSOLUTE_MAX, ERROR_BOUND_ONE, ERROR_BOUND_RELATIVE_MAX, ERROR_BOUND_TEN, - ERROR_BOUND_ZERO, }; use proptest::num; use proptest::num::f32 as ProptestValue; @@ -301,13 +294,8 @@ mod tests { // Tests for is_value_within_error_bound(). proptest! { #[test] - fn test_same_value_is_always_within_absolute_error_bound(value in ProptestValue::ANY) { - prop_assert!(is_value_within_error_bound(ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), value, value)); - } - - #[test] - fn test_same_value_is_always_within_relative_error_bound(value in ProptestValue::ANY) { - prop_assert!(is_value_within_error_bound(ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), value, value)); + fn test_same_value_is_always_within_lossless_error_bound(value in ProptestValue::ANY) { + prop_assert!(is_value_within_error_bound(ErrorBound::Lossless, value, value)); } #[test] diff --git a/crates/modelardb_compression/src/models/pmc_mean.rs b/crates/modelardb_compression/src/models/pmc_mean.rs index 26f3f7ddf..6ea8f2813 100644 --- a/crates/modelardb_compression/src/models/pmc_mean.rs +++ b/crates/modelardb_compression/src/models/pmc_mean.rs @@ -112,71 +112,31 @@ pub fn grid(value: Value, timestamps: &[Timestamp], value_builder: &mut ValueBui mod tests { use super::*; - use modelardb_test::{ - ERROR_BOUND_ABSOLUTE_MAX, ERROR_BOUND_FIVE, ERROR_BOUND_RELATIVE_MAX, ERROR_BOUND_ZERO, - }; + use modelardb_test::{ERROR_BOUND_ABSOLUTE_MAX, ERROR_BOUND_FIVE, ERROR_BOUND_RELATIVE_MAX}; use proptest::num::f32 as ProptestValue; use proptest::{prop_assert, prop_assume, proptest}; // Tests for PMCMean. proptest! { #[test] - fn test_can_fit_sequence_of_finite_value_with_absolute_error_bound_zero(value in ProptestValue::ANY) { - can_fit_sequence_of_value_within_error_bound(ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), value) + fn test_can_fit_sequence_of_finite_value_with_lossless_error_bound(value in ProptestValue::ANY) { + can_fit_sequence_of_value_within_error_bound(ErrorBound::Lossless, value) } - - #[test] - fn test_can_fit_sequence_of_finite_value_with_relative_error_bound_zero(value in ProptestValue::ANY) { - can_fit_sequence_of_value_within_error_bound(ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), value) - } - } - - #[test] - fn test_can_fit_sequence_of_positive_infinity_with_absolute_error_bound_zero() { - can_fit_sequence_of_value_within_error_bound( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - Value::INFINITY, - ) - } - - #[test] - fn test_can_fit_sequence_of_positive_infinity_with_relative_error_bound_zero() { - can_fit_sequence_of_value_within_error_bound( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - Value::INFINITY, - ) - } - - #[test] - fn test_can_fit_sequence_of_negative_infinity_with_absolute_error_bound_zero() { - can_fit_sequence_of_value_within_error_bound( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - Value::NEG_INFINITY, - ) } #[test] - fn test_can_fit_sequence_of_negative_infinity_with_relative_error_bound_zero() { - can_fit_sequence_of_value_within_error_bound( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - Value::NEG_INFINITY, - ) + fn test_can_fit_sequence_of_positive_infinity_with_lossless_error_bound() { + can_fit_sequence_of_value_within_error_bound(ErrorBound::Lossless, Value::INFINITY) } #[test] - fn test_can_fit_sequence_of_nans_with_absolute_error_bound_zero() { - can_fit_sequence_of_value_within_error_bound( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - Value::NAN, - ) + fn test_can_fit_sequence_of_negative_infinity_with_lossless_error_bound() { + can_fit_sequence_of_value_within_error_bound(ErrorBound::Lossless, Value::NEG_INFINITY) } #[test] - fn test_can_fit_sequence_of_nans_with_relative_error_bound_zero() { - can_fit_sequence_of_value_within_error_bound( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - Value::NAN, - ) + fn test_can_fit_sequence_of_nans_with_lossless_error_bound() { + can_fit_sequence_of_value_within_error_bound(ErrorBound::Lossless, Value::NAN) } fn can_fit_sequence_of_value_within_error_bound(error_bound: ErrorBound, value: Value) { @@ -194,15 +154,8 @@ mod tests { proptest! { #[test] - fn test_can_fit_one_value_with_absolute_error_bound_zero(value in ProptestValue::ANY) { - let error_bound_zero = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - prop_assert!(PMCMean::new(error_bound_zero).fit_value(value)); - } - - #[test] - fn test_can_fit_one_value_with_relative_error_bound_zero(value in ProptestValue::ANY) { - let error_bound_zero = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); - prop_assert!(PMCMean::new(error_bound_zero).fit_value(value)); + fn test_can_fit_one_value_with_lossless_error_bound(value in ProptestValue::ANY) { + prop_assert!(PMCMean::new(ErrorBound::Lossless).fit_value(value)); } #[test] @@ -315,18 +268,9 @@ mod tests { } #[test] - fn test_cannot_fit_sequence_of_different_values_with_absolute_error_bound_zero() { - let error_bound_zero = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - assert!(!fit_sequence_of_different_values_with_error_bound( - error_bound_zero - )) - } - - #[test] - fn test_cannot_fit_sequence_of_different_values_with_relative_error_bound_zero() { - let error_bound_zero = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); + fn test_cannot_fit_sequence_of_different_values_with_lossless_error_bound() { assert!(!fit_sequence_of_different_values_with_error_bound( - error_bound_zero + ErrorBound::Lossless )) } diff --git a/crates/modelardb_compression/src/models/swing.rs b/crates/modelardb_compression/src/models/swing.rs index ba622bc6c..7a136c96f 100644 --- a/crates/modelardb_compression/src/models/swing.rs +++ b/crates/modelardb_compression/src/models/swing.rs @@ -348,9 +348,7 @@ mod tests { use arrow::array::{BinaryArray, Float32Array, Int8Array}; use arrow::datatypes::{DataType, Field, Schema}; - use modelardb_test::{ - ERROR_BOUND_ABSOLUTE_MAX, ERROR_BOUND_FIVE, ERROR_BOUND_RELATIVE_MAX, ERROR_BOUND_ZERO, - }; + use modelardb_test::{ERROR_BOUND_ABSOLUTE_MAX, ERROR_BOUND_FIVE, ERROR_BOUND_RELATIVE_MAX}; use modelardb_types::schemas::COMPRESSED_SCHEMA; use modelardb_types::types::{TimestampArray, TimestampBuilder, ValueArray, ValueBuilder}; use proptest::num::f32 as ProptestValue; @@ -368,66 +366,26 @@ mod tests { // Tests for Swing. proptest! { #[test] - fn test_can_fit_sequence_of_finite_value_with_absolute_error_bound_zero(value in ProptestValue::ANY) { + fn test_can_fit_sequence_of_finite_value_with_lossless_error_bound(value in ProptestValue::ANY) { can_fit_sequence_of_value_with_error_bound( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, value) } - - #[test] - fn test_can_fit_sequence_of_finite_value_with_relative_error_bound_zero(value in ProptestValue::ANY) { - can_fit_sequence_of_value_with_error_bound( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - value) - } - } - - #[test] - fn test_can_fit_sequence_of_positive_infinity_with_absolute_error_bound_zero() { - can_fit_sequence_of_value_with_error_bound( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - Value::INFINITY, - ) - } - - #[test] - fn test_can_fit_sequence_of_positive_infinity_with_relative_error_bound_zero() { - can_fit_sequence_of_value_with_error_bound( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - Value::INFINITY, - ) - } - - #[test] - fn test_can_fit_sequence_of_negative_infinity_with_absolute_error_bound_zero() { - can_fit_sequence_of_value_with_error_bound( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - Value::NEG_INFINITY, - ) } #[test] - fn test_can_fit_sequence_of_negative_infinity_with_relative_error_bound_zero() { - can_fit_sequence_of_value_with_error_bound( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - Value::NEG_INFINITY, - ) + fn test_can_fit_sequence_of_positive_infinity_with_lossless_error_bound() { + can_fit_sequence_of_value_with_error_bound(ErrorBound::Lossless, Value::INFINITY) } #[test] - fn test_can_fit_sequence_of_nans_with_absolute_error_bound_zero() { - can_fit_sequence_of_value_with_error_bound( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - Value::NAN, - ) + fn test_can_fit_sequence_of_negative_infinity_with_lossless_error_bound() { + can_fit_sequence_of_value_with_error_bound(ErrorBound::Lossless, Value::NEG_INFINITY) } #[test] - fn test_can_fit_sequence_of_nans_with_relative_error_bound_zero() { - can_fit_sequence_of_value_with_error_bound( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - Value::NAN, - ) + fn test_can_fit_sequence_of_nans_with_lossless_error_bound() { + can_fit_sequence_of_value_with_error_bound(ErrorBound::Lossless, Value::NAN) } fn can_fit_sequence_of_value_with_error_bound(error_bound: ErrorBound, value: Value) { @@ -456,35 +414,16 @@ mod tests { proptest! { #[test] - fn test_can_fit_one_value_with_absolute_error_bound_zero(value in ProptestValue::ANY) { - let error_bound_zero = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - prop_assert!(Swing::new(error_bound_zero).fit_data_point(START_TIME, value)); - } - - #[test] - fn test_can_fit_one_value_with_relative_error_bound_zero(value in ProptestValue::ANY) { - let error_bound_zero = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); - prop_assert!(Swing::new(error_bound_zero).fit_data_point(START_TIME, value)); - } - - #[test] - fn test_can_fit_two_finite_value_with_absolute_error_bound_zero( - first_value in ProptestValue::NORMAL, - second_value in ProptestValue::NORMAL - ) { - let error_bound_zero = ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = Swing::new(error_bound_zero); - prop_assert!(model_type.fit_data_point(START_TIME, first_value)); - prop_assert!(model_type.fit_data_point(END_TIME, second_value)); + fn test_can_fit_one_value_with_lossless_error_bound(value in ProptestValue::ANY) { + prop_assert!(Swing::new(ErrorBound::Lossless).fit_data_point(START_TIME, value)); } #[test] - fn test_can_fit_two_finite_value_with_relative_error_bound_zero( + fn test_can_fit_two_finite_value_with_lossless_error_bound( first_value in ProptestValue::NORMAL, second_value in ProptestValue::NORMAL ) { - let error_bound_zero = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(); - let mut model_type = Swing::new(error_bound_zero); + let mut model_type = Swing::new(ErrorBound::Lossless); prop_assert!(model_type.fit_data_point(START_TIME, first_value)); prop_assert!(model_type.fit_data_point(END_TIME, second_value)); } @@ -599,33 +538,17 @@ mod tests { } #[test] - fn test_can_fit_sequence_of_linear_values_with_absolute_error_bound_zero() { - assert!(fit_sequence_of_values_with_error_bound( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - &[42.0, 84.0, 126.0, 168.0, 210.0], - )) - } - - #[test] - fn test_can_fit_sequence_of_linear_values_with_relative_error_bound_zero() { + fn test_can_fit_sequence_of_linear_values_with_lossless_error_bound() { assert!(fit_sequence_of_values_with_error_bound( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, &[42.0, 84.0, 126.0, 168.0, 210.0], )) } #[test] - fn test_cannot_fit_sequence_of_different_values_with_absolute_error_bound_zero() { - assert!(!fit_sequence_of_values_with_error_bound( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - &[42.0, 42.0, 42.8, 42.0, 42.0], - )) - } - - #[test] - fn test_cannot_fit_sequence_of_different_values_with_relative_error_bound_zero() { + fn test_cannot_fit_sequence_of_different_values_with_lossless_error_bound() { assert!(!fit_sequence_of_values_with_error_bound( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, &[42.0, 42.0, 42.8, 42.0, 42.0], )) } @@ -793,43 +716,20 @@ mod tests { } #[test] - fn test_can_reconstruct_sequence_of_linear_increasing_values_within_absolute_error_bound_zero() - { - assert_can_reconstruct_sequence_of_linear_values_within_error_bound( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - (42..=4200).step_by(42).map(|value| value as f32).collect(), - ) - } - - #[test] - fn test_can_reconstruct_sequence_of_linear_increasing_values_within_relative_error_bound_zero() - { + fn test_can_reconstruct_sequence_of_linear_increasing_values_within_lossless_error_bound() { assert_can_reconstruct_sequence_of_linear_values_within_error_bound( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, (42..=4200).step_by(42).map(|value| value as f32).collect(), ) } #[test] - fn test_can_reconstruct_sequence_of_linear_decreasing_values_within_absolute_error_bound_zero() - { - let mut values: Vec = (42..=4200).step_by(42).map(|value| value as f32).collect(); - values.reverse(); - - assert_can_reconstruct_sequence_of_linear_values_within_error_bound( - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - values, - ); - } - - #[test] - fn test_can_reconstruct_sequence_of_linear_decreasing_values_within_relative_error_bound_zero() - { + fn test_can_reconstruct_sequence_of_linear_decreasing_values_within_lossless_error_bound() { let mut values: Vec = (42..=4200).step_by(42).map(|value| value as f32).collect(); values.reverse(); assert_can_reconstruct_sequence_of_linear_values_within_error_bound( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, values, ); } diff --git a/crates/modelardb_compression/src/types.rs b/crates/modelardb_compression/src/types.rs index 00a3c6033..94fee9f4a 100644 --- a/crates/modelardb_compression/src/types.rs +++ b/crates/modelardb_compression/src/types.rs @@ -511,8 +511,8 @@ mod tests { use arrow::array::BinaryArray; use arrow::datatypes::{DataType, Field}; + use modelardb_test::ERROR_BOUND_TEN; use modelardb_test::data_generation::{self, ValuesStructure}; - use modelardb_test::{ERROR_BOUND_TEN, ERROR_BOUND_ZERO}; use modelardb_types::schemas::COMPRESSED_SCHEMA; use modelardb_types::types::{TimestampArray, ValueArray}; @@ -792,7 +792,7 @@ mod tests { let model = compression::fit_next_model( 0, - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, &uncompressed_timestamps, uncompressed_values, ); @@ -819,7 +819,7 @@ mod tests { CompressedSegmentBatchBuilder::new(compressed_schema, vec!["tag".to_owned()], 0, 1); model.finish( - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, residuals_end_index, &uncompressed_timestamps, uncompressed_values, diff --git a/crates/modelardb_embedded/bindings/python/modelardb/error_bound.py b/crates/modelardb_embedded/bindings/python/modelardb/error_bound.py index c719a15f7..1fc78e652 100644 --- a/crates/modelardb_embedded/bindings/python/modelardb/error_bound.py +++ b/crates/modelardb_embedded/bindings/python/modelardb/error_bound.py @@ -27,7 +27,7 @@ class AbsoluteErrorBound: def __init__(self, value: float): # Error checking is required as the C-API depends on the error bounds being valid. - if not math.isfinite(value) or value < 0: + if not math.isfinite(value) or value <= 0: raise ValueError("An absolute error bound must be a positive finite value.") self.value = value @@ -36,15 +36,15 @@ def __init__(self, value: float): class RelativeErrorBound: """A per value relative error bound. - param value: A value between 0 and 100 that specifies the error bound. + param value: A positive value that is at most 100 that specifies the error bound as a percentage. :type value: float - :raises ValueError: If `value` is not between 0 and 100. + :raises ValueError: If `value` is not a positive value that is at most 100. """ def __init__(self, value: float): # Error checking is required as the C-API depends on the error bounds being valid. - if not 0 <= value <= 100: + if not 0 < value <= 100: raise ValueError( - "A relative error bound must be a value from 0.0% to 100.0%." + "A relative error bound must be a positive value that is at most 100.0%." ) self.value = value diff --git a/crates/modelardb_embedded/bindings/python/modelardb/table.py b/crates/modelardb_embedded/bindings/python/modelardb/table.py index d61f22625..49fb49e69 100644 --- a/crates/modelardb_embedded/bindings/python/modelardb/table.py +++ b/crates/modelardb_embedded/bindings/python/modelardb/table.py @@ -43,8 +43,7 @@ class TimeSeriesTable: :type schema: Schema :param error_bounds: Absolute or relative error bounds for the field columns with type :class:`pyarrow.float32()` in `schema`. If no error bound is - specified for a column, it will be zero so the values will be stored - losslessly. + specified for a column, the values will be stored losslessly. :type error_bounds: dict[str, AbsoluteErrorBound | RelativeErrorBound], optional :param generated_columns: SQL expressions for generating field columns of diff --git a/crates/modelardb_embedded/bindings/python/tests/test_error_bound.py b/crates/modelardb_embedded/bindings/python/tests/test_error_bound.py index fdf295a9b..443e7d1e4 100644 --- a/crates/modelardb_embedded/bindings/python/tests/test_error_bound.py +++ b/crates/modelardb_embedded/bindings/python/tests/test_error_bound.py @@ -19,8 +19,8 @@ class TestErrorBound(unittest.TestCase): # Tests for AbsoluteErrorBound. - def test_can_create_absolute_error_bound_with_float_zero(self): - _ = AbsoluteErrorBound(0) + def test_cannot_create_absolute_error_bound_with_float_zero(self): + self.assertRaises(ValueError, lambda: AbsoluteErrorBound(0)) def test_can_create_absolute_error_bound_with_normal_positive_float(self): _ = AbsoluteErrorBound(1) @@ -38,8 +38,8 @@ def test_cannot_create_absolute_error_bound_with_nan(self): self.assertRaises(ValueError, lambda: AbsoluteErrorBound(float("nan"))) # Tests for RelativeErrorBound. - def test_can_create_relative_error_bound_with_float_zero(self): - _ = RelativeErrorBound(0) + def test_cannot_create_relative_error_bound_with_float_zero(self): + self.assertRaises(ValueError, lambda: RelativeErrorBound(0)) def test_can_create_relative_error_bound_with_normal_positive_float(self): _ = RelativeErrorBound(1) diff --git a/crates/modelardb_embedded/src/capi.rs b/crates/modelardb_embedded/src/capi.rs index 64f865c1a..6cee0106e 100644 --- a/crates/modelardb_embedded/src/capi.rs +++ b/crates/modelardb_embedded/src/capi.rs @@ -336,10 +336,10 @@ unsafe fn create( TOKIO_RUNTIME.block_on(modelardb.create(table_name, table_type)) } -/// Converts the [`MapArray`] in error_bounds_array` to a [`HashMap`]. If a value is zero the column -/// with that name is stored losslessly, if the value is positive it is interpreted as an absolute -/// error bound, and if the value is negative it is interpreted as a relative error bound. Assumes -/// the error bound values are finite [`f32`]s. +/// Converts the [`MapArray`] in `error_bounds_array` to a [`HashMap`]. If the value is positive, it +/// is interpreted as an absolute error bound, and if the value is negative, it is interpreted as a +/// relative error bound. If no error bound is given for a column, it is stored losslessly. +/// Assumes the error bound values are finite [`f32`]s. fn error_bounds_array_to_error_bounds( error_bounds_array: &MapArray, ) -> HashMap { diff --git a/crates/modelardb_embedded/src/operations/data_folder.rs b/crates/modelardb_embedded/src/operations/data_folder.rs index f01c0c12f..bdf677ff5 100644 --- a/crates/modelardb_embedded/src/operations/data_folder.rs +++ b/crates/modelardb_embedded/src/operations/data_folder.rs @@ -1098,6 +1098,7 @@ mod tests { .map(|error_bound| match error_bound { ErrorBound::Absolute(value) => *value, ErrorBound::Relative(value) => -*value, + ErrorBound::Lossless => 0.0, }) .collect(); diff --git a/crates/modelardb_embedded/src/operations/mod.rs b/crates/modelardb_embedded/src/operations/mod.rs index e3185c86b..a5e97627a 100644 --- a/crates/modelardb_embedded/src/operations/mod.rs +++ b/crates/modelardb_embedded/src/operations/mod.rs @@ -130,12 +130,14 @@ fn try_new_time_series_table_metadata( let schema = Arc::new(schema); let df_schema: DFSchema = schema.clone().try_into()?; - let lossless = ErrorBound::try_new_absolute(0.0)?; - let mut error_bounds_all = Vec::with_capacity(schema.fields().len()); let mut generated_columns_all = Vec::with_capacity(schema.fields().len()); for field in schema.fields() { - error_bounds_all.push(error_bounds.remove(field.name()).unwrap_or(lossless)); + error_bounds_all.push( + error_bounds + .remove(field.name()) + .unwrap_or(ErrorBound::Lossless), + ); if let Some(sql_expr) = generated_columns.get(field.name()) { let expr = tokenize_and_parse_sql_expression(sql_expr, &df_schema)?; diff --git a/crates/modelardb_storage/src/metadata/table_metadata_manager.rs b/crates/modelardb_storage/src/metadata/table_metadata_manager.rs index 98a222d97..12172e16e 100644 --- a/crates/modelardb_storage/src/metadata/table_metadata_manager.rs +++ b/crates/modelardb_storage/src/metadata/table_metadata_manager.rs @@ -355,6 +355,7 @@ impl TableMetadataManager { match time_series_table_metadata.error_bounds[schema_index] { ErrorBound::Absolute(value) => (value, false), ErrorBound::Relative(value) => (value, true), + ErrorBound::Lossless => (0.0, false), } } else { (0.0, false) @@ -540,8 +541,7 @@ impl TableMetadataManager { ); let batch = sql_and_concat(&self.session_context, &sql).await?; - let mut column_to_error_bound = - vec![ErrorBound::try_new_absolute(0.0)?; query_schema_columns]; + let mut column_to_error_bound = vec![ErrorBound::Lossless; query_schema_columns]; let column_index_array = modelardb_types::array!(batch, 0, Int16Array); let error_bound_value_array = modelardb_types::array!(batch, 1, Float32Array); @@ -552,13 +552,15 @@ impl TableMetadataManager { let error_bound_value = error_bound_value_array.value(row_index); let error_bound_is_relative = error_bound_is_relative_array.value(row_index); - let error_bound = if error_bound_is_relative { - ErrorBound::try_new_relative(error_bound_value) - } else { - ErrorBound::try_new_absolute(error_bound_value) - }?; + if error_bound_value != 0.0 { + let error_bound = if error_bound_is_relative { + ErrorBound::try_new_relative(error_bound_value) + } else { + ErrorBound::try_new_absolute(error_bound_value) + }?; - column_to_error_bound[error_bound_index as usize] = error_bound; + column_to_error_bound[error_bound_index as usize] = error_bound; + } } Ok(column_to_error_bound) @@ -609,7 +611,6 @@ mod tests { use datafusion::arrow::datatypes::DataType; use datafusion::common::ScalarValue::Int64; use datafusion::logical_expr::Expr::Literal; - use modelardb_test::ERROR_BOUND_ZERO; use modelardb_test::table::{self, TIME_SERIES_TABLE_NAME}; use modelardb_types::types::{ArrowTimestamp, ArrowValue}; use tempfile::TempDir; @@ -923,6 +924,7 @@ mod tests { .map(|error_bound| match error_bound { ErrorBound::Absolute(value) => *value, ErrorBound::Relative(value) => *value, + ErrorBound::Lossless => 0.0, }) .collect(); @@ -945,10 +947,7 @@ mod tests { Field::new("tag", DataType::Utf8, false), ])); - let error_bounds = vec![ - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(); - query_schema.fields.len() - ]; + let error_bounds = vec![ErrorBound::Lossless; query_schema.fields.len()]; let plus_one_column = Some(GeneratedColumn { expr: col("field_1") + Literal(Int64(Some(1))), diff --git a/crates/modelardb_storage/src/parser.rs b/crates/modelardb_storage/src/parser.rs index b35f43894..310b530e7 100644 --- a/crates/modelardb_storage/src/parser.rs +++ b/crates/modelardb_storage/src/parser.rs @@ -1021,30 +1021,30 @@ fn column_defs_to_time_series_table_query_schema( } /// Extract the error bounds from the columns in `column_defs`. The error bound for the timestamp -/// and tag columns will be zero so the error bound of each column can be accessed using its index. +/// and tag columns will be lossless so the error bound of each column can be accessed using its index. fn extract_error_bounds_for_all_columns( column_defs: &[ColumnDef], ) -> StdResult, ParserError> { let mut error_bounds = Vec::with_capacity(column_defs.len()); for column_def in column_defs { - let mut error_bound_value = 0.0; - let mut is_relative = false; + let mut error_bound = ErrorBound::Lossless; for column_def_option in &column_def.options { if let ColumnOption::DialectSpecific(dialect_specific_tokens) = &column_def_option.option { - (error_bound_value, is_relative) = tokens_to_error_bound(dialect_specific_tokens)?; - } - } + let (error_bound_value, is_relative) = + tokens_to_error_bound(dialect_specific_tokens)?; - let error_bound = if !is_relative { - ErrorBound::try_new_absolute(error_bound_value) - } else { - ErrorBound::try_new_relative(error_bound_value) + error_bound = if !is_relative { + ErrorBound::try_new_absolute(error_bound_value) + } else { + ErrorBound::try_new_relative(error_bound_value) + } + .map_err(|error| ParserError::ParserError(error.to_string()))?; + } } - .map_err(|error| ParserError::ParserError(error.to_string()))?; error_bounds.push(error_bound); } diff --git a/crates/modelardb_test/src/table.rs b/crates/modelardb_test/src/table.rs index a852464d7..56fccb6e2 100644 --- a/crates/modelardb_test/src/table.rs +++ b/crates/modelardb_test/src/table.rs @@ -24,7 +24,7 @@ use modelardb_types::types::{ Value, ValueArray, }; -use crate::{ERROR_BOUND_FIVE, ERROR_BOUND_ONE, ERROR_BOUND_ZERO}; +use crate::{ERROR_BOUND_FIVE, ERROR_BOUND_ONE}; /// SQL to create a normal table with a timestamp column and two floating point columns. pub const NORMAL_TABLE_SQL: &str = @@ -90,10 +90,10 @@ pub fn time_series_table_metadata() -> TimeSeriesTableMetadata { ])); let error_bounds = vec![ - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, ErrorBound::try_new_absolute(ERROR_BOUND_ONE).unwrap(), ErrorBound::try_new_relative(ERROR_BOUND_FIVE).unwrap(), - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, ]; let generated_columns = vec![None, None, None, None]; diff --git a/crates/modelardb_types/src/flight/mod.rs b/crates/modelardb_types/src/flight/mod.rs index 2fbe05202..88325f747 100644 --- a/crates/modelardb_types/src/flight/mod.rs +++ b/crates/modelardb_types/src/flight/mod.rs @@ -170,13 +170,15 @@ fn encode_error_bounds( let mut error_bounds_all = Vec::with_capacity(time_series_table_metadata.query_schema.fields().len()); - let absolute_error_bound = + let absolute_error_bound_type = protocol::table_metadata::time_series_table_metadata::error_bound::Type::Absolute as i32; - let relative_error_bound = + let relative_error_bound_type = protocol::table_metadata::time_series_table_metadata::error_bound::Type::Relative as i32; + let lossless_error_bound_type = + protocol::table_metadata::time_series_table_metadata::error_bound::Type::Lossless as i32; let lossless = protocol::table_metadata::time_series_table_metadata::ErrorBound { - r#type: absolute_error_bound, + r#type: lossless_error_bound_type, value: 0.0, }; @@ -184,8 +186,9 @@ fn encode_error_bounds( if let Ok(field_index) = time_series_table_metadata.schema.index_of(field.name()) { let (error_bound_type, value) = match time_series_table_metadata.error_bounds[field_index] { - ErrorBound::Absolute(value) => (absolute_error_bound, value), - ErrorBound::Relative(value) => (relative_error_bound, value), + ErrorBound::Absolute(value) => (absolute_error_bound_type, value), + ErrorBound::Relative(value) => (relative_error_bound_type, value), + ErrorBound::Lossless => (lossless_error_bound_type, 0.0), }; error_bounds_all.push( @@ -250,6 +253,9 @@ fn decode_error_bounds( Ok( protocol::table_metadata::time_series_table_metadata::error_bound::Type::Relative, ) => error_bounds.push(ErrorBound::Relative(error_bound.value)), + Ok( + protocol::table_metadata::time_series_table_metadata::error_bound::Type::Lossless, + ) => error_bounds.push(ErrorBound::Lossless), _ => { return Err(ModelarDbTypesError::InvalidArgument(format!( "Unknown error bound type: {}.", diff --git a/crates/modelardb_types/src/flight/protocol.proto b/crates/modelardb_types/src/flight/protocol.proto index 8d2babd82..85242ac13 100644 --- a/crates/modelardb_types/src/flight/protocol.proto +++ b/crates/modelardb_types/src/flight/protocol.proto @@ -72,6 +72,7 @@ message TableMetadata { enum Type { ABSOLUTE = 0; RELATIVE = 1; + LOSSLESS = 2; } Type type = 1; float value = 2; diff --git a/crates/modelardb_types/src/types.rs b/crates/modelardb_types/src/types.rs index 4bcfca1d2..5c3a97f7a 100644 --- a/crates/modelardb_types/src/types.rs +++ b/crates/modelardb_types/src/types.rs @@ -103,6 +103,7 @@ impl TimeSeriesTableMetadata { /// * The number of potentially generated columns does not match the number of columns. /// * A generated column includes another generated column in its expression. /// * There are more than 32767 columns. + /// * The `query_schema` includes columns with unsupported data types. /// * The `query_schema` does not include a single timestamp column. /// * The `query_schema` does not include at least one stored field column. pub fn try_new( @@ -145,6 +146,23 @@ impl TimeSeriesTableMetadata { )); } + // If the schema contains an unsupported data type that does not correspond to a timestamp, + // field, or tag column, return an error. Note that this is only possible if the schema is + // provided directly since the schema of a time series table created using SQL is always + // valid. + for field in query_schema.fields() { + let data_type = field.data_type(); + if data_type != &ArrowTimestamp::DATA_TYPE + && data_type != &ArrowValue::DATA_TYPE + && data_type != &DataType::Utf8 + { + return Err(ModelarDbTypesError::InvalidArgument(format!( + "The data type '{data_type}' of column '{}' is not supported in a time series table.", + field.name() + ))); + } + } + // Remove the generated field columns from the query schema and the error bounds as these // columns should never be provided when inserting data points into the time series table. let mut fields_without_generated = Vec::with_capacity(query_schema.fields().len()); @@ -274,20 +292,22 @@ fn compute_indices_of_columns_with_data_type(schema: &Schema, data_type: DataTyp .collect() } -/// Absolute or relative per-value error bound. +/// Absolute, relative, or lossless per-value error bound. #[derive(Debug, Copy, Clone, PartialEq)] pub enum ErrorBound { /// An error bound that guarantees each value cannot deviate more than the [`Value`]. Absolute(Value), /// An error bound that guarantees each value cannot deviate more than 0.0% to 100.0%. Relative(f32), + /// An error bound that guarantees each value is stored losslessly. + Lossless, } impl ErrorBound { /// Return an [`ErrorBound::Absolute`] with `value` as its absolute per-value bound. A - /// [`ModelarDbTypesError`] is returned if a negative or non-normal value is passed. + /// [`ModelarDbTypesError`] is returned if a non-positive or non-normal value is passed. pub fn try_new_absolute(value: f32) -> Result { - if !value.is_finite() || value < 0.0 { + if !value.is_finite() || value <= 0.0 { Err(ModelarDbTypesError::InvalidArgument( "An absolute error bound must be a positive finite value.".to_owned(), )) @@ -297,11 +317,13 @@ impl ErrorBound { } /// Return an [`ErrorBound::Relative`] with `percentage` as its relative per-value bound. A - /// [`ModelarDbTypesError`] is returned if a value below 0% or a value above 100% is passed. + /// [`ModelarDbTypesError`] is returned if a value that is 0% or below is passed or a value + /// above 100% is passed. pub fn try_new_relative(percentage: f32) -> Result { - if !(0.0..=100.0).contains(&percentage) { + if !(0.0 < percentage && percentage <= 100.0) { Err(ModelarDbTypesError::InvalidArgument( - "A relative error bound must be a value from 0.0% to 100.0%.".to_owned(), + "A relative error bound must be a positive value that is at most 100.0%." + .to_owned(), )) } else { Ok(Self::Relative(percentage)) @@ -389,8 +411,8 @@ mod tests { use proptest::num; use proptest::proptest; - use modelardb_test::ERROR_BOUND_ZERO; use modelardb_test::table::{self, TIME_SERIES_TABLE_NAME}; + use modelardb_test::{ERROR_BOUND_FIVE, ERROR_BOUND_ZERO}; // Tests for TimeSeriesTableMetadata. #[test] @@ -458,11 +480,14 @@ mod tests { fn create_simple_time_series_table_metadata( query_schema: Schema, ) -> Result { + let error_bounds = vec![ErrorBound::Lossless; query_schema.fields.len()]; + let generated_columns = vec![None; query_schema.fields.len()]; + TimeSeriesTableMetadata::try_new( TIME_SERIES_TABLE_NAME.to_owned(), Arc::new(query_schema), - vec![ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap()], - vec![None], + error_bounds, + generated_columns, ) } @@ -533,13 +558,13 @@ mod tests { Field::new("temperature", ArrowValue::DATA_TYPE, false), ])), vec![ - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(), - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), - ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(), + ErrorBound::Lossless, + ErrorBound::Lossless, + ErrorBound::Lossless, + ErrorBound::Lossless, + ErrorBound::try_new_absolute(ERROR_BOUND_FIVE).unwrap(), + ErrorBound::Lossless, + ErrorBound::try_new_relative(ERROR_BOUND_FIVE).unwrap(), ], vec![None, None, None, None, None, None, None], ) @@ -608,8 +633,8 @@ mod tests { // Tests for ErrorBound. #[test] - fn test_absolute_error_bound_can_be_zero() { - assert!(ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).is_ok()) + fn test_absolute_error_bound_cannot_be_zero() { + assert!(ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).is_err()) } proptest! { @@ -640,8 +665,8 @@ mod tests { } #[test] - fn test_relative_error_bound_can_be_zero() { - assert!(ErrorBound::try_new_relative(ERROR_BOUND_ZERO).is_ok()) + fn test_relative_error_bound_cannot_be_zero() { + assert!(ErrorBound::try_new_relative(ERROR_BOUND_ZERO).is_err()) } proptest! {