Skip to content

Add lossless error bound enum variant #350

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
CGodiksen merged 16 commits into from
Sep 10, 2025
Merged

Add lossless error bound enum variant #350

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
e9e5738
Add lossless error bound type to protocol buffers
CGodiksen Sep 7, 2025
de0c4e3
Add Lossless variant to ErrorBound enum
CGodiksen Sep 7, 2025
1fe5c1e
Use Lossless variant instead of ERROR_BOUND_ZERO in compression tests
CGodiksen Sep 7, 2025
c2f8411
Fix bug that made it possible to create a time series table with an u…
CGodiksen Sep 7, 2025
f86b19e
Use lossless error bound instead of ERROR_BOUND_ZERO in macaque tests
CGodiksen Sep 8, 2025
307544c
Merge branch 'main' into dev/lossless-error-bound
CGodiksen Sep 9, 2025
2a05c08
No longer allow absolute or relative error bounds that are 0
CGodiksen Sep 9, 2025
17e8ead
Use ErrorBound::Lossless for swing tests
CGodiksen Sep 9, 2025
25353df
Use ErrorBound::Lossless for pmc mean tests
CGodiksen Sep 9, 2025
8b29af6
Use ErrorBound::Lossless for is_value_within_error_bound() test
CGodiksen Sep 9, 2025
eb6de3c
Check directly if an error bound is lossless by the variant
CGodiksen Sep 9, 2025
5dcb71d
Use ErrorBound::Lossless in parser and Operations API
CGodiksen Sep 9, 2025
5a93194
No longer allow 0 for error bounds in Python bindings
CGodiksen Sep 9, 2025
8c672fd
Merge branch 'main' into dev/lossless-error-bound
CGodiksen Sep 9, 2025
5940eb1
Update doc comment to make it more clear that a value cannot be zero
CGodiksen Sep 9, 2025
ee1b0fc
Update based on comment from @skejserjensen
CGodiksen Sep 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
250 changes: 35 additions & 215 deletions crates/modelardb_compression/src/compression.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,8 +264,8 @@ mod tests {

use arrow::array::{ArrayBuilder, BinaryArray, Float32Array, Int8Array};
use arrow::datatypes::{DataType, Field};
use modelardb_test::ERROR_BOUND_FIVE;
use modelardb_test::data_generation::{self, ValuesStructure};
use modelardb_test::{ERROR_BOUND_FIVE, ERROR_BOUND_ZERO};
use modelardb_types::schemas::COMPRESSED_SCHEMA;
use modelardb_types::types::{TimestampBuilder, ValueBuilder};

Expand All @@ -277,11 +277,11 @@ mod tests {

// Tests for try_compress().
#[test]
fn test_try_compress_empty_time_series_within_absolute_error_bound_zero() {
fn test_try_compress_empty_time_series_within_lossless_error_bound() {
let compressed_record_batch = try_compress(
&TimestampBuilder::new().finish(),
&ValueBuilder::new().finish(),
ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(),
ErrorBound::Lossless,
compressed_schema(),
vec![TAG_VALUE.to_owned()],
0,
Expand All @@ -291,55 +291,21 @@ mod tests {
}

#[test]
fn test_try_compress_empty_time_series_within_relative_error_bound_zero() {
let compressed_record_batch = try_compress(
&TimestampBuilder::new().finish(),
&ValueBuilder::new().finish(),
ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(),
compressed_schema(),
vec![TAG_VALUE.to_owned()],
0,
)
.unwrap();
assert_eq!(0, compressed_record_batch.num_rows());
}

#[test]
fn test_try_compress_regular_constant_time_series_within_absolute_error_bound_zero() {
generate_compress_and_assert_known_segment(
false,
ValuesStructure::Constant(None),
ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(),
&[models::PMC_MEAN_ID],
);
}

#[test]
fn test_try_compress_regular_constant_time_series_within_relative_error_bound_zero() {
fn test_try_compress_regular_constant_time_series_within_lossless_error_bound() {
generate_compress_and_assert_known_segment(
false,
ValuesStructure::Constant(None),
ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(),
ErrorBound::Lossless,
&[models::PMC_MEAN_ID],
);
}

#[test]
fn test_try_compress_irregular_constant_time_series_within_absolute_error_bound_zero() {
fn test_try_compress_irregular_constant_time_series_within_lossless_error_bound() {
generate_compress_and_assert_known_segment(
true,
ValuesStructure::Constant(None),
ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(),
&[models::PMC_MEAN_ID],
);
}

#[test]
fn test_try_compress_irregular_constant_time_series_within_relative_error_bound_zero() {
generate_compress_and_assert_known_segment(
true,
ValuesStructure::Constant(None),
ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(),
ErrorBound::Lossless,
&[models::PMC_MEAN_ID],
);
}
Expand Down Expand Up @@ -385,41 +351,21 @@ mod tests {
}

#[test]
fn test_try_compress_regular_linear_time_series_within_absolute_error_bound_zero() {
fn test_try_compress_regular_linear_time_series_within_lossless_error_bound() {
generate_compress_and_assert_known_segment(
false,
ValuesStructure::Linear(None),
ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(),
&[models::SWING_ID],
);
}

#[test]
fn test_try_compress_regular_linear_time_series_within_relative_error_bound_zero() {
generate_compress_and_assert_known_segment(
false,
ValuesStructure::Linear(None),
ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(),
&[models::SWING_ID],
);
}

#[test]
fn test_try_compress_irregular_linear_time_series_within_absolute_error_bound_zero() {
generate_compress_and_assert_known_segment(
true,
ValuesStructure::Linear(None),
ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(),
ErrorBound::Lossless,
&[models::SWING_ID],
);
}

#[test]
fn test_try_compress_irregular_linear_time_series_within_relative_error_bound_zero() {
fn test_try_compress_irregular_linear_time_series_within_lossless_error_bound() {
generate_compress_and_assert_known_segment(
true,
ValuesStructure::Linear(None),
ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(),
ErrorBound::Lossless,
&[models::SWING_ID],
);
}
Expand Down Expand Up @@ -465,41 +411,21 @@ mod tests {
}

#[test]
fn test_try_compress_regular_random_time_series_within_absolute_error_bound_zero() {
generate_compress_and_assert_known_segment(
false,
ValuesStructure::largest_random_without_overflow(),
ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(),
&[models::MACAQUE_V_ID],
);
}

#[test]
fn test_try_compress_regular_random_time_series_within_relative_error_bound_zero() {
fn test_try_compress_regular_random_time_series_within_lossless_error_bound() {
generate_compress_and_assert_known_segment(
false,
ValuesStructure::largest_random_without_overflow(),
ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(),
ErrorBound::Lossless,
&[models::MACAQUE_V_ID],
);
}

#[test]
fn test_try_compress_irregular_random_time_series_within_absolute_error_bound_zero() {
fn test_try_compress_irregular_random_time_series_within_lossless_error_bound() {
generate_compress_and_assert_known_segment(
true,
ValuesStructure::largest_random_without_overflow(),
ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(),
&[models::MACAQUE_V_ID],
);
}

#[test]
fn test_try_compress_irregular_random_time_series_within_relative_error_bound_zero() {
generate_compress_and_assert_known_segment(
true,
ValuesStructure::largest_random_without_overflow(),
ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(),
ErrorBound::Lossless,
&[models::MACAQUE_V_ID],
);
}
Expand Down Expand Up @@ -534,87 +460,41 @@ mod tests {
}

#[test]
fn test_try_compress_regular_random_linear_constant_time_series_within_absolute_error_bound_zero()
{
fn test_try_compress_regular_random_linear_constant_time_series_within_lossless_error_bound() {
generate_compress_and_assert_known_time_series(
ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(),
ErrorBound::Lossless,
false,
&[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID],
&[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID],
);
}

#[test]
fn test_try_compress_regular_random_linear_constant_time_series_within_relative_error_bound_zero()
{
generate_compress_and_assert_known_time_series(
ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(),
false,
&[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID],
&[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID],
);
}

#[test]
fn test_try_compress_irregular_random_linear_constant_time_series_within_absolute_error_bound_zero()
{
generate_compress_and_assert_known_time_series(
ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(),
true,
&[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID],
&[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID],
);
}

#[test]
fn test_try_compress_irregular_random_linear_constant_time_series_within_relative_error_bound_zero()
{
fn test_try_compress_irregular_random_linear_constant_time_series_within_lossless_error_bound()
{
generate_compress_and_assert_known_time_series(
ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(),
ErrorBound::Lossless,
true,
&[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID],
&[models::MACAQUE_V_ID, models::SWING_ID, models::PMC_MEAN_ID],
);
}

#[test]
fn test_try_compress_regular_constant_linear_random_time_series_within_absolute_error_bound_zero()
{
generate_compress_and_assert_known_time_series(
ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(),
false,
&[models::PMC_MEAN_ID, models::SWING_ID, models::MACAQUE_V_ID],
&[models::PMC_MEAN_ID, models::SWING_ID],
);
}

#[test]
fn test_try_compress_regular_constant_linear_random_time_series_within_relative_error_bound_zero()
{
fn test_try_compress_regular_constant_linear_random_time_series_within_lossless_error_bound() {
generate_compress_and_assert_known_time_series(
ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(),
ErrorBound::Lossless,
false,
&[models::PMC_MEAN_ID, models::SWING_ID, models::MACAQUE_V_ID],
&[models::PMC_MEAN_ID, models::SWING_ID],
);
}

#[test]
fn test_try_compress_irregular_constant_linear_random_time_series_within_absolute_error_bound_zero()
{
generate_compress_and_assert_known_time_series(
ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(),
true,
&[models::PMC_MEAN_ID, models::SWING_ID, models::MACAQUE_V_ID],
&[models::PMC_MEAN_ID, models::SWING_ID],
);
}

#[test]
fn test_try_compress_irregular_constant_linear_random_time_series_within_relative_error_bound_zero()
{
fn test_try_compress_irregular_constant_linear_random_time_series_within_lossless_error_bound()
{
generate_compress_and_assert_known_time_series(
ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(),
ErrorBound::Lossless,
true,
&[models::PMC_MEAN_ID, models::SWING_ID, models::MACAQUE_V_ID],
&[models::PMC_MEAN_ID, models::SWING_ID],
Expand Down Expand Up @@ -707,23 +587,8 @@ mod tests {
}

#[test]
fn test_try_compress_regular_synthetic_time_series_without_noise_within_absolute_error_bound_zero()
{
generate_compress_and_assert_time_series(
ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(),
false,
None,
)
}

#[test]
fn test_try_compress_regular_synthetic_time_series_without_noise_within_relative_error_bound_zero()
{
generate_compress_and_assert_time_series(
ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(),
false,
None,
)
fn test_try_compress_regular_synthetic_time_series_without_noise_within_lossless_error_bound() {
generate_compress_and_assert_time_series(ErrorBound::Lossless, false, None)
}

#[test]
Expand All @@ -747,23 +612,8 @@ mod tests {
}

#[test]
fn test_try_compress_regular_synthetic_time_series_with_noise_within_absolute_error_bound_zero()
{
generate_compress_and_assert_time_series(
ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(),
false,
ADD_NOISE_RANGE,
)
}

#[test]
fn test_try_compress_regular_synthetic_time_series_with_noise_within_relative_error_bound_zero()
{
generate_compress_and_assert_time_series(
ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(),
false,
ADD_NOISE_RANGE,
)
fn test_try_compress_regular_synthetic_time_series_with_noise_within_lossless_error_bound() {
generate_compress_and_assert_time_series(ErrorBound::Lossless, false, ADD_NOISE_RANGE)
}

#[test]
Expand All @@ -787,23 +637,9 @@ mod tests {
}

#[test]
fn test_try_compress_irregular_synthetic_time_series_without_noise_within_absolute_error_bound_zero()
{
generate_compress_and_assert_time_series(
ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(),
true,
None,
)
}

#[test]
fn test_try_compress_irregular_synthetic_time_series_without_noise_within_relative_error_bound_zero()
{
generate_compress_and_assert_time_series(
ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(),
true,
None,
)
fn test_try_compress_irregular_synthetic_time_series_without_noise_within_lossless_error_bound()
{
generate_compress_and_assert_time_series(ErrorBound::Lossless, true, None)
}

#[test]
Expand All @@ -827,23 +663,8 @@ mod tests {
}

#[test]
fn test_try_compress_irregular_synthetic_time_series_with_noise_within_absolute_error_bound_zero()
{
generate_compress_and_assert_time_series(
ErrorBound::try_new_absolute(ERROR_BOUND_ZERO).unwrap(),
true,
ADD_NOISE_RANGE,
)
}

#[test]
fn test_try_compress_irregular_synthetic_time_series_with_noise_within_relative_error_bound_zero()
{
generate_compress_and_assert_time_series(
ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap(),
true,
ADD_NOISE_RANGE,
)
fn test_try_compress_irregular_synthetic_time_series_with_noise_within_lossless_error_bound() {
generate_compress_and_assert_time_series(ErrorBound::Lossless, true, ADD_NOISE_RANGE)
}

#[test]
Expand Down Expand Up @@ -967,7 +788,6 @@ mod tests {
// Tests for compress_and_store_residuals_in_a_separate_segment().
#[test]
fn test_compress_and_store_residuals_in_a_separate_segment() {
let error_bound = ErrorBound::try_new_relative(ERROR_BOUND_ZERO).unwrap();
let uncompressed_timestamps = TimestampArray::from_iter_values((100..=500).step_by(100));
let uncompressed_values = ValueArray::from(vec![73.0, 37.0, 37.0, 37.0, 73.0]);

Expand All @@ -979,7 +799,7 @@ mod tests {
);

compress_and_store_residuals_in_a_separate_segment(
error_bound,
ErrorBound::Lossless,
0,
uncompressed_timestamps.len() - 1,
&uncompressed_timestamps,
Expand Down
Loading
Loading