From 62fabfd1faebff6775118a6ae7cc86237b73de72 Mon Sep 17 00:00:00 2001 From: UnamedRus Date: Thu, 17 Sep 2026 15:15:02 +0300 Subject: [PATCH 1/5] Update contrib/datasketches-cpp to 5.2.0 and backport the HLL union fix Backport of ClickHouse/ClickHouse@a067567ebb179fcf59cd20ef4dbb003c6625d6c7, re-created here rather than cherry-picked so that it carries a sign-off. The submodule tracked `apache/datasketches-cpp` directly, pinned to `76edd74f` (2024-05-16), an upstream development commit. It moves onto a `ClickHouse/`-prefixed branch of our fork, the way `docs/development/contrib` asks: ClickHouse/datasketches-cpp, branch ClickHouse/5.2.0 de8553ba 5.2.0, the newest upstream release (2025-01-15) 23bd9b07 backport of apache/datasketches-cpp#512 apache/datasketches-cpp#512 is the HyperLogLog union fix. No upstream release carries it, so it is cherry-picked onto the `5.2.0` tag there. `uniqApacheHLL`, added next, needs it: without it a merged HLL sketch reports a wrong estimate and serializes a state that other DataSketches implementations read differently. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: UnamedRus --- .gitmodules | 2 +- contrib/datasketches-cpp | 2 +- ...5055_uniq_theta_compressed_state.reference | 1 + .../05055_uniq_theta_compressed_state.sql | 22 +++++++++++++++++++ 4 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05055_uniq_theta_compressed_state.reference create mode 100644 tests/queries/0_stateless/05055_uniq_theta_compressed_state.sql diff --git a/.gitmodules b/.gitmodules index 6acf8b86ad42..343e61166d95 100644 --- a/.gitmodules +++ b/.gitmodules @@ -154,7 +154,7 @@ url = https://github.com/ClickHouse/NuRaft [submodule "contrib/datasketches-cpp"] path = contrib/datasketches-cpp - url = https://github.com/apache/datasketches-cpp + url = https://github.com/ClickHouse/datasketches-cpp [submodule "contrib/yaml-cpp"] path = contrib/yaml-cpp url = https://github.com/ClickHouse/yaml-cpp diff --git a/contrib/datasketches-cpp b/contrib/datasketches-cpp index 76edd74f5db2..23bd9b070ddf 160000 --- a/contrib/datasketches-cpp +++ b/contrib/datasketches-cpp @@ -1 +1 @@ -Subproject commit 76edd74f5db286b672c170a8ded4ce39b3a8800f +Subproject commit 23bd9b070ddf7f0dcf32c87d6752ce535374697b diff --git a/tests/queries/0_stateless/05055_uniq_theta_compressed_state.reference b/tests/queries/0_stateless/05055_uniq_theta_compressed_state.reference new file mode 100644 index 000000000000..5a931d5663aa --- /dev/null +++ b/tests/queries/0_stateless/05055_uniq_theta_compressed_state.reference @@ -0,0 +1 @@ +16 16 30 diff --git a/tests/queries/0_stateless/05055_uniq_theta_compressed_state.sql b/tests/queries/0_stateless/05055_uniq_theta_compressed_state.sql new file mode 100644 index 000000000000..9668fc48c74e --- /dev/null +++ b/tests/queries/0_stateless/05055_uniq_theta_compressed_state.sql @@ -0,0 +1,22 @@ +-- Tags: no-fasttest +-- - no-fasttest -- compiled w/o datasketches + +-- Apache DataSketches can serialize a Theta sketch in a compressed form (serialization +-- version 4) that packs the retained values at a variable number of bits per entry. +-- ClickHouse always writes the uncompressed form, but it has to read the compressed one, +-- because an `AggregateFunction(uniqTheta, ...)` state can come from another +-- implementation of the library. Two of the unpacking routines decoded it incorrectly, +-- and the result was a silently too low estimate rather than an error. + +-- Both states below are canonical compressed encodings of a sketch that retains 16 values +-- with theta = 1, so the estimate is exact. The first packs its values at 33 bits per +-- entry and the second at 35; the two widths are decoded by separate routines. The two +-- sketches have exactly two values in common, so their union retains 30. + +WITH + CAST(unhex('4B01040321011ACC9310000001F4000000FA0000007D0000003E8000001F4000000FA0000007D180000000000001F4000000FA0000007D0000003E8000001F4000000FA0000007D0000003E8') AS AggregateFunction(uniqTheta, UInt64)) AS state_33_bits, + CAST(unhex('4F01040323011ACC93100000007D0000000FA1000000004000000000000007D0000000FA0000001F40000003E80000007D0000000FA1000000000000003E80000007D0000000FA0000001F40000003E8') AS AggregateFunction(uniqTheta, UInt64)) AS state_35_bits +SELECT + finalizeAggregation(state_33_bits), + finalizeAggregation(state_35_bits), + finalizeAggregation(uniqThetaUnion(state_33_bits, state_35_bits)); From 8cd058668b70adfb39dca33309142019d4905ae0 Mon Sep 17 00:00:00 2001 From: UnamedRus Date: Thu, 17 Sep 2026 15:15:03 +0300 Subject: [PATCH 2/5] Add uniqApacheHLL: Apache DataSketches HLL as a native aggregate function Ported from `vk/uniq-apache-hll` (github.com/UnamedRus/ClickHouse), which develops the function against `master`. `uniqApacheHLL` counts distinct values into an Apache DataSketches HLL sketch, and its `-State` serializes that sketch in the DataSketches format, so states can be exchanged with Java, Python and C++ services through the standard `-State`/`-Merge` combinators. Only the argument types those libraries hash the same way are accepted - integers of at most 64 bits, Enum8/16, BFloat16, Float32/64, String, FixedString, UUID, IPv4, IPv6, Date, Date32, DateTime and DateTime64 - so no state can be built here that an external consumer cannot reproduce. Two deviations from the branch this is taken from, both forced by the age of this base: - the state overrides `merge`, not `mergeImpl`. `IAggregateFunction::merge` is still the pure virtual here; the split into a non-virtual `merge` plus a `mergeImpl` override came later. - `introduced_in` says 26.6 rather than 26.9, this being the release it ships in. NOT BUILT OR TESTED on this base - only ported and checked by inspection against the 26.6 headers. CI is the first real build. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: UnamedRus --- contrib/datasketches-cpp-cmake/CMakeLists.txt | 1 + .../AggregateFunctionUniqApacheHLL.cpp | 150 +++++++++ .../AggregateFunctionUniqApacheHLL.h | 295 ++++++++++++++++++ .../registerAggregateFunctions.cpp | 6 + .../04327_uniq_apache_hll.reference | 19 ++ .../0_stateless/04327_uniq_apache_hll.sql | 60 ++++ .../05026_uniq_apache_hll_interop.reference | 23 ++ .../05026_uniq_apache_hll_interop.sql | 95 ++++++ ...5027_uniq_apache_hll_cross_scale.reference | 11 + .../05027_uniq_apache_hll_cross_scale.sql | 47 +++ ...8_uniq_apache_hll_argument_types.reference | 26 ++ .../05028_uniq_apache_hll_argument_types.sql | 54 ++++ ..._uniq_apache_hll_corrupted_state.reference | 3 + .../05136_uniq_apache_hll_corrupted_state.sh | 37 +++ 14 files changed, 827 insertions(+) create mode 100644 src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp create mode 100644 src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h create mode 100644 tests/queries/0_stateless/04327_uniq_apache_hll.reference create mode 100644 tests/queries/0_stateless/04327_uniq_apache_hll.sql create mode 100644 tests/queries/0_stateless/05026_uniq_apache_hll_interop.reference create mode 100644 tests/queries/0_stateless/05026_uniq_apache_hll_interop.sql create mode 100644 tests/queries/0_stateless/05027_uniq_apache_hll_cross_scale.reference create mode 100644 tests/queries/0_stateless/05027_uniq_apache_hll_cross_scale.sql create mode 100644 tests/queries/0_stateless/05028_uniq_apache_hll_argument_types.reference create mode 100644 tests/queries/0_stateless/05028_uniq_apache_hll_argument_types.sql create mode 100644 tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.reference create mode 100755 tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.sh diff --git a/contrib/datasketches-cpp-cmake/CMakeLists.txt b/contrib/datasketches-cpp-cmake/CMakeLists.txt index 497d6956d0ef..ff4199c17430 100644 --- a/contrib/datasketches-cpp-cmake/CMakeLists.txt +++ b/contrib/datasketches-cpp-cmake/CMakeLists.txt @@ -10,6 +10,7 @@ add_library(_datasketches INTERFACE) target_include_directories(_datasketches SYSTEM BEFORE INTERFACE "${ClickHouse_SOURCE_DIR}/contrib/datasketches-cpp/common/include" "${ClickHouse_SOURCE_DIR}/contrib/datasketches-cpp/count/include" + "${ClickHouse_SOURCE_DIR}/contrib/datasketches-cpp/hll/include" "${ClickHouse_SOURCE_DIR}/contrib/datasketches-cpp/theta/include") add_library(ch_contrib::datasketches ALIAS _datasketches) diff --git a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp new file mode 100644 index 000000000000..a9425146ea95 --- /dev/null +++ b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp @@ -0,0 +1,150 @@ +#include +#include +#include + +#if USE_DATASKETCHES + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ARGUMENT_OUT_OF_BOUND; + extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +AggregateFunctionPtr createAggregateFunctionUniqApacheHLL( + const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *) +{ + uint8_t lg_config_k = 12; + datasketches::target_hll_type target_type = datasketches::HLL_4; + + if (params.size() > 2) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Aggregate function {} accepts at most two parameters (lg_k, type).", name); + + if (!params.empty()) + { + const UInt64 lg_k_param = applyVisitor(FieldVisitorConvertToNumber(), params[0]); + if (lg_k_param < 4 || lg_k_param > 21) + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, + "Parameter lg_k for aggregate function {} is out of range: [4, 21].", name); + lg_config_k = static_cast(lg_k_param); + } + + if (params.size() == 2) + { + if (params[1].getType() != Field::Types::String) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Parameter type for aggregate function {} must be a string.", name); + + const String type_param = params[1].safeGet(); + if (type_param == "HLL_4") + target_type = datasketches::HLL_4; + else if (type_param == "HLL_6") + target_type = datasketches::HLL_6; + else if (type_param == "HLL_8") + target_type = datasketches::HLL_8; + else + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Parameter type for aggregate function {} must be one of 'HLL_4', 'HLL_6', 'HLL_8'.", name); + } + + /// Only the types a sketch hashes the same way in every language are accepted, so that no state + /// is built here that an external consumer cannot reproduce. Anything else - a decimal, a wide + /// integer, an array, a tuple, several arguments - would first need a hash only ClickHouse has. + if (argument_types.size() != 1) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Aggregate function {} requires exactly one argument, passed {}. Several arguments would " + "have to be hashed to one value by ClickHouse first, which no producer outside ClickHouse " + "can reproduce. To count distinct combinations of several columns, use uniq, uniqCombined " + "or uniqHLL12.", + name, argument_types.size()); + + const IDataType & argument_type = *argument_types[0]; + WhichDataType which(argument_type); + + /// Backed by a decimal, but accepted unlike one: it holds the epoch time a caller elsewhere + /// passes to `update(long)`. + if (which.isDateTime64()) + return std::make_shared>(lg_config_k, target_type, argument_types, params); + + /// `createWithNumericType` also covers the 128 and 256 bit integers, for which no byte order is + /// agreed on, so they have to be refused before it is reached. + if (!which.isInt128() && !which.isInt256() && !which.isUInt128() && !which.isUInt256()) + { + AggregateFunctionPtr res(createWithNumericType( + argument_type, lg_config_k, target_type, argument_types, params)); + if (res) + return res; + } + + if (which.isDate()) + return std::make_shared>(lg_config_k, target_type, argument_types, params); + if (which.isDate32()) + return std::make_shared>(lg_config_k, target_type, argument_types, params); + if (which.isDateTime()) + return std::make_shared>(lg_config_k, target_type, argument_types, params); + if (which.isStringOrFixedString()) + return std::make_shared>(lg_config_k, target_type, argument_types, params); + if (which.isUUID()) + return std::make_shared>(lg_config_k, target_type, argument_types, params); + if (which.isIPv4()) + return std::make_shared>(lg_config_k, target_type, argument_types, params); + if (which.isIPv6()) + return std::make_shared>(lg_config_k, target_type, argument_types, params); + + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Aggregate function {} does not support the argument type {}: only the types an Apache " + "DataSketches HLL sketch hashes the same way outside ClickHouse are supported: integers of " + "at most 64 bits, Enum8, Enum16, BFloat16, Float32, Float64, String, FixedString, UUID, " + "IPv4, IPv6, Date, Date32, DateTime and DateTime64. To count distinct values of any other " + "type, use uniq, uniqCombined or uniqHLL12.", + name, argument_type.getName()); +} + +void registerAggregateFunctionUniqApacheHLL(AggregateFunctionFactory & factory); +void registerAggregateFunctionUniqApacheHLL(AggregateFunctionFactory & factory) +{ + FunctionDocumentation::Description description = R"( +Calculates the approximate number of different argument values using an [Apache DataSketches](https://datasketches.apache.org/docs/HLL/HllSketches.html) HyperLogLog sketch. + +The serialized state produced by the `-State` combinator carries the sketch in the Apache DataSketches HLL format, framed with a varint length prefix, so sketches can be exchanged with external services (Java, Python, C++) using the standard `-State`/`-Merge` combinators. For example, a sketch built by an upstream service can be merged with `uniqApacheHLLMerge`, and a sketch built in ClickHouse can be exported with `uniqApacheHLLState`. + +Only the types that a sketch hashes the same way in every implementation are accepted, so that every state ClickHouse produces can be reproduced elsewhere. Integers of at most 64 bits are hashed as their 8-byte representation (matching DataSketches `update(long)`), floating-point values as an IEEE-754 double, strings as their raw bytes, `UUID`s as their canonical 16 bytes and `IPv6` addresses in network order. + +`Date`, `Date32`, `DateTime` and `DateTime64` are hashed as the integer they hold, which for `DateTime64(3)` is the epoch milliseconds a caller elsewhere would pass to `update(long)`. The unit belongs to the column type rather than to the value, so both sides have to agree on it. + +Every other type is rejected, because a sketch over it would have to be built from a hash that only ClickHouse can compute: the 128 and 256 bit integers have no agreed byte order across implementations, decimals have no representation any DataSketches binding accepts, and an array or a tuple has none either. The same goes for a call with more than one argument. Use `uniq`, `uniqCombined` or `uniqHLL12` to count distinct values of those. + +An estimate obtained by merging sketches is not the same number as one computed in a single pass over the same values, even though both are derived from identical registers: DataSketches reports the HIP estimator for a sketch that has only been updated and the composite estimator for one produced by a union. The result therefore depends on how the aggregation was partitioned across threads, parts and shards, and is slightly less accurate once any merge has taken place. + +The resolution of a merged sketch is the smallest `lg_k` among its inputs, not the `lg_k` named by the type. Merging a sketch that was built with a lower `lg_k` - for example one produced by another service - permanently lowers the resolution of both the estimate and the state written back. + )"; + FunctionDocumentation::Syntax syntax = "uniqApacheHLL([lg_k, [type]])(x)"; + FunctionDocumentation::Arguments arguments = { + {"x", "Column to compute the number of distinct values of.", {"(U)Int8/16/32/64", "Enum8", "Enum16", "BFloat16", "Float32", "Float64", "String", "FixedString", "UUID", "IPv4", "IPv6", "Date", "Date32", "DateTime", "DateTime64"}}, + }; + FunctionDocumentation::Parameters parameters = { + {"lg_k", "Optional. Log-base-2 of the number of buckets, in range [4, 21]. Higher means better accuracy and more memory. Default: 12.", {"UInt8"}}, + {"type", "Optional. Storage format of the sketch: 'HLL_4', 'HLL_6', or 'HLL_8'. Default: 'HLL_4'.", {"String"}}, + }; + FunctionDocumentation::ReturnedValue returned_value = {"Returns the approximate number of distinct values.", {"UInt64"}}; + FunctionDocumentation::Examples examples = { + {"Basic usage", "SELECT uniqApacheHLL(number) FROM numbers(1000)", "1000"}, + {"With parameters", "SELECT uniqApacheHLL(14, 'HLL_8')(number) FROM numbers(1000)", "1000"}, + }; + FunctionDocumentation::IntroducedIn introduced_in = {26, 6}; + FunctionDocumentation::Category category = FunctionDocumentation::Category::AggregateFunction; + FunctionDocumentation documentation = {description, syntax, arguments, parameters, returned_value, examples, introduced_in, category}; + + AggregateFunctionProperties properties = { .returns_default_when_only_null = true, .is_order_dependent = false }; + + factory.registerFunction("uniqApacheHLL", {createAggregateFunctionUniqApacheHLL, documentation, properties}); +} + +} + +#endif diff --git a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h new file mode 100644 index 000000000000..b223440e1e1b --- /dev/null +++ b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h @@ -0,0 +1,295 @@ +#pragma once + +#include "config.h" + +#if USE_DATASKETCHES + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; +} + + +/** An Apache DataSketches HLL sketch as an aggregate function state, serialized in the + * DataSketches HLL format so that `-State`/`-Merge` interoperate with external services. + * + * `lg_config_k` and the target type are owned by the aggregate function and passed in, so the + * state stores the sketch alone. + */ +class HllSketchData : private boost::noncopyable +{ +private: + /// Used for insertions. + std::unique_ptr sk_update; + /// Used for merging. + std::unique_ptr sk_union; + + datasketches::hll_sketch * getSkUpdate(uint8_t lg_config_k, datasketches::target_hll_type tgt_type) + { + if (!sk_update) + sk_update = std::make_unique(lg_config_k, tgt_type); + return sk_update.get(); + } + + datasketches::hll_union * getSkUnion(uint8_t lg_config_k) + { + if (!sk_union) + { + /// `hll_union` takes `lg_max_k` in [7, 21] while a sketch may use [4, 21], hence the + /// floor of 7. It does not inflate the result: `get_result` downsamples to the smallest + /// `lg_config_k` the union has seen. So the resolution of a merged state is the minimum + /// over its inputs, not the `lg_config_k` of the type, and merging a coarser sketch from + /// elsewhere lowers both the estimate and the state written back. Merged from empty + /// states only there is no minimum, and it serializes as `lg_config_k = 7`. + sk_union = std::make_unique(std::max(lg_config_k, 7)); + } + return sk_union.get(); + } + + /// Fold a sketch updated after the union was allocated into it, so merges see one state. + void foldUpdateIntoUnionIfNeeded() + { + if (sk_union && sk_update) + { + sk_union->update(*sk_update); + sk_update.reset(nullptr); + } + } + +public: + HllSketchData() = default; + ~HllSketchData() = default; + + template + void insert(T value, uint8_t lg_config_k, datasketches::target_hll_type tgt_type) + { + getSkUpdate(lg_config_k, tgt_type)->update(value); + foldUpdateIntoUnionIfNeeded(); + } + + void insertData(const char * data, size_t size, uint8_t lg_config_k, datasketches::target_hll_type tgt_type) + { + getSkUpdate(lg_config_k, tgt_type)->update(static_cast(data), size); + foldUpdateIntoUnionIfNeeded(); + } + + UInt64 size(datasketches::target_hll_type tgt_type) const + { + /// Round rather than truncate: `get_estimate` returns a `double`, and `999.9999` for an + /// exactly-known cardinality must not become `999`. + if (sk_union) + return static_cast(std::llround(sk_union->get_result(tgt_type).get_estimate())); + if (sk_update) + return static_cast(std::llround(sk_update->get_estimate())); + return 0; + } + + void merge(const HllSketchData & rhs, uint8_t lg_config_k, datasketches::target_hll_type tgt_type) + { + datasketches::hll_union * u = getSkUnion(lg_config_k); + + if (sk_update) + { + u->update(*sk_update); + sk_update.reset(nullptr); + } + + if (rhs.sk_update) + u->update(*rhs.sk_update); + else if (rhs.sk_union) + u->update(rhs.sk_union->get_result(tgt_type)); + } + + /// You can only call this for an empty object. + void read(ReadBuffer & in, uint8_t lg_config_k) + { + datasketches::hll_sketch::vector_bytes bytes; + readVectorBinary(bytes, in); + if (bytes.empty()) + return; + + try + { + auto sk = datasketches::hll_sketch::deserialize(bytes.data(), bytes.size()); + getSkUnion(lg_config_k)->update(std::move(sk)); + } + catch (const DB::Exception &) + { + throw; + } + catch (const std::bad_alloc &) + { + /// Memory pressure, not corrupted data. + throw; + } + catch (const std::exception & e) + { + /// `datasketches` reports malformed input as `std::invalid_argument` / `std::out_of_range`. + /// Not being `DB::Exception`, those escape `SerializationAggregateFunction`'s + /// `catch (...)` and abort as a logical error, so translate them here. + throw Exception(ErrorCodes::CORRUPTED_DATA, "Cannot deserialize HLL sketch state: {}", e.what()); + } + } + + void write(WriteBuffer & out, datasketches::target_hll_type tgt_type) const + { + if (sk_update) + { + auto bytes = sk_update->serialize_compact(); + writeVectorBinary(bytes, out); + } + else if (sk_union) + { + auto bytes = sk_union->get_result(tgt_type).serialize_compact(); + writeVectorBinary(bytes, out); + } + else + { + datasketches::hll_sketch::vector_bytes bytes; + writeVectorBinary(bytes, out); + } + } +}; + + +/** `uniqApacheHLL` over a single column of a type the sketch can hash directly. + * + * The value takes one of the three shapes the DataSketches API accepts - an 8-byte integer, an + * IEEE-754 double, or raw bytes - so that an external producer reaches the same sketch. + * + * `lg_config_k` and the target type live here rather than in the state, which is what lets states + * of different parameterisations share one binary representation. + */ +template +class AggregateFunctionUniqApacheHLL final : public IAggregateFunctionDataHelper> +{ + using Base = IAggregateFunctionDataHelper>; + + uint8_t lg_config_k; + datasketches::target_hll_type target_type; + +public: + AggregateFunctionUniqApacheHLL( + uint8_t lg_config_k_, + datasketches::target_hll_type target_type_, + const DataTypes & argument_types_, + const Array & params_) + : Base(argument_types_, params_, std::make_shared()) + , lg_config_k(lg_config_k_) + , target_type(target_type_) + { + } + + String getName() const override { return "uniqApacheHLL"; } + + bool allocatesMemoryInArena() const override { return false; } + + void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const override + { + auto & data = this->data(place); + + if constexpr (std::is_same_v) + { + const auto value = columns[0]->getDataAt(row_num); + data.insertData(value.data(), value.size(), lg_config_k, target_type); + } + else + { + const auto & value = assert_cast &>(*columns[0]).getData()[row_num]; + + if constexpr (std::is_same_v) + { + /// ClickHouse holds a UUID as two 64-bit halves in host order, so its bytes in + /// memory are not the canonical 16 an external producer works from. + const UInt64 halves[2] = { + std::byteswap(UUIDHelpers::getHighBytes(value)), + std::byteswap(UUIDHelpers::getLowBytes(value)), + }; + data.insertData(reinterpret_cast(halves), sizeof(halves), lg_config_k, target_type); + } + else if constexpr (std::is_same_v) + /// Already held in network order, which is the canonical form. + data.insertData(reinterpret_cast(&value), sizeof(value), lg_config_k, target_type); + else if constexpr (is_decimal) + /// `DateTime64(3)` holds the epoch milliseconds a caller elsewhere passes to + /// `update(long)`. The scale belongs to the type, so both sides must agree on it. + data.insert(static_cast(value.value), lg_config_k, target_type); + else if constexpr (std::is_same_v) + data.insert(static_cast(value.toUnderType()), lg_config_k, target_type); + else if constexpr (std::is_same_v || std::is_floating_point_v) + data.insert(static_cast(value), lg_config_k, target_type); + else if constexpr (std::is_signed_v) + data.insert(static_cast(value), lg_config_k, target_type); + else + data.insert(static_cast(value), lg_config_k, target_type); + } + } + /// A serialized sketch describes its own configuration, so states of different parameterisations + /// are interchangeable. Merging across them takes the resolution of the coarsest input. + bool haveSameStateRepresentationImpl(const IAggregateFunction & rhs) const override + { + return getName() == rhs.getName() && this->haveEqualArgumentTypes(rhs); + } + + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override + { + this->data(place).merge(this->data(rhs), lg_config_k, target_type); + } + + void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override + { + this->data(place).write(buf, target_type); + } + + void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena *) const override + { + this->data(place).read(buf, lg_config_k); + } + + void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override + { + assert_cast(to).getData().push_back(this->data(place).size(target_type)); + } + +}; + + +/// `uniqApacheHLL([lg_k, [type]])(x)`, with `lg_k` in [4, 21] (default 12) and `type` one of +/// 'HLL_4', 'HLL_6', 'HLL_8' (default 'HLL_4'). +AggregateFunctionPtr createAggregateFunctionUniqApacheHLL( + const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *); + +} + +#endif diff --git a/src/AggregateFunctions/registerAggregateFunctions.cpp b/src/AggregateFunctions/registerAggregateFunctions.cpp index 46997750a403..670180d87a46 100644 --- a/src/AggregateFunctions/registerAggregateFunctions.cpp +++ b/src/AggregateFunctions/registerAggregateFunctions.cpp @@ -61,6 +61,9 @@ void registerAggregateFunctionSumMap(AggregateFunctionFactory &); void registerAggregateFunctionsUniq(AggregateFunctionFactory &); void registerAggregateFunctionUniqCombined(AggregateFunctionFactory &); void registerAggregateFunctionUniqUpTo(AggregateFunctionFactory &); +#if USE_DATASKETCHES +void registerAggregateFunctionUniqApacheHLL(AggregateFunctionFactory &); +#endif void registerAggregateFunctionTopK(AggregateFunctionFactory &); void registerAggregateFunctionsBitwise(AggregateFunctionFactory &); void registerAggregateFunctionsBitmap(AggregateFunctionFactory &); @@ -174,6 +177,9 @@ void registerAggregateFunctions() registerAggregateFunctionsUniq(factory); registerAggregateFunctionUniqCombined(factory); registerAggregateFunctionUniqUpTo(factory); +#if USE_DATASKETCHES + registerAggregateFunctionUniqApacheHLL(factory); +#endif registerAggregateFunctionTopK(factory); registerAggregateFunctionsBitwise(factory); registerAggregateFunctionCramersV(factory); diff --git a/tests/queries/0_stateless/04327_uniq_apache_hll.reference b/tests/queries/0_stateless/04327_uniq_apache_hll.reference new file mode 100644 index 000000000000..f2059d5f29fa --- /dev/null +++ b/tests/queries/0_stateless/04327_uniq_apache_hll.reference @@ -0,0 +1,19 @@ +accuracy +1 +1 +1 +1 +empty and single +0 +1 +argument types +1 +1 +1 +1 +state/merge roundtrip is native +1 +1 +1 +AggregateFunction(uniqApacheHLL(14, \'HLL_8\'), UInt64) +parameter validation diff --git a/tests/queries/0_stateless/04327_uniq_apache_hll.sql b/tests/queries/0_stateless/04327_uniq_apache_hll.sql new file mode 100644 index 000000000000..584ba4c067c2 --- /dev/null +++ b/tests/queries/0_stateless/04327_uniq_apache_hll.sql @@ -0,0 +1,60 @@ +-- Tags: no-fasttest +-- ^ DataSketches is not built in fast-test builds. + +SELECT 'accuracy'; +-- HLL is approximate but deterministic; assert the estimate is within the expected error band. +SELECT abs(toInt64(uniqApacheHLL(number)) - 1000) < 30 FROM numbers(1000); +SELECT abs(toInt64(uniqApacheHLL(number)) - 100000) < 3000 FROM numbers(100000); +-- Higher lg_k -> better accuracy. +SELECT abs(toInt64(uniqApacheHLL(14)(number)) - 100000) < 1500 FROM numbers(100000); +-- Storage type does not change the estimate. +SELECT uniqApacheHLL(12, 'HLL_4')(number) = uniqApacheHLL(12, 'HLL_8')(number) FROM numbers(1000); + +SELECT 'empty and single'; +SELECT uniqApacheHLL(number) FROM numbers(0); +SELECT uniqApacheHLL(number) FROM numbers(1); + +SELECT 'argument types'; +SELECT abs(toInt64(uniqApacheHLL(toInt32(number))) - 500) < 20 FROM numbers(500); +SELECT abs(toInt64(uniqApacheHLL(toFloat64(number))) - 500) < 20 FROM numbers(500); +SELECT abs(toInt64(uniqApacheHLL(toString(number))) - 500) < 20 FROM numbers(500); +SELECT abs(toInt64(uniqApacheHLL(toDate('2020-01-01') + number)) - 500) < 20 FROM numbers(500); + +SELECT 'state/merge roundtrip is native'; +-- Merging per-group `-State` values covers exactly the same set as a direct aggregate, so the two +-- estimates agree to within the error of the sketch. They are not required to be equal: DataSketches +-- reports the HIP estimate for a sketch that has only been updated and the composite estimate for +-- one that came out of a union, so a merged result is not the same number as a directly built one +-- even though both are computed from identical registers. +SELECT + abs(toInt64(uniqApacheHLLMerge(s)) - toInt64((SELECT uniqApacheHLL(number) FROM numbers(100000)))) < 3000 +FROM +( + SELECT uniqApacheHLLState(number) AS s + FROM numbers(100000) + GROUP BY number % 17 +); + +-- Merging is independent of how the input was partitioned: the registers of the union do not depend +-- on the grouping, and both sides come out of a union, so two different partitionings of the same +-- set agree exactly. +SELECT + (SELECT uniqApacheHLLMerge(s) FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(100000) GROUP BY number % 17)) + = (SELECT uniqApacheHLLMerge(s) FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(100000) GROUP BY number % 13)); + +-- The same holds for a single state. +SELECT + abs(toInt64(uniqApacheHLLMerge(s)) - toInt64((SELECT uniqApacheHLL(number) FROM numbers(1000)))) < 30 +FROM +( + SELECT uniqApacheHLLState(number) AS s FROM numbers(1000) +); + +-- The state type carries the sketch parameters. +SELECT toTypeName(uniqApacheHLLState(14, 'HLL_8')(number)) FROM numbers(1); + +SELECT 'parameter validation'; +SELECT uniqApacheHLL(3)(number) FROM numbers(1); -- { serverError ARGUMENT_OUT_OF_BOUND } +SELECT uniqApacheHLL(22)(number) FROM numbers(1); -- { serverError ARGUMENT_OUT_OF_BOUND } +SELECT uniqApacheHLL(12, 'HLL_9')(number) FROM numbers(1); -- { serverError BAD_ARGUMENTS } +SELECT uniqApacheHLL(12, 'HLL_4', 1)(number) FROM numbers(1); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } diff --git a/tests/queries/0_stateless/05026_uniq_apache_hll_interop.reference b/tests/queries/0_stateless/05026_uniq_apache_hll_interop.reference new file mode 100644 index 000000000000..ac07e5cd8a1c --- /dev/null +++ b/tests/queries/0_stateless/05026_uniq_apache_hll_interop.reference @@ -0,0 +1,23 @@ +import sketches built outside ClickHouse +5 +5 +8 +8 +3 +84 +5 +0 +types wider than the 8 bytes update(long) takes +1 +5 +1 +5 +DateTime64 is hashed as the epoch time it counts +1 +export sketches for consumption outside ClickHouse +1 +1 +1 +states survive a write/read cycle through storage +20 +1 diff --git a/tests/queries/0_stateless/05026_uniq_apache_hll_interop.sql b/tests/queries/0_stateless/05026_uniq_apache_hll_interop.sql new file mode 100644 index 000000000000..890d74aa89b0 --- /dev/null +++ b/tests/queries/0_stateless/05026_uniq_apache_hll_interop.sql @@ -0,0 +1,95 @@ +-- Tags: no-fasttest +-- ^ DataSketches is not built in fast-test builds. + +-- The state of `uniqApacheHLL` carries an Apache DataSketches HLL sketch in its native serialized +-- form, framed with the varint length prefix that `writeVectorBinary` adds. Every blob below was +-- produced by the Apache DataSketches C++ library directly rather than by ClickHouse, so these +-- queries pin the wire contract with external producers in both directions. A change that breaks +-- interoperability has to change one of these literals. + +SELECT 'import sketches built outside ClickHouse'; + +-- lg_k = 12, HLL_4, `hll_sketch::update(uint64_t)` over 0..4. Coupon list mode. +SELECT finalizeAggregation(CAST(unhex('1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06'), 'AggregateFunction(uniqApacheHLL, UInt64)')); + +-- The same sketch reached through `-Merge`. +SELECT uniqApacheHLLMerge(s) FROM (SELECT CAST(unhex('1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06'), 'AggregateFunction(uniqApacheHLL, UInt64)') AS s); + +-- Two external sketches over 0..4 and 3..7 union to 8 distinct values. +SELECT uniqApacheHLLMerge(s) FROM +( + SELECT CAST(unhex('1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06'), 'AggregateFunction(uniqApacheHLL, UInt64)') AS s + UNION ALL + SELECT CAST(unhex('1C0201070C030805007581660781BC5D067B65E608FC2D420AC1E91705'), 'AggregateFunction(uniqApacheHLL, UInt64)') AS s +); + +-- An external sketch merged with one built by ClickHouse: the two hash identically, so 0..4 and +-- 3..7 overlap on 3 and 4 and the union is 8. +SELECT uniqApacheHLLMerge(s) FROM +( + SELECT CAST(unhex('1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06'), 'AggregateFunction(uniqApacheHLL, UInt64)') AS s + UNION ALL + SELECT uniqApacheHLLState(number) AS s FROM numbers(3, 5) +); + +-- lg_k = 12, HLL_4, raw bytes of 'alpha', 'beta', 'gamma'. +SELECT finalizeAggregation(CAST(unhex('140201070C03080300BD3A090A8E5A62115168C90A'), 'AggregateFunction(uniqApacheHLL, UInt64)')); + +-- lg_k = 4, HLL_4, 100 distinct values. Dense HLL mode rather than a coupon list. +SELECT finalizeAggregation(CAST(unhex('300A0107040008020215EB1DC787F15440000000000000FB3F000000000000000003000000000000000251214121031025'), 'AggregateFunction(uniqApacheHLL(4), UInt64)')); + +-- lg_k = 14, HLL_8: the non-default parameters are carried by the state type. +SELECT finalizeAggregation(CAST(unhex('1C0201070E03080508CBD7C2042BF2FB06862FF90D7581660781BC5D06'), 'AggregateFunction(uniqApacheHLL(14, \'HLL_8\'), UInt64)')); + +-- An externally produced empty sketch. +SELECT finalizeAggregation(CAST(unhex('080201070C030C0000'), 'AggregateFunction(uniqApacheHLL, UInt64)')); + +SELECT 'types wider than the 8 bytes update(long) takes'; + +-- lg_k = 12, HLL_4 over the canonical 16 bytes of 00000000-0000-0000-0000-00000000000{0..4}. ClickHouse +-- holds a UUID as two 64-bit halves in host order, so hashing its bytes in memory would not agree with +-- an external producer working from the textual form; the canonical order is hashed instead. +SELECT hex(toString(uniqApacheHLLState(toUUID(concat('00000000-0000-0000-0000-00000000000', toString(number)))))) + = '1C0201070C0308050050C94D05854BD10ADB8CBD053C56FB07F8FDB206' +FROM numbers(5) SETTINGS max_threads = 1; +SELECT finalizeAggregation(CAST(unhex('1C0201070C0308050050C94D05854BD10ADB8CBD053C56FB07F8FDB206'), 'AggregateFunction(uniqApacheHLL, UUID)')); + +-- The same for 2001:db8::1 .. ::5, which ClickHouse already holds in network order. +SELECT hex(toString(uniqApacheHLLState(toIPv6(concat('2001:db8::', hex(number + 1)))))) + = '1C0201070C0308050018216E09FAB4750F52D5BB07DFBDE30A79BC9D0B' +FROM numbers(5) SETTINGS max_threads = 1; +SELECT finalizeAggregation(CAST(unhex('1C0201070C0308050018216E09FAB4750F52D5BB07DFBDE30A79BC9D0B'), 'AggregateFunction(uniqApacheHLL, IPv6)')); + +SELECT 'DateTime64 is hashed as the epoch time it counts'; + +-- A `DateTime64(3)` is hashed as its epoch milliseconds, which is the `long` a caller elsewhere +-- passes for the same instant. These five are 2020-01-01 00:00:00.000 and the next four seconds. +SELECT hex(toString(uniqApacheHLLState(toDateTime64('2020-01-01 00:00:00.000', 3, 'UTC') + number))) + = '1C0201070C0308050001D4B019CBDD6F1059310D0E833938083897B304' +FROM numbers(5) SETTINGS max_threads = 1; + +SELECT 'export sketches for consumption outside ClickHouse'; + +-- `max_threads` is pinned because a state that was merged from several partial states may lay its +-- coupons out in a different order than a state built by a single thread. +SELECT hex(toString(uniqApacheHLLState(number))) = '1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06' FROM numbers(5) SETTINGS max_threads = 1; +SELECT hex(toString(uniqApacheHLLState(14, 'HLL_8')(number))) = '1C0201070E03080508CBD7C2042BF2FB06862FF90D7581660781BC5D06' FROM numbers(5) SETTINGS max_threads = 1; + +-- Importing an external sketch and exporting it again must reproduce it byte for byte. +SELECT hex(toString(uniqApacheHLLMergeState(s))) = '1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06' +FROM (SELECT CAST(unhex('1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06'), 'AggregateFunction(uniqApacheHLL, UInt64)') AS s) +SETTINGS max_threads = 1; + +SELECT 'states survive a write/read cycle through storage'; + +DROP TABLE IF EXISTS hll_interop_states; +CREATE TABLE hll_interop_states (k UInt8, s AggregateFunction(uniqApacheHLL, UInt64)) ENGINE = AggregatingMergeTree ORDER BY k; +INSERT INTO hll_interop_states SELECT number % 4 AS k, uniqApacheHLLState(number) FROM numbers(20) GROUP BY k; +OPTIMIZE TABLE hll_interop_states FINAL; +-- Small enough to stay in coupon mode, so the union is exact regardless of how it was partitioned. +SELECT uniqApacheHLLMerge(s) FROM hll_interop_states; +-- A state read back from disk is still a valid sketch for an external consumer. +SELECT countDistinct(hex(toString(s))) = 4 FROM hll_interop_states; +DROP TABLE hll_interop_states; + +-- Malformed states are rejected as `CORRUPTED_DATA`; see `05136_uniq_apache_hll_corrupted_state.sh`. diff --git a/tests/queries/0_stateless/05027_uniq_apache_hll_cross_scale.reference b/tests/queries/0_stateless/05027_uniq_apache_hll_cross_scale.reference new file mode 100644 index 000000000000..2fc0b7c7d810 --- /dev/null +++ b/tests/queries/0_stateless/05027_uniq_apache_hll_cross_scale.reference @@ -0,0 +1,11 @@ +the two types remain distinct +AggregateFunction(uniqApacheHLL, UInt64) +AggregateFunction(uniqApacheHLL(8), UInt64) +merging states built with a different lg_k +1 +1 +relabelling with CAST does not rescale +1 1 +a state of one lg_k can be stored in a column declared with another +20 +only the parameters are interchangeable diff --git a/tests/queries/0_stateless/05027_uniq_apache_hll_cross_scale.sql b/tests/queries/0_stateless/05027_uniq_apache_hll_cross_scale.sql new file mode 100644 index 000000000000..43588e9d133a --- /dev/null +++ b/tests/queries/0_stateless/05027_uniq_apache_hll_cross_scale.sql @@ -0,0 +1,47 @@ +-- Tags: no-fasttest +-- ^ DataSketches is not built in fast-test builds. + +-- `lg_k` and the sketch type configure the sketch but are held by the aggregate function rather than +-- by its state, whose layout is the same whatever they are, and a serialized sketch records its own +-- `lg_k`. `uniqApacheHLL` therefore reports states of different parameterisations as having one +-- binary representation, which lets `-Merge` read a state built with one `lg_k` under a function +-- declared with another, and lets `CAST` relabel a column between the two types. + +SELECT 'the two types remain distinct'; +SELECT toTypeName(uniqApacheHLLState(number)) FROM numbers(1); +SELECT toTypeName(uniqApacheHLLState(8)(number)) FROM numbers(1); + +SELECT 'merging states built with a different lg_k'; +-- Downsampling during a union produces the same registers as building at the target `lg_k` from the +-- start, so rescaling 17 states of `lg_k` 12 agrees exactly with merging 17 built at 8. Both sides +-- come out of a union and so use the same estimator. +SELECT + (SELECT uniqApacheHLLMerge(8)(s) FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(100000) GROUP BY number % 17)) + = (SELECT uniqApacheHLLMerge(8)(s) FROM (SELECT uniqApacheHLLState(8)(number) AS s FROM numbers(100000) GROUP BY number % 17)); + +-- Merging is lossy in one direction only: the union takes the resolution of its coarsest input, so a +-- rescaled state is much smaller than the states it was built from. +SELECT + length(toString(uniqApacheHLLMergeState(8)(s))) < length(toString(uniqApacheHLLMergeState(s))) / 8 +FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(100000) GROUP BY number % 17); + +SELECT 'relabelling with CAST does not rescale'; +-- `CAST` between the two types re-associates the column with the other function without touching its +-- data, so the sketch keeps the resolution it was built with and the estimate does not change. +SELECT + finalizeAggregation(CAST(s, 'AggregateFunction(uniqApacheHLL(8), UInt64)')) = finalizeAggregation(s), + length(toString(CAST(s, 'AggregateFunction(uniqApacheHLL(8), UInt64)'))) = length(toString(s)) +FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(1000)); + +SELECT 'a state of one lg_k can be stored in a column declared with another'; +DROP TABLE IF EXISTS hll_cross_scale; +CREATE TABLE hll_cross_scale (s AggregateFunction(uniqApacheHLL, UInt64)) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO hll_cross_scale SELECT uniqApacheHLLState(8)(number) FROM numbers(20); +SELECT uniqApacheHLLMerge(s) FROM hll_cross_scale; +DROP TABLE hll_cross_scale; + +SELECT 'only the parameters are interchangeable'; +-- The argument types and the function itself must still match: `haveEqualArgumentTypes` and the name +-- comparison keep these apart even though the parameters no longer do. +SELECT CAST(uniqApacheHLLState(toString(number)), 'AggregateFunction(uniqApacheHLL, UInt64)') FROM numbers(10); -- { serverError CANNOT_CONVERT_TYPE } +SELECT CAST(uniqThetaState(number), 'AggregateFunction(uniqApacheHLL, UInt64)') FROM numbers(10); -- { serverError CANNOT_CONVERT_TYPE } diff --git a/tests/queries/0_stateless/05028_uniq_apache_hll_argument_types.reference b/tests/queries/0_stateless/05028_uniq_apache_hll_argument_types.reference new file mode 100644 index 000000000000..6478f4dcb46e --- /dev/null +++ b/tests/queries/0_stateless/05028_uniq_apache_hll_argument_types.reference @@ -0,0 +1,26 @@ +types the sketch hashes directly +20 +20 +20 +20 +20 +20 +20 +20 +20 +20 +20 +20 +20 +20 +3 +the wrappers of an accepted type are accepted +20 +20 +a DateTime64 is the integer a caller elsewhere would hash +1 +types no other implementation can reproduce are rejected +one argument only +the parameters still apply +20 +AggregateFunction(uniqApacheHLL(8, \'HLL_6\'), UInt64) diff --git a/tests/queries/0_stateless/05028_uniq_apache_hll_argument_types.sql b/tests/queries/0_stateless/05028_uniq_apache_hll_argument_types.sql new file mode 100644 index 000000000000..6b133e97f64c --- /dev/null +++ b/tests/queries/0_stateless/05028_uniq_apache_hll_argument_types.sql @@ -0,0 +1,54 @@ +-- Tags: no-fasttest +-- ^ DataSketches is not built in fast-test builds. + +-- `uniqApacheHLL` accepts only the types a sketch hashes the same way in every implementation, as +-- an 8-byte integer, an IEEE-754 double or their raw bytes. Anything else would have to be hashed +-- to a single value by ClickHouse first, which no producer outside ClickHouse could reproduce. +-- All counts below are small enough for the sketch to stay in coupon mode, where it is exact. + +SELECT 'types the sketch hashes directly'; +SELECT uniqApacheHLL(toUInt64(number)) FROM numbers(20); +SELECT uniqApacheHLL(toInt32(number)) FROM numbers(20); +SELECT uniqApacheHLL(toBFloat16(number)) FROM numbers(20); +SELECT uniqApacheHLL(toFloat32(number)) FROM numbers(20); +SELECT uniqApacheHLL(toFloat64(number)) FROM numbers(20); +SELECT uniqApacheHLL(toString(number)) FROM numbers(20); +SELECT uniqApacheHLL(toFixedString(toString(number), 8)) FROM numbers(20); +SELECT uniqApacheHLL(toDate('2020-01-01') + number) FROM numbers(20); +SELECT uniqApacheHLL(toDate32('2020-01-01') + number) FROM numbers(20); +SELECT uniqApacheHLL(toDateTime('2020-01-01 00:00:00') + number) FROM numbers(20); +SELECT uniqApacheHLL(toDateTime64('2020-01-01 00:00:00.000', 3) + number) FROM numbers(20); +SELECT uniqApacheHLL(reinterpretAsUUID(toUInt128(number))) FROM numbers(20); +SELECT uniqApacheHLL(toIPv4('1.2.3.0') + number) FROM numbers(20); +SELECT uniqApacheHLL(toIPv6(concat('2001:db8::', hex(number + 1)))) FROM numbers(20); +SELECT uniqApacheHLL(CAST(number % 3, 'Enum8(\'a\' = 0, \'b\' = 1, \'c\' = 2)')) FROM numbers(20); + +SELECT 'the wrappers of an accepted type are accepted'; +SELECT uniqApacheHLL(toNullable(number)) FROM numbers(20); +SELECT uniqApacheHLL(toLowCardinality(toString(number))) FROM numbers(20); + +SELECT 'a DateTime64 is the integer a caller elsewhere would hash'; +-- `DateTime64(3)` counts epoch milliseconds, so it produces the sketch of those milliseconds. +SELECT hex(toString(uniqApacheHLLState(toDateTime64('2020-01-01 00:00:00.000', 3, 'UTC') + number))) + = hex(toString(uniqApacheHLLState(toInt64(1577836800000) + number * 1000))) +FROM numbers(5) SETTINGS max_threads = 1; + +SELECT 'types no other implementation can reproduce are rejected'; +-- No byte order for the wide integers is agreed on across implementations. +SELECT uniqApacheHLL(toInt128(number)) FROM numbers(20); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT uniqApacheHLL(toUInt256(number)) FROM numbers(20); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +-- No DataSketches binding accepts a decimal, and the scale is not part of the sketch. +SELECT uniqApacheHLL(toDecimal64(number, 2)) FROM numbers(20); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT uniqApacheHLL(toDecimal128(number, 4)) FROM numbers(20); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +-- A composite value would have to be hashed by ClickHouse first. +SELECT uniqApacheHLL(materialize([number])) FROM numbers(20); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT uniqApacheHLL((number, number + 1)) FROM numbers(20); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } + +SELECT 'one argument only'; +SELECT uniqApacheHLL(number, number + 1) FROM numbers(20); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT uniqApacheHLL(toString(number), number) FROM numbers(20); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT uniqApacheHLL() FROM numbers(1); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +SELECT 'the parameters still apply'; +SELECT uniqApacheHLL(8)(reinterpretAsUUID(toUInt128(number))) FROM numbers(20); +SELECT toTypeName(uniqApacheHLLState(8, 'HLL_6')(number)) FROM numbers(1); diff --git a/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.reference b/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.reference new file mode 100644 index 000000000000..f70b0368a421 --- /dev/null +++ b/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.reference @@ -0,0 +1,3 @@ +OK unknown type +OK bad payload +OK rowbinary diff --git a/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.sh b/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.sh new file mode 100755 index 000000000000..18ad78792ac1 --- /dev/null +++ b/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest -- compiled w/o datasketches + +# A `uniqApacheHLL` state that is not a sketch must be rejected as `CORRUPTED_DATA`. +# +# `datasketches` reports a payload it cannot parse as `std::invalid_argument` or +# `std::out_of_range`, neither of which is a `DB::Exception`, so without the +# translation in `HllSketchData::read` they would escape +# `SerializationAggregateFunction`'s `catch (...)` and be reported as a logical error. +# +# This is a shell test rather than a `.sql` one with a `serverError` hint because the +# client prints an extra exception with a stack trace to stderr for this class of error, +# which a `.sql` test counts as a failure. The same applies to `uniqTheta`, see +# `04307_uniqTheta_corrupted_state_106259.sh`. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A well-formed varint length prefix followed by eight bytes that are not a sketch: +# the first byte of the payload is the preamble length, which no HLL sketch uses. +$CLICKHOUSE_CLIENT --query \ + "SELECT finalizeAggregation(CAST(unhex('08FFFFFFFFFFFFFFFF'), 'AggregateFunction(uniqApacheHLL, UInt64)'))" 2>&1 \ + | grep -q -F 'CORRUPTED_DATA' && echo 'OK unknown type' || echo 'FAIL unknown type' + +$CLICKHOUSE_CLIENT --query \ + "SELECT finalizeAggregation(CAST(unhex('0801020304050607FF'), 'AggregateFunction(uniqApacheHLL, UInt64)'))" 2>&1 \ + | grep -q -F 'CORRUPTED_DATA' && echo 'OK bad payload' || echo 'FAIL bad payload' + +# The `RowBinary` path reaches `HllSketchData::read` through `deserializeBinary`: +# the leading 0x03 claims a three-byte payload, shorter than any HLL sketch. +printf '\x03\x03\x03\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' \ + | $CLICKHOUSE_LOCAL --input-format=RowBinary \ + --structure='x AggregateFunction(uniqApacheHLL, IPv6)' \ + --query='SELECT x FROM table' 2>&1 \ + | grep -q -F 'CORRUPTED_DATA' && echo 'OK rowbinary' || echo 'FAIL rowbinary' From 9c9c7fbb49f510a5ecd72c41c14ab5952d1d4d58 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Tue, 29 Sep 2026 21:26:57 +0200 Subject: [PATCH 3/5] Simplify `uniqApacheHLL` implementation and tests Reuse `assertUnary`, consolidate state handling, and shorten repetitive comments and documentation. Merge overlapping HLL tests while retaining interoperability, cross-resolution, corruption, and dependency coverage. Use the canonical `Enum` documentation type to avoid a logical-error exception when reading `system.functions`. Validation: rebuilt ClickHouse and passed all four remaining PR tests with the pinned DataSketches revision; no tests skipped. Related: https://github.com/Altinity/ClickHouse/pull/2398 --- .../AggregateFunctionUniqApacheHLL.cpp | 45 ++++------- .../AggregateFunctionUniqApacheHLL.h | 80 ++++--------------- .../04327_uniq_apache_hll.reference | 29 ++++--- .../0_stateless/04327_uniq_apache_hll.sql | 65 ++++++++------- .../05026_uniq_apache_hll_interop.reference | 14 ++-- .../05026_uniq_apache_hll_interop.sql | 65 +++++++-------- ...5027_uniq_apache_hll_cross_scale.reference | 11 --- .../05027_uniq_apache_hll_cross_scale.sql | 47 ----------- ...8_uniq_apache_hll_argument_types.reference | 26 ------ .../05028_uniq_apache_hll_argument_types.sql | 54 ------------- .../05055_uniq_theta_compressed_state.sql | 14 +--- ..._uniq_apache_hll_corrupted_state.reference | 1 - .../05136_uniq_apache_hll_corrupted_state.sh | 22 +---- 13 files changed, 130 insertions(+), 343 deletions(-) delete mode 100644 tests/queries/0_stateless/05027_uniq_apache_hll_cross_scale.reference delete mode 100644 tests/queries/0_stateless/05027_uniq_apache_hll_cross_scale.sql delete mode 100644 tests/queries/0_stateless/05028_uniq_apache_hll_argument_types.reference delete mode 100644 tests/queries/0_stateless/05028_uniq_apache_hll_argument_types.sql diff --git a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp index a9425146ea95..8da3fc42092a 100644 --- a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp +++ b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #if USE_DATASKETCHES @@ -15,7 +16,7 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } -AggregateFunctionPtr createAggregateFunctionUniqApacheHLL( +static AggregateFunctionPtr createAggregateFunctionUniqApacheHLL( const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *) { uint8_t lg_config_k = 12; @@ -52,27 +53,16 @@ AggregateFunctionPtr createAggregateFunctionUniqApacheHLL( "Parameter type for aggregate function {} must be one of 'HLL_4', 'HLL_6', 'HLL_8'.", name); } - /// Only the types a sketch hashes the same way in every language are accepted, so that no state - /// is built here that an external consumer cannot reproduce. Anything else - a decimal, a wide - /// integer, an array, a tuple, several arguments - would first need a hash only ClickHouse has. - if (argument_types.size() != 1) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Aggregate function {} requires exactly one argument, passed {}. Several arguments would " - "have to be hashed to one value by ClickHouse first, which no producer outside ClickHouse " - "can reproduce. To count distinct combinations of several columns, use uniq, uniqCombined " - "or uniqHLL12.", - name, argument_types.size()); + assertUnary(name, argument_types); const IDataType & argument_type = *argument_types[0]; WhichDataType which(argument_type); - /// Backed by a decimal, but accepted unlike one: it holds the epoch time a caller elsewhere - /// passes to `update(long)`. + /// Unlike other decimals, `DateTime64` has an interoperable representation as epoch ticks. if (which.isDateTime64()) return std::make_shared>(lg_config_k, target_type, argument_types, params); - /// `createWithNumericType` also covers the 128 and 256 bit integers, for which no byte order is - /// agreed on, so they have to be refused before it is reached. + /// Exclude wide integers: DataSketches has no portable representation for them. if (!which.isInt128() && !which.isInt256() && !which.isUInt128() && !which.isUInt256()) { AggregateFunctionPtr res(createWithNumericType( @@ -97,12 +87,8 @@ AggregateFunctionPtr createAggregateFunctionUniqApacheHLL( return std::make_shared>(lg_config_k, target_type, argument_types, params); throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Aggregate function {} does not support the argument type {}: only the types an Apache " - "DataSketches HLL sketch hashes the same way outside ClickHouse are supported: integers of " - "at most 64 bits, Enum8, Enum16, BFloat16, Float32, Float64, String, FixedString, UUID, " - "IPv4, IPv6, Date, Date32, DateTime and DateTime64. To count distinct values of any other " - "type, use uniq, uniqCombined or uniqHLL12.", - name, argument_type.getName()); + "Illegal type {} of argument for aggregate function {}. Use uniq, uniqCombined or uniqHLL12 for unsupported types.", + argument_type.getName(), name); } void registerAggregateFunctionUniqApacheHLL(AggregateFunctionFactory & factory); @@ -111,21 +97,18 @@ void registerAggregateFunctionUniqApacheHLL(AggregateFunctionFactory & factory) FunctionDocumentation::Description description = R"( Calculates the approximate number of different argument values using an [Apache DataSketches](https://datasketches.apache.org/docs/HLL/HllSketches.html) HyperLogLog sketch. -The serialized state produced by the `-State` combinator carries the sketch in the Apache DataSketches HLL format, framed with a varint length prefix, so sketches can be exchanged with external services (Java, Python, C++) using the standard `-State`/`-Merge` combinators. For example, a sketch built by an upstream service can be merged with `uniqApacheHLLMerge`, and a sketch built in ClickHouse can be exported with `uniqApacheHLLState`. - -Only the types that a sketch hashes the same way in every implementation are accepted, so that every state ClickHouse produces can be reproduced elsewhere. Integers of at most 64 bits are hashed as their 8-byte representation (matching DataSketches `update(long)`), floating-point values as an IEEE-754 double, strings as their raw bytes, `UUID`s as their canonical 16 bytes and `IPv6` addresses in network order. - -`Date`, `Date32`, `DateTime` and `DateTime64` are hashed as the integer they hold, which for `DateTime64(3)` is the epoch milliseconds a caller elsewhere would pass to `update(long)`. The unit belongs to the column type rather than to the value, so both sides have to agree on it. - -Every other type is rejected, because a sketch over it would have to be built from a hash that only ClickHouse can compute: the 128 and 256 bit integers have no agreed byte order across implementations, decimals have no representation any DataSketches binding accepts, and an array or a tuple has none either. The same goes for a call with more than one argument. Use `uniq`, `uniqCombined` or `uniqHLL12` to count distinct values of those. +The `-State` and `-Merge` combinators exchange Apache DataSketches HLL sketches with a varint length prefix. -An estimate obtained by merging sketches is not the same number as one computed in a single pass over the same values, even though both are derived from identical registers: DataSketches reports the HIP estimator for a sketch that has only been updated and the composite estimator for one produced by a union. The result therefore depends on how the aggregation was partitioned across threads, parts and shards, and is slightly less accurate once any merge has taken place. +For interoperability, integers of at most 64 bits are hashed as 8-byte integers, floating-point values as IEEE-754 doubles, strings as raw bytes, `UUID` values as canonical 16 bytes, and `IPv6` addresses in network order. +`Date`, `Date32`, `DateTime` and `DateTime64` use their underlying integer values. External producers must use the same unit; for example, `DateTime64(3)` uses epoch milliseconds. +Unsupported types and multiple arguments are rejected. Use `uniq`, `uniqCombined` or `uniqHLL12` for those inputs. -The resolution of a merged sketch is the smallest `lg_k` among its inputs, not the `lg_k` named by the type. Merging a sketch that was built with a lower `lg_k` - for example one produced by another service - permanently lowers the resolution of both the estimate and the state written back. +Merging can switch from the HIP estimator to the less accurate composite estimator, so results can depend on partitioning across threads, parts and shards. +Merging a lower-resolution sketch permanently lowers the result's resolution, regardless of the declared `lg_k`. )"; FunctionDocumentation::Syntax syntax = "uniqApacheHLL([lg_k, [type]])(x)"; FunctionDocumentation::Arguments arguments = { - {"x", "Column to compute the number of distinct values of.", {"(U)Int8/16/32/64", "Enum8", "Enum16", "BFloat16", "Float32", "Float64", "String", "FixedString", "UUID", "IPv4", "IPv6", "Date", "Date32", "DateTime", "DateTime64"}}, + {"x", "Column to compute the number of distinct values of.", {"(U)Int8/16/32/64", "Enum", "BFloat16", "Float32", "Float64", "String", "FixedString", "UUID", "IPv4", "IPv6", "Date", "Date32", "DateTime", "DateTime64"}}, }; FunctionDocumentation::Parameters parameters = { {"lg_k", "Optional. Log-base-2 of the number of buckets, in range [4, 21]. Higher means better accuracy and more memory. Default: 12.", {"UInt8"}}, diff --git a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h index b223440e1e1b..718398fe97ae 100644 --- a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h +++ b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h @@ -25,7 +25,6 @@ #include #include -#include #include #include @@ -41,18 +40,11 @@ namespace ErrorCodes } -/** An Apache DataSketches HLL sketch as an aggregate function state, serialized in the - * DataSketches HLL format so that `-State`/`-Merge` interoperate with external services. - * - * `lg_config_k` and the target type are owned by the aggregate function and passed in, so the - * state stores the sketch alone. - */ -class HllSketchData : private boost::noncopyable +/// Keeps insertion and union states separate to preserve the estimator used before merging. +class HllSketchData { private: - /// Used for insertions. std::unique_ptr sk_update; - /// Used for merging. std::unique_ptr sk_union; datasketches::hll_sketch * getSkUpdate(uint8_t lg_config_k, datasketches::target_hll_type tgt_type) @@ -66,31 +58,22 @@ class HllSketchData : private boost::noncopyable { if (!sk_union) { - /// `hll_union` takes `lg_max_k` in [7, 21] while a sketch may use [4, 21], hence the - /// floor of 7. It does not inflate the result: `get_result` downsamples to the smallest - /// `lg_config_k` the union has seen. So the resolution of a merged state is the minimum - /// over its inputs, not the `lg_config_k` of the type, and merging a coarser sketch from - /// elsewhere lowers both the estimate and the state written back. Merged from empty - /// states only there is no minimum, and it serializes as `lg_config_k = 7`. + /// `hll_union` requires at least 7, but preserves the lower resolution of its inputs. sk_union = std::make_unique(std::max(lg_config_k, 7)); } return sk_union.get(); } - /// Fold a sketch updated after the union was allocated into it, so merges see one state. void foldUpdateIntoUnionIfNeeded() { if (sk_union && sk_update) { sk_union->update(*sk_update); - sk_update.reset(nullptr); + sk_update.reset(); } } public: - HllSketchData() = default; - ~HllSketchData() = default; - template void insert(T value, uint8_t lg_config_k, datasketches::target_hll_type tgt_type) { @@ -106,8 +89,7 @@ class HllSketchData : private boost::noncopyable UInt64 size(datasketches::target_hll_type tgt_type) const { - /// Round rather than truncate: `get_estimate` returns a `double`, and `999.9999` for an - /// exactly-known cardinality must not become `999`. + /// Rounding preserves exact cardinalities despite floating-point error. if (sk_union) return static_cast(std::llround(sk_union->get_result(tgt_type).get_estimate())); if (sk_update) @@ -119,11 +101,7 @@ class HllSketchData : private boost::noncopyable { datasketches::hll_union * u = getSkUnion(lg_config_k); - if (sk_update) - { - u->update(*sk_update); - sk_update.reset(nullptr); - } + foldUpdateIntoUnionIfNeeded(); if (rhs.sk_update) u->update(*rhs.sk_update); @@ -150,47 +128,27 @@ class HllSketchData : private boost::noncopyable } catch (const std::bad_alloc &) { - /// Memory pressure, not corrupted data. throw; } catch (const std::exception & e) { - /// `datasketches` reports malformed input as `std::invalid_argument` / `std::out_of_range`. - /// Not being `DB::Exception`, those escape `SerializationAggregateFunction`'s - /// `catch (...)` and abort as a logical error, so translate them here. + /// Translate malformed input to avoid a logical exception in `SerializationAggregateFunction`. throw Exception(ErrorCodes::CORRUPTED_DATA, "Cannot deserialize HLL sketch state: {}", e.what()); } } void write(WriteBuffer & out, datasketches::target_hll_type tgt_type) const { + datasketches::hll_sketch::vector_bytes bytes; if (sk_update) - { - auto bytes = sk_update->serialize_compact(); - writeVectorBinary(bytes, out); - } + bytes = sk_update->serialize_compact(); else if (sk_union) - { - auto bytes = sk_union->get_result(tgt_type).serialize_compact(); - writeVectorBinary(bytes, out); - } - else - { - datasketches::hll_sketch::vector_bytes bytes; - writeVectorBinary(bytes, out); - } + bytes = sk_union->get_result(tgt_type).serialize_compact(); + writeVectorBinary(bytes, out); } }; -/** `uniqApacheHLL` over a single column of a type the sketch can hash directly. - * - * The value takes one of the three shapes the DataSketches API accepts - an 8-byte integer, an - * IEEE-754 double, or raw bytes - so that an external producer reaches the same sketch. - * - * `lg_config_k` and the target type live here rather than in the state, which is what lets states - * of different parameterisations share one binary representation. - */ template class AggregateFunctionUniqApacheHLL final : public IAggregateFunctionDataHelper> { @@ -230,8 +188,7 @@ class AggregateFunctionUniqApacheHLL final : public IAggregateFunctionDataHelper if constexpr (std::is_same_v) { - /// ClickHouse holds a UUID as two 64-bit halves in host order, so its bytes in - /// memory are not the canonical 16 an external producer works from. + /// Convert the two host-order halves of a `UUID` to canonical bytes. const UInt64 halves[2] = { std::byteswap(UUIDHelpers::getHighBytes(value)), std::byteswap(UUIDHelpers::getLowBytes(value)), @@ -242,8 +199,7 @@ class AggregateFunctionUniqApacheHLL final : public IAggregateFunctionDataHelper /// Already held in network order, which is the canonical form. data.insertData(reinterpret_cast(&value), sizeof(value), lg_config_k, target_type); else if constexpr (is_decimal) - /// `DateTime64(3)` holds the epoch milliseconds a caller elsewhere passes to - /// `update(long)`. The scale belongs to the type, so both sides must agree on it. + /// Hash `DateTime64` as epoch ticks; external producers must use the same scale. data.insert(static_cast(value.value), lg_config_k, target_type); else if constexpr (std::is_same_v) data.insert(static_cast(value.toUnderType()), lg_config_k, target_type); @@ -255,8 +211,8 @@ class AggregateFunctionUniqApacheHLL final : public IAggregateFunctionDataHelper data.insert(static_cast(value), lg_config_k, target_type); } } - /// A serialized sketch describes its own configuration, so states of different parameterisations - /// are interchangeable. Merging across them takes the resolution of the coarsest input. + + /// Serialized sketches carry their configuration, so parameters need not match. bool haveSameStateRepresentationImpl(const IAggregateFunction & rhs) const override { return getName() == rhs.getName() && this->haveEqualArgumentTypes(rhs); @@ -284,12 +240,6 @@ class AggregateFunctionUniqApacheHLL final : public IAggregateFunctionDataHelper }; - -/// `uniqApacheHLL([lg_k, [type]])(x)`, with `lg_k` in [4, 21] (default 12) and `type` one of -/// 'HLL_4', 'HLL_6', 'HLL_8' (default 'HLL_4'). -AggregateFunctionPtr createAggregateFunctionUniqApacheHLL( - const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *); - } #endif diff --git a/tests/queries/0_stateless/04327_uniq_apache_hll.reference b/tests/queries/0_stateless/04327_uniq_apache_hll.reference index f2059d5f29fa..ef814b89b951 100644 --- a/tests/queries/0_stateless/04327_uniq_apache_hll.reference +++ b/tests/queries/0_stateless/04327_uniq_apache_hll.reference @@ -1,19 +1,30 @@ accuracy 1 1 -1 -1 +1 1 empty and single 0 1 -argument types -1 -1 -1 -1 -state/merge roundtrip is native -1 +state and merge 1 1 AggregateFunction(uniqApacheHLL(14, \'HLL_8\'), UInt64) parameter validation +argument types +20 +20 +20 +20 +20 +20 +20 +20 +20 +20 +20 +3 +nullable and low cardinality +20 +20 +unsupported types +one argument only diff --git a/tests/queries/0_stateless/04327_uniq_apache_hll.sql b/tests/queries/0_stateless/04327_uniq_apache_hll.sql index 584ba4c067c2..1de290b47236 100644 --- a/tests/queries/0_stateless/04327_uniq_apache_hll.sql +++ b/tests/queries/0_stateless/04327_uniq_apache_hll.sql @@ -2,30 +2,20 @@ -- ^ DataSketches is not built in fast-test builds. SELECT 'accuracy'; --- HLL is approximate but deterministic; assert the estimate is within the expected error band. -SELECT abs(toInt64(uniqApacheHLL(number)) - 1000) < 30 FROM numbers(1000); SELECT abs(toInt64(uniqApacheHLL(number)) - 100000) < 3000 FROM numbers(100000); --- Higher lg_k -> better accuracy. SELECT abs(toInt64(uniqApacheHLL(14)(number)) - 100000) < 1500 FROM numbers(100000); -- Storage type does not change the estimate. -SELECT uniqApacheHLL(12, 'HLL_4')(number) = uniqApacheHLL(12, 'HLL_8')(number) FROM numbers(1000); +SELECT + uniqApacheHLL(12, 'HLL_4')(number) = uniqApacheHLL(12, 'HLL_6')(number), + uniqApacheHLL(12, 'HLL_4')(number) = uniqApacheHLL(12, 'HLL_8')(number) +FROM numbers(1000); SELECT 'empty and single'; SELECT uniqApacheHLL(number) FROM numbers(0); SELECT uniqApacheHLL(number) FROM numbers(1); -SELECT 'argument types'; -SELECT abs(toInt64(uniqApacheHLL(toInt32(number))) - 500) < 20 FROM numbers(500); -SELECT abs(toInt64(uniqApacheHLL(toFloat64(number))) - 500) < 20 FROM numbers(500); -SELECT abs(toInt64(uniqApacheHLL(toString(number))) - 500) < 20 FROM numbers(500); -SELECT abs(toInt64(uniqApacheHLL(toDate('2020-01-01') + number)) - 500) < 20 FROM numbers(500); - -SELECT 'state/merge roundtrip is native'; --- Merging per-group `-State` values covers exactly the same set as a direct aggregate, so the two --- estimates agree to within the error of the sketch. They are not required to be equal: DataSketches --- reports the HIP estimate for a sketch that has only been updated and the composite estimate for --- one that came out of a union, so a merged result is not the same number as a directly built one --- even though both are computed from identical registers. +SELECT 'state and merge'; +-- Direct and merged sketches may use different estimators. SELECT abs(toInt64(uniqApacheHLLMerge(s)) - toInt64((SELECT uniqApacheHLL(number) FROM numbers(100000)))) < 3000 FROM @@ -35,22 +25,11 @@ FROM GROUP BY number % 17 ); --- Merging is independent of how the input was partitioned: the registers of the union do not depend --- on the grouping, and both sides come out of a union, so two different partitionings of the same --- set agree exactly. +-- Different partitions must produce the same union estimate. SELECT (SELECT uniqApacheHLLMerge(s) FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(100000) GROUP BY number % 17)) = (SELECT uniqApacheHLLMerge(s) FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(100000) GROUP BY number % 13)); --- The same holds for a single state. -SELECT - abs(toInt64(uniqApacheHLLMerge(s)) - toInt64((SELECT uniqApacheHLL(number) FROM numbers(1000)))) < 30 -FROM -( - SELECT uniqApacheHLLState(number) AS s FROM numbers(1000) -); - --- The state type carries the sketch parameters. SELECT toTypeName(uniqApacheHLLState(14, 'HLL_8')(number)) FROM numbers(1); SELECT 'parameter validation'; @@ -58,3 +37,33 @@ SELECT uniqApacheHLL(3)(number) FROM numbers(1); -- { serverError ARGUMENT_OUT_O SELECT uniqApacheHLL(22)(number) FROM numbers(1); -- { serverError ARGUMENT_OUT_OF_BOUND } SELECT uniqApacheHLL(12, 'HLL_9')(number) FROM numbers(1); -- { serverError BAD_ARGUMENTS } SELECT uniqApacheHLL(12, 'HLL_4', 1)(number) FROM numbers(1); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +SELECT 'argument types'; +-- Small counts stay in coupon mode, where the estimate is exact. +SELECT uniqApacheHLL(toUInt64(number)) FROM numbers(20); +SELECT uniqApacheHLL(toInt32(number)) FROM numbers(20); +SELECT uniqApacheHLL(toBFloat16(number)) FROM numbers(20); +SELECT uniqApacheHLL(toFloat32(number)) FROM numbers(20); +SELECT uniqApacheHLL(toFloat64(number)) FROM numbers(20); +SELECT uniqApacheHLL(toString(number)) FROM numbers(20); +SELECT uniqApacheHLL(toFixedString(toString(number), 8)) FROM numbers(20); +SELECT uniqApacheHLL(toDate('2020-01-01') + number) FROM numbers(20); +SELECT uniqApacheHLL(toDate32('2020-01-01') + number) FROM numbers(20); +SELECT uniqApacheHLL(toDateTime('2020-01-01 00:00:00') + number) FROM numbers(20); +SELECT uniqApacheHLL(toIPv4('1.2.3.0') + number) FROM numbers(20); +SELECT uniqApacheHLL(CAST(number % 3, 'Enum8(\'a\' = 0, \'b\' = 1, \'c\' = 2)')) FROM numbers(20); + +SELECT 'nullable and low cardinality'; +SELECT uniqApacheHLL(toNullable(number)) FROM numbers(20); +SELECT uniqApacheHLL(toLowCardinality(toString(number))) FROM numbers(20); + +SELECT 'unsupported types'; +SELECT uniqApacheHLL(toInt128(number)) FROM numbers(20); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT uniqApacheHLL(toUInt256(number)) FROM numbers(20); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT uniqApacheHLL(toDecimal64(number, 2)) FROM numbers(20); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT uniqApacheHLL(materialize([number])) FROM numbers(20); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT uniqApacheHLL((number, number + 1)) FROM numbers(20); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } + +SELECT 'one argument only'; +SELECT uniqApacheHLL(number, number + 1) FROM numbers(20); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT uniqApacheHLL() FROM numbers(1); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } diff --git a/tests/queries/0_stateless/05026_uniq_apache_hll_interop.reference b/tests/queries/0_stateless/05026_uniq_apache_hll_interop.reference index ac07e5cd8a1c..70c97654e417 100644 --- a/tests/queries/0_stateless/05026_uniq_apache_hll_interop.reference +++ b/tests/queries/0_stateless/05026_uniq_apache_hll_interop.reference @@ -1,23 +1,23 @@ import sketches built outside ClickHouse 5 -5 -8 -8 -3 +11 84 5 0 types wider than the 8 bytes update(long) takes 1 -5 1 -5 DateTime64 is hashed as the epoch time it counts 1 export sketches for consumption outside ClickHouse 1 1 1 -states survive a write/read cycle through storage +store and merge states with a different lg_k 20 +merging states built with a different lg_k +1 1 +relabelling with CAST does not rescale +1 1 +only the parameters are interchangeable diff --git a/tests/queries/0_stateless/05026_uniq_apache_hll_interop.sql b/tests/queries/0_stateless/05026_uniq_apache_hll_interop.sql index 890d74aa89b0..f29084519fac 100644 --- a/tests/queries/0_stateless/05026_uniq_apache_hll_interop.sql +++ b/tests/queries/0_stateless/05026_uniq_apache_hll_interop.sql @@ -1,40 +1,24 @@ -- Tags: no-fasttest -- ^ DataSketches is not built in fast-test builds. --- The state of `uniqApacheHLL` carries an Apache DataSketches HLL sketch in its native serialized --- form, framed with the varint length prefix that `writeVectorBinary` adds. Every blob below was --- produced by the Apache DataSketches C++ library directly rather than by ClickHouse, so these --- queries pin the wire contract with external producers in both directions. A change that breaks --- interoperability has to change one of these literals. +-- Fixtures were generated by Apache DataSketches C++, with a varint length prefix. +-- These checks pin compatibility with external producers and consumers. SELECT 'import sketches built outside ClickHouse'; --- lg_k = 12, HLL_4, `hll_sketch::update(uint64_t)` over 0..4. Coupon list mode. +-- lg_k = 12, HLL_4, `hll_sketch::update` on `uint64_t` values over 0..4. Coupon list mode. SELECT finalizeAggregation(CAST(unhex('1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06'), 'AggregateFunction(uniqApacheHLL, UInt64)')); --- The same sketch reached through `-Merge`. -SELECT uniqApacheHLLMerge(s) FROM (SELECT CAST(unhex('1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06'), 'AggregateFunction(uniqApacheHLL, UInt64)') AS s); - --- Two external sketches over 0..4 and 3..7 union to 8 distinct values. +-- External sketches over 0..4 and 3..7 overlap with ClickHouse values 6..10. SELECT uniqApacheHLLMerge(s) FROM ( SELECT CAST(unhex('1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06'), 'AggregateFunction(uniqApacheHLL, UInt64)') AS s UNION ALL SELECT CAST(unhex('1C0201070C030805007581660781BC5D067B65E608FC2D420AC1E91705'), 'AggregateFunction(uniqApacheHLL, UInt64)') AS s -); - --- An external sketch merged with one built by ClickHouse: the two hash identically, so 0..4 and --- 3..7 overlap on 3 and 4 and the union is 8. -SELECT uniqApacheHLLMerge(s) FROM -( - SELECT CAST(unhex('1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06'), 'AggregateFunction(uniqApacheHLL, UInt64)') AS s UNION ALL - SELECT uniqApacheHLLState(number) AS s FROM numbers(3, 5) + SELECT uniqApacheHLLState(number) AS s FROM numbers(6, 5) ); --- lg_k = 12, HLL_4, raw bytes of 'alpha', 'beta', 'gamma'. -SELECT finalizeAggregation(CAST(unhex('140201070C03080300BD3A090A8E5A62115168C90A'), 'AggregateFunction(uniqApacheHLL, UInt64)')); - -- lg_k = 4, HLL_4, 100 distinct values. Dense HLL mode rather than a coupon list. SELECT finalizeAggregation(CAST(unhex('300A0107040008020215EB1DC787F15440000000000000FB3F000000000000000003000000000000000251214121031025'), 'AggregateFunction(uniqApacheHLL(4), UInt64)')); @@ -46,32 +30,26 @@ SELECT finalizeAggregation(CAST(unhex('080201070C030C0000'), 'AggregateFunction( SELECT 'types wider than the 8 bytes update(long) takes'; --- lg_k = 12, HLL_4 over the canonical 16 bytes of 00000000-0000-0000-0000-00000000000{0..4}. ClickHouse --- holds a UUID as two 64-bit halves in host order, so hashing its bytes in memory would not agree with --- an external producer working from the textual form; the canonical order is hashed instead. +-- Canonical bytes of `UUID` values 00000000-0000-0000-0000-00000000000{0..4}. SELECT hex(toString(uniqApacheHLLState(toUUID(concat('00000000-0000-0000-0000-00000000000', toString(number)))))) = '1C0201070C0308050050C94D05854BD10ADB8CBD053C56FB07F8FDB206' FROM numbers(5) SETTINGS max_threads = 1; -SELECT finalizeAggregation(CAST(unhex('1C0201070C0308050050C94D05854BD10ADB8CBD053C56FB07F8FDB206'), 'AggregateFunction(uniqApacheHLL, UUID)')); -- The same for 2001:db8::1 .. ::5, which ClickHouse already holds in network order. SELECT hex(toString(uniqApacheHLLState(toIPv6(concat('2001:db8::', hex(number + 1)))))) = '1C0201070C0308050018216E09FAB4750F52D5BB07DFBDE30A79BC9D0B' FROM numbers(5) SETTINGS max_threads = 1; -SELECT finalizeAggregation(CAST(unhex('1C0201070C0308050018216E09FAB4750F52D5BB07DFBDE30A79BC9D0B'), 'AggregateFunction(uniqApacheHLL, IPv6)')); SELECT 'DateTime64 is hashed as the epoch time it counts'; --- A `DateTime64(3)` is hashed as its epoch milliseconds, which is the `long` a caller elsewhere --- passes for the same instant. These five are 2020-01-01 00:00:00.000 and the next four seconds. +-- Epoch milliseconds for 2020-01-01 00:00:00.000 UTC and the next four seconds. SELECT hex(toString(uniqApacheHLLState(toDateTime64('2020-01-01 00:00:00.000', 3, 'UTC') + number))) = '1C0201070C0308050001D4B019CBDD6F1059310D0E833938083897B304' FROM numbers(5) SETTINGS max_threads = 1; SELECT 'export sketches for consumption outside ClickHouse'; --- `max_threads` is pinned because a state that was merged from several partial states may lay its --- coupons out in a different order than a state built by a single thread. +-- Use one thread to keep coupon ordering deterministic. SELECT hex(toString(uniqApacheHLLState(number))) = '1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06' FROM numbers(5) SETTINGS max_threads = 1; SELECT hex(toString(uniqApacheHLLState(14, 'HLL_8')(number))) = '1C0201070E03080508CBD7C2042BF2FB06862FF90D7581660781BC5D06' FROM numbers(5) SETTINGS max_threads = 1; @@ -80,16 +58,33 @@ SELECT hex(toString(uniqApacheHLLMergeState(s))) = '1C0201070C03080500CBD7C2042B FROM (SELECT CAST(unhex('1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06'), 'AggregateFunction(uniqApacheHLL, UInt64)') AS s) SETTINGS max_threads = 1; -SELECT 'states survive a write/read cycle through storage'; +SELECT 'store and merge states with a different lg_k'; DROP TABLE IF EXISTS hll_interop_states; CREATE TABLE hll_interop_states (k UInt8, s AggregateFunction(uniqApacheHLL, UInt64)) ENGINE = AggregatingMergeTree ORDER BY k; -INSERT INTO hll_interop_states SELECT number % 4 AS k, uniqApacheHLLState(number) FROM numbers(20) GROUP BY k; +INSERT INTO hll_interop_states SELECT number % 4 AS k, uniqApacheHLLState(8)(number) FROM numbers(20) GROUP BY k; OPTIMIZE TABLE hll_interop_states FINAL; -- Small enough to stay in coupon mode, so the union is exact regardless of how it was partitioned. SELECT uniqApacheHLLMerge(s) FROM hll_interop_states; --- A state read back from disk is still a valid sketch for an external consumer. -SELECT countDistinct(hex(toString(s))) = 4 FROM hll_interop_states; DROP TABLE hll_interop_states; --- Malformed states are rejected as `CORRUPTED_DATA`; see `05136_uniq_apache_hll_corrupted_state.sh`. +SELECT 'merging states built with a different lg_k'; +-- Downsampling and building at the lower resolution must produce the same union estimate. +SELECT + (SELECT uniqApacheHLLMerge(8)(s) FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(100000) GROUP BY number % 17)) + = (SELECT uniqApacheHLLMerge(8)(s) FROM (SELECT uniqApacheHLLState(8)(number) AS s FROM numbers(100000) GROUP BY number % 17)); + +-- Downsampling must also reduce the serialized state size. +SELECT + length(toString(uniqApacheHLLMergeState(8)(s))) < length(toString(uniqApacheHLLMergeState(s))) / 8 +FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(100000) GROUP BY number % 17); + +SELECT 'relabelling with CAST does not rescale'; +SELECT + finalizeAggregation(CAST(s, 'AggregateFunction(uniqApacheHLL(8), UInt64)')) = finalizeAggregation(s), + length(toString(CAST(s, 'AggregateFunction(uniqApacheHLL(8), UInt64)'))) = length(toString(s)) +FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(1000)); + +SELECT 'only the parameters are interchangeable'; +SELECT CAST(uniqApacheHLLState(toString(number)), 'AggregateFunction(uniqApacheHLL, UInt64)') FROM numbers(10); -- { serverError CANNOT_CONVERT_TYPE } +SELECT CAST(uniqThetaState(number), 'AggregateFunction(uniqApacheHLL, UInt64)') FROM numbers(10); -- { serverError CANNOT_CONVERT_TYPE } diff --git a/tests/queries/0_stateless/05027_uniq_apache_hll_cross_scale.reference b/tests/queries/0_stateless/05027_uniq_apache_hll_cross_scale.reference deleted file mode 100644 index 2fc0b7c7d810..000000000000 --- a/tests/queries/0_stateless/05027_uniq_apache_hll_cross_scale.reference +++ /dev/null @@ -1,11 +0,0 @@ -the two types remain distinct -AggregateFunction(uniqApacheHLL, UInt64) -AggregateFunction(uniqApacheHLL(8), UInt64) -merging states built with a different lg_k -1 -1 -relabelling with CAST does not rescale -1 1 -a state of one lg_k can be stored in a column declared with another -20 -only the parameters are interchangeable diff --git a/tests/queries/0_stateless/05027_uniq_apache_hll_cross_scale.sql b/tests/queries/0_stateless/05027_uniq_apache_hll_cross_scale.sql deleted file mode 100644 index 43588e9d133a..000000000000 --- a/tests/queries/0_stateless/05027_uniq_apache_hll_cross_scale.sql +++ /dev/null @@ -1,47 +0,0 @@ --- Tags: no-fasttest --- ^ DataSketches is not built in fast-test builds. - --- `lg_k` and the sketch type configure the sketch but are held by the aggregate function rather than --- by its state, whose layout is the same whatever they are, and a serialized sketch records its own --- `lg_k`. `uniqApacheHLL` therefore reports states of different parameterisations as having one --- binary representation, which lets `-Merge` read a state built with one `lg_k` under a function --- declared with another, and lets `CAST` relabel a column between the two types. - -SELECT 'the two types remain distinct'; -SELECT toTypeName(uniqApacheHLLState(number)) FROM numbers(1); -SELECT toTypeName(uniqApacheHLLState(8)(number)) FROM numbers(1); - -SELECT 'merging states built with a different lg_k'; --- Downsampling during a union produces the same registers as building at the target `lg_k` from the --- start, so rescaling 17 states of `lg_k` 12 agrees exactly with merging 17 built at 8. Both sides --- come out of a union and so use the same estimator. -SELECT - (SELECT uniqApacheHLLMerge(8)(s) FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(100000) GROUP BY number % 17)) - = (SELECT uniqApacheHLLMerge(8)(s) FROM (SELECT uniqApacheHLLState(8)(number) AS s FROM numbers(100000) GROUP BY number % 17)); - --- Merging is lossy in one direction only: the union takes the resolution of its coarsest input, so a --- rescaled state is much smaller than the states it was built from. -SELECT - length(toString(uniqApacheHLLMergeState(8)(s))) < length(toString(uniqApacheHLLMergeState(s))) / 8 -FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(100000) GROUP BY number % 17); - -SELECT 'relabelling with CAST does not rescale'; --- `CAST` between the two types re-associates the column with the other function without touching its --- data, so the sketch keeps the resolution it was built with and the estimate does not change. -SELECT - finalizeAggregation(CAST(s, 'AggregateFunction(uniqApacheHLL(8), UInt64)')) = finalizeAggregation(s), - length(toString(CAST(s, 'AggregateFunction(uniqApacheHLL(8), UInt64)'))) = length(toString(s)) -FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(1000)); - -SELECT 'a state of one lg_k can be stored in a column declared with another'; -DROP TABLE IF EXISTS hll_cross_scale; -CREATE TABLE hll_cross_scale (s AggregateFunction(uniqApacheHLL, UInt64)) ENGINE = MergeTree ORDER BY tuple(); -INSERT INTO hll_cross_scale SELECT uniqApacheHLLState(8)(number) FROM numbers(20); -SELECT uniqApacheHLLMerge(s) FROM hll_cross_scale; -DROP TABLE hll_cross_scale; - -SELECT 'only the parameters are interchangeable'; --- The argument types and the function itself must still match: `haveEqualArgumentTypes` and the name --- comparison keep these apart even though the parameters no longer do. -SELECT CAST(uniqApacheHLLState(toString(number)), 'AggregateFunction(uniqApacheHLL, UInt64)') FROM numbers(10); -- { serverError CANNOT_CONVERT_TYPE } -SELECT CAST(uniqThetaState(number), 'AggregateFunction(uniqApacheHLL, UInt64)') FROM numbers(10); -- { serverError CANNOT_CONVERT_TYPE } diff --git a/tests/queries/0_stateless/05028_uniq_apache_hll_argument_types.reference b/tests/queries/0_stateless/05028_uniq_apache_hll_argument_types.reference deleted file mode 100644 index 6478f4dcb46e..000000000000 --- a/tests/queries/0_stateless/05028_uniq_apache_hll_argument_types.reference +++ /dev/null @@ -1,26 +0,0 @@ -types the sketch hashes directly -20 -20 -20 -20 -20 -20 -20 -20 -20 -20 -20 -20 -20 -20 -3 -the wrappers of an accepted type are accepted -20 -20 -a DateTime64 is the integer a caller elsewhere would hash -1 -types no other implementation can reproduce are rejected -one argument only -the parameters still apply -20 -AggregateFunction(uniqApacheHLL(8, \'HLL_6\'), UInt64) diff --git a/tests/queries/0_stateless/05028_uniq_apache_hll_argument_types.sql b/tests/queries/0_stateless/05028_uniq_apache_hll_argument_types.sql deleted file mode 100644 index 6b133e97f64c..000000000000 --- a/tests/queries/0_stateless/05028_uniq_apache_hll_argument_types.sql +++ /dev/null @@ -1,54 +0,0 @@ --- Tags: no-fasttest --- ^ DataSketches is not built in fast-test builds. - --- `uniqApacheHLL` accepts only the types a sketch hashes the same way in every implementation, as --- an 8-byte integer, an IEEE-754 double or their raw bytes. Anything else would have to be hashed --- to a single value by ClickHouse first, which no producer outside ClickHouse could reproduce. --- All counts below are small enough for the sketch to stay in coupon mode, where it is exact. - -SELECT 'types the sketch hashes directly'; -SELECT uniqApacheHLL(toUInt64(number)) FROM numbers(20); -SELECT uniqApacheHLL(toInt32(number)) FROM numbers(20); -SELECT uniqApacheHLL(toBFloat16(number)) FROM numbers(20); -SELECT uniqApacheHLL(toFloat32(number)) FROM numbers(20); -SELECT uniqApacheHLL(toFloat64(number)) FROM numbers(20); -SELECT uniqApacheHLL(toString(number)) FROM numbers(20); -SELECT uniqApacheHLL(toFixedString(toString(number), 8)) FROM numbers(20); -SELECT uniqApacheHLL(toDate('2020-01-01') + number) FROM numbers(20); -SELECT uniqApacheHLL(toDate32('2020-01-01') + number) FROM numbers(20); -SELECT uniqApacheHLL(toDateTime('2020-01-01 00:00:00') + number) FROM numbers(20); -SELECT uniqApacheHLL(toDateTime64('2020-01-01 00:00:00.000', 3) + number) FROM numbers(20); -SELECT uniqApacheHLL(reinterpretAsUUID(toUInt128(number))) FROM numbers(20); -SELECT uniqApacheHLL(toIPv4('1.2.3.0') + number) FROM numbers(20); -SELECT uniqApacheHLL(toIPv6(concat('2001:db8::', hex(number + 1)))) FROM numbers(20); -SELECT uniqApacheHLL(CAST(number % 3, 'Enum8(\'a\' = 0, \'b\' = 1, \'c\' = 2)')) FROM numbers(20); - -SELECT 'the wrappers of an accepted type are accepted'; -SELECT uniqApacheHLL(toNullable(number)) FROM numbers(20); -SELECT uniqApacheHLL(toLowCardinality(toString(number))) FROM numbers(20); - -SELECT 'a DateTime64 is the integer a caller elsewhere would hash'; --- `DateTime64(3)` counts epoch milliseconds, so it produces the sketch of those milliseconds. -SELECT hex(toString(uniqApacheHLLState(toDateTime64('2020-01-01 00:00:00.000', 3, 'UTC') + number))) - = hex(toString(uniqApacheHLLState(toInt64(1577836800000) + number * 1000))) -FROM numbers(5) SETTINGS max_threads = 1; - -SELECT 'types no other implementation can reproduce are rejected'; --- No byte order for the wide integers is agreed on across implementations. -SELECT uniqApacheHLL(toInt128(number)) FROM numbers(20); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } -SELECT uniqApacheHLL(toUInt256(number)) FROM numbers(20); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } --- No DataSketches binding accepts a decimal, and the scale is not part of the sketch. -SELECT uniqApacheHLL(toDecimal64(number, 2)) FROM numbers(20); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } -SELECT uniqApacheHLL(toDecimal128(number, 4)) FROM numbers(20); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } --- A composite value would have to be hashed by ClickHouse first. -SELECT uniqApacheHLL(materialize([number])) FROM numbers(20); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } -SELECT uniqApacheHLL((number, number + 1)) FROM numbers(20); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } - -SELECT 'one argument only'; -SELECT uniqApacheHLL(number, number + 1) FROM numbers(20); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } -SELECT uniqApacheHLL(toString(number), number) FROM numbers(20); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } -SELECT uniqApacheHLL() FROM numbers(1); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } - -SELECT 'the parameters still apply'; -SELECT uniqApacheHLL(8)(reinterpretAsUUID(toUInt128(number))) FROM numbers(20); -SELECT toTypeName(uniqApacheHLLState(8, 'HLL_6')(number)) FROM numbers(1); diff --git a/tests/queries/0_stateless/05055_uniq_theta_compressed_state.sql b/tests/queries/0_stateless/05055_uniq_theta_compressed_state.sql index 9668fc48c74e..848cfc356e9f 100644 --- a/tests/queries/0_stateless/05055_uniq_theta_compressed_state.sql +++ b/tests/queries/0_stateless/05055_uniq_theta_compressed_state.sql @@ -1,17 +1,9 @@ -- Tags: no-fasttest -- - no-fasttest -- compiled w/o datasketches --- Apache DataSketches can serialize a Theta sketch in a compressed form (serialization --- version 4) that packs the retained values at a variable number of bits per entry. --- ClickHouse always writes the uncompressed form, but it has to read the compressed one, --- because an `AggregateFunction(uniqTheta, ...)` state can come from another --- implementation of the library. Two of the unpacking routines decoded it incorrectly, --- and the result was a silently too low estimate rather than an error. - --- Both states below are canonical compressed encodings of a sketch that retains 16 values --- with theta = 1, so the estimate is exact. The first packs its values at 33 bits per --- entry and the second at 35; the two widths are decoded by separate routines. The two --- sketches have exactly two values in common, so their union retains 30. +-- Compressed Theta states (serialization version 4) exercise separate 33- and 35-bit +-- unpacking routines that previously underestimated cardinality. Each retains 16 values +-- with theta = 1; two shared values give a union of 30. WITH CAST(unhex('4B01040321011ACC9310000001F4000000FA0000007D0000003E8000001F4000000FA0000007D180000000000001F4000000FA0000007D0000003E8000001F4000000FA0000007D0000003E8') AS AggregateFunction(uniqTheta, UInt64)) AS state_33_bits, diff --git a/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.reference b/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.reference index f70b0368a421..d7c73d3f18be 100644 --- a/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.reference +++ b/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.reference @@ -1,3 +1,2 @@ OK unknown type -OK bad payload OK rowbinary diff --git a/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.sh b/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.sh index 18ad78792ac1..80c16c7e17f6 100755 --- a/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.sh +++ b/tests/queries/0_stateless/05136_uniq_apache_hll_corrupted_state.sh @@ -2,34 +2,20 @@ # Tags: no-fasttest # no-fasttest -- compiled w/o datasketches -# A `uniqApacheHLL` state that is not a sketch must be rejected as `CORRUPTED_DATA`. -# -# `datasketches` reports a payload it cannot parse as `std::invalid_argument` or -# `std::out_of_range`, neither of which is a `DB::Exception`, so without the -# translation in `HllSketchData::read` they would escape -# `SerializationAggregateFunction`'s `catch (...)` and be reported as a logical error. -# -# This is a shell test rather than a `.sql` one with a `serverError` hint because the -# client prints an extra exception with a stack trace to stderr for this class of error, -# which a `.sql` test counts as a failure. The same applies to `uniqTheta`, see +# Malformed states must raise `CORRUPTED_DATA`, not a logical exception. +# A shell test tolerates the client's extra stack trace, as in # `04307_uniqTheta_corrupted_state_106259.sh`. CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -# A well-formed varint length prefix followed by eight bytes that are not a sketch: -# the first byte of the payload is the preamble length, which no HLL sketch uses. +# Valid length prefix, invalid sketch preamble. $CLICKHOUSE_CLIENT --query \ "SELECT finalizeAggregation(CAST(unhex('08FFFFFFFFFFFFFFFF'), 'AggregateFunction(uniqApacheHLL, UInt64)'))" 2>&1 \ | grep -q -F 'CORRUPTED_DATA' && echo 'OK unknown type' || echo 'FAIL unknown type' -$CLICKHOUSE_CLIENT --query \ - "SELECT finalizeAggregation(CAST(unhex('0801020304050607FF'), 'AggregateFunction(uniqApacheHLL, UInt64)'))" 2>&1 \ - | grep -q -F 'CORRUPTED_DATA' && echo 'OK bad payload' || echo 'FAIL bad payload' - -# The `RowBinary` path reaches `HllSketchData::read` through `deserializeBinary`: -# the leading 0x03 claims a three-byte payload, shorter than any HLL sketch. +# The `RowBinary` payload is shorter than any valid HLL sketch. printf '\x03\x03\x03\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' \ | $CLICKHOUSE_LOCAL --input-format=RowBinary \ --structure='x AggregateFunction(uniqApacheHLL, IPv6)' \ From 562352fabff053515ba52306a096db61a726dbf6 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Wed, 30 Sep 2026 13:53:03 +0200 Subject: [PATCH 4/5] Fix `uniqApacheHLL` merge resolution below 7 and empty state serialization - Construct `hll_union` with the requested `lg_k` instead of flooring it at 7, so `uniqApacheHLLMerge(4..6)` produces sketches with the requested `lg_k`. `hll_union` accepts `lg_max_k` in [4, 21]. - Serialize an untouched state as the library's compact empty sketch instead of a zero-length vector, so external DataSketches consumers can read it. Related: https://github.com/Altinity/ClickHouse/pull/2398#discussion_r4142688238 Related: https://github.com/Altinity/ClickHouse/pull/2398#discussion_r4142688245 --- .../AggregateFunctionUniqApacheHLL.h | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h index 718398fe97ae..a6325e9f0ca8 100644 --- a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h +++ b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.h @@ -26,7 +26,6 @@ #include -#include #include #include #include @@ -57,10 +56,7 @@ class HllSketchData datasketches::hll_union * getSkUnion(uint8_t lg_config_k) { if (!sk_union) - { - /// `hll_union` requires at least 7, but preserves the lower resolution of its inputs. - sk_union = std::make_unique(std::max(lg_config_k, 7)); - } + sk_union = std::make_unique(lg_config_k); return sk_union.get(); } @@ -137,13 +133,15 @@ class HllSketchData } } - void write(WriteBuffer & out, datasketches::target_hll_type tgt_type) const + void write(WriteBuffer & out, uint8_t lg_config_k, datasketches::target_hll_type tgt_type) const { datasketches::hll_sketch::vector_bytes bytes; if (sk_update) bytes = sk_update->serialize_compact(); else if (sk_union) bytes = sk_union->get_result(tgt_type).serialize_compact(); + else + bytes = datasketches::hll_sketch(lg_config_k, tgt_type).serialize_compact(); writeVectorBinary(bytes, out); } }; @@ -225,7 +223,7 @@ class AggregateFunctionUniqApacheHLL final : public IAggregateFunctionDataHelper void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override { - this->data(place).write(buf, target_type); + this->data(place).write(buf, lg_config_k, target_type); } void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena *) const override From 81fb83c627ce91272e5ae780bb8b7a663e41e25b Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:12:51 +0200 Subject: [PATCH 5/5] Fix `uniqApacheHLL` on an all-NULL argument and merge its tests For `Nullable(Nothing)` the factory creates the nested function before the `Null` combinator replaces it with `nothing`, so `uniqApacheHLL(NULL)` threw `ILLEGAL_TYPE_OF_ARGUMENT` instead of returning 0 like `uniq` and `uniqTheta`. Merge `05026_uniq_apache_hll_interop` into `04327_uniq_apache_hll` and add the all-NULL cases there. `05136_uniq_apache_hll_corrupted_state.sh` stays a shell test because `CORRUPTED_DATA` is always logged with a stack trace to stderr. https://github.com/Altinity/ClickHouse/pull/2398#issuecomment-5925895693 --- .../AggregateFunctionUniqApacheHLL.cpp | 4 + .../04327_uniq_apache_hll.reference | 27 ++++++ .../0_stateless/04327_uniq_apache_hll.sql | 93 +++++++++++++++++++ .../05026_uniq_apache_hll_interop.reference | 23 ----- .../05026_uniq_apache_hll_interop.sql | 90 ------------------ 5 files changed, 124 insertions(+), 113 deletions(-) delete mode 100644 tests/queries/0_stateless/05026_uniq_apache_hll_interop.reference delete mode 100644 tests/queries/0_stateless/05026_uniq_apache_hll_interop.sql diff --git a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp index 8da3fc42092a..21fb8363e155 100644 --- a/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp +++ b/src/AggregateFunctions/AggregateFunctionUniqApacheHLL.cpp @@ -86,6 +86,10 @@ static AggregateFunctionPtr createAggregateFunctionUniqApacheHLL( if (which.isIPv6()) return std::make_shared>(lg_config_k, target_type, argument_types, params); + /// For `Nullable(Nothing)` the `Null` combinator replaces this function with `nothing`, but it must be created first. + if (argument_type.onlyNull()) + return std::make_shared>(lg_config_k, target_type, argument_types, params); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument for aggregate function {}. Use uniq, uniqCombined or uniqHLL12 for unsupported types.", argument_type.getName(), name); diff --git a/tests/queries/0_stateless/04327_uniq_apache_hll.reference b/tests/queries/0_stateless/04327_uniq_apache_hll.reference index ef814b89b951..3939bfc12e81 100644 --- a/tests/queries/0_stateless/04327_uniq_apache_hll.reference +++ b/tests/queries/0_stateless/04327_uniq_apache_hll.reference @@ -26,5 +26,32 @@ argument types nullable and low cardinality 20 20 +0 +0 +0 +UInt64 0 unsupported types one argument only +import sketches built outside ClickHouse +5 +11 +84 +5 +0 +types wider than the 8 bytes update(long) takes +1 +1 +DateTime64 is hashed as the epoch time it counts +1 +export sketches for consumption outside ClickHouse +1 +1 +1 +store and merge states with a different lg_k +20 +merging states built with a different lg_k +1 +1 +relabelling with CAST does not rescale +1 1 +only the parameters are interchangeable diff --git a/tests/queries/0_stateless/04327_uniq_apache_hll.sql b/tests/queries/0_stateless/04327_uniq_apache_hll.sql index 1de290b47236..4d2a4bf5d477 100644 --- a/tests/queries/0_stateless/04327_uniq_apache_hll.sql +++ b/tests/queries/0_stateless/04327_uniq_apache_hll.sql @@ -56,6 +56,11 @@ SELECT uniqApacheHLL(CAST(number % 3, 'Enum8(\'a\' = 0, \'b\' = 1, \'c\' = 2)')) SELECT 'nullable and low cardinality'; SELECT uniqApacheHLL(toNullable(number)) FROM numbers(20); SELECT uniqApacheHLL(toLowCardinality(toString(number))) FROM numbers(20); +-- All-NULL argument returns 0, as `uniq` does. +SELECT uniqApacheHLL(NULL); +SELECT uniqApacheHLL(14, 'HLL_8')(NULL); +SELECT uniqApacheHLL(NULL) FROM numbers(10); +SELECT toTypeName(uniqApacheHLLState(NULL)), uniqApacheHLLState(NULL); SELECT 'unsupported types'; SELECT uniqApacheHLL(toInt128(number)) FROM numbers(20); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } @@ -67,3 +72,91 @@ SELECT uniqApacheHLL((number, number + 1)) FROM numbers(20); -- { serverError IL SELECT 'one argument only'; SELECT uniqApacheHLL(number, number + 1) FROM numbers(20); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } SELECT uniqApacheHLL() FROM numbers(1); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +-- Fixtures were generated by Apache DataSketches C++, with a varint length prefix. +-- These checks pin compatibility with external producers and consumers. + +SELECT 'import sketches built outside ClickHouse'; + +-- lg_k = 12, HLL_4, `hll_sketch::update` on `uint64_t` values over 0..4. Coupon list mode. +SELECT finalizeAggregation(CAST(unhex('1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06'), 'AggregateFunction(uniqApacheHLL, UInt64)')); + +-- External sketches over 0..4 and 3..7 overlap with ClickHouse values 6..10. +SELECT uniqApacheHLLMerge(s) FROM +( + SELECT CAST(unhex('1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06'), 'AggregateFunction(uniqApacheHLL, UInt64)') AS s + UNION ALL + SELECT CAST(unhex('1C0201070C030805007581660781BC5D067B65E608FC2D420AC1E91705'), 'AggregateFunction(uniqApacheHLL, UInt64)') AS s + UNION ALL + SELECT uniqApacheHLLState(number) AS s FROM numbers(6, 5) +); + +-- lg_k = 4, HLL_4, 100 distinct values. Dense HLL mode rather than a coupon list. +SELECT finalizeAggregation(CAST(unhex('300A0107040008020215EB1DC787F15440000000000000FB3F000000000000000003000000000000000251214121031025'), 'AggregateFunction(uniqApacheHLL(4), UInt64)')); + +-- lg_k = 14, HLL_8: the non-default parameters are carried by the state type. +SELECT finalizeAggregation(CAST(unhex('1C0201070E03080508CBD7C2042BF2FB06862FF90D7581660781BC5D06'), 'AggregateFunction(uniqApacheHLL(14, \'HLL_8\'), UInt64)')); + +-- An externally produced empty sketch. +SELECT finalizeAggregation(CAST(unhex('080201070C030C0000'), 'AggregateFunction(uniqApacheHLL, UInt64)')); + +SELECT 'types wider than the 8 bytes update(long) takes'; + +-- Canonical bytes of `UUID` values 00000000-0000-0000-0000-00000000000{0..4}. +SELECT hex(toString(uniqApacheHLLState(toUUID(concat('00000000-0000-0000-0000-00000000000', toString(number)))))) + = '1C0201070C0308050050C94D05854BD10ADB8CBD053C56FB07F8FDB206' +FROM numbers(5) SETTINGS max_threads = 1; + +-- The same for 2001:db8::1 .. ::5, which ClickHouse already holds in network order. +SELECT hex(toString(uniqApacheHLLState(toIPv6(concat('2001:db8::', hex(number + 1)))))) + = '1C0201070C0308050018216E09FAB4750F52D5BB07DFBDE30A79BC9D0B' +FROM numbers(5) SETTINGS max_threads = 1; + +SELECT 'DateTime64 is hashed as the epoch time it counts'; + +-- Epoch milliseconds for 2020-01-01 00:00:00.000 UTC and the next four seconds. +SELECT hex(toString(uniqApacheHLLState(toDateTime64('2020-01-01 00:00:00.000', 3, 'UTC') + number))) + = '1C0201070C0308050001D4B019CBDD6F1059310D0E833938083897B304' +FROM numbers(5) SETTINGS max_threads = 1; + +SELECT 'export sketches for consumption outside ClickHouse'; + +-- Use one thread to keep coupon ordering deterministic. +SELECT hex(toString(uniqApacheHLLState(number))) = '1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06' FROM numbers(5) SETTINGS max_threads = 1; +SELECT hex(toString(uniqApacheHLLState(14, 'HLL_8')(number))) = '1C0201070E03080508CBD7C2042BF2FB06862FF90D7581660781BC5D06' FROM numbers(5) SETTINGS max_threads = 1; + +-- Importing an external sketch and exporting it again must reproduce it byte for byte. +SELECT hex(toString(uniqApacheHLLMergeState(s))) = '1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06' +FROM (SELECT CAST(unhex('1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06'), 'AggregateFunction(uniqApacheHLL, UInt64)') AS s) +SETTINGS max_threads = 1; + +SELECT 'store and merge states with a different lg_k'; + +DROP TABLE IF EXISTS hll_interop_states; +CREATE TABLE hll_interop_states (k UInt8, s AggregateFunction(uniqApacheHLL, UInt64)) ENGINE = AggregatingMergeTree ORDER BY k; +INSERT INTO hll_interop_states SELECT number % 4 AS k, uniqApacheHLLState(8)(number) FROM numbers(20) GROUP BY k; +OPTIMIZE TABLE hll_interop_states FINAL; +-- Small enough to stay in coupon mode, so the union is exact regardless of how it was partitioned. +SELECT uniqApacheHLLMerge(s) FROM hll_interop_states; +DROP TABLE hll_interop_states; + +SELECT 'merging states built with a different lg_k'; +-- Downsampling and building at the lower resolution must produce the same union estimate. +SELECT + (SELECT uniqApacheHLLMerge(8)(s) FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(100000) GROUP BY number % 17)) + = (SELECT uniqApacheHLLMerge(8)(s) FROM (SELECT uniqApacheHLLState(8)(number) AS s FROM numbers(100000) GROUP BY number % 17)); + +-- Downsampling must also reduce the serialized state size. +SELECT + length(toString(uniqApacheHLLMergeState(8)(s))) < length(toString(uniqApacheHLLMergeState(s))) / 8 +FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(100000) GROUP BY number % 17); + +SELECT 'relabelling with CAST does not rescale'; +SELECT + finalizeAggregation(CAST(s, 'AggregateFunction(uniqApacheHLL(8), UInt64)')) = finalizeAggregation(s), + length(toString(CAST(s, 'AggregateFunction(uniqApacheHLL(8), UInt64)'))) = length(toString(s)) +FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(1000)); + +SELECT 'only the parameters are interchangeable'; +SELECT CAST(uniqApacheHLLState(toString(number)), 'AggregateFunction(uniqApacheHLL, UInt64)') FROM numbers(10); -- { serverError CANNOT_CONVERT_TYPE } +SELECT CAST(uniqThetaState(number), 'AggregateFunction(uniqApacheHLL, UInt64)') FROM numbers(10); -- { serverError CANNOT_CONVERT_TYPE } diff --git a/tests/queries/0_stateless/05026_uniq_apache_hll_interop.reference b/tests/queries/0_stateless/05026_uniq_apache_hll_interop.reference deleted file mode 100644 index 70c97654e417..000000000000 --- a/tests/queries/0_stateless/05026_uniq_apache_hll_interop.reference +++ /dev/null @@ -1,23 +0,0 @@ -import sketches built outside ClickHouse -5 -11 -84 -5 -0 -types wider than the 8 bytes update(long) takes -1 -1 -DateTime64 is hashed as the epoch time it counts -1 -export sketches for consumption outside ClickHouse -1 -1 -1 -store and merge states with a different lg_k -20 -merging states built with a different lg_k -1 -1 -relabelling with CAST does not rescale -1 1 -only the parameters are interchangeable diff --git a/tests/queries/0_stateless/05026_uniq_apache_hll_interop.sql b/tests/queries/0_stateless/05026_uniq_apache_hll_interop.sql deleted file mode 100644 index f29084519fac..000000000000 --- a/tests/queries/0_stateless/05026_uniq_apache_hll_interop.sql +++ /dev/null @@ -1,90 +0,0 @@ --- Tags: no-fasttest --- ^ DataSketches is not built in fast-test builds. - --- Fixtures were generated by Apache DataSketches C++, with a varint length prefix. --- These checks pin compatibility with external producers and consumers. - -SELECT 'import sketches built outside ClickHouse'; - --- lg_k = 12, HLL_4, `hll_sketch::update` on `uint64_t` values over 0..4. Coupon list mode. -SELECT finalizeAggregation(CAST(unhex('1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06'), 'AggregateFunction(uniqApacheHLL, UInt64)')); - --- External sketches over 0..4 and 3..7 overlap with ClickHouse values 6..10. -SELECT uniqApacheHLLMerge(s) FROM -( - SELECT CAST(unhex('1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06'), 'AggregateFunction(uniqApacheHLL, UInt64)') AS s - UNION ALL - SELECT CAST(unhex('1C0201070C030805007581660781BC5D067B65E608FC2D420AC1E91705'), 'AggregateFunction(uniqApacheHLL, UInt64)') AS s - UNION ALL - SELECT uniqApacheHLLState(number) AS s FROM numbers(6, 5) -); - --- lg_k = 4, HLL_4, 100 distinct values. Dense HLL mode rather than a coupon list. -SELECT finalizeAggregation(CAST(unhex('300A0107040008020215EB1DC787F15440000000000000FB3F000000000000000003000000000000000251214121031025'), 'AggregateFunction(uniqApacheHLL(4), UInt64)')); - --- lg_k = 14, HLL_8: the non-default parameters are carried by the state type. -SELECT finalizeAggregation(CAST(unhex('1C0201070E03080508CBD7C2042BF2FB06862FF90D7581660781BC5D06'), 'AggregateFunction(uniqApacheHLL(14, \'HLL_8\'), UInt64)')); - --- An externally produced empty sketch. -SELECT finalizeAggregation(CAST(unhex('080201070C030C0000'), 'AggregateFunction(uniqApacheHLL, UInt64)')); - -SELECT 'types wider than the 8 bytes update(long) takes'; - --- Canonical bytes of `UUID` values 00000000-0000-0000-0000-00000000000{0..4}. -SELECT hex(toString(uniqApacheHLLState(toUUID(concat('00000000-0000-0000-0000-00000000000', toString(number)))))) - = '1C0201070C0308050050C94D05854BD10ADB8CBD053C56FB07F8FDB206' -FROM numbers(5) SETTINGS max_threads = 1; - --- The same for 2001:db8::1 .. ::5, which ClickHouse already holds in network order. -SELECT hex(toString(uniqApacheHLLState(toIPv6(concat('2001:db8::', hex(number + 1)))))) - = '1C0201070C0308050018216E09FAB4750F52D5BB07DFBDE30A79BC9D0B' -FROM numbers(5) SETTINGS max_threads = 1; - -SELECT 'DateTime64 is hashed as the epoch time it counts'; - --- Epoch milliseconds for 2020-01-01 00:00:00.000 UTC and the next four seconds. -SELECT hex(toString(uniqApacheHLLState(toDateTime64('2020-01-01 00:00:00.000', 3, 'UTC') + number))) - = '1C0201070C0308050001D4B019CBDD6F1059310D0E833938083897B304' -FROM numbers(5) SETTINGS max_threads = 1; - -SELECT 'export sketches for consumption outside ClickHouse'; - --- Use one thread to keep coupon ordering deterministic. -SELECT hex(toString(uniqApacheHLLState(number))) = '1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06' FROM numbers(5) SETTINGS max_threads = 1; -SELECT hex(toString(uniqApacheHLLState(14, 'HLL_8')(number))) = '1C0201070E03080508CBD7C2042BF2FB06862FF90D7581660781BC5D06' FROM numbers(5) SETTINGS max_threads = 1; - --- Importing an external sketch and exporting it again must reproduce it byte for byte. -SELECT hex(toString(uniqApacheHLLMergeState(s))) = '1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06' -FROM (SELECT CAST(unhex('1C0201070C03080500CBD7C2042BF2FB06862FF90D7581660781BC5D06'), 'AggregateFunction(uniqApacheHLL, UInt64)') AS s) -SETTINGS max_threads = 1; - -SELECT 'store and merge states with a different lg_k'; - -DROP TABLE IF EXISTS hll_interop_states; -CREATE TABLE hll_interop_states (k UInt8, s AggregateFunction(uniqApacheHLL, UInt64)) ENGINE = AggregatingMergeTree ORDER BY k; -INSERT INTO hll_interop_states SELECT number % 4 AS k, uniqApacheHLLState(8)(number) FROM numbers(20) GROUP BY k; -OPTIMIZE TABLE hll_interop_states FINAL; --- Small enough to stay in coupon mode, so the union is exact regardless of how it was partitioned. -SELECT uniqApacheHLLMerge(s) FROM hll_interop_states; -DROP TABLE hll_interop_states; - -SELECT 'merging states built with a different lg_k'; --- Downsampling and building at the lower resolution must produce the same union estimate. -SELECT - (SELECT uniqApacheHLLMerge(8)(s) FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(100000) GROUP BY number % 17)) - = (SELECT uniqApacheHLLMerge(8)(s) FROM (SELECT uniqApacheHLLState(8)(number) AS s FROM numbers(100000) GROUP BY number % 17)); - --- Downsampling must also reduce the serialized state size. -SELECT - length(toString(uniqApacheHLLMergeState(8)(s))) < length(toString(uniqApacheHLLMergeState(s))) / 8 -FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(100000) GROUP BY number % 17); - -SELECT 'relabelling with CAST does not rescale'; -SELECT - finalizeAggregation(CAST(s, 'AggregateFunction(uniqApacheHLL(8), UInt64)')) = finalizeAggregation(s), - length(toString(CAST(s, 'AggregateFunction(uniqApacheHLL(8), UInt64)'))) = length(toString(s)) -FROM (SELECT uniqApacheHLLState(number) AS s FROM numbers(1000)); - -SELECT 'only the parameters are interchangeable'; -SELECT CAST(uniqApacheHLLState(toString(number)), 'AggregateFunction(uniqApacheHLL, UInt64)') FROM numbers(10); -- { serverError CANNOT_CONVERT_TYPE } -SELECT CAST(uniqThetaState(number), 'AggregateFunction(uniqApacheHLL, UInt64)') FROM numbers(10); -- { serverError CANNOT_CONVERT_TYPE }