diff --git a/CMakeLists.txt b/CMakeLists.txt index b149e904..663e2fb5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,7 @@ set(EXTENSION_SOURCES src/core/parser_plan_helpers.cpp src/core/parser_sql_extractors.cpp src/core/sql_utils.cpp + src/core/time_travel_pins.cpp src/core/refresh_metadata.cpp src/core/incremental_checker.cpp src/core/plan_rewrite.cpp @@ -77,7 +78,8 @@ set(EXTENSION_SOURCES ${LPTS_DIR}/src/lpts_ast_renderer.cpp ${LPTS_DIR}/src/lpts_ast_builder.cpp ${LPTS_DIR}/src/lpts_ast_flattener.cpp - ${LPTS_DIR}/src/dialect_function_map.cpp) + ${LPTS_DIR}/src/dialect_function_map.cpp + ${LPTS_DIR}/src/spark_scalar_functions.cpp) build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) diff --git a/README.md b/README.md index 694560ef..89115ca9 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ MVs can be created using any SQL construct. Unsupported operators automatically | `openivm_regular_nterm` | BOOLEAN | `true` | Use N-term telescoping for eligible regular-table inner joins compiled for external engines | [Inner join](docs/operators/inner-join.md#regular-table-n-term-compilation) | | `openivm_profile_refresh` | BOOLEAN | `false` | Record per-step refresh timings in `openivm_refresh_profile` | [Automatic refresh](docs/refresh/automatic-refresh.md) | | `openivm_files_path` | VARCHAR | — | Directory for compiled SQL reference files | [Internals](docs/internals/delta-tables.md) | +| `openivm_input_dialect` | VARCHAR | `duckdb` | Dialect of incoming `CREATE MATERIALIZED VIEW` bodies (e.g. `spark` for `VERSION AS OF`) | [Parser](docs/internals/parser.md#input-dialect) | ## Pragmas diff --git a/docs/internals/parser.md b/docs/internals/parser.md index 271f67c5..eb67517b 100644 --- a/docs/internals/parser.md +++ b/docs/internals/parser.md @@ -43,6 +43,38 @@ CREATE MATERIALIZED VIEW mv REFRESH EVERY '5 minutes' AS The parsed interval (300 seconds) is stored in the `refresh_interval` column of `openivm_views`. When omitted, `refresh_interval` is `NULL` (manual refresh only). See [Automatic Refresh](../refresh/automatic-refresh.md) for how the daemon uses this. +## Input dialect + +`openivm_input_dialect` (default `duckdb`) declares the dialect the *incoming* `CREATE MATERIALIZED VIEW` body is written in. When it is not `duckdb`, the body is run through LPTS' `NormalizeInputSqlToDuckDB` before `Parser::ParseQuery`, so backtick identifiers, dialect casts, interval literals and time-travel clauses are translated into DuckDB syntax first. The setting is mirrored onto the parser extension's `ParserExtensionInfo` because `parser_override` is not given a `ClientContext`. + +```sql +SET openivm_input_dialect='spark'; +CREATE MATERIALIZED VIEW mv AS + SELECT region, SUM(amount) FROM sales VERSION AS OF 366 GROUP BY region; +``` + +## Time-travel pins + +A Spark/Delta pin (`VERSION AS OF n`, `TIMESTAMP AS OF '...'`) normalizes to DuckDB's `AT (VERSION => n)` / `AT (TIMESTAMP => '...')` qualifier. OpenIVM registers the sources of a compiled view as plain in-memory stand-in tables whose catalog reports `SupportsTimeTravel() == false`, so a pinned scan cannot be bound directly. + +`src/core/time_travel_pins.cpp` handles this by *peeling* the qualifier off each `BaseTableRef` whose catalog cannot honour it, planning against the pin-less stand-in, and *restoring* the qualifier onto the matching `AstGetNode::table_name` after `LogicalPlanToAst`. Pins are keyed by relation, so aliases, repeated scans, CTEs and joins all keep their own pin, and two relations pinned to different snapshots stay distinct. A catalog that does support time travel (DuckLake) is never peeled and binds natively. + +Because pin restoration is keyed by relation, a view that pins the same relation ambiguously is refused with `NotImplementedException` rather than compiled with a guessed snapshot: + +- the same relation pinned to two different snapshots, +- the same pin naming two differently qualified relations, +- the same relation scanned both pinned and unpinned. + +The first is a real limitation rather than a policy choice. DuckDB resolves the `AT (...)` qualifier during *catalog lookup* — it selects which snapshot of the catalog entry to bind — so neither `LogicalGet` nor `AstGetNode` carries a per-scan pin. Two scans of one relation at two versions are indistinguishable in the bound plan except by `table_index`, and pairing those back to parse-tree references would rest on the binder's index-allocation order, whose failure mode is silently reading the wrong snapshot. Repeated scans that share a pin, including across CTEs, are supported normally. + +The pin is stored in the view SQL in `openivm_views.sql_string` so it survives a restart. Every site that binds or locally executes that SQL peels it first, and refresh source qualification drops it, since the local stand-in catalog holds no snapshots. Foreign-dialect output re-attaches it: Spark renders `VERSION AS OF n`, DuckDB keeps `AT (...)`, and any dialect LPTS has no verified time-travel syntax for raises `LPTS_UNSUPPORTED_TIME_TRAVEL` instead of silently reading the latest snapshot. + +### Alias association + +Spark writes the pin *between* a relation and its alias (`FROM t VERSION AS OF n p`) where DuckDB wants it after both (`FROM t AS p AT (VERSION => n)`), so normalization has to carry the alias across the rewrite. That is the one step where a pin could land on a neighbouring relation, attach to the wrong alias, or be dropped outright — each of which reads a different snapshot while still compiling cleanly. + +The association is therefore checked rather than trusted. `CollectSourceSnapshotBindings` reads every `{relation, alias, qualifier}` triple straight off the *source* text before normalization, and `VerifySnapshotBindings` re-checks them against the parse tree DuckDB produced, raising `NotImplementedException` on any pin that did not come through on the same relation and alias. + ## IVM compatibility classification After rewriting, the parser plans the query and walks the logical plan to classify the view into a refresh type: diff --git a/src/core/ivm_delta_model.cpp b/src/core/ivm_delta_model.cpp index 97364445..f47bec21 100644 --- a/src/core/ivm_delta_model.cpp +++ b/src/core/ivm_delta_model.cpp @@ -29,6 +29,23 @@ static void AddStringCI(vector &values, const string &candidate) { } } +static string WindowPartitionOutputColumn(const string &partition_column) { + auto separator = partition_column.find('='); + return separator == string::npos ? partition_column : partition_column.substr(0, separator); +} + +static vector ProjectedWindowPartitionColumns(const vector &partition_columns, + const vector &output_names) { + vector projected; + for (auto &partition_column : partition_columns) { + auto output_column = WindowPartitionOutputColumn(partition_column); + if (!IncrementalTableNames::IsInternalColumn(output_column) && ContainsStringCI(output_names, output_column)) { + AddStringCI(projected, partition_column); + } + } + return projected; +} + static void AddOutputColumnName(const string &name, vector &visible, vector &hidden) { if (name.empty()) { return; @@ -633,9 +650,21 @@ void PopulateDeltaViewModelLineage(DeltaViewModel &model, const CreateMVPlanFact } const auto &analysis = facts.analysis; if (model.type == RefreshType::WINDOW_PARTITION) { + const auto lineage_partition_columns = model.window_partition_columns; vector direct_lineage_ops; - bool has_lineage = BuildWindowPartitionLineageOps(facts, model.window_partition_columns, - model.window_lineage_ops, &direct_lineage_ops); + bool has_lineage = BuildWindowPartitionLineageOps(facts, lineage_partition_columns, model.window_lineage_ops, + &direct_lineage_ops); + model.window_partition_columns = ProjectedWindowPartitionColumns(lineage_partition_columns, output_names); + for (auto &node : model.nodes) { + if (node.kind == DeltaModelNodeKind::WINDOW) { + node.affected_key_columns = model.window_partition_columns; + } + } + if (model.window_partition_columns.empty()) { + model.features.erase( + std::remove(model.features.begin(), model.features.end(), DeltaModelFeature::WINDOW_AFFECTED_PARTITION), + model.features.end()); + } if (analysis.found_asof_join && (!has_lineage || AsofWindowPartitionReadsRightSideDirectly(direct_lineage_ops, model) || !WindowLineageCoversAllSources(model.window_lineage_ops, facts))) { @@ -644,7 +673,7 @@ void PopulateDeltaViewModelLineage(DeltaViewModel &model, const CreateMVPlanFact ValidateDeltaViewModelInvariants(model); return; } - if (has_lineage) { + if (has_lineage && !model.window_partition_columns.empty()) { DeltaAffectedDomain domain; domain.kind = DeltaAffectedDomainKind::WINDOW_PARTITION; domain.node_id = FindFirstNodeId(model, DeltaModelNodeKind::WINDOW); diff --git a/src/core/parser.cpp b/src/core/parser.cpp index 8e413307..0162b19b 100644 --- a/src/core/parser.cpp +++ b/src/core/parser.cpp @@ -13,6 +13,7 @@ #include "core/ivm_view_classifier.hpp" #include "lpts_pipeline.hpp" #include "core/sql_utils.hpp" +#include "core/time_travel_pins.hpp" #include "rules/column_hider.hpp" #include "upsert/refresh_compiler.hpp" #include "duckdb/common/printer.hpp" @@ -24,7 +25,10 @@ #include "duckdb/main/settings.hpp" #include "duckdb/parser/parser.hpp" #include "duckdb/parser/qualified_name.hpp" +#include "duckdb/parser/parsed_data/create_table_info.hpp" +#include "duckdb/parser/statement/create_statement.hpp" #include "duckdb/parser/statement/drop_statement.hpp" +#include "duckdb/parser/statement/select_statement.hpp" #include "duckdb/planner/expression/bound_aggregate_expression.hpp" #include "duckdb/planner/expression/bound_columnref_expression.hpp" #include "duckdb/planner/operator/logical_top_n.hpp" @@ -405,6 +409,14 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC // re-derives what it needs from the plan. unordered_set table_names; auto table_names_start = create_profile_now(); + // Peel time-travel pins the catalog cannot bind before anything plans this statement, keeping + // each pin keyed by its relation so it can be re-attached to the generated SQL below. + auto time_travel_pins = openivm::TimeTravelPins::Peel(context, *statement); + // SQL OpenIVM binds or executes itself runs against the very catalog that cannot honour the pin, + // so those copies drop it. The stored view SQL keeps it, and refresh re-attaches it when + // rendering for a foreign dialect. + auto &select_statement = *statement->Cast().info->Cast().query; + auto local_view_query = time_travel_pins.Empty() ? original_view_query : select_statement.ToString(); try { table_names = con.GetTableNames(statement->query); } catch (const std::exception &e) { @@ -436,10 +448,8 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC bool has_computed_sum_aggregate_projection = false; { auto select_parse_plan_start = create_profile_now(); - Parser select_parser; - select_parser.ParseQuery(original_view_query); Planner select_planner(context); - select_planner.CreatePlan(std::move(select_parser.statements[0])); + select_planner.CreatePlan(select_statement.Copy()); auto select_plan = std::move(select_planner.plan); visible_output_count = select_planner.names.size(); for (auto &name : select_planner.names) { @@ -536,7 +546,7 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC // CREATE MATERIALIZED VIEW always stores the view body in DuckDB's own dialect. // Refresh-time target dialects are selected per CompileFacts. SqlDialect dialect = SqlDialect::DUCKDB; - auto ast = LogicalPlanToAst(context, select_plan, dialect); + auto ast = LogicalPlanToAst(context, select_plan, dialect, time_travel_pins.Resolver()); auto cte_list = AstToCteList(*ast, dialect); view_query = cte_list->ToQuery(true, output_names); if (!view_query.empty() && view_query.back() == ';') { @@ -665,7 +675,7 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC FilteredGroupCountExtract filtered_group_count_extract; FilteredGroupCountAuxRequirement filtered_group_count_aux_candidate; if (analysis.found_nested_aggregate && - ExtractFilteredGroupCount(original_view_query, output_names, filtered_group_count_extract)) { + ExtractFilteredGroupCount(local_view_query, output_names, filtered_group_count_extract)) { string aux_table = "openivm_filtered_group_count_" + view_name; string group_q = KeywordHelper::WriteOptionallyQuoted(filtered_group_count_extract.group_col); string sum_q = KeywordHelper::WriteOptionallyQuoted(filtered_group_count_extract.sum_col); @@ -679,7 +689,7 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC } if (analysis.found_semi_anti_join && !analysis.found_aggregation) { - if (ExtractSemiAntiQuery(original_view_query, semi_anti_extract)) { + if (ExtractSemiAntiQuery(local_view_query, semi_anti_extract)) { string left_table_name = SqlUtils::LastIdentifierPart(semi_anti_extract.left_table); auto col_result = con.Query("SELECT column_name FROM information_schema.columns WHERE " "lower(table_name) = lower('" + @@ -805,7 +815,7 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC if (aux_enabled && single_source) { vector dcols; string d_input_sql, d_source, d_filter; - if (!ExtractInnerDistinct(original_view_query, dcols, d_input_sql, d_source, d_filter)) { + if (!ExtractInnerDistinct(local_view_query, dcols, d_input_sql, d_source, d_filter)) { OPENIVM_DEBUG_PRINT("[CREATE MV] DISTINCT_INCREMENTAL extractor failed — demoting to " "GROUP_RECOMPUTE\n"); } else { @@ -893,7 +903,7 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC candidate_group_columns.push_back(output_names[i]); } CountDistinctExtract cd_extract; - if (ExtractCountDistinctAggregate(original_view_query, candidate_group_columns, output_names, cd_extract)) { + if (ExtractCountDistinctAggregate(local_view_query, candidate_group_columns, output_names, cd_extract)) { count_distinct_aux_candidate = { "openivm_aux_" + view_name, SqlUtils::LastIdentifierPart(cd_extract.source), candidate_group_columns, cd_extract.group_exprs, @@ -1359,7 +1369,8 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC if (!current_catalog.empty() && current_catalog != default_db) { con.Query("USE " + current_catalog_schema); } - string initial_load_statement = "CREATE TABLE " + initial_load_target + " AS " + view_query; + string local_initial_load_query = time_travel_pins.StripFrom(context, view_query); + string initial_load_statement = "CREATE TABLE " + initial_load_target + " AS " + local_initial_load_query; string diagnostic; diagnostic += "\n[OpenIVM initial-load diagnostic]\n"; diagnostic += "view_name: " + view_name + "\n"; @@ -1369,8 +1380,8 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC diagnostic += "initial_load_statement:\n" + initial_load_statement + "\n\n"; diagnostic += "original_view_query:\n" + original_view_query + "\n\n"; diagnostic += "generated_view_query:\n" + view_query + "\n\n"; - diagnostic += ExplainInitialLoadQuery(con, "EXPLAIN original_view_query:", original_view_query); - diagnostic += ExplainInitialLoadQuery(con, "EXPLAIN generated_view_query:", view_query); + diagnostic += ExplainInitialLoadQuery(con, "EXPLAIN original_view_query:", local_view_query); + diagnostic += ExplainInitialLoadQuery(con, "EXPLAIN generated_view_query:", local_initial_load_query); diagnostic += ExplainInitialLoadQuery(con, "EXPLAIN initial_load_statement:", initial_load_statement); Printer::Print(diagnostic); @@ -1403,7 +1414,7 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC ddl.push_back(BuildSemiAntiInitialDataSQL(initial_load_target, aux_target, meta.join_type, meta.left_cols, meta.output_cols, meta.null_aware, meta.null_aware_left_col)); } else { - ddl.push_back("create table " + initial_load_target + " as " + view_query); + ddl.push_back("create table " + initial_load_target + " as " + time_travel_pins.StripFrom(context, view_query)); } if (staged_cross_catalog_replace) { // DuckDB cannot make the DuckLake objects and native metadata atomic @@ -1523,10 +1534,9 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC add_cleanup("DROP TABLE IF EXISTS " + qdv); // --- Index DDL (for aggregate group queries) --- - // DuckLake source scans and DuckLake-backed MV state do not support this optional - // native index. + // The index belongs to the MV state, not its sources. DuckLake-backed state cannot have it. if ((refresh_type == RefreshType::AGGREGATE_GROUP || refresh_type == RefreshType::AGGREGATE_HAVING) && - !aggregate_columns.empty() && ducklake_tables.empty() && view_catalog_prefix.empty()) { + !aggregate_columns.empty() && !target_is_ducklake && view_catalog_prefix.empty()) { add_profile_marker("create_view_index", "columns=" + to_string(aggregate_columns.size())); string index_name = KeywordHelper::WriteOptionallyQuoted(data_table + openivm::INDEX_SUFFIX); ddl.push_back("create unique index " + index_name + " on " + qdt + "(" + @@ -1645,7 +1655,7 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC string MaterializedViewLifecycleQuery(ClientContext &context, const FunctionParameters ¶meters) { auto query = StringValue::Get(parameters.values[0]); - auto parse_result = MaterializedViewParserExtension::ParseFunction(nullptr, query); + auto parse_result = ParseMaterializedViewStatement(query, OpenIvmInputDialect(context)); if (parse_result.type != ParserExtensionResultType::PARSE_SUCCESSFUL) { throw ParserException("OpenIVM could not parse the materialized-view lifecycle statement"); } diff --git a/src/core/parser_parse.cpp b/src/core/parser_parse.cpp index 0efec8fc..04147a3b 100644 --- a/src/core/parser_parse.cpp +++ b/src/core/parser_parse.cpp @@ -3,11 +3,14 @@ #include "core/openivm_constants.hpp" #include "core/openivm_debug.hpp" #include "core/sql_utils.hpp" +#include "duckdb/main/config.hpp" +#include "duckdb/main/extension_callback_manager.hpp" #include "duckdb/parser/expression/constant_expression.hpp" #include "duckdb/parser/parser.hpp" #include "duckdb/parser/qualified_name.hpp" #include "duckdb/parser/statement/drop_statement.hpp" #include "duckdb/parser/statement/pragma_statement.hpp" +#include "lpts_parser.hpp" #include @@ -20,10 +23,37 @@ static unique_ptr BuildInternalPragma(const string &name, const st return std::move(statement); } +SqlDialect OpenIvmInputDialect(ClientContext &context) { + Value setting_value; + if (!context.TryGetCurrentSetting(OPENIVM_INPUT_DIALECT_SETTING, setting_value) || setting_value.IsNull()) { + return SqlDialect::DUCKDB; + } + return ParseSqlDialectSetting(setting_value.ToString(), OPENIVM_INPUT_DIALECT_SETTING); +} + +void SetOpenIvmInputDialect(ClientContext &context, SetScope scope, Value ¶meter) { + auto dialect = parameter.IsNull() ? SqlDialect::DUCKDB + : ParseSqlDialectSetting(parameter.ToString(), OPENIVM_INPUT_DIALECT_SETTING); + // `parser_override` and `parse_function` are handed the extension info, not a ClientContext, so + // mirror the resolved dialect there. The info is owned by the DBConfig, so the mirror stays + // scoped to this database. + for (auto &extension : ExtensionCallbackManager::Get(context).ParserExtensions()) { + auto info = dynamic_cast(extension.parser_info.get()); + if (info) { + info->SetInputDialect(dialect); + } + } +} + +static SqlDialect InputDialectFromInfo(ParserExtensionInfo *info) { + auto materialized_view_info = dynamic_cast(info); + return materialized_view_info ? materialized_view_info->InputDialect() : SqlDialect::DUCKDB; +} + ParserOverrideResult MaterializedViewParserExtension::OverrideFunction(ParserExtensionInfo *info, const string &query, ParserOptions &options) { try { - auto extension_result = ParseFunction(info, query); + auto extension_result = ParseMaterializedViewStatement(query, InputDialectFromInfo(info)); if (extension_result.type == ParserExtensionResultType::PARSE_SUCCESSFUL) { vector> statements; statements.push_back(BuildInternalPragma("openivm_materialized_view_lifecycle", query)); @@ -55,6 +85,10 @@ ParserOverrideResult MaterializedViewParserExtension::OverrideFunction(ParserExt ParserExtensionParseResult MaterializedViewParserExtension::ParseFunction(ParserExtensionInfo *info, const string &query) { + return ParseMaterializedViewStatement(query, InputDialectFromInfo(info)); +} + +ParserExtensionParseResult ParseMaterializedViewStatement(const string &query, SqlDialect input_dialect) { auto query_lower = SqlUtils::SQLToLowercase(StringUtil::Replace(query, ";", "")); StringUtil::Trim(query_lower); // Strip SQL line comments (-- to end of line) before whitespace normalization. @@ -126,6 +160,13 @@ ParserExtensionParseResult MaterializedViewParserExtension::ParseFunction(Parser // at the plan level in PlanFunction via PlanRewrite + LPTS. OPENIVM_DEBUG_PRINT("[CREATE MV] After structural rewrite: %s\n", query_lower.c_str()); + if (input_dialect != SqlDialect::DUCKDB) { + // LPTS owns input syntax; the parsed AT clause below carries the snapshot and its relation. + query_lower = NormalizeInputSqlToDuckDB(query_lower, input_dialect); + OPENIVM_DEBUG_PRINT("[CREATE MV] After %s input normalization: %s\n", SqlDialectToString(input_dialect).c_str(), + query_lower.c_str()); + } + Parser p; p.ParseQuery(query_lower); diff --git a/src/core/sql_utils.cpp b/src/core/sql_utils.cpp index 56b86e4e..653c9e59 100644 --- a/src/core/sql_utils.cpp +++ b/src/core/sql_utils.cpp @@ -34,7 +34,7 @@ static bool StartsWithKeyword(const string &sql, size_t pos, const string &keywo return pos + keyword.size() == sql.size() || !IsIdentifierChar(sql[pos + keyword.size()]); } -static string TrimSQLFragment(const string &input) { +string SqlUtils::TrimSQLFragment(const string &input) { idx_t start = 0; while (start < input.size() && std::isspace(static_cast(input[start]))) { start++; diff --git a/src/core/time_travel_pins.cpp b/src/core/time_travel_pins.cpp new file mode 100644 index 00000000..1b1ea6e0 --- /dev/null +++ b/src/core/time_travel_pins.cpp @@ -0,0 +1,543 @@ +#include "core/time_travel_pins.hpp" + +#include "core/openivm_debug.hpp" +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/parser/expression/subquery_expression.hpp" +#include "duckdb/parser/parsed_expression_iterator.hpp" +#include "duckdb/parser/query_node/cte_node.hpp" +#include "duckdb/parser/query_node/recursive_cte_node.hpp" +#include "duckdb/parser/query_node/select_node.hpp" +#include "duckdb/parser/query_node/set_operation_node.hpp" +#include "duckdb/parser/statement/create_statement.hpp" +#include "duckdb/parser/statement/insert_statement.hpp" +#include "duckdb/parser/statement/select_statement.hpp" +#include "duckdb/parser/tableref/basetableref.hpp" +#include "duckdb/parser/tableref/joinref.hpp" +#include "duckdb/parser/tableref/pivotref.hpp" +#include "duckdb/parser/tableref/subqueryref.hpp" +#include "duckdb/parser/tableref/table_function_ref.hpp" +#include "duckdb/parser/parsed_data/create_table_info.hpp" +#include "duckdb/parser/parser.hpp" +#include "duckdb/planner/binder.hpp" +#include "lpts_helpers.hpp" +#include "lpts_sql_scanner.hpp" + +#include + +namespace duckdb { +namespace openivm { + +using BaseTableRefCallback = std::function; + +// A `BaseTableRef` whose unqualified name matches a CTE visible at that point is a reference to +// the CTE, not a scan of a same-named relation, so it must never be treated as an unpinned scan. +struct RefVisitor { + const BaseTableRefCallback &callback; + case_insensitive_set_t cte_names; +}; + +static void VisitQueryNode(QueryNode &node, const RefVisitor &visitor); + +static void VisitExpression(ParsedExpression &expression, const RefVisitor &visitor) { + if (expression.GetExpressionClass() == ExpressionClass::SUBQUERY) { + auto &subquery = expression.Cast(); + if (subquery.subquery && subquery.subquery->node) { + VisitQueryNode(*subquery.subquery->node, visitor); + } + } + ParsedExpressionIterator::EnumerateChildren(expression, + [&](ParsedExpression &child) { VisitExpression(child, visitor); }); +} + +static void VisitTableRef(TableRef &ref, const RefVisitor &visitor) { + switch (ref.type) { + case TableReferenceType::BASE_TABLE: { + auto &base_table = ref.Cast(); + bool unqualified = base_table.catalog_name.empty() && base_table.schema_name.empty(); + if (unqualified && visitor.cte_names.find(base_table.table_name) != visitor.cte_names.end()) { + break; + } + visitor.callback(base_table); + break; + } + case TableReferenceType::JOIN: { + auto &join = ref.Cast(); + if (join.left) { + VisitTableRef(*join.left, visitor); + } + if (join.right) { + VisitTableRef(*join.right, visitor); + } + if (join.condition) { + VisitExpression(*join.condition, visitor); + } + break; + } + case TableReferenceType::SUBQUERY: { + auto &subquery = ref.Cast(); + if (subquery.subquery && subquery.subquery->node) { + VisitQueryNode(*subquery.subquery->node, visitor); + } + break; + } + case TableReferenceType::PIVOT: { + auto &pivot = ref.Cast(); + if (pivot.source) { + VisitTableRef(*pivot.source, visitor); + } + break; + } + case TableReferenceType::TABLE_FUNCTION: { + auto &table_function = ref.Cast(); + if (table_function.function) { + VisitExpression(*table_function.function, visitor); + } + break; + } + default: + break; + } +} + +static void VisitQueryNode(QueryNode &node, const RefVisitor &visitor) { + // Each CTE body sees preceding siblings, but not its own name. + RefVisitor scoped {visitor.callback, visitor.cte_names}; + for (auto &cte : node.cte_map.map) { + if (cte.second->query && cte.second->query->node) { + VisitQueryNode(*cte.second->query->node, scoped); + } + scoped.cte_names.insert(cte.first); + } + switch (node.type) { + case QueryNodeType::SELECT_NODE: { + auto &select = node.Cast(); + if (select.from_table) { + VisitTableRef(*select.from_table, scoped); + } + auto visit = [&](unique_ptr &expression) { + if (expression) { + VisitExpression(*expression, scoped); + } + }; + for (auto &expression : select.select_list) { + visit(expression); + } + for (auto &expression : select.groups.group_expressions) { + visit(expression); + } + visit(select.where_clause); + visit(select.having); + visit(select.qualify); + break; + } + case QueryNodeType::SET_OPERATION_NODE: { + auto &set_operation = node.Cast(); + for (auto &child : set_operation.children) { + if (child) { + VisitQueryNode(*child, scoped); + } + } + break; + } + case QueryNodeType::RECURSIVE_CTE_NODE: { + auto &recursive_cte = node.Cast(); + RefVisitor recursive_scope {visitor.callback, scoped.cte_names}; + recursive_scope.cte_names.insert(recursive_cte.ctename); + if (recursive_cte.left) { + VisitQueryNode(*recursive_cte.left, recursive_scope); + } + if (recursive_cte.right) { + VisitQueryNode(*recursive_cte.right, recursive_scope); + } + break; + } + case QueryNodeType::CTE_NODE: { + auto &cte = node.Cast(); + if (cte.query) { + VisitQueryNode(*cte.query, scoped); + } + if (cte.child) { + RefVisitor child_scope {visitor.callback, scoped.cte_names}; + child_scope.cte_names.insert(cte.ctename); + VisitQueryNode(*cte.child, child_scope); + } + break; + } + default: + break; + } + ParsedExpressionIterator::EnumerateQueryNodeModifiers(node, [&](unique_ptr &expression) { + if (expression) { + VisitExpression(*expression, scoped); + } + }); +} + +static void VisitStatement(SQLStatement &statement, const RefVisitor &visitor) { + switch (statement.type) { + case StatementType::SELECT_STATEMENT: { + auto &select = statement.Cast(); + if (select.node) { + VisitQueryNode(*select.node, visitor); + } + break; + } + case StatementType::CREATE_STATEMENT: { + auto &create = statement.Cast(); + if (create.info && create.info->type == CatalogType::TABLE_ENTRY) { + auto &table_info = create.info->Cast(); + if (table_info.query && table_info.query->node) { + VisitQueryNode(*table_info.query->node, visitor); + } + } + break; + } + case StatementType::INSERT_STATEMENT: { + auto &insert = statement.Cast(); + if (insert.select_statement && insert.select_statement->node) { + VisitQueryNode(*insert.select_statement->node, visitor); + } + break; + } + default: + break; + } +} + +// Resolve without the AT clause; DuckDB owns lookup errors for unresolved relations. +static optional_ptr LookupPinRelation(ClientContext &context, BaseTableRef &ref) { + QueryErrorContext error_context; + EntryLookupInfo lookup(CatalogType::TABLE_ENTRY, ref.table_name, error_context); + try { + auto catalog = ref.catalog_name; + auto schema = ref.schema_name; + Binder::BindSchemaOrCatalog(context, catalog, schema); + return Catalog::GetEntry(context, catalog, schema, lookup, OnEntryNotFound::RETURN_NULL); + } catch (const std::exception &) { + return nullptr; + } +} + +[[noreturn]] static void ThrowAmbiguousPin(const string &table_name, const string &reason) { + throw NotImplementedException( + "OpenIVM cannot compile a materialized view that pins relation '%s' ambiguously: %s. DuckDB resolves a " + "time-travel qualifier during catalog lookup, so the bound plan keeps no per-scan record of it and " + "re-attaching one can only be keyed by relation — conflating scans that must read different snapshots is " + "not something OpenIVM will do silently. Give every scan of the relation the same pin, or split them into " + "separate views.", + table_name, reason); +} + +TimeTravelPins TimeTravelPins::Peel(ClientContext &context, SQLStatement &statement, + const BaseTableRefCallback &qualify_source) { + TimeTravelPins result; + case_insensitive_set_t unpinned; + BaseTableRefCallback callback = [&](BaseTableRef &ref) { + if (qualify_source) { + qualify_source(ref); + } + if (!ref.at_clause) { + // Track native scans too: the pin map is keyed by relation name, so mixed pinned and + // unpinned uses of that name cannot be represented safely. + unpinned.insert(ref.table_name); + return; + } + Pin pin; + pin.catalog = ref.catalog_name; + pin.schema = ref.schema_name; + auto relation = LookupPinRelation(context, ref); + pin.binds_natively = !relation || relation->ParentCatalog().SupportsTimeTravel(); + if (relation) { + pin.catalog = relation->ParentCatalog().GetName(); + pin.schema = relation->ParentSchema().name; + } + // Binding loses the explicit AT clause even when it names the current native snapshot. + // Preserve it for LPTS, but only remove foreign pins from the locally executable query. + pin.snapshot = pin.binds_natively ? ref.at_clause->Copy() : std::move(ref.at_clause); + auto existing = result.pins.find(ref.table_name); + if (existing != result.pins.end()) { + if (!AtClause::Equals(existing->second.snapshot, pin.snapshot)) { + ThrowAmbiguousPin(ref.table_name, "it is pinned to both '" + existing->second.snapshot->ToString() + + "' and '" + pin.snapshot->ToString() + "'"); + } + if (existing->second.catalog != pin.catalog || existing->second.schema != pin.schema) { + ThrowAmbiguousPin(ref.table_name, "the same pin names two differently qualified relations"); + } + } + OPENIVM_DEBUG_PRINT("[TIME TRAVEL] Recorded pin '%s' on relation '%s' (native=%d)\n", + pin.snapshot->ToString().c_str(), ref.table_name.c_str(), pin.binds_natively); + result.pins[ref.table_name] = std::move(pin); + }; + RefVisitor visitor {callback, case_insensitive_set_t()}; + VisitStatement(statement, visitor); + for (auto &entry : result.pins) { + if (unpinned.find(entry.first) != unpinned.end()) { + ThrowAmbiguousPin(entry.first, "it is scanned both pinned and unpinned"); + } + } + return result; +} + +SnapshotResolver TimeTravelPins::Resolver() const { + if (pins.empty()) { + return {}; + } + return [this](const TableCatalogEntry &table) -> unique_ptr { + auto entry = pins.find(table.name); + if (entry == pins.end()) { + return nullptr; + } + auto &pin = entry->second; + if ((!pin.catalog.empty() && !StringUtil::CIEquals(pin.catalog, table.ParentCatalog().GetName())) || + (!pin.schema.empty() && !StringUtil::CIEquals(pin.schema, table.schema.name))) { + return nullptr; + } + return pin.snapshot->Copy(); + }; +} + +// Copy the quoted run starting at `sql[start]` (whose delimiter is `sql[start]`) into `result`, +// returning the index just past the closing delimiter. Doubled delimiters escape. +static idx_t CopyQuotedRun(const string &sql, idx_t start, string &result) { + char quote = sql[start]; + result += quote; + idx_t i = start + 1; + while (i < sql.size()) { + if (sql[i] == quote) { + if (i + 1 < sql.size() && sql[i + 1] == quote) { + result += quote; + result += quote; + i += 2; + continue; + } + result += quote; + return i + 1; + } + result += sql[i]; + i++; + } + return i; +} + +string TimeTravelPins::StripFrom(ClientContext &context, const string &sql) const { + if (pins.empty()) { + return sql; + } + Parser parser(context.GetParserOptions()); + parser.ParseQuery(sql); + if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::SELECT_STATEMENT) { + throw InternalException("Expected one view query while stripping time-travel pins"); + } + BaseTableRefCallback callback = [&](BaseTableRef &ref) { + auto entry = pins.find(ref.table_name); + if (entry == pins.end() || entry->second.binds_natively) { + return; + } + auto &pin = entry->second; + auto relation = LookupPinRelation(context, ref); + if (relation && StringUtil::CIEquals(pin.catalog, relation->ParentCatalog().GetName()) && + StringUtil::CIEquals(pin.schema, relation->ParentSchema().name)) { + ref.at_clause.reset(); + } + }; + VisitStatement(*parser.statements[0], RefVisitor {callback, {}}); + return parser.statements[0]->ToString(); +} + +// Copy the `[catalog.][schema.]relation` chain starting at `sql[start]` into `result`, honouring +// quoted components. Reports the unquoted final component — the relation name pins are keyed by — +// and whether the chain carried a catalog/schema prefix, which a CTE reference never does. +static idx_t CopyQualifiedIdentifierChain(const string &sql, idx_t start, string &result, string &final_component, + bool &qualified) { + idx_t i = start; + qualified = false; + while (true) { + if (i < sql.size() && (sql[i] == '"' || sql[i] == '`')) { + idx_t quoted_start = result.size(); + i = CopyQuotedRun(sql, i, result); + final_component = result.substr(quoted_start + 1, result.size() - quoted_start - 2); + } else if (i < sql.size() && IsIdentStart(sql[i])) { + idx_t end = i; + while (end < sql.size() && (IsIdentPart(sql[end]) || sql[end] == '$')) { + end++; + } + final_component = sql.substr(i, end - i); + result.append(sql, i, end - i); + i = end; + } else { + break; + } + if (i < sql.size() && sql[i] == '.') { + result += '.'; + i++; + qualified = true; + continue; + } + break; + } + return i; +} + +// Whether `sql` already carries `qualifier` at `pos` (ignoring how its whitespace is spelled), so an +// AST-rendered scan is never given a second copy of its own pin. +static bool CarriesQualifierAt(const string &sql, idx_t pos, const string &qualifier) { + idx_t sql_pos = pos; + idx_t qualifier_pos = 0; + while (qualifier_pos < qualifier.size()) { + if (std::isspace(static_cast(qualifier[qualifier_pos]))) { + bool sql_has_space = sql_pos < sql.size() && std::isspace(static_cast(sql[sql_pos])); + while (qualifier_pos < qualifier.size() && + std::isspace(static_cast(qualifier[qualifier_pos]))) { + qualifier_pos++; + } + while (sql_pos < sql.size() && std::isspace(static_cast(sql[sql_pos]))) { + sql_pos++; + } + if (!sql_has_space) { + return false; + } + continue; + } + if (sql_pos >= sql.size() || std::tolower(static_cast(sql[sql_pos])) != + std::tolower(static_cast(qualifier[qualifier_pos]))) { + return false; + } + sql_pos++; + qualifier_pos++; + } + return true; +} + +// Advance past whitespace and comments so a qualifier written behind either is still found. +static idx_t SkipIgnorableSpan(const string &sql, idx_t pos) { + while (pos < sql.size()) { + pos = SkipWhitespace(sql, pos); + idx_t end; + if (pos == sql.size() || (sql[pos] != '-' && sql[pos] != '/') || !TryReadSkippableSqlSpan(sql, pos, end)) { + break; + } + pos = end; + } + return pos; +} + +// Words that close a FROM list, so a comma past them separates something other than relations. +static bool EndsFromList(const string &token) { + static const char *const TERMINATORS[] = {"where", "group", "having", "qualify", "window", "order", + "limit", "offset", "union", "except", "intersect", "select", + "values", "returning", "set", "insert", "update", "delete"}; + for (auto terminator : TERMINATORS) { + if (StringUtil::CIEquals(token, terminator)) { + return true; + } + } + return false; +} + +// Whether the parenthesis that ends at `pos` opens a derived table rather than a parenthesized join +// list. `(SELECT ...)`, `(WITH ...)`, `(VALUES ...)`, `(TABLE t)` and DuckDB's `(FROM t ...)` all +// start a query of their own; anything else in table position is a relation, and a nested +// parenthesis just defers the question one level. +static bool OpensDerivedTable(const string &sql, idx_t pos) { + idx_t cursor = SkipIgnorableSpan(sql, pos); + while (cursor < sql.size() && sql[cursor] == '(') { + cursor = SkipIgnorableSpan(sql, cursor + 1); + } + idx_t token_end; + string token; + if (!TryReadIdentifierToken(sql, cursor, token_end, token)) { + return false; + } + return StringUtil::CIEquals(token, "select") || StringUtil::CIEquals(token, "with") || + StringUtil::CIEquals(token, "values") || StringUtil::CIEquals(token, "table") || + StringUtil::CIEquals(token, "from"); +} + +string TimeTravelPins::RestoreIntoSql(const string &sql, SqlDialect dialect) const { + if (pins.empty()) { + return sql; + } + string result; + result.reserve(sql.size()); + // Only a relation directly behind FROM, JOIN or a FROM-list comma is a scan; anything else naming + // the relation is a column reference, a delta/metadata table or a literal, none of which take a + // pin. Each parenthesis nests its own FROM list so a subquery never leaks the enclosing one. + bool expect_relation = false; + vector from_list_open; + from_list_open.push_back(false); + idx_t i = 0; + while (i < sql.size()) { + char c = sql[i]; + idx_t end; + // Quoted identifiers must reach relation matching below; only literals/comments are skipped. + if (c != '"' && TryReadSkippableSqlSpan(sql, i, end)) { + result.append(sql, i, end - i); + i = end; + if (c == '\'') { + expect_relation = false; + } + continue; + } + if (IsIdentStart(c) || c == '"' || c == '`') { + string final_component; + bool qualified; + i = CopyQualifiedIdentifierChain(sql, i, result, final_component, qualified); + if (expect_relation) { + expect_relation = false; + auto entry = pins.find(final_component); + // PrepareViewQuerySources qualifies base scans in the parsed tree, not CTE references. + // Plan-rendered scans already carry their pins, even with output qualification overrides. + if (!qualified || entry == pins.end() || entry->second.binds_natively) { + continue; + } + auto dialect_suffix = RenderSnapshotSuffix(entry->second.snapshot.get(), dialect); + if (CarriesQualifierAt(sql, i, dialect_suffix)) { + continue; + } + result += dialect_suffix; + OPENIVM_DEBUG_PRINT("[TIME TRAVEL] Restored pin '%s' onto rendered scan of '%s'\n", + dialect_suffix.c_str(), final_component.c_str()); + continue; + } + if (StringUtil::CIEquals(final_component, "from")) { + expect_relation = true; + from_list_open.back() = true; + } else if (StringUtil::CIEquals(final_component, "join")) { + expect_relation = true; + } else if (EndsFromList(final_component)) { + from_list_open.back() = false; + } + continue; + } + result += c; + i++; + if (c == '(') { + // In table position a parenthesis opens either a derived table, which starts its own + // query, or a parenthesized join list, whose first element is still a scan that needs its + // pin. Only the query keywords tell the two apart; a table function's argument list never + // reaches here because its own name already consumed the table position. + bool table_list = expect_relation && !OpensDerivedTable(sql, i); + from_list_open.push_back(table_list); + expect_relation = table_list; + } else if (c == ')') { + if (from_list_open.size() > 1) { + from_list_open.pop_back(); + } + expect_relation = false; + } else if (c == ',') { + // An implicit cross join: the next relation is a scan of its own and needs its own pin. + expect_relation = from_list_open.back(); + } else if (c == ';') { + from_list_open.assign(1, false); + expect_relation = false; + } else if (!std::isspace(static_cast(c))) { + expect_relation = false; + } + } + return result; +} + +} // namespace openivm +} // namespace duckdb diff --git a/src/delta/operators/join.cpp b/src/delta/operators/join.cpp index e9b6cc84..1ae456e1 100644 --- a/src/delta/operators/join.cpp +++ b/src/delta/operators/join.cpp @@ -301,6 +301,7 @@ struct JoinPlanFacts { string ducklake_fallback_reason; bool has_outer_join = false; bool only_inner_joins = true; + bool only_inner_or_left_joins = true; }; // Collect join leaves and the properties needed to select a compilation strategy. Non-join wrappers remain leaves, @@ -346,6 +347,9 @@ static void CollectJoinPlanFacts(LogicalOperator *node, vector &path, bo if (!join || join->join_type != JoinType::INNER) { facts.only_inner_joins = false; } + if (!join || (join->join_type != JoinType::INNER && join->join_type != JoinType::LEFT)) { + facts.only_inner_or_left_joins = false; + } } if ((collect_leaf || collect_ducklake_leaf) && is_join_tree_node) { bool left_is_nullable = is_right_of_left; @@ -924,16 +928,31 @@ void AppendMultiplicityToAncestorProjectionMaps(unique_ptr &ter // A collected join path contains only child 0 or 1; <= would admit the invalid size() index. if (join && child_side < join->children.size()) { // mull-ignore: cxx_lt_to_le auto &proj_map = (child_side == 0) ? join->left_projection_map : join->right_projection_map; + bool immediate_parent = depth + 1 == leaf_path.size(); + // A binary join's sibling is 1-child_side. Addition is either the same index for side 0 or the invalid + // index 2 for side 1; it cannot describe another valid planner shape. + bool preserve_full_child = + preserve_constant_sibling_child_outputs && immediate_parent && ancestors[depth]->children.size() == 2 && + IsConstantLeafSubtree(ancestors[depth]->children[1 - child_side].get()); // mull-ignore: cxx_sub_to_add + auto child_bindings = ancestors[depth]->children[child_side]->GetColumnBindings(); + auto shift_parent_projection = [&](idx_t insertion_idx, idx_t added) { + if (added == 0 || depth == 0) { + return; + } + size_t parent_side = leaf_path[depth - 1]; + auto *parent_join = dynamic_cast(ancestors[depth - 1]); + if (!parent_join || parent_side >= parent_join->children.size()) { + return; + } + auto &parent_map = + (parent_side == 0) ? parent_join->left_projection_map : parent_join->right_projection_map; + for (auto &parent_idx : parent_map) { + if (parent_idx >= insertion_idx) { + parent_idx += added; + } + } + }; if (!proj_map.empty()) { - bool immediate_parent = depth + 1 == leaf_path.size(); - // A binary join's sibling is 1-child_side. Addition is either the same index for side 0 or the invalid - // index 2 for side 1; it cannot describe another valid planner shape. - bool preserve_full_child = - preserve_constant_sibling_child_outputs && immediate_parent && - ancestors[depth]->children.size() == 2 && - IsConstantLeafSubtree( - ancestors[depth]->children[1 - child_side].get()); // mull-ignore: cxx_sub_to_add - auto child_bindings = ancestors[depth]->children[child_side]->GetColumnBindings(); for (auto projected_idx : proj_map) { // Equality is the first invalid binding index and is handled by this exception. // mull-ignore-next: cxx_ge_to_gt @@ -963,23 +982,69 @@ void AppendMultiplicityToAncestorProjectionMaps(unique_ptr &ter if (mul_idx == DConstants::INVALID_INDEX) { continue; } + idx_t old_width = proj_map.size(); + idx_t insertion_idx = old_width; + if (child_side == 1) { + auto left_bindings = join->children[0]->GetColumnBindings(); + insertion_idx += + join->left_projection_map.empty() ? left_bindings.size() : join->left_projection_map.size(); + } + // Appending to this join's own left_projection_map grows its LEFT + // contribution width, which shifts the absolute position where its RIGHT + // contribution starts within its own combined GetColumnBindings(). A + // grandparent ancestor may already have a projection map entry referencing + // (by that now-stale absolute position) a column from this join's right + // side; left as-is, that entry would silently start pointing at the + // newly-inserted column instead, dropping the real column it used to + // select. Appending to right_projection_map never has this effect: right + // contributions are always placed last, so a new entry there only ever + // extends the combined output with a brand-new highest index. if (preserve_full_child) { idx_t projectable_count = MinValue(mul_idx + 1, child_bindings.size()); + idx_t added = 0; for (idx_t binding_idx = 0; binding_idx < projectable_count; binding_idx++) { if (std::find(proj_map.begin(), proj_map.end(), binding_idx) != proj_map.end()) { continue; } proj_map.push_back(binding_idx); + added++; OPENIVM_DEBUG_PRINT("[%s] Preserved child col %lu in immediate %s proj_map\n", context_label, (unsigned long)binding_idx, child_side == 0 ? "left" : "right"); } - } else if (std::find(proj_map.begin(), proj_map.end(), mul_idx) == proj_map.end()) { - proj_map.push_back(mul_idx); - OPENIVM_DEBUG_PRINT("[%s] Added mul col %lu to ancestor %s proj_map\n", context_label, - (unsigned long)mul_idx, child_side == 0 ? "left" : "right"); + shift_parent_projection(insertion_idx, added); + } else { + // proj_map entries are positions into the child's *current* combined + // GetColumnBindings(). Testing raw index membership of mul_idx against + // proj_map can alias onto an unrelated pre-existing entry that now shares + // the same numeric position after a deeper level's own map grew. Compare + // by column identity against what this ancestor currently exposes instead + // of trusting the raw index. + auto exposed = join->GetColumnBindings(); + if (std::find(exposed.begin(), exposed.end(), mul_binding) == exposed.end()) { + proj_map.push_back(mul_idx); + shift_parent_projection(insertion_idx, 1); + OPENIVM_DEBUG_PRINT("[%s] Added mul col %lu to ancestor %s proj_map\n", context_label, + (unsigned long)mul_idx, child_side == 0 ? "left" : "right"); + } } join->ResolveOperatorTypes(); + continue; } + + auto mul_binding_it = std::find(child_bindings.begin(), child_bindings.end(), mul_binding); + if (mul_binding_it == child_bindings.end()) { + continue; + } + idx_t insertion_idx = idx_t(mul_binding_it - child_bindings.begin()); + if (child_side == 1) { + auto left_bindings = join->children[0]->GetColumnBindings(); + insertion_idx += + join->left_projection_map.empty() ? left_bindings.size() : join->left_projection_map.size(); + } + // An empty map passes the child's bindings through unchanged, so the + // multiplicity is inserted in-place and shifts every later binding. + shift_parent_projection(insertion_idx, 1); + join->ResolveOperatorTypes(); } } } @@ -1660,9 +1725,20 @@ BuildInclusionExclusionTerms(DeltaOperatorInput input, ClientContext &context, B for (size_t i = 0; i < N; i++) { if (mask & (1ULL << i)) { if (leaves[i].get) { - DeltaGetResult delta_i = CreateDeltaGetNode(context, binder, leaves[i].get, input.context.view); + // leaves[] was collected once on the ORIGINAL input.plan, before this + // mask's own renumber_and_rebind_subtree pass, so leaves[i].get is a + // stale pointer carrying the pre-renumbering table_index. The rest of + // `term` (join conditions, transitioning-key guards, etc.) was rebound + // to the FRESH per-term index, so the replacement delta node -- which + // reuses old_get->table_index verbatim -- must be built from term's own, + // already-renumbered GET at this leaf's (renumbering-invariant) path, + // not from leaves[i].get, or every reference elsewhere in `term` to this + // leaf's fresh index is left dangling. + auto &leaf_node_ref = GetNodeAtPath(term, leaves[i].path); + auto &term_local_get = leaf_node_ref->Cast(); + DeltaGetResult delta_i = CreateDeltaGetNode(context, binder, &term_local_get, input.context.view); mul_bindings.push_back(delta_i.mul_binding); - GetNodeAtPath(term, leaves[i].path) = std::move(delta_i.node); + leaf_node_ref = std::move(delta_i.node); UpdateParentProjectionMap(term, leaves[i], delta_i.mul_binding); } else { auto &subtree_ref = GetNodeAtPath(term, leaves[i].path); @@ -1861,7 +1937,7 @@ static DeltaPlanFragment CompileRegularLeafDelta(const DeltaOperatorInput &input static vector> BuildRegularJoinTerms(DeltaOperatorInput input, ClientContext &context, Binder &binder, - const vector &leaves, uint64_t unchanged_mask, + const vector &leaves, uint64_t unchanged_mask, bool has_left_join, const vector &existing_multiplicity_bindings) { vector> terms; // Base scans see post-DML state. Term i uses current state before i, delta i, and reconstructs old state after i as @@ -1892,6 +1968,15 @@ BuildRegularJoinTerms(DeltaOperatorInput input, ClientContext &context, Binder & auto renumbered = renumber_and_rebind_subtree(std::move(term), binder); term = std::move(renumbered.op); LogicalOperator *term_root = term.get(); + // LEFT-JOIN telescoping: demote only the outer join(s) whose NULL-supplying + // subtree contains this term's single delta leaf, mirroring the DuckLake + // N-term path (DemoteLeftJoinsForMask). Preserved outer joins elsewhere keep + // their NULL-padded rows; the upsert layer's key-based partial recompute + // (BuildLeftJoinProjectionRefresh) fixes NULL<->match transition rows. + if (has_left_join) { + DemoteLeftJoinsForMask(term.get(), leaves, (1ULL << delta_leaf)); + } + vector mul_bindings; for (size_t leaf = 0; leaf < leaves.size(); leaf++) { @@ -2047,10 +2132,22 @@ DeltaPlanFragment CompileJoinDelta(DeltaOperatorInput input) { } auto compile_facts = openivm::CompileFactsContextSlot::Get(context); auto unchanged_mask = ComputeFactsUnchangedMask(compile_facts, leaves); - bool regular_nterm = !all_ducklake && compile_facts.compile_only && !has_left_join && - input.context.model.type == RefreshType::SIMPLE_PROJECTION && join_facts.only_inner_joins && - RegularNtermPreservesFKPruning(context, compile_facts, leaves, input.plan.get()) && - SqlUtils::GetBoolSetting(context, "openivm_regular_nterm", true); + bool regular_nterm_base = !all_ducklake && compile_facts.compile_only && + input.context.model.type == RefreshType::SIMPLE_PROJECTION && + SqlUtils::GetBoolSetting(context, "openivm_regular_nterm", true); + bool regular_nterm; + if (has_left_join) { + // LEFT-JOIN telescoping: the regular N-term delta extends to LEFT joins via + // per-term demotion of only the outer join whose NULL-supplying side carries + // that term's delta (see BuildRegularJoinTerms). FULL OUTER / RIGHT shapes and + // the inclusion-exclusion FK-pruning path are out of scope; NULL-padded row + // correctness is completed by BuildLeftJoinProjectionRefresh in the upsert layer. + regular_nterm = regular_nterm_base && join_facts.only_inner_or_left_joins && + SqlUtils::GetBoolSetting(context, "openivm_regular_nterm_left", true); + } else { + regular_nterm = regular_nterm_base && join_facts.only_inner_joins && + RegularNtermPreservesFKPruning(context, compile_facts, leaves, input.plan.get()); + } if (regular_nterm) { for (auto &leaf : leaves) { if (!SupportsRegularNtermLeaf(leaf)) { @@ -2068,7 +2165,7 @@ DeltaPlanFragment CompileJoinDelta(DeltaOperatorInput input) { if (all_ducklake) { terms = BuildDuckLakeJoinTerms(input, context, binder, leaves, has_left_join, flattened_ducklake); } else if (regular_nterm) { - terms = BuildRegularJoinTerms(input, context, binder, leaves, unchanged_mask, + terms = BuildRegularJoinTerms(input, context, binder, leaves, unchanged_mask, has_left_join, join_facts.existing_multiplicity_bindings); } else { terms = BuildInclusionExclusionTerms(input, context, binder, leaves, has_left_join, diff --git a/src/include/core/parser.hpp b/src/include/core/parser.hpp index bebef42a..7fbe8b85 100644 --- a/src/include/core/parser.hpp +++ b/src/include/core/parser.hpp @@ -2,18 +2,40 @@ #define OPENIVM_PARSER_HPP #include "duckdb.hpp" +#include "duckdb/main/setting_info.hpp" #include "duckdb/parser/parser_extension.hpp" +#include "sql_dialect.hpp" +#include #include namespace duckdb { +//! Name of the setting that declares which SQL dialect the caller writes materialized-view bodies in. +constexpr const char *OPENIVM_INPUT_DIALECT_SETTING = "openivm_input_dialect"; + +//! Parser-extension state. `parse_function` and `parser_override` run before any ClientContext +//! exists, so the input dialect is mirrored here from the `openivm_input_dialect` setting: it +//! decides whether a materialized-view body must be normalized out of its source dialect before +//! DuckDB's parser ever sees it. +struct MaterializedViewParserExtensionInfo : ParserExtensionInfo { + std::atomic input_dialect {static_cast(SqlDialect::DUCKDB)}; + + SqlDialect InputDialect() const { + return static_cast(input_dialect.load()); + } + void SetInputDialect(SqlDialect dialect) { + input_dialect.store(static_cast(dialect)); + } +}; + class MaterializedViewParserExtension : public ParserExtension { public: explicit MaterializedViewParserExtension() { parse_function = ParseFunction; plan_function = PlanFunction; parser_override = OverrideFunction; + parser_info = make_shared_ptr(); } static ParserExtensionParseResult ParseFunction(ParserExtensionInfo *info, const string &query); @@ -23,6 +45,15 @@ class MaterializedViewParserExtension : public ParserExtension { unique_ptr parse_data); }; +//! Parse a materialized-view lifecycle statement written in `input_dialect`. +ParserExtensionParseResult ParseMaterializedViewStatement(const string &query, SqlDialect input_dialect); + +//! The dialect materialized-view bodies arrive in for this session (`openivm_input_dialect`). +SqlDialect OpenIvmInputDialect(ClientContext &context); + +//! `openivm_input_dialect` set callback: validates the value and mirrors it onto the parser extension. +void SetOpenIvmInputDialect(ClientContext &context, SetScope scope, Value ¶meter); + string MaterializedViewLifecycleQuery(ClientContext &context, const FunctionParameters ¶meters); string MaterializedViewDropQuery(ClientContext &context, const FunctionParameters ¶meters); diff --git a/src/include/core/sql_utils.hpp b/src/include/core/sql_utils.hpp index 39cd2a67..fdaa9072 100644 --- a/src/include/core/sql_utils.hpp +++ b/src/include/core/sql_utils.hpp @@ -24,6 +24,8 @@ class SqlUtils { static void RemoveRedundantWhitespaces(string &query); /// Strip SQL line comments (-- to end of line) while respecting single-quoted string literals. static void StripLineComments(string &query); + /// Trim whitespace without stripping non-ASCII identifier bytes on signed-char builds. + static string TrimSQLFragment(const string &input); static vector SplitSQLStatements(const string &sql); static string SQLStatementPreview(const string &statement); static string DeltaName(const string &name); diff --git a/src/include/core/time_travel_pins.hpp b/src/include/core/time_travel_pins.hpp new file mode 100644 index 00000000..20af6f58 --- /dev/null +++ b/src/include/core/time_travel_pins.hpp @@ -0,0 +1,66 @@ +#ifndef OPENIVM_TIME_TRAVEL_PINS_HPP +#define OPENIVM_TIME_TRAVEL_PINS_HPP + +#pragma once + +#include "duckdb.hpp" +#include "duckdb/common/case_insensitive_map.hpp" +#include "lpts_pipeline.hpp" +#include "sql_dialect.hpp" + +namespace duckdb { + +class SQLStatement; +class BaseTableRef; + +namespace openivm { + +// Foreign snapshots cannot bind against the Spark bridge's local schema-only tables. Keep their +// parsed AT clauses while binding those stand-ins, then resolve them during LPTS's scan walk. +// Native time-travel catalogs retain their clauses. Record these too: a pin to the current native +// snapshot is indistinguishable from an unpinned scan in the bound plan. +class TimeTravelPins { +public: + // Record relation -> qualifier, peeling only unbindable pins out of `statement`. Throws when a + // relation cannot be given one unambiguous pin (two different pins, or pinned in one scan and + // unpinned in another), because re-attaching by relation would conflate the two. + // Refresh can qualify each source before pin lookup, reusing the same CTE-aware traversal. + static TimeTravelPins Peel(ClientContext &context, SQLStatement &statement, + const std::function &qualify_source = {}); + + bool Empty() const { + return pins.empty(); + } + + // Resolve pins during LPTS's existing scan walk, using the original catalog identity. + SnapshotResolver Resolver() const; + + // Strip pins from a SELECT for local stand-in execution; stored metadata keeps the pinned query. + string StripFrom(ClientContext &context, const string &sql) const; + + // Re-attach foreign qualifiers, in `dialect`'s own spelling, to the scans of that relation + // in already-rendered `sql`. Refresh programs for several view shapes (min/max aggregates, + // group recompute, interrupted-refresh recovery, ...) are assembled as SQL text rather than + // through the AST, so the scan resolver never sees them; without this they would ship to the target + // engine reading the latest snapshot instead of the pinned one. Foreign DuckDB pins must already + // be peeled from the source query. Scans with target-dialect pins are left alone. Throws + // through LPTS for dialects with no verified time-travel syntax rather than emitting an unpinned + // scan. + string RestoreIntoSql(const string &sql, SqlDialect dialect) const; + +private: + struct Pin { + string catalog; + string schema; + unique_ptr snapshot; + bool binds_natively; + }; + + // Qualifiers are checked against the original catalog entry, before LPTS output overrides. + case_insensitive_map_t pins; +}; + +} // namespace openivm +} // namespace duckdb + +#endif // OPENIVM_TIME_TRAVEL_PINS_HPP diff --git a/src/include/upsert/refresh_compiler.hpp b/src/include/upsert/refresh_compiler.hpp index ee06f309..115ebbb3 100644 --- a/src/include/upsert/refresh_compiler.hpp +++ b/src/include/upsert/refresh_compiler.hpp @@ -56,7 +56,10 @@ string CompileWindowRecompute(const string &view_name, const string &view_query_ const vector &partition_delta_specs = {}, bool emit_cascade_delta = false, const string &affected_keys_sql = "", const vector &column_names = {}, bool running_window_incremental = false); -string CompileFullRecompute(const string &view_name, const string &view_query_sql, const string &catalog_prefix = ""); +/// Full recompute, optionally emitting new_bag - old_bag into the view's delta table. +/// Unscopable group/window refreshes must preserve the requested cascade delta for downstream MVs. +string CompileFullRecompute(const string &view_name, const string &view_query_sql, const string &catalog_prefix = "", + bool emit_cascade_delta = false); /// Group-level partial recompute, used by `RefreshType::GROUP_RECOMPUTE` /// (inner-DISTINCT under aggregate). For each base table T_i with a non-empty diff --git a/src/include/upsert/refresh_internal.hpp b/src/include/upsert/refresh_internal.hpp index 1ac0f3a6..be472007 100644 --- a/src/include/upsert/refresh_internal.hpp +++ b/src/include/upsert/refresh_internal.hpp @@ -3,6 +3,7 @@ #include "compile_facts.hpp" #include "core/refresh_metadata.hpp" +#include "core/time_travel_pins.hpp" #include "duckdb.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" #include "duckdb/function/table_function.hpp" @@ -214,10 +215,11 @@ string BuildDuckLakeSnapshotQuery(RefreshMetadata &metadata, Connection &con, co const string &view_query_sql, const vector &delta_table_names, const string &view_catalog_name, const string &view_schema_name, const string &attached_db_catalog_name, const string &attached_db_schema_name); -string QualifyViewQuerySources(RefreshMetadata &metadata, Connection &con, const string &view_name, - const string &view_query_sql, const vector &delta_sources, - const string &view_catalog_name, const string &view_schema_name, - const string &attached_db_catalog_name, const string &attached_db_schema_name); +openivm::TimeTravelPins PrepareViewQuerySources(Connection &con, const string &view_name, string &view_query_sql, + const vector &delta_sources, + const string &view_catalog_name, const string &view_schema_name, + const string &attached_db_catalog_name, + const string &attached_db_schema_name); string DuckLakeSnapshotPlaceholder(const string &catalog_name); DeltaFastPathFlags ResolveDeltaFastPathFlags(ClientContext &context, RefreshMetadata &metadata, Connection &con, diff --git a/src/openivm_extension.cpp b/src/openivm_extension.cpp index 2a7fa02e..33d25682 100644 --- a/src/openivm_extension.cpp +++ b/src/openivm_extension.cpp @@ -2,11 +2,13 @@ #include "core/openivm_extension.hpp" #include "compile_facts.hpp" +#include "spark_scalar_functions.hpp" #include "core/openivm_constants.hpp" #include "core/refresh_metadata.hpp" #include "core/refresh_daemon.hpp" #include "core/refresh_locks.hpp" #include "core/sql_utils.hpp" +#include "core/time_travel_pins.hpp" #include "rules/column_hider.hpp" #include "upsert/refresh_cost_model.hpp" #include "upsert/refresh.hpp" @@ -125,9 +127,9 @@ static duckdb::unique_ptr ComputeDeltaBind(ClientContext &context, Parser parser; parser.ParseQuery(view_query); - auto statement = parser.statements[0].get(); + duckdb::openivm::TimeTravelPins::Peel(context, *parser.statements[0]); Planner planner(context); - planner.CreatePlan(statement->Copy()); + planner.CreatePlan(std::move(parser.statements[0])); OPENIVM_DEBUG_PRINT("[ComputeDelta Bind] Plan:\n%s\n", planner.plan->ToString().c_str()); auto result = make_uniq(); @@ -166,6 +168,8 @@ static void LoadInternal(ExtensionLoader &loader) { // statement OpenIVM does not recognize with DuckDB's native parser. db_config.SetOption(AllowParserOverrideExtensionSetting::SettingIndex, Value("fallback")); + RegisterSparkScalarFunctions(loader); + db_config.AddExtensionOption("openivm_files_path", "path for compiled SQL reference files", LogicalType::VARCHAR); db_config.AddExtensionOption("openivm_refresh_mode", "refresh strategy: incremental, full, or auto", LogicalType::VARCHAR, Value("incremental")); @@ -197,11 +201,17 @@ static void LoadInternal(ExtensionLoader &loader) { LogicalType::BOOLEAN, Value::BOOLEAN(true)); db_config.AddExtensionOption("openivm_regular_nterm", "use N-term telescoping for compile-only regular inner joins", LogicalType::BOOLEAN, Value::BOOLEAN(true)); + db_config.AddExtensionOption("openivm_regular_nterm_left", + "extend compile-only N-term telescoping to LEFT-join projection views", + LogicalType::BOOLEAN, Value::BOOLEAN(true)); db_config.AddExtensionOption("openivm_fk_pruning", "prune inclusion-exclusion join terms using FK constraints", LogicalType::BOOLEAN, Value::BOOLEAN(true)); db_config.AddExtensionOption("openivm_emit_spark_hints", "emit Spark optimizer hints in target_dialect=spark compiled refresh SQL", LogicalType::BOOLEAN, Value::BOOLEAN(false)); + db_config.AddExtensionOption(duckdb::OPENIVM_INPUT_DIALECT_SETTING, + "SQL dialect materialized-view bodies are written in: duckdb (default) or spark", + LogicalType::VARCHAR, Value("duckdb"), duckdb::SetOpenIvmInputDialect); db_config.AddExtensionOption("openivm_skip_aggregate_delete", "skip zero-row DELETE for grouped aggregates when deltas are insert-only", LogicalType::BOOLEAN, Value::BOOLEAN(true)); diff --git a/src/rules/incremental_rewrite_rule.cpp b/src/rules/incremental_rewrite_rule.cpp index d927a99e..ee29f39f 100644 --- a/src/rules/incremental_rewrite_rule.cpp +++ b/src/rules/incremental_rewrite_rule.cpp @@ -6,6 +6,7 @@ #include "core/parser_plan_helpers.hpp" #include "core/scoped_optimizer_settings.hpp" #include "core/sql_utils.hpp" +#include "core/time_travel_pins.hpp" #include "delta/delta_compiler.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" #include "duckdb/optimizer/optimizer.hpp" @@ -96,10 +97,10 @@ void IncrementalRewriteRule::IncrementalRewriteRuleFunction(OptimizerExtensionIn if (parser.statements.empty()) { throw Exception(ExceptionType::PARSER, "IVM: empty view definition for '" + view + "'"); } - auto statement = parser.statements[0].get(); + openivm::TimeTravelPins::Peel(input.context, *parser.statements[0]); OPENIVM_DEBUG_PRINT("[REWRITE] About to CreatePlan for view query\n"); - planner.CreatePlan(statement->Copy()); + planner.CreatePlan(std::move(parser.statements[0])); OPENIVM_DEBUG_PRINT("[REWRITE] CreatePlan done\n"); #if OPENIVM_DEBUG OPENIVM_DEBUG_PRINT("Unoptimized plan: \n%s\n", planner.plan->ToString().c_str()); diff --git a/src/upsert/refresh_compiler.cpp b/src/upsert/refresh_compiler.cpp index 66ecb0db..f784f135 100644 --- a/src/upsert/refresh_compiler.cpp +++ b/src/upsert/refresh_compiler.cpp @@ -1450,9 +1450,28 @@ string CompileProjectionsFilters(const string &view_name, const vector & return delete_query + insert_query; } -string CompileFullRecompute(const string &view_name, const string &view_query_sql, const string &catalog_prefix) { +string CompileFullRecompute(const string &view_name, const string &view_query_sql, const string &catalog_prefix, + bool emit_cascade_delta) { string data_table = catalog_prefix + SqlUtils::QuoteIdentifier(IncrementalTableNames::DataTableName(view_name)); - return SqlUtils::BuildFullRecomputeSQL(data_table, view_query_sql); + if (!emit_cascade_delta) { + return SqlUtils::BuildFullRecomputeSQL(data_table, view_query_sql); + } + string delta_table = catalog_prefix + SqlUtils::QuoteIdentifier(SqlUtils::DeltaName(view_name)); + string old_temp_table = SqlUtils::QuoteIdentifier(string(openivm::TEMP_TABLE_PREFIX) + view_name); + string new_temp_table = SqlUtils::QuoteIdentifier(string("openivm_new_") + view_name); + + string sql; + sql += "CREATE OR REPLACE TEMP TABLE " + old_temp_table + " AS\nSELECT * FROM " + data_table + " openivm_old;\n\n"; + sql += "CREATE OR REPLACE TEMP TABLE " + new_temp_table + " AS\nSELECT * FROM (" + view_query_sql + + ") openivm_recompute;\n\n"; + sql += SqlUtils::BuildFullRecomputeSQL(data_table, "SELECT * FROM " + new_temp_table); + sql += "\n" + BuildSignedMultisetDeltaInsertSQL(delta_table, old_temp_table, new_temp_table); + sql += "DROP TABLE IF EXISTS " + old_temp_table + ";\n"; + sql += "DROP TABLE IF EXISTS " + new_temp_table + ";\n"; + OPENIVM_DEBUG_PRINT("[CompileFullRecompute] unscopable recompute for '%s' — emitting signed " + "whole-view cascade delta\n", + view_name.c_str()); + return sql; } string CompileGroupRecompute(const string &view_name, const string &view_query_sql, const vector &group_columns, @@ -1462,8 +1481,10 @@ string CompileGroupRecompute(const string &view_name, const string &view_query_s string data_table = catalog_prefix + SqlUtils::QuoteIdentifier(IncrementalTableNames::DataTableName(view_name)); // No GROUP BY columns or no source deltas registered → can't scope; fall back to full. + // A cascade delta was still requested, so emit the whole-view signed delta rather than + // silently producing a program with no `openivm_delta_` rows. if (group_columns.empty() || delta_table_specs.empty()) { - return CompileFullRecompute(view_name, view_query_sql, catalog_prefix); + return CompileFullRecompute(view_name, view_query_sql, catalog_prefix, emit_cascade_delta); } string group_csv = SqlUtils::JoinQuotedColumns(group_columns); diff --git a/src/upsert/refresh_compiler_aux.cpp b/src/upsert/refresh_compiler_aux.cpp index a1fe3c0d..278706f8 100644 --- a/src/upsert/refresh_compiler_aux.cpp +++ b/src/upsert/refresh_compiler_aux.cpp @@ -20,26 +20,8 @@ static string DeltaSourceRef(const string &source, const string &catalog_prefix) return catalog_prefix + SqlUtils::QuoteIdentifier(source); } -static string TrimCopy(const string &input) { - idx_t start = 0; - while (start < input.size() && std::isspace(static_cast(input[start]))) { - start++; - } - idx_t end = input.size(); - while (end > start && std::isspace(static_cast(input[end - 1]))) { - end--; - } - return input.substr(start, end - start); -} - -static string LowerCopy(string input) { - std::transform(input.begin(), input.end(), input.begin(), - [](unsigned char c) { return static_cast(std::tolower(c)); }); - return input; -} - static string StripIdentifierQuotes(string input) { - input = TrimCopy(input); + input = SqlUtils::TrimSQLFragment(input); if (input.size() >= 2 && input.front() == '"' && input.back() == '"') { return input.substr(1, input.size() - 2); } @@ -97,16 +79,16 @@ static vector SplitTopLevelComma(const string &input) { } else if (c == ')' && depth > 0) { depth--; } else if (c == ',' && depth == 0) { - parts.push_back(TrimCopy(input.substr(start, i - start))); + parts.push_back(SqlUtils::TrimSQLFragment(input.substr(start, i - start))); start = i + 1; } } - parts.push_back(TrimCopy(input.substr(start))); + parts.push_back(SqlUtils::TrimSQLFragment(input.substr(start))); return parts; } static idx_t FindTopLevelFrom(const string &input) { - string lower = LowerCopy(input); + string lower = StringUtil::Lower(input); int depth = 0; for (idx_t i = 0; i + 6 <= lower.size(); i++) { char c = lower[i]; @@ -124,7 +106,7 @@ static idx_t FindTopLevelFrom(const string &input) { } static idx_t FindTopLevelKeyword(const string &input, const string &keyword, idx_t start = 0) { - string lower = LowerCopy(input); + string lower = StringUtil::Lower(input); string needle = " " + keyword + " "; int depth = 0; for (idx_t i = start; i + needle.size() <= lower.size(); i++) { @@ -169,7 +151,7 @@ static bool ParsePassthroughProjection(const string &item, pair string expr = item; string output; if (std::regex_match(item, match, alias_regex)) { - expr = TrimCopy(match[1].str()); + expr = SqlUtils::TrimSQLFragment(match[1].str()); output = StripIdentifierQuotes(match[2].str()); } else { output = StripIdentifierQuotes(expr); @@ -183,16 +165,16 @@ static bool ParsePassthroughProjection(const string &item, pair } static bool ParseWindowSpec(const string &spec_input, string &partition_col, string &order_col) { - string spec = TrimCopy(spec_input); - string spec_lower = LowerCopy(spec); + string spec = SqlUtils::TrimSQLFragment(spec_input); + string spec_lower = StringUtil::Lower(spec); auto part_pos = spec_lower.find("partition by "); auto order_pos = spec_lower.find(" order by "); if (part_pos == string::npos || order_pos == string::npos || order_pos <= part_pos) { return false; } - string part_expr = TrimCopy(spec.substr(part_pos + 13, order_pos - (part_pos + 13))); - string order_expr = TrimCopy(spec.substr(order_pos + 10)); - string order_lower = LowerCopy(order_expr); + string part_expr = SqlUtils::TrimSQLFragment(spec.substr(part_pos + 13, order_pos - (part_pos + 13))); + string order_expr = SqlUtils::TrimSQLFragment(spec.substr(order_pos + 10)); + string order_lower = StringUtil::Lower(order_expr); auto frame_pos = order_lower.find(" rows "); if (frame_pos == string::npos) { frame_pos = order_lower.find(" range "); @@ -202,19 +184,19 @@ static bool ParseWindowSpec(const string &spec_input, string &partition_col, str if (frame.find("unbounded preceding") == string::npos || frame.find("current row") == string::npos) { return false; } - order_expr = TrimCopy(order_expr.substr(0, frame_pos)); + order_expr = SqlUtils::TrimSQLFragment(order_expr.substr(0, frame_pos)); } - auto nulls_pos = LowerCopy(order_expr).find(" nulls "); + auto nulls_pos = StringUtil::Lower(order_expr).find(" nulls "); if (nulls_pos != string::npos) { - order_expr = TrimCopy(order_expr.substr(0, nulls_pos)); + order_expr = SqlUtils::TrimSQLFragment(order_expr.substr(0, nulls_pos)); } auto order_space = order_expr.find(' '); if (order_space != string::npos) { - string suffix = LowerCopy(TrimCopy(order_expr.substr(order_space + 1))); + string suffix = StringUtil::Lower(SqlUtils::TrimSQLFragment(order_expr.substr(order_space + 1))); if (suffix != "asc") { return false; } - order_expr = TrimCopy(order_expr.substr(0, order_space)); + order_expr = SqlUtils::TrimSQLFragment(order_expr.substr(0, order_space)); } string parsed_part = StripIdentifierQuotes(part_expr); string parsed_order = StripIdentifierQuotes(order_expr); @@ -241,7 +223,8 @@ static bool ParseNamedWindows(const string &tail, std::map &name if (!std::regex_match(item, match, named_regex)) { return false; } - named_windows[StringUtil::Lower(StripIdentifierQuotes(match[1].str()))] = TrimCopy(match[2].str()); + named_windows[StringUtil::Lower(StripIdentifierQuotes(match[1].str()))] = + SqlUtils::TrimSQLFragment(match[2].str()); } return !named_windows.empty(); } @@ -254,7 +237,7 @@ static bool ParseRunningWindowProjection(const string &item, const std::mapsecond; } else { - spec = TrimCopy(window_match[3].str()); + spec = SqlUtils::TrimSQLFragment(window_match[3].str()); } if (!ParseWindowSpec(spec, partition_col, order_col)) { return false; } - out.function_name = LowerCopy(window_match[1].str()); + out.function_name = StringUtil::Lower(window_match[1].str()); out.argument = StripIdentifierQuotes(window_match[2].str()); return true; } @@ -291,10 +274,10 @@ static bool ParseRunningWindowExpression(const string &expr, RunningWindowExpr & if (!std::regex_match(expr, window_match, window_regex)) { return false; } - if (!ParseWindowSpec(TrimCopy(window_match[3].str()), partition_col, order_col)) { + if (!ParseWindowSpec(SqlUtils::TrimSQLFragment(window_match[3].str()), partition_col, order_col)) { return false; } - out.function_name = LowerCopy(window_match[1].str()); + out.function_name = StringUtil::Lower(window_match[1].str()); out.argument = StripIdentifierQuotes(window_match[2].str()); return true; } @@ -323,8 +306,8 @@ static bool LooksLikeLptsAlias(const string &expr) { static bool TryParseRunningWindowPlan(const string &view_query_sql, const vector &partition_columns, const vector &column_names, RunningWindowPlan &plan) { - string query = TrimCopy(view_query_sql); - if (!StringUtil::StartsWith(LowerCopy(query), "select ")) { + string query = SqlUtils::TrimSQLFragment(view_query_sql); + if (!StringUtil::StartsWith(StringUtil::Lower(query), "select ")) { return false; } auto from_pos = FindTopLevelFrom(query); @@ -332,12 +315,12 @@ static bool TryParseRunningWindowPlan(const string &view_query_sql, const vector return false; } string select_list = query.substr(7, from_pos - 7); - string from_tail = TrimCopy(query.substr(from_pos + 6)); + string from_tail = SqlUtils::TrimSQLFragment(query.substr(from_pos + 6)); std::map named_windows; auto window_pos = FindTopLevelKeyword(" " + from_tail, "window"); if (window_pos != string::npos) { - string source_tail = TrimCopy(from_tail.substr(0, window_pos - 1)); - string window_tail = TrimCopy(from_tail.substr(window_pos + 7)); + string source_tail = SqlUtils::TrimSQLFragment(from_tail.substr(0, window_pos - 1)); + string window_tail = SqlUtils::TrimSQLFragment(from_tail.substr(window_pos + 7)); if (!ParseNamedWindows(window_tail, named_windows)) { return false; } @@ -350,7 +333,7 @@ static bool TryParseRunningWindowPlan(const string &view_query_sql, const vector auto parsed_partition = partition_columns.empty() ? "" : PartitionOutputColumn(partition_columns[0]); string parsed_order; for (auto &item : SplitTopLevelComma(select_list)) { - if (LowerCopy(item).find(" over ") != string::npos) { + if (StringUtil::Lower(item).find(" over ") != string::npos) { RunningWindowExpr expr; if (!ParseRunningWindowProjection(item, named_windows, expr, parsed_partition, parsed_order)) { return false; @@ -423,7 +406,7 @@ static string NormalizeLptsRunningWindowSql(const string &sql) { static bool TryParseLptsRunningWindowPlan(const string &raw_view_query_sql, const vector &partition_columns, const vector &column_names, RunningWindowPlan &plan) { string view_query_sql = NormalizeLptsRunningWindowSql(raw_view_query_sql); - string lower = LowerCopy(view_query_sql); + string lower = StringUtil::Lower(view_query_sql); // The first CTE in a running-window LPTS program is the base table scan. Locate it via the // leading WITH rather than the CTE name, which the refactor changed from "scan_0" to "t0_scan". auto scan_pos = lower.find("with"); @@ -538,7 +521,7 @@ static bool TryParseLptsRunningWindowPlan(const string &raw_view_query_sql, cons auto &item = cte_items[item_idx]; string output_alias = StripIdentifierQuotes(cte_aliases[item_idx]); string output_key = StringUtil::Lower(output_alias); - if (LowerCopy(item).find(" over ") == string::npos) { + if (StringUtil::Lower(item).find(" over ") == string::npos) { string source_key = StringUtil::Lower(StripIdentifierQuotes(item)); auto passthrough = alias_to_output.find(source_key); if (passthrough != alias_to_output.end()) { @@ -550,7 +533,7 @@ static bool TryParseLptsRunningWindowPlan(const string &raw_view_query_sql, cons next_alias_to_output[output_key] = scan_passthrough->second; continue; } - string item_lower = LowerCopy(TrimCopy(item)); + string item_lower = StringUtil::Lower(SqlUtils::TrimSQLFragment(item)); if (StringUtil::StartsWith(item_lower, "case ")) { RunningDerivedExpr derived; derived.output_column = output_alias; @@ -1600,7 +1583,10 @@ string CompileWindowRecompute(const string &view_name, const string &view_query_ bool running_window_incremental) { bool have_affected_keys = !affected_keys_sql.empty(); if (!have_affected_keys && (partition_columns.empty() || partition_delta_specs.empty())) { - return CompileFullRecompute(view_name, view_query_sql, catalog_prefix); + // No PARTITION BY (global surrogate-key window) or no partition key resolvable in any + // source delta table → nothing to scope the recompute to. Keep the cascade delta the + // caller asked for so downstream MVs stay incremental. + return CompileFullRecompute(view_name, view_query_sql, catalog_prefix, emit_cascade_delta); } if (running_window_incremental) { auto suffix_sql = BuildRunningWindowSuffixRefreshSQL(view_name, view_query_sql, delta_ts_filter, catalog_prefix, diff --git a/src/upsert/refresh_cost_model.cpp b/src/upsert/refresh_cost_model.cpp index d7e98acc..3cff2378 100644 --- a/src/upsert/refresh_cost_model.cpp +++ b/src/upsert/refresh_cost_model.cpp @@ -6,6 +6,7 @@ #include "core/openivm_constants.hpp" #include "core/refresh_metadata.hpp" #include "core/sql_utils.hpp" +#include "core/time_travel_pins.hpp" #include "core/openivm_debug.hpp" #include "rules/column_hider.hpp" #include "storage/ducklake_scan.hpp" @@ -907,7 +908,8 @@ string RefreshCostQuery(ClientContext &context, const FunctionParameters ¶me throw ParserException("View '" + view_name + "' has an empty IVM metadata query"); } Planner planner(con_ctx); - planner.CreatePlan(p.statements[0]->Copy()); + openivm::TimeTravelPins::Peel(con_ctx, *p.statements[0]); + planner.CreatePlan(std::move(p.statements[0])); Optimizer optimizer(*planner.binder, con_ctx); auto plan = optimizer.Optimize(std::move(planner.plan)); diff --git a/src/upsert/refresh_helpers.cpp b/src/upsert/refresh_helpers.cpp index e28060ae..7e958ecc 100644 --- a/src/upsert/refresh_helpers.cpp +++ b/src/upsert/refresh_helpers.cpp @@ -10,6 +10,8 @@ #include "duckdb/main/client_context.hpp" #include "duckdb/main/client_data.hpp" #include "duckdb/main/connection.hpp" +#include "duckdb/parser/parser.hpp" +#include "duckdb/parser/tableref/basetableref.hpp" #include "duckdb/planner/operator/logical_aggregate.hpp" #include "duckdb/planner/operator/logical_join.hpp" @@ -1285,11 +1287,12 @@ string BuildDuckLakeSnapshotQuery(RefreshMetadata &metadata, Connection &con, co return snapshot_query; } -string QualifyViewQuerySources(RefreshMetadata &metadata, Connection &con, const string &view_name, - const string &view_query_sql, const vector &delta_sources, - const string &view_catalog_name, const string &view_schema_name, - const string &attached_db_catalog_name, const string &attached_db_schema_name) { - string qualified_query = view_query_sql; +openivm::TimeTravelPins PrepareViewQuerySources(Connection &con, const string &view_name, string &view_query_sql, + const vector &delta_sources, + const string &view_catalog_name, const string &view_schema_name, + const string &attached_db_catalog_name, + const string &attached_db_schema_name) { + case_insensitive_map_t locations; OPENIVM_DEBUG_PRINT("[UPSERT] Qualifying %zu sources for %s from one metadata snapshot\n", delta_sources.size(), view_name.c_str()); for (auto &source : delta_sources) { @@ -1306,10 +1309,27 @@ string QualifyViewQuerySources(RefreshMetadata &metadata, Connection &con, const continue; } string base_name = BaseTableNameFromDeltaKey(source.table_name); - qualified_query = SqlUtils::ReplaceTableReferences(qualified_query, base_name, - SqlUtils::FullName(catalog_name, schema_name, base_name)); - } - return qualified_query; + locations[base_name] = {catalog_name, schema_name}; + } + Parser parser(con.context->GetParserOptions()); + parser.ParseQuery(view_query_sql); + bool qualified = false; + openivm::TimeTravelPins pins; + con.context->RunFunctionInTransaction([&]() { + pins = openivm::TimeTravelPins::Peel(*con.context, *parser.statements.at(0), [&](BaseTableRef &ref) { + auto location = locations.find(ref.table_name); + if (location != locations.end() && (ref.catalog_name != location->second.catalog_name || + ref.schema_name != location->second.schema_name)) { + ref.catalog_name = location->second.catalog_name; + ref.schema_name = location->second.schema_name; + qualified = true; + } + }); + if (qualified || !pins.Empty()) { + view_query_sql = parser.statements[0]->ToString(); + } + }); + return pins; } static string HexEncodeToken(const string &input) { diff --git a/src/upsert/refresh_sql.cpp b/src/upsert/refresh_sql.cpp index 4a379226..6e027347 100644 --- a/src/upsert/refresh_sql.cpp +++ b/src/upsert/refresh_sql.cpp @@ -5,6 +5,7 @@ #include "core/openivm_debug.hpp" #include "core/scoped_optimizer_settings.hpp" #include "core/sql_utils.hpp" +#include "core/time_travel_pins.hpp" #include "rules/column_hider.hpp" #include "upsert/refresh_compiler.hpp" #include "upsert/refresh_cost_model.hpp" @@ -45,20 +46,27 @@ static string SparkPortableRefreshSQL(string sql) { return sql; } -static string RenderStoredViewQueryForDialect(ClientContext &context, const string &view_query_sql, - const vector &output_names, SqlDialect dialect) { - Parser parser(context.GetParserOptions()); - parser.ParseQuery(view_query_sql); - if (parser.statements.size() != 1) { - throw ParserException("Expected one stored view query, found %llu", - static_cast(parser.statements.size())); - } - Planner planner(context); - planner.CreatePlan(parser.statements[0]->Copy()); - auto plan = std::move(planner.plan); - auto ast = LogicalPlanToAst(context, plan, dialect); - auto cte_list = AstToCteList(*ast, dialect); - auto rendered = cte_list->ToQuery(true, output_names); +static string RenderStoredViewQueryForDialect(Connection &con, const string &view_query_sql, + const vector &output_names, SqlDialect dialect, + const openivm::TimeTravelPins &time_travel_pins) { + string rendered; + con.context->RunFunctionInTransaction([&]() { + auto &context = *con.context; + Parser parser(context.GetParserOptions()); + parser.ParseQuery(view_query_sql); + if (parser.statements.size() != 1) { + throw ParserException("Expected one stored view query, found %llu", + static_cast(parser.statements.size())); + } + Planner planner(context); + // Source qualification already stripped foreign pins; native catalog snapshots remain bindable. + planner.CreatePlan(std::move(parser.statements[0])); + auto plan = std::move(planner.plan); + auto ast = LogicalPlanToAst(context, plan, dialect, + dialect == SqlDialect::DUCKDB ? SnapshotResolver() : time_travel_pins.Resolver()); + auto cte_list = AstToCteList(*ast, dialect); + rendered = cte_list->ToQuery(true, output_names); + }); if (!rendered.empty() && rendered.back() == ';') { rendered.pop_back(); } @@ -227,8 +235,8 @@ static void PropagateRefreshPlanningSettings(ClientContext &from, ClientContext // session-scoped planning settings still need to be mirrored onto the fresh // planning connection. static const char *PLANNING_SETTINGS[] = { - "openivm_adaptive_refresh", "openivm_cost_decay", "openivm_skip_empty_deltas", - "openivm_fk_pruning", "openivm_ducklake_nterm", "openivm_regular_nterm", + "openivm_adaptive_refresh", "openivm_cost_decay", "openivm_skip_empty_deltas", "openivm_fk_pruning", + "openivm_ducklake_nterm", "openivm_regular_nterm", "openivm_regular_nterm_left", }; for (auto setting_name : PLANNING_SETTINGS) { CopyOpenIvmSetting(from, to, setting_name); @@ -494,8 +502,17 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam "; delta_tables=" + to_string(delta_table_names.size()) + "; target_ducklake=" + string(target_is_ducklake ? "true" : "false")); auto qualify_start = profile_now(); - view_query_sql = QualifyViewQuerySources(metadata, con, view_name, view_query_sql, delta_sources, view_catalog_name, - view_schema_name, attached_db_catalog_name, attached_db_schema_name); + auto view_time_travel_pins = + PrepareViewQuerySources(con, view_name, view_query_sql, delta_sources, view_catalog_name, view_schema_name, + attached_db_catalog_name, attached_db_schema_name); + // Text-only refresh paths still need restoration. Plan-based paths already carry typed snapshots; + // DuckDB output uses the unpinned local source query and must not acquire foreign pins. + auto finalize_refresh_sql = [&](string refresh_sql) { + if (view_time_travel_pins.Empty() || active_facts.target_dialect == SqlDialect::DUCKDB) { + return refresh_sql; + } + return view_time_travel_pins.RestoreIntoSql(refresh_sql, active_facts.target_dialect); + }; add_profile_step("generate_refresh_sql.qualify_sources", qualify_start, "query_bytes=" + to_string(view_query_sql.size())); auto recovery_start = profile_now(); @@ -505,8 +522,13 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam if (!flag_result->HasError() && flag_result->RowCount() > 0 && !flag_result->GetValue(0, 0).IsNull() && flag_result->GetValue(0, 0).GetValue()) { Printer::Print("Warning: recovering '" + view_name + "' from interrupted refresh via full recompute."); + auto recovery_source_sql = view_query_sql; + if (!view_time_travel_pins.Empty() && active_facts.target_dialect != SqlDialect::DUCKDB) { + recovery_source_sql = RenderStoredViewQueryForDialect( + con, view_query_sql, vector(), active_facts.target_dialect, view_time_travel_pins); + } auto recovery_query = - BuildRecomputeQuery(metadata, view_name, view_query_sql, cross_system, attached_db_catalog_name, + BuildRecomputeQuery(metadata, view_name, recovery_source_sql, cross_system, attached_db_catalog_name, attached_db_schema_name, internal_catalog_prefix, metadata_prefix, out_post_meta); if (cross_system) { metadata.SetRefreshInProgress(view_name, false); @@ -515,7 +537,7 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam " SET refresh_in_progress = false WHERE view_name = '" + SqlUtils::EscapeValue(view_name) + "';\n"; } - return recovery_query; + return finalize_refresh_sql(recovery_query); } } add_profile_step("generate_refresh_sql.recovery_check", recovery_start); @@ -577,7 +599,7 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam Parser cost_parser; cost_parser.ParseQuery(view_query_sql); Planner cost_planner(planning_context); - cost_planner.CreatePlan(cost_parser.statements[0]->Copy()); + cost_planner.CreatePlan(std::move(cost_parser.statements[0])); Optimizer cost_optimizer(*cost_planner.binder, planning_context); auto cost_plan = cost_optimizer.Optimize(std::move(cost_planner.plan)); @@ -608,15 +630,20 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam if (use_full_recompute && !full_recompute_needs_cascade_delta) { auto full_refresh_start = profile_now(); + string recompute_source_sql = view_query_sql; + if (!view_time_travel_pins.Empty() && active_facts.target_dialect != SqlDialect::DUCKDB) { + recompute_source_sql = RenderStoredViewQueryForDialect(con, view_query_sql, vector(), + active_facts.target_dialect, view_time_travel_pins); + } auto recompute_query = - BuildRecomputeQuery(metadata, view_name, view_query_sql, cross_system, attached_db_catalog_name, + BuildRecomputeQuery(metadata, view_name, recompute_source_sql, cross_system, attached_db_catalog_name, attached_db_schema_name, internal_catalog_prefix, metadata_prefix, out_post_meta); add_profile_step("generate_refresh_sql.dispatch", full_refresh_start, "full_recompute=true; metadata_requires_full_refresh=" + string(metadata_requires_full_refresh ? "true" : "false") + "; adaptive_recompute=" + string(adaptive_recompute ? "true" : "false") + "; sql_bytes=" + to_string(recompute_query.size())); - return recompute_query; + return finalize_refresh_sql(recompute_query); } RefreshType dispatch_refresh_type = use_full_recompute ? RefreshType::FULL_REFRESH : view_query_type; refresh_plan.refresh_type = dispatch_refresh_type; @@ -1152,15 +1179,8 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam output_names.push_back(column_name); } } - con.BeginTransaction(); - try { - full_recompute_query = RenderStoredViewQueryForDialect(planning_context, view_query_sql, output_names, - active_facts.target_dialect); - con.Rollback(); - } catch (...) { - con.Rollback(); - throw; - } + full_recompute_query = RenderStoredViewQueryForDialect(con, view_query_sql, output_names, + active_facts.target_dialect, view_time_travel_pins); } upsert_query = CompileFullRecompute(view_name, full_recompute_query, internal_catalog_prefix); OPENIVM_DEBUG_PRINT("[UPSERT] Compiling upsert for type: %s\n", RefreshTypeName(dispatch_refresh_type)); @@ -1405,7 +1425,9 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam try { auto lpts_start = profile_now(); SqlDialect dialect = active_facts.target_dialect; - auto ast = LogicalPlanToAst(con_ctx, plan, dialect); + auto ast = LogicalPlanToAst(con_ctx, plan, dialect, + dialect == SqlDialect::DUCKDB ? SnapshotResolver() + : view_time_travel_pins.Resolver()); bool emit_spark_hints = dialect == SqlDialect::SPARK && (active_facts.emit_spark_hints || SqlUtils::GetBoolSetting(con_ctx, "openivm_emit_spark_hints", false)); @@ -1642,6 +1664,7 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam } else { clean_query = meta_pre_sql + data_sql + meta_post_sql; } + clean_query = finalize_refresh_sql(std::move(clean_query)); Value files_path_val; if (write_query_file && context.TryGetCurrentSetting("openivm_files_path", files_path_val) && !files_path_val.IsNull()) { diff --git a/src/upsert/refresh_window.cpp b/src/upsert/refresh_window.cpp index 1ce999ed..b645e0a6 100644 --- a/src/upsert/refresh_window.cpp +++ b/src/upsert/refresh_window.cpp @@ -677,7 +677,7 @@ string BuildWindowPartitionRefresh(RefreshMetadata &metadata, Connection &con, c if (any_ducklake) { OPENIVM_DEBUG_PRINT( "[UPSERT] Compiling upsert for type: WINDOW_PARTITION (DuckLake, full recompute fallback)\n"); - return "DELETE FROM " + data_table + ";\n" + "INSERT INTO " + data_table + " " + view_query_sql + ";\n"; + return CompileFullRecompute(view_name, view_query_sql, internal_catalog_prefix, emit_cascade_delta); } auto lineage_result = BuildLineageStandardAffectedKeysSQL( metadata, con, view_name, delta_table_names, partition_cols, delta_ts_filter, view_catalog_name, @@ -685,7 +685,7 @@ string BuildWindowPartitionRefresh(RefreshMetadata &metadata, Connection &con, c if (lineage_result == LineageAffectedKeysResult::UNSAFE) { OPENIVM_DEBUG_PRINT("[UPSERT] WINDOW_PARTITION lineage is unsafe for '%s' — full recompute fallback\n", view_name.c_str()); - return CompileFullRecompute(view_name, view_query_sql, internal_catalog_prefix); + return CompileFullRecompute(view_name, view_query_sql, internal_catalog_prefix, emit_cascade_delta); } have_lineage_affected_keys = lineage_result == LineageAffectedKeysResult::AVAILABLE; if (!have_lineage_affected_keys && delta_table_names.size() > 1 && @@ -693,7 +693,7 @@ string BuildWindowPartitionRefresh(RefreshMetadata &metadata, Connection &con, c OPENIVM_DEBUG_PRINT("[UPSERT] WINDOW_PARTITION lineage incomplete for '%s' (%zu sources) — full recompute " "fallback\n", view_name.c_str(), delta_table_names.size()); - return CompileFullRecompute(view_name, view_query_sql, internal_catalog_prefix); + return CompileFullRecompute(view_name, view_query_sql, internal_catalog_prefix, emit_cascade_delta); } OPENIVM_DEBUG_PRINT("[UPSERT] Compiling upsert for type: WINDOW_PARTITION (%zu partition cols, lineage keys: %s)\n", partition_cols.size(), have_lineage_affected_keys ? "yes" : "no"); diff --git a/test/integration/test_regular_nterm_compiled.py b/test/integration/test_regular_nterm_compiled.py index 2eb9479c..e7d0428a 100644 --- a/test/integration/test_regular_nterm_compiled.py +++ b/test/integration/test_regular_nterm_compiled.py @@ -23,8 +23,8 @@ def run_duckdb(binary: Path, database: Path, sql: str) -> str: def bag_equality_sql(view_name: str, base_query: str) -> str: return f""" SELECT - (SELECT COUNT(*) FROM (SELECT * FROM {view_name} EXCEPT ALL {base_query})) + - (SELECT COUNT(*) FROM ({base_query} EXCEPT ALL SELECT * FROM {view_name})); + (SELECT COUNT(*) FROM ((SELECT * FROM {view_name}) EXCEPT ALL ({base_query}))) + + (SELECT COUNT(*) FROM (({base_query}) EXCEPT ALL (SELECT * FROM {view_name}))); """ @@ -48,9 +48,14 @@ def run_scenario( compile_sql = f""" SET openivm_files_path='{output_path}'; {setup_sql} -SELECT COUNT(*) FROM openivm_compile_with_facts('{view_name}', '{facts_json}'); +SELECT DISTINCT 'openivm_refresh_type=' || refresh_type_name +FROM openivm_compile_with_facts('{view_name}', '{facts_json}') +WHERE stmt_kind = 'data'; """ - run_duckdb(binary, database, compile_sql) + compile_output = run_duckdb(binary, database, compile_sql) + classifications = [line for line in compile_output.splitlines() if line.startswith("openivm_refresh_type=")] + if classifications != ["openivm_refresh_type=SIMPLE_PROJECTION"]: + raise AssertionError(f"{name}: compiled join must stay incremental, got {classifications!r}") program_path = output_dir / f"openivm_upsert_queries_{view_name}.sql" if not program_path.exists(): @@ -196,6 +201,51 @@ def main(): AND prev.event_ts < cur.event_ts JOIN self_account account ON cur.account_id = account.account_id""", ) + run_scenario( + binary, + root, + "left_join_passthrough_mixed", + """ +CREATE TABLE left_fact(id INTEGER, k1 INTEGER, k2 INTEGER); +CREATE TABLE left_one(k INTEGER, label VARCHAR); +CREATE TABLE left_two(k INTEGER, label VARCHAR); +INSERT INTO left_fact VALUES (1, 10, 20), (2, 11, 21), (3, 10, 21), (4, 12, 22); +INSERT INTO left_one VALUES (10, 'a'), (10, 'duplicate'), (12, 'c'); +INSERT INTO left_two VALUES (20, 'old'), (21, 'removed'), (22, 'kept'); +CREATE MATERIALIZED VIEW left_mv AS +WITH projected AS ( + SELECT f.id, f.k2, d1.label + FROM left_fact f LEFT JOIN left_one d1 ON f.k1 = d1.k +) +SELECT p.id, d2.label +FROM projected p LEFT JOIN left_two d2 ON p.k2 = d2.k; +INSERT INTO left_one VALUES (11, 'late'), (11, 'late_duplicate'); +DELETE FROM left_one WHERE label = 'duplicate'; +UPDATE left_two SET label = 'changed' WHERE k = 20; +DELETE FROM left_two WHERE k = 21; +INSERT INTO left_two VALUES (23, 'new'); +UPDATE left_fact SET k2 = 23 WHERE id = 1; +DELETE FROM left_fact WHERE id = 4; +INSERT INTO left_fact VALUES (5, 11, 20); +""", + "left_mv", + { + "target_dialect": "duckdb", + "compile_only": True, + "force_view_delta_cascade": True, + "delta_shape": { + "left_fact": "MIXED", + "left_one": "MIXED", + "left_two": "MIXED", + }, + }, + """WITH projected AS ( + SELECT f.id, f.k2, d1.label + FROM left_fact f LEFT JOIN left_one d1 ON f.k1 = d1.k +) +SELECT p.id, d2.label +FROM projected p LEFT JOIN left_two d2 ON p.k2 = d2.k""", + ) print("regular N-term compiled SQL integration tests passed") diff --git a/test/sql/auto_refresh.test b/test/sql/auto_refresh.test index 30490eb7..7eb598c0 100644 --- a/test/sql/auto_refresh.test +++ b/test/sql/auto_refresh.test @@ -881,6 +881,14 @@ SELECT count(*) FROM openivm_refresh_profile WHERE view_name = 'mv_lt'; ---- 0 +# Force deterministic incremental path for the profiling assertions below. +# The adaptive cost model (still enabled from Test 18 setup) can nondeterministically +# choose full recompute vs incremental depending on prior refresh durations, which +# changes the statement count (6 vs 7). The profiling feature is path-independent; +# the adaptive cost model is already validated by the history assertions above. +statement ok +SET openivm_adaptive_refresh = false; + statement ok SET openivm_profile_refresh = true; @@ -895,15 +903,22 @@ SELECT count(*) FROM openivm_refresh_profile WHERE view_name = 'mv_lt' AND step_ ---- 3 -# 6, not 4: the full-recompute path for AGGREGATE_GROUP views emits four statements -# (CREATE TEMP TABLE / DELETE vanished keys / INSERT OR REPLACE / DROP) instead of a plain -# DELETE + INSERT, because their data table carries a UNIQUE index and DuckDB's on-disk unique -# index rejects re-inserting a key deleted earlier in the same transaction. See -# test/sql/group_recompute_persistent_unique_index.test. +# 7 profiled statements per refresh for an AGGREGATE_GROUP MERGE view: +# 1. SET refresh_in_progress = true +# 2. INSERT INTO openivm_delta_ (delta computation) +# 3. MERGE INTO openivm_data_ (applies delta via CTE from the delta view) +# 4. DELETE FROM openivm_delta_ (view delta cleanup) +# 5. DELETE FROM openivm_delta_ (source delta cleanup) +# 6. UPDATE openivm_delta_tables SET last_update (metadata timestamp) +# 7. SET refresh_in_progress = false query I SELECT count(*) FROM openivm_refresh_profile WHERE view_name = 'mv_lt' AND step_name = 'execute_refresh_sql_stmt'; ---- -6 +7 + +# Re-enable adaptive refresh so subsequent refreshes record history for Test 20. +statement ok +SET openivm_adaptive_refresh = true; query I SELECT CASE WHEN count(*) >= 13 THEN 1 ELSE 0 END FROM openivm_refresh_profile WHERE view_name = 'mv_lt' AND duration_ms >= 0; @@ -1010,10 +1025,13 @@ SET openivm_skip_empty_deltas = true; # Test 20: Learned cost model — history cleanup on REPLACE # ========================================== +# 5 history rows: 3 from Test 18 setup + 2 from profiling retention tests. +# The profiled refresh itself ran with openivm_adaptive_refresh = false (for deterministic +# statement count), so it did not record a history entry. query I SELECT count(*) FROM openivm_refresh_history WHERE view_name = 'mv_lt'; ---- -6 +5 statement ok CREATE OR REPLACE MATERIALIZED VIEW mv_lt AS SELECT grp, sum(val) as total FROM lt GROUP BY grp; diff --git a/test/sql/cascade_simple_projection_join.test b/test/sql/cascade_simple_projection_join.test index 8de65d6c..147e136b 100644 --- a/test/sql/cascade_simple_projection_join.test +++ b/test/sql/cascade_simple_projection_join.test @@ -565,3 +565,345 @@ SELECT COUNT(*) FROM ( ); ---- 0 + +# Reduced "arc_machine_status_transaction" shape: a CTE (INNER JOIN + LEFT +# JOIN) feeding an outer query with two further chained LEFT JOINs. Before +# src/delta/operators/join.cpp's AppendMultiplicityToAncestorProjectionMaps +# fix, a deeper join's own left_projection_map growing (to append a new +# per-leaf multiplicity column) silently shifted where its right-side +# contribution begins in its own combined column numbering. A grandparent +# join's pre-existing projection-map entry, fixed before that growth +# happened, could then alias onto the just-added multiplicity column +# instead of the real business column (`cust_key`) it used to select, +# permanently dropping it and surfacing as an internal arity mismatch. +statement ok +CREATE TABLE cspj_arcm_msf(id INT, coll_key INT); + +statement ok +CREATE TABLE cspj_arcm_acd(coll_key INT, sub_id INT, arm_id VARCHAR); + +statement ok +CREATE TABLE cspj_arcm_hw(arm_id VARCHAR, provider VARCHAR); + +statement ok +CREATE TABLE cspj_arcm_tpid(sub_id INT, tp_id INT); + +statement ok +CREATE TABLE cspj_arcm_cust(tp_id INT, cust_key INT); + +statement ok +INSERT INTO cspj_arcm_msf VALUES (1, 100), (2, 100); + +statement ok +INSERT INTO cspj_arcm_acd VALUES (100, 10, 'arm-1'); + +statement ok +INSERT INTO cspj_arcm_hw VALUES ('arm-1', 'azure'); + +statement ok +INSERT INTO cspj_arcm_tpid VALUES (10, 200); + +statement ok +INSERT INTO cspj_arcm_cust VALUES (200, 21); + +statement ok +CREATE MATERIALIZED VIEW cspj_arcm_mv AS + WITH bff AS ( + SELECT msf.id, acd.sub_id, + coalesce(hw.provider, 'N/A') AS provider + FROM cspj_arcm_msf msf + INNER JOIN cspj_arcm_acd acd ON msf.coll_key = acd.coll_key + LEFT JOIN cspj_arcm_hw hw ON acd.arm_id = hw.arm_id + ) + SELECT bff.id, + coalesce(cust.cust_key, 1) AS cust_key + FROM bff + LEFT JOIN cspj_arcm_tpid tpid ON bff.sub_id = tpid.sub_id + LEFT JOIN cspj_arcm_cust cust ON coalesce(tpid.tp_id, -1) = cust.tp_id; + +statement ok +SELECT COUNT(*) FROM openivm_compile_with_facts( + 'cspj_arcm_mv', + '{"target_dialect":"duckdb","compile_only":true}' +); + +query I +SELECT CASE + WHEN contains(content, 'INSERT INTO openivm_delta_cspj_arcm_mv') AND + contains(content, 'openivm_delta_cspj_arcm_msf') AND + contains(content, 'openivm_delta_cspj_arcm_acd') AND + contains(content, 'openivm_delta_cspj_arcm_hw') AND + contains(content, 'openivm_delta_cspj_arcm_tpid') AND + contains(content, 'openivm_delta_cspj_arcm_cust') AND + NOT contains(content, 'SELECT NULL::') + THEN 1 ELSE 0 + END +FROM read_text('__TEST_DIR__/openivm_upsert_queries_cspj_arcm_mv.sql'); +---- +1 + +statement ok +INSERT INTO cspj_arcm_msf VALUES (3, 300); + +statement ok +INSERT INTO cspj_arcm_acd VALUES (300, 30, 'arm-2'); + +statement ok +INSERT INTO cspj_arcm_hw VALUES ('arm-2', 'gcp'); + +statement ok +INSERT INTO cspj_arcm_tpid VALUES (30, 400); + +statement ok +DELETE FROM cspj_arcm_msf WHERE id = 2; + +statement ok +UPDATE cspj_arcm_cust SET cust_key = 99 WHERE tp_id = 200; + +statement ok +PRAGMA refresh('cspj_arcm_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM cspj_arcm_mv + EXCEPT ALL + SELECT bff.id, coalesce(cust.cust_key, 1) AS cust_key + FROM ( + SELECT msf.id, acd.sub_id, coalesce(hw.provider, 'N/A') AS provider + FROM cspj_arcm_msf msf + INNER JOIN cspj_arcm_acd acd ON msf.coll_key = acd.coll_key + LEFT JOIN cspj_arcm_hw hw ON acd.arm_id = hw.arm_id + ) bff + LEFT JOIN cspj_arcm_tpid tpid ON bff.sub_id = tpid.sub_id + LEFT JOIN cspj_arcm_cust cust ON coalesce(tpid.tp_id, -1) = cust.tp_id +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT bff.id, coalesce(cust.cust_key, 1) AS cust_key + FROM ( + SELECT msf.id, acd.sub_id, coalesce(hw.provider, 'N/A') AS provider + FROM cspj_arcm_msf msf + INNER JOIN cspj_arcm_acd acd ON msf.coll_key = acd.coll_key + LEFT JOIN cspj_arcm_hw hw ON acd.arm_id = hw.arm_id + ) bff + LEFT JOIN cspj_arcm_tpid tpid ON bff.sub_id = tpid.sub_id + LEFT JOIN cspj_arcm_cust cust ON coalesce(tpid.tp_id, -1) = cust.tp_id + EXCEPT ALL + SELECT * FROM cspj_arcm_mv +); +---- +0 + +# Reduced "int_instance_status_transaction" shape: a UNION ALL of two LEFT +# JOIN branches partitioned by an IS NOT NULL / IS NULL predicate on the +# joined side. Before the `third_party/lpts` pin was advanced to commit +# 77ed5e5 (fork PR #18, "Fix set-op column binding remaps"), remapping +# duplicated UNION ALL key-projection output aliases could leave a stale +# reference to a child alias that no longer existed in the rewritten plan, +# surfacing as a runtime Binder Error ("Referenced column ... not found in +# FROM clause"). +statement ok +CREATE TABLE cspj_inti_isf(id INT, k INT); + +statement ok +CREATE TABLE cspj_inti_ml(k INT, res VARCHAR); + +statement ok +INSERT INTO cspj_inti_isf VALUES (1, 100), (2, 200); + +statement ok +INSERT INTO cspj_inti_ml VALUES (100, 'machine-1'); + +statement ok +CREATE MATERIALIZED VIEW cspj_inti_mv AS + SELECT isf.id, isf.k FROM cspj_inti_isf isf LEFT JOIN cspj_inti_ml ml ON isf.k = ml.k WHERE ml.res IS NOT NULL + UNION ALL + SELECT isf.id, isf.k FROM cspj_inti_isf isf LEFT JOIN cspj_inti_ml ml ON isf.k = ml.k WHERE ml.res IS NULL; + +statement ok +SELECT COUNT(*) FROM openivm_compile_with_facts( + 'cspj_inti_mv', + '{"target_dialect":"duckdb","compile_only":true}' +); + +query I +SELECT CASE + WHEN contains(content, 'INSERT INTO openivm_delta_cspj_inti_mv') AND + contains(content, 'openivm_delta_cspj_inti_isf') AND + contains(content, 'openivm_delta_cspj_inti_ml') AND + contains(content, 'UNION ALL') AND + NOT contains(content, 'SELECT NULL::') + THEN 1 ELSE 0 + END +FROM read_text('__TEST_DIR__/openivm_upsert_queries_cspj_inti_mv.sql'); +---- +1 + +statement ok +INSERT INTO cspj_inti_isf VALUES (3, 300), (4, 400); + +statement ok +INSERT INTO cspj_inti_ml VALUES (300, 'machine-2'), (400, NULL); + +statement ok +DELETE FROM cspj_inti_isf WHERE id = 2; + +statement ok +UPDATE cspj_inti_ml SET res = NULL WHERE k = 100; + +statement ok +PRAGMA refresh('cspj_inti_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM cspj_inti_mv + EXCEPT ALL + ( + SELECT isf.id, isf.k FROM cspj_inti_isf isf LEFT JOIN cspj_inti_ml ml ON isf.k = ml.k WHERE ml.res IS NOT NULL + UNION ALL + SELECT isf.id, isf.k FROM cspj_inti_isf isf LEFT JOIN cspj_inti_ml ml ON isf.k = ml.k WHERE ml.res IS NULL + ) +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + ( + SELECT isf.id, isf.k FROM cspj_inti_isf isf LEFT JOIN cspj_inti_ml ml ON isf.k = ml.k WHERE ml.res IS NOT NULL + UNION ALL + SELECT isf.id, isf.k FROM cspj_inti_isf isf LEFT JOIN cspj_inti_ml ml ON isf.k = ml.k WHERE ml.res IS NULL + ) + EXCEPT ALL + SELECT * FROM cspj_inti_mv +); +---- +0 + +# Reduced "machine-status left-deep UNION" shape: a literal three-way +# (left-deep) UNION ALL of LEFT JOIN branches, each partitioned by a mutually +# exclusive predicate on the joined side (active / inactive / unmatched). +# Internally this compiles to a nested union whose own union is itself one +# arm of a further union (`(A UNION ALL B) UNION ALL C`), carrying duplicated +# `openivm_left_key`/`openivm_multiplicity` output columns at every level of +# the chain. This matches the shape of upstream `third_party/lpts` commit +# 754c797's own `test/sql/union.test` regression ("OpenIVM join-delta UNION +# terms carry ... a duplicated hidden left key and multiplicity. A left-deep +# UNION must preserve and bind all ... positions"); the fix retains a +# trailing rewritten UNION binding as an alias of its physical multiplicity +# output rather than losing it. Compiled with force_view_delta_cascade under +# target_dialect=spark, this must classify SIMPLE_PROJECTION -- never +# COMPILE_FAILED/FULL_REFRESH -- and a real batched multi-table +# CREATE + INSERT/DELETE/UPDATE + PRAGMA refresh must stay in bidirectional +# bag-equality with the view definition. +statement ok +CREATE TABLE cspj_ms_terms(event_time INT, machine_arm_id VARCHAR, customer_key INT); + +statement ok +CREATE TABLE cspj_ms_status(machine_arm_id VARCHAR, status_label VARCHAR); + +statement ok +INSERT INTO cspj_ms_terms VALUES (10, 'arm-1', 1), (20, 'arm-2', 6); + +statement ok +INSERT INTO cspj_ms_status VALUES ('arm-1', 'active'); + +statement ok +CREATE MATERIALIZED VIEW cspj_ms_mv AS + SELECT t.event_time, t.machine_arm_id, t.customer_key + FROM cspj_ms_terms t LEFT JOIN cspj_ms_status s ON t.machine_arm_id = s.machine_arm_id + WHERE s.status_label = 'active' + UNION ALL + SELECT t.event_time, t.machine_arm_id, t.customer_key + FROM cspj_ms_terms t LEFT JOIN cspj_ms_status s ON t.machine_arm_id = s.machine_arm_id + WHERE s.status_label = 'inactive' + UNION ALL + SELECT t.event_time, t.machine_arm_id, t.customer_key + FROM cspj_ms_terms t LEFT JOIN cspj_ms_status s ON t.machine_arm_id = s.machine_arm_id + WHERE s.status_label IS NULL; + +statement ok +INSERT INTO cspj_ms_terms VALUES (30, 'arm-3', 11); + +query IT +SELECT refresh_type, refresh_type_name +FROM openivm_compile_with_facts( + 'cspj_ms_mv', + '{"target_dialect":"spark","compile_only":true,"force_view_delta_cascade":true}' +) +LIMIT 1; +---- +2 SIMPLE_PROJECTION + +query I +SELECT CASE + WHEN contains(content, 'INSERT INTO openivm_delta_cspj_ms_mv') AND + contains(content, 'openivm_delta_cspj_ms_terms') AND + contains(content, 'openivm_delta_cspj_ms_status') AND + contains(content, 'UNION ALL') AND + NOT contains(content, 'SELECT NULL::') + THEN 1 ELSE 0 + END +FROM read_text('__TEST_DIR__/openivm_upsert_queries_cspj_ms_mv.sql'); +---- +1 + +statement ok +INSERT INTO cspj_ms_terms VALUES (40, 'arm-4', 16); + +statement ok +INSERT INTO cspj_ms_status VALUES ('arm-3', 'active'), ('arm-4', NULL); + +statement ok +DELETE FROM cspj_ms_terms WHERE event_time = 20; + +statement ok +UPDATE cspj_ms_status SET status_label = 'inactive' WHERE machine_arm_id = 'arm-1'; + +statement ok +PRAGMA refresh('cspj_ms_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM cspj_ms_mv + EXCEPT ALL + ( + SELECT t.event_time, t.machine_arm_id, t.customer_key + FROM cspj_ms_terms t LEFT JOIN cspj_ms_status s ON t.machine_arm_id = s.machine_arm_id + WHERE s.status_label = 'active' + UNION ALL + SELECT t.event_time, t.machine_arm_id, t.customer_key + FROM cspj_ms_terms t LEFT JOIN cspj_ms_status s ON t.machine_arm_id = s.machine_arm_id + WHERE s.status_label = 'inactive' + UNION ALL + SELECT t.event_time, t.machine_arm_id, t.customer_key + FROM cspj_ms_terms t LEFT JOIN cspj_ms_status s ON t.machine_arm_id = s.machine_arm_id + WHERE s.status_label IS NULL + ) +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + ( + SELECT t.event_time, t.machine_arm_id, t.customer_key + FROM cspj_ms_terms t LEFT JOIN cspj_ms_status s ON t.machine_arm_id = s.machine_arm_id + WHERE s.status_label = 'active' + UNION ALL + SELECT t.event_time, t.machine_arm_id, t.customer_key + FROM cspj_ms_terms t LEFT JOIN cspj_ms_status s ON t.machine_arm_id = s.machine_arm_id + WHERE s.status_label = 'inactive' + UNION ALL + SELECT t.event_time, t.machine_arm_id, t.customer_key + FROM cspj_ms_terms t LEFT JOIN cspj_ms_status s ON t.machine_arm_id = s.machine_arm_id + WHERE s.status_label IS NULL + ) + EXCEPT ALL + SELECT * FROM cspj_ms_mv +); +---- +0 diff --git a/test/sql/cascade_window_unscopable_delta.test b/test/sql/cascade_window_unscopable_delta.test new file mode 100644 index 00000000..38f93446 --- /dev/null +++ b/test/sql/cascade_window_unscopable_delta.test @@ -0,0 +1,529 @@ +# name: test/sql/cascade_window_unscopable_delta.test +# description: WINDOW_PARTITION plans whose partition keys cannot be scoped to a source delta still emit signed cascade deltas when facts.force_view_delta_cascade=true +# group: [sql] + +require openivm + +statement ok +SET openivm_files_path='__TEST_DIR__'; + +# ========================================================================== +# Case 1 - global surrogate key: ROW_NUMBER() OVER (ORDER BY ...) with no +# PARTITION BY. The classifier still selects WINDOW_PARTITION, but there are +# no partition columns to scope an affected-key set with, so the compiler +# falls back to a full recompute of the view body. That fallback must still +# honour an explicitly requested cascade view delta - otherwise every +# downstream materialized view observes "no delta" and gets demoted to a +# full refresh. +# ========================================================================== + +statement ok +CREATE TABLE uwd_src (id INT, grp VARCHAR, val INT); + +statement ok +INSERT INTO uwd_src VALUES + (1, 'a', 10), + (2, 'a', 20), + (3, 'b', 5), + (4, 'b', 15); + +statement ok +CREATE MATERIALIZED VIEW uwd_dim AS + SELECT CAST(ROW_NUMBER() OVER (ORDER BY grp, id) AS INT) AS dim_key, id, grp, val + FROM uwd_src; + +# The refresh type is unchanged by the cascade request: no relabeling, no +# demotion to FULL_REFRESH. +query IT +SELECT refresh_type, refresh_type_name +FROM openivm_compile_with_facts( + 'uwd_dim', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":false}' +) +LIMIT 1; +---- +5 WINDOW_PARTITION + +query IT +SELECT refresh_type, refresh_type_name +FROM openivm_compile_with_facts( + 'uwd_dim', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":true}' +) +LIMIT 1; +---- +5 WINDOW_PARTITION + +# Without a cascade request the program is the plain in-place recompute and +# must not write into the view's own delta table. +query I +SELECT CASE WHEN + string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%DELETE FROM %openivm_data_uwd_dim%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%INSERT INTO %openivm_data_uwd_dim%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%INSERT INTO openivm_delta_uwd_dim%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'uwd_dim', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":false}' +) +WHERE stmt_kind = 'data'; +---- +1 + +# With a cascade request the same recompute must be bracketed by old/new +# snapshots and publish the signed whole-view delta. +query I +SELECT CASE WHEN + string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%CREATE OR REPLACE TEMP TABLE openivm_old_uwd_dim AS%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%CREATE OR REPLACE TEMP TABLE openivm_new_uwd_dim AS%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%DELETE FROM %openivm_data_uwd_dim%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%INSERT INTO %openivm_data_uwd_dim%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%INSERT INTO openivm_delta_uwd_dim%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%CAST%-1%INTEGER%openivm_old_uwd_dim%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%UNION ALL%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%CAST%1%INTEGER%openivm_new_uwd_dim%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%DROP TABLE IF EXISTS openivm_old_uwd_dim%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%DROP TABLE IF EXISTS openivm_new_uwd_dim%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'uwd_dim', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":true}' +) +WHERE stmt_kind = 'data'; +---- +1 + +# ========================================================================== +# Case 2 - computed partition key: PARTITION BY lower(a) || '_' || lower(b) +# is not a column of any source delta table, so no partition delta spec can +# be built. Same requirement. +# ========================================================================== + +statement ok +CREATE TABLE uwd_os_src (os_name VARCHAR, os_sku VARCHAR); + +statement ok +INSERT INTO uwd_os_src VALUES ('linux', 'a'), ('windows', 'b'), ('linux', 'c'); + +statement ok +CREATE MATERIALIZED VIEW uwd_os_dim AS + SELECT CAST(ROW_NUMBER() OVER (ORDER BY os_full) AS INT) AS os_key, os_full, os_name, os_sku + FROM ( + SELECT lower(os_name) || '_' || lower(os_sku) AS os_full, os_name, os_sku, + ROW_NUMBER() OVER (PARTITION BY lower(os_name) || '_' || lower(os_sku) ORDER BY os_name) AS rn + FROM uwd_os_src + ) t + WHERE rn = 1; + +query IT +SELECT refresh_type, refresh_type_name +FROM openivm_compile_with_facts( + 'uwd_os_dim', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":true}' +) +LIMIT 1; +---- +5 WINDOW_PARTITION + +query I +SELECT CASE WHEN + string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%INSERT INTO openivm_delta_uwd_os_dim%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'uwd_os_dim', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":false}' +) +WHERE stmt_kind = 'data'; +---- +1 + +query I +SELECT CASE WHEN + string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%CREATE OR REPLACE TEMP TABLE openivm_old_uwd_os_dim AS%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%CREATE OR REPLACE TEMP TABLE openivm_new_uwd_os_dim AS%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%INSERT INTO openivm_delta_uwd_os_dim%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%CAST%-1%INTEGER%openivm_old_uwd_os_dim%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%CAST%1%INTEGER%openivm_new_uwd_os_dim%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%DROP TABLE IF EXISTS openivm_new_uwd_os_dim%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'uwd_os_dim', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":true}' +) +WHERE stmt_kind = 'data'; +---- +1 + +# ========================================================================== +# Case 3 - multi-source join whose partition key is a computed expression: +# the window partition lineage cannot cover every source delta table. +# ========================================================================== + +statement ok +CREATE TABLE uwd_inst (instance_id INT, sub_id INT, region VARCHAR); + +statement ok +CREATE TABLE uwd_sub (sub_id INT, sub_name VARCHAR); + +statement ok +INSERT INTO uwd_inst VALUES (1, 10, 'eus'), (2, 10, 'wus'), (3, 11, 'eus'); + +statement ok +INSERT INTO uwd_sub VALUES (10, 's10'), (11, 's11'); + +statement ok +CREATE MATERIALIZED VIEW uwd_target AS + SELECT CAST(ROW_NUMBER() OVER (PARTITION BY upper(i.region) ORDER BY i.instance_id) AS INT) AS rn, + i.instance_id, i.region, s.sub_name + FROM uwd_inst i JOIN uwd_sub s ON i.sub_id = s.sub_id; + +query IT +SELECT refresh_type, refresh_type_name +FROM openivm_compile_with_facts( + 'uwd_target', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":true}' +) +LIMIT 1; +---- +5 WINDOW_PARTITION + +query I +SELECT CASE WHEN + string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%CREATE OR REPLACE TEMP TABLE openivm_old_uwd_target AS%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%CREATE OR REPLACE TEMP TABLE openivm_new_uwd_target AS%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%INSERT INTO openivm_delta_uwd_target%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%CAST%-1%INTEGER%openivm_old_uwd_target%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%CAST%1%INTEGER%openivm_new_uwd_target%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'uwd_target', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":true}' +) +WHERE stmt_kind = 'data'; +---- +1 + +# ========================================================================== +# End-to-end: run the emitted cascade program for uwd_dim after a batch of +# conflicting DML (insert + update + delete applied before a single refresh) +# and prove (a) the signed delta reconstructs the new view content from the +# old content under bag semantics, and (b) a downstream materialized view +# refreshes incrementally off that delta and stays bidirectionally equal to +# its base query. PRAGMA refresh() uses the default CompileFacts (cascade +# off), so the cascade program is executed explicitly here exactly as the +# engine driver executes it. +# ========================================================================== + +statement ok +CREATE MATERIALIZED VIEW uwd_down AS + SELECT dim_key, grp, val FROM uwd_dim WHERE val > 6; + +statement ok +INSERT INTO uwd_src VALUES (5, 'a', 15), (6, 'c', 1); + +statement ok +DELETE FROM uwd_src WHERE id = 1; + +statement ok +UPDATE uwd_src SET val = 12 WHERE id = 3; + +statement ok +CREATE TABLE uwd_old_snapshot AS SELECT * FROM openivm_data_uwd_dim; + +statement ok +CREATE OR REPLACE TEMP TABLE openivm_old_uwd_dim AS +SELECT * FROM openivm_data_uwd_dim openivm_old; + +statement ok +CREATE OR REPLACE TEMP TABLE openivm_new_uwd_dim AS +SELECT * FROM ( + SELECT CAST(ROW_NUMBER() OVER (ORDER BY grp, id) AS INT) AS dim_key, id, grp, val + FROM uwd_src +) openivm_recompute; + +statement ok +DELETE FROM openivm_data_uwd_dim; + +statement ok +INSERT INTO openivm_data_uwd_dim +SELECT * FROM openivm_new_uwd_dim; + +statement ok +INSERT INTO openivm_delta_uwd_dim +SELECT *, CAST(-1 AS INTEGER), CURRENT_TIMESTAMP FROM openivm_old_uwd_dim +UNION ALL +SELECT *, CAST(1 AS INTEGER), CURRENT_TIMESTAMP FROM openivm_new_uwd_dim; + +statement ok +DROP TABLE IF EXISTS openivm_old_uwd_dim; + +statement ok +DROP TABLE IF EXISTS openivm_new_uwd_dim; + +# The retraction leg is exactly the pre-refresh content (bag equality). +query I +SELECT COUNT(*) FROM ( + SELECT dim_key, id, grp, val FROM openivm_delta_uwd_dim WHERE openivm_multiplicity = -1 + EXCEPT ALL + SELECT dim_key, id, grp, val FROM uwd_old_snapshot +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT dim_key, id, grp, val FROM uwd_old_snapshot + EXCEPT ALL + SELECT dim_key, id, grp, val FROM openivm_delta_uwd_dim WHERE openivm_multiplicity = -1 +); +---- +0 + +# Applying the signed delta to the old content reproduces the new content +# exactly, in both directions. +query I +WITH applied AS ( + ( + SELECT dim_key, id, grp, val FROM uwd_old_snapshot + UNION ALL + SELECT dim_key, id, grp, val FROM openivm_delta_uwd_dim WHERE openivm_multiplicity = 1 + ) + EXCEPT ALL + SELECT dim_key, id, grp, val FROM openivm_delta_uwd_dim WHERE openivm_multiplicity = -1 +) +SELECT COUNT(*) FROM ( + SELECT dim_key, id, grp, val FROM applied + EXCEPT ALL + SELECT dim_key, id, grp, val FROM openivm_data_uwd_dim +); +---- +0 + +query I +WITH applied AS ( + ( + SELECT dim_key, id, grp, val FROM uwd_old_snapshot + UNION ALL + SELECT dim_key, id, grp, val FROM openivm_delta_uwd_dim WHERE openivm_multiplicity = 1 + ) + EXCEPT ALL + SELECT dim_key, id, grp, val FROM openivm_delta_uwd_dim WHERE openivm_multiplicity = -1 +) +SELECT COUNT(*) FROM ( + SELECT dim_key, id, grp, val FROM openivm_data_uwd_dim + EXCEPT ALL + SELECT dim_key, id, grp, val FROM applied +); +---- +0 + +# The refreshed view itself matches its base query. +query I +SELECT COUNT(*) FROM ( + SELECT dim_key, id, grp, val FROM openivm_data_uwd_dim + EXCEPT ALL + SELECT CAST(ROW_NUMBER() OVER (ORDER BY grp, id) AS INT) AS dim_key, id, grp, val FROM uwd_src +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT CAST(ROW_NUMBER() OVER (ORDER BY grp, id) AS INT) AS dim_key, id, grp, val FROM uwd_src + EXCEPT ALL + SELECT dim_key, id, grp, val FROM openivm_data_uwd_dim +); +---- +0 + +# The downstream view consumes the cascade delta incrementally and is +# bidirectionally equal to its base query. +statement ok +PRAGMA refresh('uwd_down'); + +query I +SELECT COUNT(*) FROM ( + SELECT dim_key, grp, val FROM openivm_data_uwd_down + EXCEPT ALL + SELECT dim_key, grp, val FROM ( + SELECT CAST(ROW_NUMBER() OVER (ORDER BY grp, id) AS INT) AS dim_key, id, grp, val FROM uwd_src + ) d + WHERE val > 6 +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT dim_key, grp, val FROM ( + SELECT CAST(ROW_NUMBER() OVER (ORDER BY grp, id) AS INT) AS dim_key, id, grp, val FROM uwd_src + ) d + WHERE val > 6 + EXCEPT ALL + SELECT dim_key, grp, val FROM openivm_data_uwd_down +); +---- +0 + +# ========================================================================== +# Duplicate-heavy view: the signed delta must carry exact multiplicities, +# not a de-duplicated set. +# ========================================================================== + +statement ok +CREATE TABLE uwd_bag_src (grp VARCHAR, val INT); + +statement ok +INSERT INTO uwd_bag_src VALUES ('a', 1), ('a', 1), ('a', 1), ('b', 2), ('b', 2); + +statement ok +CREATE MATERIALIZED VIEW uwd_bag_mv AS + SELECT grp, val, CAST(SUM(val) OVER () AS INT) AS total FROM uwd_bag_src; + +query IT +SELECT refresh_type, refresh_type_name +FROM openivm_compile_with_facts( + 'uwd_bag_mv', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":true}' +) +LIMIT 1; +---- +5 WINDOW_PARTITION + +query I +SELECT CASE WHEN + string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%INSERT INTO openivm_delta_uwd_bag_mv%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%CAST%-1%INTEGER%openivm_old_uwd_bag_mv%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%CAST%1%INTEGER%openivm_new_uwd_bag_mv%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'uwd_bag_mv', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":true}' +) +WHERE stmt_kind = 'data'; +---- +1 + +statement ok +INSERT INTO uwd_bag_src VALUES ('a', 1), ('c', 3); + +statement ok +CREATE TABLE uwd_bag_old AS SELECT * FROM openivm_data_uwd_bag_mv; + +statement ok +CREATE OR REPLACE TEMP TABLE openivm_old_uwd_bag_mv AS +SELECT * FROM openivm_data_uwd_bag_mv openivm_old; + +statement ok +CREATE OR REPLACE TEMP TABLE openivm_new_uwd_bag_mv AS +SELECT * FROM ( + SELECT grp, val, CAST(SUM(val) OVER () AS INT) AS total FROM uwd_bag_src +) openivm_recompute; + +statement ok +DELETE FROM openivm_data_uwd_bag_mv; + +statement ok +INSERT INTO openivm_data_uwd_bag_mv +SELECT * FROM openivm_new_uwd_bag_mv; + +statement ok +INSERT INTO openivm_delta_uwd_bag_mv +SELECT *, CAST(-1 AS INTEGER), CURRENT_TIMESTAMP FROM openivm_old_uwd_bag_mv +UNION ALL +SELECT *, CAST(1 AS INTEGER), CURRENT_TIMESTAMP FROM openivm_new_uwd_bag_mv; + +statement ok +DROP TABLE IF EXISTS openivm_old_uwd_bag_mv; + +statement ok +DROP TABLE IF EXISTS openivm_new_uwd_bag_mv; + +# Three identical ('a', 1, 5) rows must be retracted three times, and four +# identical ('a', 1, 9) rows must be inserted four times. +query III +SELECT grp, val, COUNT(*) FROM openivm_delta_uwd_bag_mv +WHERE openivm_multiplicity = -1 AND grp = 'a' +GROUP BY grp, val; +---- +a 1 3 + +query III +SELECT grp, val, COUNT(*) FROM openivm_delta_uwd_bag_mv +WHERE openivm_multiplicity = 1 AND grp = 'a' +GROUP BY grp, val; +---- +a 1 4 + +query I +WITH applied AS ( + ( + SELECT grp, val, total FROM uwd_bag_old + UNION ALL + SELECT grp, val, total FROM openivm_delta_uwd_bag_mv WHERE openivm_multiplicity = 1 + ) + EXCEPT ALL + SELECT grp, val, total FROM openivm_delta_uwd_bag_mv WHERE openivm_multiplicity = -1 +) +SELECT COUNT(*) FROM ( + SELECT grp, val, total FROM applied + EXCEPT ALL + SELECT grp, val, total FROM openivm_data_uwd_bag_mv +); +---- +0 + +query I +WITH applied AS ( + ( + SELECT grp, val, total FROM uwd_bag_old + UNION ALL + SELECT grp, val, total FROM openivm_delta_uwd_bag_mv WHERE openivm_multiplicity = 1 + ) + EXCEPT ALL + SELECT grp, val, total FROM openivm_delta_uwd_bag_mv WHERE openivm_multiplicity = -1 +) +SELECT COUNT(*) FROM ( + SELECT grp, val, total FROM openivm_data_uwd_bag_mv + EXCEPT ALL + SELECT grp, val, total FROM applied +); +---- +0 + +# ========================================================================== +# Requesting a cascade delta must not relabel or rescue an unsupported plan. +# A window view whose predicate is non-deterministic (the same guard that +# demotes production predicates built on current_date()) stays FULL_REFRESH, +# and an unknown view still fails explicitly. +# ========================================================================== + +statement ok +CREATE TABLE uwd_vol_src (id INT, event_date DATE, val INT); + +statement ok +INSERT INTO uwd_vol_src VALUES (1, DATE '2024-01-01', 10); + +statement ok +CREATE MATERIALIZED VIEW uwd_vol_mv AS + SELECT CAST(ROW_NUMBER() OVER (ORDER BY id) AS INT) AS k, id, event_date, val + FROM uwd_vol_src + WHERE val > random(); + +query IT +SELECT refresh_type, refresh_type_name +FROM openivm_compile_with_facts( + 'uwd_vol_mv', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":true}' +) +LIMIT 1; +---- +3 FULL_REFRESH + +statement error +SELECT * FROM openivm_compile_with_facts( + 'uwd_does_not_exist', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":true}' +); +---- +materialized view 'uwd_does_not_exist' not found diff --git a/test/sql/compile_refresh.test b/test/sql/compile_refresh.test index 531b4181..95afb325 100644 --- a/test/sql/compile_refresh.test +++ b/test/sql/compile_refresh.test @@ -387,6 +387,212 @@ WHERE stmt_kind = 'data'; ---- 1 +# ========================================== +# Test 10: reduced "arc_machine_status_transaction" shape — a CTE joining an +# INNER JOIN with a LEFT JOIN, whose result feeds an outer query with two +# further chained LEFT JOINs. Before src/delta/operators/join.cpp's +# AppendMultiplicityToAncestorProjectionMaps fix, a deeper join's own +# left_projection_map growing (to carry a new per-leaf multiplicity column up +# to the root) silently shifted where its right-side contribution begins in +# its own combined column numbering. A grandparent join's pre-existing +# projection-map entry — fixed before that growth happened — could then +# alias onto the just-added multiplicity column instead of the real column +# it used to select, permanently dropping a business column and surfacing +# as an internal arity mismatch ("union lhs column ref not in column_map"). +# ========================================== + +statement ok +CREATE TABLE arcm_msf(id INT, coll_key INT); + +statement ok +CREATE TABLE arcm_acd(coll_key INT, sub_id INT, arm_id VARCHAR); + +statement ok +CREATE TABLE arcm_hw(arm_id VARCHAR, provider VARCHAR); + +statement ok +CREATE TABLE arcm_tpid(sub_id INT, tp_id INT); + +statement ok +CREATE TABLE arcm_cust(tp_id INT, cust_key INT); + +statement ok +INSERT INTO arcm_msf VALUES (1, 100); + +statement ok +INSERT INTO arcm_acd VALUES (100, 10, 'arm-1'); + +statement ok +INSERT INTO arcm_hw VALUES ('arm-1', 'azure'); + +statement ok +INSERT INTO arcm_tpid VALUES (10, 200); + +statement ok +INSERT INTO arcm_cust VALUES (200, 21); + +statement ok +CREATE MATERIALIZED VIEW mv_arcm AS + WITH bff AS ( + SELECT msf.id, acd.sub_id, + coalesce(hw.provider, 'N/A') AS provider + FROM arcm_msf msf + INNER JOIN arcm_acd acd ON msf.coll_key = acd.coll_key + LEFT JOIN arcm_hw hw ON acd.arm_id = hw.arm_id + ) + SELECT bff.id, + coalesce(cust.cust_key, 1) AS cust_key + FROM bff + LEFT JOIN arcm_tpid tpid ON bff.sub_id = tpid.sub_id + LEFT JOIN arcm_cust cust ON coalesce(tpid.tp_id, -1) = cust.tp_id; + +statement ok +INSERT INTO arcm_msf VALUES (2, 100); + +# Must compile to a real incremental cascade delta (SIMPLE_PROJECTION), never +# FULL_REFRESH, and must not lose the `cust_key` column from the final insert. +query IT +SELECT refresh_type, refresh_type_name +FROM openivm_compile_with_facts( + 'mv_arcm', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":true}' +) +LIMIT 1; +---- +2 SIMPLE_PROJECTION + +query I +SELECT CASE WHEN + string_agg(sql, ' ' ORDER BY stmt_order) + LIKE '%INSERT INTO openivm_delta_mv_arcm (id, cust_key, openivm_multiplicity)%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%openivm_delta_arcm_msf%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%openivm_delta_arcm_acd%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%openivm_delta_arcm_hw%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%openivm_delta_arcm_tpid%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%openivm_delta_arcm_cust%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'mv_arcm', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":true}' +) +WHERE stmt_kind = 'data'; +---- +1 + +# Batched multi-leaf delta variant of the same shape: simultaneous +# inserts/delete/update spread across ALL FIVE base tables (not just a +# single-row insert into one leaf). This is the exact reduction that exposed +# a second, deeper defect in src/delta/operators/join.cpp's +# BuildInclusionExclusionTerms: leaf substitution reused a stale pointer into +# the ORIGINAL (pre-mask-renumbering) plan instead of the mask's own +# freshly-renumbered leaf, so a leaf feeding two joins (msf join acd AND acd +# join hw) got its delta substituted at the wrong table_index, leaving the +# second join's condition dangling and surfacing as +# LPTS_UNSUPPORTED_COLUMN_REF once more than one leaf changed at once. +statement ok +INSERT INTO arcm_msf VALUES (3, 300); + +statement ok +INSERT INTO arcm_acd VALUES (300, 30, 'arm-2'); + +statement ok +INSERT INTO arcm_hw VALUES ('arm-2', 'gcp'); + +statement ok +INSERT INTO arcm_tpid VALUES (30, 400); + +statement ok +DELETE FROM arcm_msf WHERE id = 2; + +statement ok +UPDATE arcm_cust SET cust_key = 99 WHERE tp_id = 200; + +query IT +SELECT refresh_type, refresh_type_name +FROM openivm_compile_with_facts( + 'mv_arcm', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":true}' +) +LIMIT 1; +---- +2 SIMPLE_PROJECTION + +query I +SELECT CASE WHEN + string_agg(sql, ' ' ORDER BY stmt_order) + LIKE '%INSERT INTO openivm_delta_mv_arcm (id, cust_key, openivm_multiplicity)%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%openivm_delta_arcm_msf%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%openivm_delta_arcm_acd%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%openivm_delta_arcm_hw%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%openivm_delta_arcm_tpid%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%openivm_delta_arcm_cust%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'mv_arcm', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":true}' +) +WHERE stmt_kind = 'data'; +---- +1 + +# ========================================== +# Test 11: reduced "int_instance_status_transaction" shape — a UNION ALL of +# two LEFT JOIN branches partitioned by an IS NOT NULL / IS NULL predicate on +# the joined side. Before the `third_party/lpts` pin was advanced to commit +# 77ed5e5 (fork PR #18, "Fix set-op column binding remaps"), remapping +# duplicated UNION ALL key-projection output aliases could leave a stale +# reference to a child alias that no longer existed in the rewritten plan, +# surfacing as a runtime Binder Error ("Referenced column ... not found in +# FROM clause"). +# ========================================== + +statement ok +CREATE TABLE inti_isf(id INT, k INT); + +statement ok +CREATE TABLE inti_ml(k INT, res VARCHAR); + +statement ok +INSERT INTO inti_isf VALUES (1, 100), (2, 200); + +statement ok +INSERT INTO inti_ml VALUES (100, 'machine-1'); + +statement ok +CREATE MATERIALIZED VIEW mv_inti AS + SELECT isf.id, isf.k FROM inti_isf isf LEFT JOIN inti_ml ml ON isf.k = ml.k WHERE ml.res IS NOT NULL + UNION ALL + SELECT isf.id, isf.k FROM inti_isf isf LEFT JOIN inti_ml ml ON isf.k = ml.k WHERE ml.res IS NULL; + +statement ok +INSERT INTO inti_isf VALUES (3, 300); + +query IT +SELECT refresh_type, refresh_type_name +FROM openivm_compile_with_facts( + 'mv_inti', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":true}' +) +LIMIT 1; +---- +2 SIMPLE_PROJECTION + +query I +SELECT CASE WHEN + string_agg(sql, ' ' ORDER BY stmt_order) + LIKE '%INSERT INTO openivm_delta_mv_inti (id, k, openivm_left_key, openivm_multiplicity)%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%openivm_delta_inti_isf%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%openivm_delta_inti_ml%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%UNION ALL%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'mv_inti', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":true}' +) +WHERE stmt_kind = 'data'; +---- +1 + # ========================================== # Test 9: openivm_compile_with_facts on a non-existent view fails cleanly # ========================================== diff --git a/test/sql/compile_spark_dialect_hardening.test b/test/sql/compile_spark_dialect_hardening.test index 246eefd3..8d4b180b 100644 --- a/test/sql/compile_spark_dialect_hardening.test +++ b/test/sql/compile_spark_dialect_hardening.test @@ -195,3 +195,260 @@ WHERE stmt_kind = 'data'; statement ok SET openivm_refresh_mode = 'incremental'; + +# ========================================== +# Spark hardening for the reduced "arc_machine_status_transaction" shape: a +# CTE (INNER JOIN + LEFT JOIN) feeding an outer query with two further +# chained LEFT JOINs must still compile to a real incremental cascade delta +# under target_dialect=spark, with a spark-portable (no `::`) cast and +# backtick-quoted identifiers, and without dropping the `cust_key` column +# from the final signed insert — the regression this shape covers is in +# src/delta/operators/join.cpp's AppendMultiplicityToAncestorProjectionMaps, +# not dialect-specific, but the original failure was first observed against +# a spark-targeted compile. +# ========================================== +statement ok +CREATE TABLE sph_arcm_msf(id INT, coll_key INT); + +statement ok +CREATE TABLE sph_arcm_acd(coll_key INT, sub_id INT, arm_id VARCHAR); + +statement ok +CREATE TABLE sph_arcm_hw(arm_id VARCHAR, provider VARCHAR); + +statement ok +CREATE TABLE sph_arcm_tpid(sub_id INT, tp_id INT); + +statement ok +CREATE TABLE sph_arcm_cust(tp_id INT, cust_key INT); + +statement ok +INSERT INTO sph_arcm_msf VALUES (1, 100); + +statement ok +INSERT INTO sph_arcm_acd VALUES (100, 10, 'arm-1'); + +statement ok +INSERT INTO sph_arcm_hw VALUES ('arm-1', 'azure'); + +statement ok +INSERT INTO sph_arcm_tpid VALUES (10, 200); + +statement ok +INSERT INTO sph_arcm_cust VALUES (200, 21); + +statement ok +CREATE MATERIALIZED VIEW sph_arcm_mv AS + WITH bff AS ( + SELECT msf.id, acd.sub_id, + coalesce(hw.provider, 'N/A') AS provider + FROM sph_arcm_msf msf + INNER JOIN sph_arcm_acd acd ON msf.coll_key = acd.coll_key + LEFT JOIN sph_arcm_hw hw ON acd.arm_id = hw.arm_id + ) + SELECT bff.id, + coalesce(cust.cust_key, 1) AS cust_key + FROM bff + LEFT JOIN sph_arcm_tpid tpid ON bff.sub_id = tpid.sub_id + LEFT JOIN sph_arcm_cust cust ON coalesce(tpid.tp_id, -1) = cust.tp_id; + +statement ok +INSERT INTO sph_arcm_msf VALUES (2, 100); + +query IT +SELECT refresh_type, refresh_type_name +FROM openivm_compile_with_facts( + 'sph_arcm_mv', + '{"target_dialect":"spark","compile_only":true,"force_view_delta_cascade":true}' +) +LIMIT 1; +---- +2 SIMPLE_PROJECTION + +query I +SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%::%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%`%' + AND string_agg(sql, ' ' ORDER BY stmt_order) + LIKE '%INSERT INTO openivm_delta_sph_arcm_mv (id, cust_key, openivm_multiplicity)%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'sph_arcm_mv', + '{"target_dialect":"spark","compile_only":true,"force_view_delta_cascade":true}' +) +WHERE stmt_kind = 'data'; +---- +1 + +# ========================================== +# Spark hardening for the reduced "int_instance_status_transaction" shape: a +# UNION ALL of two LEFT JOIN branches partitioned by an IS NOT NULL / IS NULL +# predicate on the joined side must still compile cleanly under +# target_dialect=spark without a stale child-alias reference leaking into the +# emitted SQL (the failure this shape covers, fixed upstream in +# `third_party/lpts` at commit 77ed5e5, surfaced as a Binder Error referencing +# a column that no longer existed in the rewritten plan). +# ========================================== +statement ok +CREATE TABLE sph_inti_isf(id INT, k INT); + +statement ok +CREATE TABLE sph_inti_ml(k INT, res VARCHAR); + +statement ok +INSERT INTO sph_inti_isf VALUES (1, 100), (2, 200); + +statement ok +INSERT INTO sph_inti_ml VALUES (100, 'machine-1'); + +statement ok +CREATE MATERIALIZED VIEW sph_inti_mv AS + SELECT isf.id, isf.k FROM sph_inti_isf isf LEFT JOIN sph_inti_ml ml ON isf.k = ml.k WHERE ml.res IS NOT NULL + UNION ALL + SELECT isf.id, isf.k FROM sph_inti_isf isf LEFT JOIN sph_inti_ml ml ON isf.k = ml.k WHERE ml.res IS NULL; + +statement ok +INSERT INTO sph_inti_isf VALUES (3, 300); + +query IT +SELECT refresh_type, refresh_type_name +FROM openivm_compile_with_facts( + 'sph_inti_mv', + '{"target_dialect":"spark","compile_only":true,"force_view_delta_cascade":true}' +) +LIMIT 1; +---- +2 SIMPLE_PROJECTION + +query I +SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%::%' + AND string_agg(sql, ' ' ORDER BY stmt_order) + LIKE '%INSERT INTO openivm_delta_sph_inti_mv (id, k, openivm_left_key, openivm_multiplicity)%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%UNION ALL%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'sph_inti_mv', + '{"target_dialect":"spark","compile_only":true,"force_view_delta_cascade":true}' +) +WHERE stmt_kind = 'data'; +---- +1 + +# ========================================== +# Spark hardening for the "bounded HUGEINT" canary: a plain (non-aggregate) +# projection that widens a BIGINT to HUGEINT via COALESCE(CAST(...), 0) must +# still compile to a real incremental cascade delta under +# target_dialect=spark. Before `third_party/lpts` was advanced to commit +# 754c797 ("Fix bounded Spark HUGEINT and UNION aliases"), rendering ANY cast +# to HUGEINT unconditionally raised LPTS_UNSUPPORTED_TYPE for Spark, even +# when the source type (BIGINT here) provably fits within Spark's +# DECIMAL(38,0); the fix maps a provably-bounded HUGEINT cast/literal to +# DECIMAL(38,0) instead of failing the compile. Note a literal +# COALESCE(SUM(...), 0) is classified GROUP_RECOMPUTE by OpenIVM (bypassing +# LPTS entirely, echoing native SQL), so this canary uses an equivalent +# non-aggregate COALESCE(CAST(... AS HUGEINT), 0) shape that is classified +# SIMPLE_PROJECTION and does go through LPTS. +# ========================================== +statement ok +CREATE TABLE sph_hugeint_src(id INT, amount BIGINT); + +statement ok +INSERT INTO sph_hugeint_src VALUES (1, 5000000000), (2, NULL); + +statement ok +CREATE MATERIALIZED VIEW sph_hugeint_mv AS + SELECT id, COALESCE(CAST(amount AS HUGEINT), 0) AS wide_amount + FROM sph_hugeint_src; + +statement ok +INSERT INTO sph_hugeint_src VALUES (3, 2000000000); + +query IT +SELECT refresh_type, refresh_type_name +FROM openivm_compile_with_facts( + 'sph_hugeint_mv', + '{"target_dialect":"spark","compile_only":true,"force_view_delta_cascade":true}' +) +LIMIT 1; +---- +2 SIMPLE_PROJECTION + +query I +SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%DECIMAL(38,0)%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '% HUGEINT%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'sph_hugeint_mv', + '{"target_dialect":"spark","compile_only":true,"force_view_delta_cascade":true}' +) +WHERE stmt_kind = 'data'; +---- +1 + +# ========================================== +# Spark hardening for the "machine-status left-deep UNION" canary: a literal +# three-way (left-deep) UNION ALL of LEFT JOIN branches, each partitioned by a +# mutually exclusive predicate on the joined side, must still compile to a +# real incremental cascade delta under target_dialect=spark, correctly +# binding every column -- including the trailing duplicated +# `openivm_left_key`/`openivm_multiplicity` output columns -- at every level +# of the left-deep chain (internally: a nested union node whose own union is +# itself one arm of a further union, e.g. `(A UNION ALL B) UNION ALL C`). +# This is upstream `third_party/lpts` commit 754c797's second fix ("retain +# trailing rewritten UNION bindings as aliases of their physical multiplicity +# output"); unlike the two-way `sph_inti_mv` shape above, this exercises an +# actual left-deep union chain, matching the shape of LPTS's own +# `test/sql/union.test` regression (a left-deep "machine status" UNION ALL +# carrying duplicated key/multiplicity output columns). +# ========================================== +statement ok +CREATE TABLE sph_ms_terms(event_time INT, machine_arm_id VARCHAR, customer_key INT); + +statement ok +CREATE TABLE sph_ms_status(machine_arm_id VARCHAR, status_label VARCHAR); + +statement ok +INSERT INTO sph_ms_terms VALUES (10, 'arm-1', 1), (20, 'arm-2', 6); + +statement ok +INSERT INTO sph_ms_status VALUES ('arm-1', 'active'); + +statement ok +CREATE MATERIALIZED VIEW sph_ms_mv AS + SELECT t.event_time, t.machine_arm_id, t.customer_key + FROM sph_ms_terms t LEFT JOIN sph_ms_status s ON t.machine_arm_id = s.machine_arm_id + WHERE s.status_label = 'active' + UNION ALL + SELECT t.event_time, t.machine_arm_id, t.customer_key + FROM sph_ms_terms t LEFT JOIN sph_ms_status s ON t.machine_arm_id = s.machine_arm_id + WHERE s.status_label = 'inactive' + UNION ALL + SELECT t.event_time, t.machine_arm_id, t.customer_key + FROM sph_ms_terms t LEFT JOIN sph_ms_status s ON t.machine_arm_id = s.machine_arm_id + WHERE s.status_label IS NULL; + +statement ok +INSERT INTO sph_ms_terms VALUES (30, 'arm-3', 11); + +query IT +SELECT refresh_type, refresh_type_name +FROM openivm_compile_with_facts( + 'sph_ms_mv', + '{"target_dialect":"spark","compile_only":true,"force_view_delta_cascade":true}' +) +LIMIT 1; +---- +2 SIMPLE_PROJECTION + +query I +SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%::%' + AND string_agg(sql, ' ' ORDER BY stmt_order) + LIKE '%INSERT INTO openivm_delta_sph_ms_mv (event_time, machine_arm_id, customer_key, openivm_left_key, openivm_multiplicity)%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%UNION ALL%UNION ALL%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'sph_ms_mv', + '{"target_dialect":"spark","compile_only":true,"force_view_delta_cascade":true}' +) +WHERE stmt_kind = 'data'; +---- +1 diff --git a/test/sql/left_join_regular_nterm.test b/test/sql/left_join_regular_nterm.test new file mode 100644 index 00000000..5b4177d3 --- /dev/null +++ b/test/sql/left_join_regular_nterm.test @@ -0,0 +1,161 @@ +# name: test/sql/left_join_regular_nterm.test +# description: Compile-only N-term telescoping delta for LEFT-JOIN SIMPLE_PROJECTION views (openivm_regular_nterm_left) +# group: [sql] + +# A deep LEFT-join star that would blow up under 2^N inclusion-exclusion must +# compile to a linear delta and stay bag-correct. + +require openivm + +statement ok +SET openivm_files_path='__TEST_DIR__'; + +statement ok +SET openivm_regular_nterm_left=true; + +# ========================================== +# Star schema: 1 fact + 5 LEFT-joined dimensions, projection (no aggregation). +# ========================================== + +statement ok +CREATE TABLE fact(id INTEGER, amount INTEGER, k1 INTEGER, k2 INTEGER, k3 INTEGER, k4 INTEGER, k5 INTEGER); + +statement ok +CREATE TABLE d1(k INTEGER, v VARCHAR); + +statement ok +CREATE TABLE d2(k INTEGER, v VARCHAR); + +statement ok +CREATE TABLE d3(k INTEGER, v VARCHAR); + +statement ok +CREATE TABLE d4(k INTEGER, v VARCHAR); + +statement ok +CREATE TABLE d5(k INTEGER, v VARCHAR); + +statement ok +INSERT INTO d1 VALUES (11, 'd1a'), (21, 'd1b'); + +statement ok +INSERT INTO d2 VALUES (12, 'd2a'), (22, 'd2b'); + +statement ok +INSERT INTO d3 VALUES (13, 'd3a'), (23, 'd3b'); + +statement ok +INSERT INTO d4 VALUES (14, 'd4a'), (24, 'd4b'); + +statement ok +INSERT INTO d5 VALUES (15, 'd5a'), (25, 'd5b'); + +# Row 2 has k1=91 which has NO matching d1 yet (NULL-padded until d1 gets 91). +statement ok +INSERT INTO fact VALUES + (1, 100, 11, 12, 13, 14, 15), + (2, 200, 91, 12, 13, 14, 15), + (3, 300, 21, 22, 23, 24, 25), + (4, 400, 11, 22, 13, 24, 15); + +statement ok +CREATE MATERIALIZED VIEW mv AS + SELECT f.id, f.amount, + d1.v AS v1, d2.v AS v2, d3.v AS v3, d4.v AS v4, d5.v AS v5 + FROM fact f + LEFT JOIN d1 ON f.k1 = d1.k + LEFT JOIN d2 ON f.k2 = d2.k + LEFT JOIN d3 ON f.k3 = d3.k + LEFT JOIN d4 ON f.k4 = d4.k + LEFT JOIN d5 ON f.k5 = d5.k; + +# Snapshot the MV so we can prove openivm_compile_with_facts does not mutate it. +statement ok +CREATE TABLE mv_before AS SELECT * FROM mv; + +# ========================================== +# Mixed DML batch: NULL->match (d1 gains key 91), match->NULL (d2 loses key 22), +# a dimension value update, and fact insert/update/delete. +# ========================================== + +statement ok +INSERT INTO d1 VALUES (91, 'd1_late'); + +statement ok +DELETE FROM d2 WHERE k = 22; + +statement ok +UPDATE d3 SET v = 'd3_upd' WHERE k = 13; + +statement ok +UPDATE fact SET amount = amount + 5 WHERE id = 1; + +statement ok +INSERT INTO fact VALUES (5, 500, 91, 22, 13, 14, 15); + +statement ok +DELETE FROM fact WHERE id = 3; + +# ========================================== +# Test 1: compile-only path emits a SIMPLE_PROJECTION join delta into +# openivm_delta_mv without exploding (a LEFT star that would be 2^6-1 under +# inclusion-exclusion compiles to a linear N-term telescoping delta). +# ========================================== + +query I +SELECT COUNT(*) > 0 +FROM openivm_compile_with_facts( + 'mv', + '{"target_dialect":"duckdb","compile_only":true, + "delta_shape":{"fact":"MIXED","d1":"INSERT_ONLY","d2":"MIXED","d3":"MIXED","d4":"UNCHANGED","d5":"UNCHANGED"}}' +) +WHERE stmt_kind = 'data' AND refresh_type_name = 'SIMPLE_PROJECTION' + AND sql LIKE '%openivm_delta_mv%'; +---- +true + +# Test 2: the compile call left the materialized view untouched. +query I +SELECT (SELECT count(*) FROM ((SELECT * FROM mv) EXCEPT ALL (SELECT * FROM mv_before))) = 0 AND + (SELECT count(*) FROM ((SELECT * FROM mv_before) EXCEPT ALL (SELECT * FROM mv))) = 0 AS unchanged; +---- +true + +# ========================================== +# Test 3: a real incremental refresh applies the batched deltas and the MV is +# bag-equal to the fully recomputed base query in BOTH directions (full IVM +# correctness cross-check, including the NULL<->match transition rows). +# ========================================== + +statement ok +PRAGMA refresh('mv'); + +query I +SELECT count(*) FROM ( + SELECT f.id, f.amount, d1.v AS v1, d2.v AS v2, d3.v AS v3, d4.v AS v4, d5.v AS v5 + FROM fact f + LEFT JOIN d1 ON f.k1 = d1.k + LEFT JOIN d2 ON f.k2 = d2.k + LEFT JOIN d3 ON f.k3 = d3.k + LEFT JOIN d4 ON f.k4 = d4.k + LEFT JOIN d5 ON f.k5 = d5.k + EXCEPT ALL + SELECT id, amount, v1, v2, v3, v4, v5 FROM mv +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, amount, v1, v2, v3, v4, v5 FROM mv + EXCEPT ALL + SELECT f.id, f.amount, d1.v AS v1, d2.v AS v2, d3.v AS v3, d4.v AS v4, d5.v AS v5 + FROM fact f + LEFT JOIN d1 ON f.k1 = d1.k + LEFT JOIN d2 ON f.k2 = d2.k + LEFT JOIN d3 ON f.k3 = d3.k + LEFT JOIN d4 ON f.k4 = d4.k + LEFT JOIN d5 ON f.k5 = d5.k +); +---- +0 diff --git a/test/sql/projected_window_partition.test b/test/sql/projected_window_partition.test new file mode 100644 index 00000000..8f0cff6f --- /dev/null +++ b/test/sql/projected_window_partition.test @@ -0,0 +1,94 @@ +# name: test/sql/projected_window_partition.test +# description: Window keys omitted from the MV output never become target predicates +# group: [sql] + +require openivm + +statement ok +SET openivm_files_path='__TEST_DIR__'; + +statement ok +CREATE TABLE pwp_src(id INTEGER, grp INTEGER, score INTEGER); + +statement ok +INSERT INTO pwp_src VALUES (1, 10, 20), (2, 10, 30), (3, 20, 10), (4, 20, 40); + +statement ok +CREATE MATERIALIZED VIEW pwp_mv AS +SELECT id, score, ROW_NUMBER() OVER (PARTITION BY grp ORDER BY score, id) AS rn +FROM pwp_src; + +query IT +SELECT refresh_type, refresh_type_name +FROM openivm_compile_with_facts( + 'pwp_mv', + '{"target_dialect":"spark","compile_only":true,"force_view_delta_cascade":true}' +) +LIMIT 1; +---- +5 WINDOW_PARTITION + +query I +SELECT bool_or(sql LIKE '%INSERT INTO openivm_delta_pwp_mv%') +FROM openivm_compile_with_facts( + 'pwp_mv', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":true}' +) +WHERE stmt_kind = 'data'; +---- +true + +statement ok +CREATE MATERIALIZED VIEW pwp_down AS SELECT id, score FROM pwp_mv WHERE rn = 1; + +statement ok +INSERT INTO pwp_src VALUES (5, 10, 5), (6, 20, 25); + +statement ok +UPDATE pwp_src SET grp = 20, score = 15 WHERE id = 2; + +statement ok +DELETE FROM pwp_src WHERE id = 3; + +statement ok +PRAGMA refresh('pwp_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM pwp_mv + EXCEPT ALL + SELECT id, score, ROW_NUMBER() OVER (PARTITION BY grp ORDER BY score, id) AS rn FROM pwp_src +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT id, score, ROW_NUMBER() OVER (PARTITION BY grp ORDER BY score, id) AS rn FROM pwp_src + EXCEPT ALL + SELECT * FROM pwp_mv +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM pwp_down + EXCEPT ALL + SELECT id, score FROM ( + SELECT id, score, ROW_NUMBER() OVER (PARTITION BY grp ORDER BY score, id) AS rn FROM pwp_src + ) ranked WHERE rn = 1 +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT id, score FROM ( + SELECT id, score, ROW_NUMBER() OVER (PARTITION BY grp ORDER BY score, id) AS rn FROM pwp_src + ) ranked WHERE rn = 1 + EXCEPT ALL + SELECT * FROM pwp_down +); +---- +0 diff --git a/test/sql/spark_add_months.test b/test/sql/spark_add_months.test new file mode 100644 index 00000000..38265715 --- /dev/null +++ b/test/sql/spark_add_months.test @@ -0,0 +1,98 @@ +# name: test/sql/spark_add_months.test +# description: add_months incremental MV maintenance through the openivm compiler +# group: [sql] + +# +# Exhaustive scalar-correctness coverage for add_months lives in lpts +# (third_party/lpts test/sql/spark_add_months.test). openivm consumes the +# function from lpts; this test covers openivm's concern: that add_months +# resolves in the compiler and drives a real SIMPLE_PROJECTION delta rather +# than a silent FULL_REFRESH demotion. + +require openivm + +statement ok +SET openivm_files_path='__TEST_DIR__'; + +# add_months resolves in an openivm session (registered via lpts source) +query I +SELECT add_months(DATE '2015-01-31', 1) = DATE '2015-02-28'; +---- +true + +query I +SELECT add_months(DATE '2015-02-28', 1) = DATE '2015-03-28'; +---- +true + +# --- Incremental materialized-view maintenance using add_months --- + +statement ok +CREATE TABLE am_item (id INT, d DATE, n INT); + +statement ok +INSERT INTO am_item VALUES + (1, DATE '2015-01-31', 1), + (2, DATE '2016-02-29', 12), + (3, DATE '2015-01-15', 13); + +statement ok +CREATE MATERIALIZED VIEW am_mv AS + SELECT id, d, n, add_months(d, n) AS shifted + FROM am_item; + +statement ok +SELECT COUNT(*) FROM openivm_compile_with_facts( + 'am_mv', + '{"target_dialect":"duckdb","compile_only":true}' +); + +# Real SIMPLE_PROJECTION delta emitted (no full-refresh demotion) +query I +SELECT CASE + WHEN contains(content, 'INSERT INTO openivm_delta_am_mv') AND + contains(content, 'openivm_delta_am_item') AND + NOT contains(content, 'SELECT NULL::') + THEN 1 ELSE 0 + END +FROM read_text('__TEST_DIR__/openivm_upsert_queries_am_mv.sql'); +---- +1 + +statement ok +UPDATE am_item SET d = DATE '2015-02-28', n = 1 WHERE id = 1; + +statement ok +INSERT INTO am_item VALUES (4, DATE '2015-04-30', 1); + +statement ok +DELETE FROM am_item WHERE id = 3; + +statement ok +PRAGMA refresh('am_mv'); + +query IT rowsort +SELECT id, shifted FROM am_mv; +---- +1 2015-03-28 +2 2017-02-28 +4 2015-05-30 + +# Incrementally maintained MV matches full recomputation, both directions +query I +SELECT COUNT(*) FROM ( + SELECT * FROM am_mv + EXCEPT ALL + SELECT id, d, n, add_months(d, n) AS shifted FROM am_item +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT id, d, n, add_months(d, n) AS shifted FROM am_item + EXCEPT ALL + SELECT * FROM am_mv +); +---- +0 diff --git a/test/sql/time_travel.test b/test/sql/time_travel.test new file mode 100644 index 00000000..120a8bc2 --- /dev/null +++ b/test/sql/time_travel.test @@ -0,0 +1,1019 @@ +# name: test/sql/time_travel.test +# description: Spark/Delta time-travel pins survive parsing, binding and dialect rendering. +# group: [sql] + +require openivm + +statement ok +SET openivm_files_path='__TEST_DIR__'; + +statement ok +CREATE TABLE tt_orders(o_id INT, c_id INT, amount INT); + +statement ok +CREATE TABLE tt_customers(c_id INT, region VARCHAR); + +statement ok +INSERT INTO tt_orders VALUES (1, 1, 10), (2, 2, 20); + +statement ok +INSERT INTO tt_customers VALUES (1, 'us'), (2, 'eu'); + +# ========================================== +# Input parsing: Spark syntax is only accepted when the input dialect says so +# ========================================== + +# The default input dialect is DuckDB, so raw Spark time-travel syntax must still +# be a parse error rather than being silently reinterpreted. +statement error +CREATE MATERIALIZED VIEW tt_rejected AS + SELECT c_id, SUM(amount) AS total FROM tt_orders VERSION AS OF 366 GROUP BY c_id; +---- +syntax error + +statement error +SET openivm_input_dialect='klingon'; +---- +openivm_input_dialect + +statement ok +SET openivm_input_dialect='spark'; + +# ========================================== +# Binding: a pinned scan binds against a plain in-memory table and loads data +# ========================================== + +statement ok +CREATE MATERIALIZED VIEW tt_single AS + SELECT c_id, SUM(amount) AS total FROM tt_orders VERSION AS OF 366 GROUP BY c_id; + +query II +SELECT c_id, total FROM tt_single ORDER BY c_id; +---- +1 10 +2 20 + +# The pin is preserved on the stored view SQL, in DuckDB spelling, on the pinned relation. +query I +SELECT CASE WHEN sql_string LIKE '%tt_orders AT (VERSION => 366)%' THEN 1 ELSE 0 END +FROM openivm_views WHERE view_name = 'tt_single'; +---- +1 + +# A TIMESTAMP pin is represented the same way. +statement ok +CREATE MATERIALIZED VIEW tt_stamp AS + SELECT c_id, SUM(amount) AS total + FROM tt_orders TIMESTAMP AS OF '2024-01-01 00:00:00' + GROUP BY c_id; + +query I +SELECT CASE WHEN sql_string LIKE '%tt_orders AT (TIMESTAMP => ''2024-01-01 00:00:00'')%' THEN 1 ELSE 0 END +FROM openivm_views WHERE view_name = 'tt_stamp'; +---- +1 + +# ========================================== +# Relation/pin association: aliases, joins, two differently pinned relations +# ========================================== + +statement ok +CREATE MATERIALIZED VIEW tt_join AS + SELECT c.region AS region, SUM(o.amount) AS total + FROM tt_orders VERSION AS OF 366 o + JOIN tt_customers VERSION AS OF 12 c ON o.c_id = c.c_id + GROUP BY c.region; + +query II +SELECT region, total FROM tt_join ORDER BY region; +---- +eu 20 +us 10 + +# Each pin must land on its own relation - never conflated, never swapped. +query I +SELECT CASE WHEN sql_string LIKE '%tt_orders AT (VERSION => 366)%' + AND sql_string LIKE '%tt_customers AT (VERSION => 12)%' + AND sql_string NOT LIKE '%tt_orders AT (VERSION => 12)%' + AND sql_string NOT LIKE '%tt_customers AT (VERSION => 366)%' + THEN 1 ELSE 0 END +FROM openivm_views WHERE view_name = 'tt_join'; +---- +1 + +# Repeated scans of the same relation inside CTEs keep the pin on every scan. +statement ok +CREATE MATERIALIZED VIEW tt_cte AS + WITH a AS (SELECT c_id, amount FROM tt_orders VERSION AS OF 366), + b AS (SELECT c_id, amount FROM tt_orders VERSION AS OF 366) + SELECT a.c_id AS c_id, SUM(a.amount + b.amount) AS total + FROM a JOIN b ON a.c_id = b.c_id + GROUP BY a.c_id; + +query II +SELECT (length(sql_string) - length(replace(sql_string, 'tt_orders AT (VERSION => 366)', ''))) / + length('tt_orders AT (VERSION => 366)'), + (length(sql_string) - length(replace(sql_string, 'tt_orders', ''))) / length('tt_orders') +FROM openivm_views WHERE view_name = 'tt_cte'; +---- +2 2 + +# Explicit AS aliases and Spark backtick identifiers both survive input normalization with the +# pin attached to the right relation. +statement ok +CREATE MATERIALIZED VIEW tt_backtick AS + SELECT c.region AS region, SUM(o.amount) AS total + FROM `tt_orders` VERSION AS OF 366 AS o + JOIN `tt_customers` VERSION AS OF 12 AS c ON o.c_id = c.c_id + GROUP BY c.region; + +query I +SELECT CASE WHEN sql_string LIKE '%tt_orders AT (VERSION => 366)%' + AND sql_string LIKE '%tt_customers AT (VERSION => 12)%' + THEN 1 ELSE 0 END +FROM openivm_views WHERE view_name = 'tt_backtick'; +---- +1 + +query II +SELECT region, total FROM tt_backtick ORDER BY region; +---- +eu 20 +us 10 + +# A CTE that shadows a relation name is not a scan of that relation, so it must not be mistaken +# for an unpinned scan of the pinned table. +statement ok +CREATE MATERIALIZED VIEW tt_shadow AS + WITH tt_orders AS (SELECT c_id, amount FROM tt_orders VERSION AS OF 366) + SELECT c_id, SUM(amount) AS total FROM tt_orders GROUP BY c_id; + +query I +SELECT CASE WHEN sql_string LIKE '%tt_orders AT (VERSION => 366)%' THEN 1 ELSE 0 END +FROM openivm_views WHERE view_name = 'tt_shadow'; +---- +1 + +# ========================================== +# Alias-to-pin association +# ========================================== +# +# Spark writes the alias *after* the pin (`FROM t VERSION AS OF n p`) where DuckDB wants it before +# (`FROM t AS p AT (VERSION => n)`), so normalization has to carry the alias across the rewrite. +# That must keep every alias bound to the relation it was written against: an alias may never +# migrate to a neighbouring relation, and a pin may never migrate to a neighbouring alias or be +# dropped. LPTS normalizes the syntax and the parsed table reference carries each pin. + +# A bare (no `AS`) trailing alias is the exact shape Spark produces. +statement ok +CREATE MATERIALIZED VIEW tt_bare_alias AS + SELECT p.c_id AS c_id, SUM(p.amount) AS total + FROM tt_orders VERSION AS OF 2 p + GROUP BY p.c_id; + +query I +SELECT CASE WHEN sql_string LIKE '%tt_orders AT (VERSION => 2)%' THEN 1 ELSE 0 END +FROM openivm_views WHERE view_name = 'tt_bare_alias'; +---- +1 + +query II +SELECT c_id, total FROM tt_bare_alias ORDER BY c_id; +---- +1 10 +2 20 + +# Aliases that deliberately collide with the *other* relation's name: the pin belongs to the +# relation, not to whatever the scan happens to be called. +statement ok +CREATE MATERIALIZED VIEW tt_alias_swap AS + SELECT tt_orders.region AS region, SUM(tt_customers.amount) AS total + FROM tt_orders VERSION AS OF 366 tt_customers + JOIN tt_customers VERSION AS OF 12 tt_orders ON tt_customers.c_id = tt_orders.c_id + GROUP BY tt_orders.region; + +query I +SELECT CASE WHEN sql_string LIKE '%tt_orders AT (VERSION => 366)%' + AND sql_string LIKE '%tt_customers AT (VERSION => 12)%' + AND sql_string NOT LIKE '%tt_orders AT (VERSION => 12)%' + AND sql_string NOT LIKE '%tt_customers AT (VERSION => 366)%' + THEN 1 ELSE 0 END +FROM openivm_views WHERE view_name = 'tt_alias_swap'; +---- +1 + +query II +SELECT region, total FROM tt_alias_swap ORDER BY region; +---- +eu 20 +us 10 + +# Two pinned relations, each with its own bare alias, joined: neither alias nor pin may cross over. +statement ok +CREATE MATERIALIZED VIEW tt_two_bare_aliases AS + SELECT q.region AS region, SUM(p.amount) AS total + FROM tt_orders VERSION AS OF 2 p + JOIN tt_customers VERSION AS OF 7 q ON p.c_id = q.c_id + GROUP BY q.region; + +query I +SELECT CASE WHEN sql_string LIKE '%tt_orders AT (VERSION => 2)%' + AND sql_string LIKE '%tt_customers AT (VERSION => 7)%' + AND sql_string NOT LIKE '%tt_orders AT (VERSION => 7)%' + AND sql_string NOT LIKE '%tt_customers AT (VERSION => 2)%' + THEN 1 ELSE 0 END +FROM openivm_views WHERE view_name = 'tt_two_bare_aliases'; +---- +1 + +query II +SELECT region, total FROM tt_two_bare_aliases ORDER BY region; +---- +eu 20 +us 10 + +# A clause keyword directly after the pin is not an alias, so nothing may be consumed from the +# WHERE that follows it. +statement ok +CREATE MATERIALIZED VIEW tt_no_alias AS + SELECT c_id, SUM(amount) AS total + FROM tt_orders VERSION AS OF 366 + WHERE amount > 5 + GROUP BY c_id; + +query I +SELECT CASE WHEN sql_string LIKE '%tt_orders AT (VERSION => 366)%' THEN 1 ELSE 0 END +FROM openivm_views WHERE view_name = 'tt_no_alias'; +---- +1 + +query II +SELECT c_id, total FROM tt_no_alias ORDER BY c_id; +---- +1 10 +2 20 + +# A column-alias list belongs to the alias, so it has to travel with it across the pin. +statement ok +CREATE MATERIALIZED VIEW tt_column_alias AS + SELECT p.p_c_id AS c_id, SUM(p.p_amount) AS total + FROM tt_orders VERSION AS OF 366 AS p (p_o_id, p_c_id, p_amount) + GROUP BY p.p_c_id; + +query I +SELECT CASE WHEN sql_string LIKE '%tt_orders AT (VERSION => 366)%' THEN 1 ELSE 0 END +FROM openivm_views WHERE view_name = 'tt_column_alias'; +---- +1 + +query II +SELECT c_id, total FROM tt_column_alias ORDER BY c_id; +---- +1 10 +2 20 + +# A pinned relation joined to an unpinned one keeps the pin on its own side only. +statement ok +CREATE MATERIALIZED VIEW tt_half_pinned AS + SELECT c.region AS region, SUM(o.amount) AS total + FROM tt_orders VERSION AS OF 366 o + JOIN tt_customers c ON o.c_id = c.c_id + GROUP BY c.region; + +query I +SELECT CASE WHEN sql_string LIKE '%tt_orders AT (VERSION => 366)%' + AND sql_string NOT LIKE '%tt_customers AT (%' + THEN 1 ELSE 0 END +FROM openivm_views WHERE view_name = 'tt_half_pinned'; +---- +1 + +# ========================================== +# Ambiguity is refused, never silently conflated +# ========================================== + +statement error +CREATE MATERIALIZED VIEW tt_two_pins AS + SELECT a.c_id AS c_id, SUM(a.amount + b.amount) AS total + FROM tt_orders VERSION AS OF 366 a JOIN tt_orders VERSION AS OF 12 b ON a.c_id = b.c_id + GROUP BY a.c_id; +---- +pins relation 'tt_orders' ambiguously + +statement error +CREATE MATERIALIZED VIEW tt_mixed_pins AS + SELECT a.c_id AS c_id, SUM(a.amount + b.amount) AS total + FROM tt_orders VERSION AS OF 366 a JOIN tt_orders b ON a.c_id = b.c_id + GROUP BY a.c_id; +---- +scanned both pinned and unpinned + +# ========================================== +# Emitted SQL: Spark re-emits VERSION AS OF, DuckDB drops it, others fail loudly +# ========================================== + +statement ok +INSERT INTO tt_orders VALUES (3, 1, 5); + +# The incremental delta program still scans the pinned dimension, so Spark output +# must carry that relation's pin in Spark spelling - not the DuckDB AT (...) form. +query I +SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%tt_customers` VERSION AS OF 12%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%AT (VERSION%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts('tt_join', '{"target_dialect":"spark","compile_only":true}') +WHERE stmt_kind = 'data'; +---- +1 + +statement ok +SET openivm_refresh_mode='full'; + +# Full recompute for Spark renders both pins on their own relations. +query I +SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%tt_orders` VERSION AS OF 366%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%tt_customers` VERSION AS OF 12%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%AT (VERSION%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts('tt_join', '{"target_dialect":"spark","compile_only":true}') +WHERE stmt_kind = 'data'; +---- +1 + +# DuckDB-dialect output runs against this catalog, which holds no snapshots, so the +# pin must be gone entirely rather than emitted as unbindable syntax. +query I +SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%VERSION AS OF%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%AT (VERSION%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts('tt_join', '{"target_dialect":"duckdb","compile_only":true}') +WHERE stmt_kind = 'data'; +---- +1 + +# A dialect with no verified time-travel syntax must fail explicitly instead of +# quietly reading the latest snapshot. +statement error +SELECT sql FROM openivm_compile_with_facts('tt_join', '{"target_dialect":"postgres","compile_only":true}'); +---- +LPTS_UNSUPPORTED_TIME_TRAVEL + +statement ok +SET openivm_refresh_mode='auto'; + +# ========================================== +# Refresh still maintains a pinned view against this catalog +# ========================================== + +statement ok +PRAGMA refresh('tt_join'); + +query II +SELECT region, total FROM tt_join ORDER BY region; +---- +eu 20 +us 15 + +query I +SELECT COUNT(*) FROM ( + SELECT region, total FROM tt_join + EXCEPT ALL + SELECT c.region AS region, SUM(o.amount) AS total + FROM tt_orders o JOIN tt_customers c ON o.c_id = c.c_id + GROUP BY c.region +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT c.region AS region, SUM(o.amount) AS total + FROM tt_orders o JOIN tt_customers c ON o.c_id = c.c_id + GROUP BY c.region + EXCEPT ALL + SELECT region, total FROM tt_join +); +---- +0 + +# ========================================== +# Pins survive the paths that do not go through the AST +# ========================================== +# +# Bodies that never reach the AST keep their pin in the normalized *text*, where the alias sits +# between the relation and its `AT (...)` qualifier. Stripping the pin for local execution has to +# find it there too, or the view is executed against this catalog with a qualifier it cannot bind. +statement ok +CREATE MATERIALIZED VIEW tt_fallback_alias AS + SELECT p.c_id AS c_id, SUM(p.amount) AS total + FROM tt_orders VERSION AS OF 366 p + WHERE p.c_id IN (SELECT c_id FROM tt_customers EXCEPT SELECT c_id FROM tt_customers WHERE region = 'zz') + GROUP BY p.c_id; + +query II +SELECT c_id, total FROM tt_fallback_alias ORDER BY c_id; +---- +1 15 +2 20 + +query I +SELECT CASE WHEN sql_string LIKE '%tt_orders p AT (VERSION => 366)%' THEN 1 ELSE 0 END +FROM openivm_views WHERE view_name = 'tt_fallback_alias'; +---- +1 + +# Refresh programs that are assembled as SQL text rather than rendered from the AST (min/max +# aggregates, group recompute, recovery) must still pin every scan they emit: reading the latest +# snapshot instead of the pinned one has to be impossible, not merely unlikely. +statement ok +CREATE MATERIALIZED VIEW tt_minmax AS + SELECT c_id, MIN(amount) AS lo, MAX(amount) AS hi + FROM tt_orders VERSION AS OF 366 + GROUP BY c_id; + +query III +SELECT c_id, lo, hi FROM tt_minmax ORDER BY c_id; +---- +1 5 10 +2 20 20 + +statement ok +INSERT INTO tt_orders VALUES (5, 1, 3); + +query I +SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%VERSION AS OF 366%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%AT (VERSION%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts('tt_minmax', '{"target_dialect":"spark","compile_only":true}') +WHERE stmt_kind = 'data'; +---- +1 + +# An interrupted refresh recovers by recomputing from source, and that recovery program is built as +# text as well, so both pins have to survive it. +statement ok +UPDATE openivm_views SET refresh_in_progress = true WHERE view_name = 'tt_join'; + +query I +SELECT CASE WHEN (string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%tt_orders VERSION AS OF 366%' + OR string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%tt_orders` VERSION AS OF 366%') + AND (string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%tt_customers VERSION AS OF 12%' + OR string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%tt_customers` VERSION AS OF 12%') + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%tt_orders VERSION AS OF 12%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%tt_customers VERSION AS OF 366%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%AT (VERSION%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts('tt_join', '{"target_dialect":"spark","compile_only":true}') +WHERE stmt_kind = 'data'; +---- +1 + +# The same recovery program for DuckDB drops the pins instead, since this catalog holds no snapshots. +query I +SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%VERSION AS OF%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%AT (VERSION%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts('tt_join', '{"target_dialect":"duckdb","compile_only":true}') +WHERE stmt_kind = 'data'; +---- +1 + +statement ok +UPDATE openivm_views SET refresh_in_progress = false WHERE view_name = 'tt_join'; + +# ========================================== +# Text-carried pins: the qualifier is translated in place, never duplicated +# ========================================== +# +# A body LPTS cannot serialize keeps its pin in the normalized text, where DuckDB spells it *after* +# the alias (`t p AT (VERSION => 366)`). Every other dialect wants the pin directly behind the +# relation, so the raw clause has to be replaced rather than left standing next to a translated one: +# `t VERSION AS OF 366 p AT (VERSION => 366)` is not valid anywhere. + +# An unpinned relation of its own, so the set operation that forces the fallback never scans a +# pinned relation unpinned (which would be refused as ambiguous instead). +statement ok +CREATE TABLE tt_regions(region VARCHAR); + +statement ok +INSERT INTO tt_regions VALUES ('us'), ('eu'); + +statement ok +CREATE MATERIALIZED VIEW tt_text_join AS + SELECT c.region AS region, SUM(o.amount) AS total + FROM tt_orders VERSION AS OF 366 o + JOIN tt_customers VERSION AS OF 12 c ON o.c_id = c.c_id + WHERE c.region IN (SELECT region FROM tt_regions EXCEPT SELECT region FROM tt_regions WHERE region = 'zz') + GROUP BY c.region; + +query II +SELECT region, total FROM tt_text_join ORDER BY region; +---- +eu 20 +us 18 + +# The stored SQL is the text shape: alias between the relation and its qualifier, one per relation. +query I +SELECT CASE WHEN sql_string LIKE '%tt_orders o AT (VERSION => 366)%' + AND sql_string LIKE '%tt_customers c AT (VERSION => 12)%' + THEN 1 ELSE 0 END +FROM openivm_views WHERE view_name = 'tt_text_join'; +---- +1 + +statement ok +CREATE MATERIALIZED VIEW tt_text_comma AS + SELECT c.region AS region, MIN(o.amount) AS lo, MAX(o.amount) AS hi + FROM tt_orders VERSION AS OF 366 o, tt_customers VERSION AS OF 12 c + WHERE o.c_id = c.c_id + AND c.region IN (SELECT region FROM tt_regions EXCEPT SELECT region FROM tt_regions WHERE region = 'zz') + GROUP BY c.region; + +query III +SELECT region, lo, hi FROM tt_text_comma ORDER BY region; +---- +eu 20 20 +us 3 10 + +statement ok +INSERT INTO tt_orders VALUES (6, 2, 4); + +# Ordinary incremental/group-recompute output for Spark: both relations pinned in Spark spelling, +# aliases intact, and no DuckDB qualifier left behind on either. +query I +SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%tt_orders% VERSION AS OF 366%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%tt_customers% VERSION AS OF 12%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%AT (VERSION%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts('tt_text_comma', '{"target_dialect":"spark","compile_only":true}') +WHERE stmt_kind = 'data'; +---- +1 + +# Compiling the same view twice must produce the same SQL: translating a pin is idempotent. +query I +SELECT CASE WHEN (SELECT string_agg(sql, ' ' ORDER BY stmt_order) + FROM openivm_compile_with_facts('tt_text_comma', '{"target_dialect":"spark","compile_only":true}') + WHERE stmt_kind = 'data') = + (SELECT string_agg(sql, ' ' ORDER BY stmt_order) + FROM openivm_compile_with_facts('tt_text_comma', '{"target_dialect":"spark","compile_only":true}') + WHERE stmt_kind = 'data') + THEN 1 ELSE 0 END; +---- +1 + +statement ok +UPDATE openivm_views SET refresh_in_progress = true WHERE view_name = 'tt_text_join'; + +statement ok +UPDATE openivm_views SET refresh_in_progress = true WHERE view_name = 'tt_text_comma'; + +# Recovery renders through LPTS: original aliases become CTE bindings, and each qualified +# source scan must retain its own pin in the target dialect. +query I +SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%`memory`.`main`.`tt_orders` VERSION AS OF 366%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%`memory`.`main`.`tt_customers` VERSION AS OF 12%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%AT (VERSION%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts('tt_text_join', '{"target_dialect":"spark","compile_only":true}') +WHERE stmt_kind = 'data'; +---- +1 + +# ... and so does a comma FROM list, where every implicit cross join is a scan of its own. +query I +SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%`memory`.`main`.`tt_orders` VERSION AS OF 366%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%`memory`.`main`.`tt_customers` VERSION AS OF 12%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%AT (VERSION%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts('tt_text_comma', '{"target_dialect":"spark","compile_only":true}') +WHERE stmt_kind = 'data'; +---- +1 + +# A dialect with no verified time-travel syntax still refuses rather than dropping the qualifier. +statement error +SELECT sql FROM openivm_compile_with_facts('tt_text_comma', '{"target_dialect":"postgres","compile_only":true}'); +---- +LPTS_UNSUPPORTED_TIME_TRAVEL + +# DuckDB output executes against this catalog, which holds no snapshots, so recovery drops the +# qualifier entirely instead of handing the binder syntax it cannot resolve. +query I +SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%VERSION AS OF%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%AT (VERSION%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts('tt_text_join', '{"target_dialect":"duckdb","compile_only":true}') +WHERE stmt_kind = 'data'; +---- +1 + +statement ok +PRAGMA refresh('tt_text_join'); + +statement ok +PRAGMA refresh('tt_text_comma'); + +query II +SELECT region, total FROM tt_text_join ORDER BY region; +---- +eu 24 +us 18 + +query III +SELECT region, lo, hi FROM tt_text_comma ORDER BY region; +---- +eu 4 20 +us 3 10 + +# Qualify and pin base scans, never the CTE that shadows their name. A nonempty EXCEPT operand +# makes bypassing the CTE observably wrong, even if the resulting SQL still binds. +statement ok +CREATE TABLE tt_spare(c_id INT, amount INT); + +statement ok +INSERT INTO tt_spare VALUES (1, 3), (1, 3), (2, 20); + +statement ok +CREATE MATERIALIZED VIEW tt_text_shadow AS + WITH tt_orders(c_id, amount) AS ( + SELECT c_id, amount FROM tt_orders VERSION AS OF 366 + EXCEPT ALL + SELECT c_id, amount FROM tt_spare + ) + SELECT c_id, SUM(amount) AS total FROM tt_orders GROUP BY c_id; + +statement ok +UPDATE openivm_views SET refresh_in_progress = true WHERE view_name = 'tt_text_shadow'; + +query I +SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%`memory`.`main`.`tt_orders` VERSION AS OF 366%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%AT (VERSION%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%VERSION AS OF 366 VERSION AS OF%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%FROM tt_orders VERSION AS OF%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts('tt_text_shadow', '{"target_dialect":"spark","compile_only":true}') +WHERE stmt_kind = 'data'; +---- +1 + +statement ok +PRAGMA refresh('tt_text_shadow'); + +query II +WITH expected AS ( + SELECT c_id, SUM(amount) AS total FROM ( + SELECT c_id, amount FROM tt_orders EXCEPT ALL SELECT c_id, amount FROM tt_spare + ) GROUP BY c_id +) +SELECT (SELECT COUNT(*) FROM (SELECT * FROM tt_text_shadow EXCEPT ALL SELECT * FROM expected)), + (SELECT COUNT(*) FROM (SELECT * FROM expected EXCEPT ALL SELECT * FROM tt_text_shadow)); +---- +0 0 + +statement ok +INSERT INTO tt_spare VALUES (1, 10), (2, 4); + +statement ok +UPDATE tt_spare SET amount = 3 WHERE c_id = 2; + +statement ok +DELETE FROM tt_spare WHERE c_id = 1 AND amount = 3; + +statement ok +INSERT INTO tt_spare VALUES (1, 3), (1, 3); + +statement ok +PRAGMA refresh('tt_text_shadow'); + +query II +WITH expected AS ( + SELECT c_id, SUM(amount) AS total FROM ( + SELECT c_id, amount FROM tt_orders EXCEPT ALL SELECT c_id, amount FROM tt_spare + ) GROUP BY c_id +) +SELECT (SELECT COUNT(*) FROM (SELECT * FROM tt_text_shadow EXCEPT ALL SELECT * FROM expected)), + (SELECT COUNT(*) FROM (SELECT * FROM expected EXCEPT ALL SELECT * FROM tt_text_shadow)); +---- +0 0 + +# A parenthesized join list is still table position: its first element is a scan and needs its pin, +# where a derived table starts a query of its own and takes none. +statement ok +CREATE MATERIALIZED VIEW tt_text_paren AS + SELECT c.region AS region, SUM(o.amount) AS total + FROM (tt_orders VERSION AS OF 366 o JOIN tt_customers VERSION AS OF 12 c ON o.c_id = c.c_id) + WHERE c.region IN (SELECT region FROM tt_regions EXCEPT SELECT region FROM tt_regions WHERE region = 'zz') + GROUP BY c.region; + +query II +SELECT region, total FROM tt_text_paren ORDER BY region; +---- +eu 24 +us 18 + +statement ok +CREATE MATERIALIZED VIEW tt_text_paren_nested AS + SELECT c.region AS region, MIN(o.amount) AS lo, MAX(o.amount) AS hi + FROM ((tt_orders VERSION AS OF 366 o JOIN tt_customers VERSION AS OF 12 c ON o.c_id = c.c_id)) + WHERE c.region IN (SELECT region FROM tt_regions EXCEPT SELECT region FROM tt_regions WHERE region = 'zz') + GROUP BY c.region; + +query III +SELECT region, lo, hi FROM tt_text_paren_nested ORDER BY region; +---- +eu 4 20 +us 3 10 + +# A derived table is not a relation, so the pin belongs to the scan inside it and to nothing else. +statement ok +CREATE MATERIALIZED VIEW tt_text_derived AS + SELECT d.c_id AS c_id, SUM(d.amount) AS total + FROM (SELECT c_id, amount FROM tt_orders VERSION AS OF 366 + EXCEPT ALL + SELECT c_id, amount FROM tt_spare) d + GROUP BY d.c_id; + +query II +SELECT c_id, total FROM tt_text_derived ORDER BY c_id; +---- +1 5 +2 24 + +statement ok +INSERT INTO tt_orders VALUES (7, 1, 2); + +query I +SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%tt_orders% VERSION AS OF 366%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%tt_customers% VERSION AS OF 12%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%AT (VERSION%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts('tt_text_paren', '{"target_dialect":"spark","compile_only":true}') +WHERE stmt_kind = 'data'; +---- +1 + +statement ok +UPDATE openivm_views SET refresh_in_progress = true WHERE view_name = 'tt_text_paren'; + +statement ok +UPDATE openivm_views SET refresh_in_progress = true WHERE view_name = 'tt_text_paren_nested'; + +statement ok +UPDATE openivm_views SET refresh_in_progress = true WHERE view_name = 'tt_text_derived'; + +# LPTS renders the parenthesized join through CTEs; pins stay on the correct qualified scans. +query I +SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%`memory`.`main`.`tt_orders` VERSION AS OF 366%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%`memory`.`main`.`tt_customers` VERSION AS OF 12%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%AT (VERSION%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts('tt_text_paren', '{"target_dialect":"spark","compile_only":true}') +WHERE stmt_kind = 'data'; +---- +1 + +# Nested parentheses defer the question one level, so the innermost element is still a scan. +query I +SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%`memory`.`main`.`tt_orders` VERSION AS OF 366%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%`memory`.`main`.`tt_customers` VERSION AS OF 12%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%AT (VERSION%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts('tt_text_paren_nested', '{"target_dialect":"spark","compile_only":true}') +WHERE stmt_kind = 'data'; +---- +1 + +# The derived table keeps exactly one pin - on the scan inside it, never on the subquery itself. +query I +SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%`memory`.`main`.`tt_orders` VERSION AS OF 366%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%) VERSION AS OF%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%AT (VERSION%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts('tt_text_derived', '{"target_dialect":"spark","compile_only":true}') +WHERE stmt_kind = 'data'; +---- +1 + +statement ok +PRAGMA refresh('tt_text_paren'); + +statement ok +PRAGMA refresh('tt_text_paren_nested'); + +statement ok +PRAGMA refresh('tt_text_derived'); + +query II +SELECT region, total FROM tt_text_paren ORDER BY region; +---- +eu 24 +us 20 + +query II +SELECT c_id, total FROM tt_text_derived ORDER BY c_id; +---- +1 7 +2 24 + +# ========================================== +# Unpinned views are untouched by the Spark input dialect +# ========================================== + +statement ok +CREATE MATERIALIZED VIEW tt_plain AS + SELECT c_id, SUM(amount) AS total FROM tt_orders GROUP BY c_id; + +query I +SELECT CASE WHEN sql_string NOT LIKE '%AT (%' THEN 1 ELSE 0 END +FROM openivm_views WHERE view_name = 'tt_plain'; +---- +1 + +statement ok +INSERT INTO tt_orders VALUES (4, 2, 7); + +statement ok +PRAGMA refresh('tt_plain'); + +query I +SELECT COUNT(*) FROM ( + SELECT c_id, total FROM tt_plain + EXCEPT ALL + SELECT c_id, SUM(amount) AS total FROM tt_orders GROUP BY c_id +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT c_id, SUM(amount) AS total FROM tt_orders GROUP BY c_id + EXCEPT ALL + SELECT c_id, total FROM tt_plain +); +---- +0 + +statement ok +SET openivm_input_dialect='duckdb'; + +# Foreign pins use resolved catalog identity too: memory.table is not schema.table. +statement ok +CREATE TABLE tt_identity(k INTEGER, v INTEGER); + +statement ok +INSERT INTO tt_identity VALUES (1, 10), (1, 10), (2, 20); + +statement ok +SET openivm_input_dialect='spark'; + +statement ok +CREATE MATERIALIZED VIEW tt_identity_mv AS +SELECT a.k, SUM(a.v + b.v) AS total +FROM memory.tt_identity VERSION AS OF 7 a +JOIN memory.main.tt_identity VERSION AS OF 7 b ON a.k = b.k GROUP BY a.k; + +query I +SELECT sql_string LIKE '%AT (VERSION => 7)%' FROM openivm_views WHERE view_name='tt_identity_mv'; +---- +true + +query I +SELECT COUNT(*) > 0 AND COUNT(*) FILTER (WHERE sql LIKE '%`tt_identity` VERSION AS OF 7%') > 0 + AND COUNT(*) FILTER (WHERE sql LIKE '%AT (VERSION%') = 0 +FROM openivm_compile_with_facts('tt_identity_mv', '{"target_dialect":"spark","compile_only":true}') +WHERE stmt_kind='data'; +---- +true + +statement ok +INSERT INTO tt_identity VALUES (1, 5), (2, 30); + +statement ok +UPDATE tt_identity SET v = v + 1 WHERE k = 1; + +statement ok +DELETE FROM tt_identity WHERE k = 2; + +statement ok +INSERT INTO tt_identity VALUES (2, 7), (2, 7); + +statement ok +PRAGMA refresh('tt_identity_mv'); + +query II +WITH expected AS (SELECT a.k, SUM(a.v + b.v) AS total FROM tt_identity a JOIN tt_identity b ON a.k=b.k GROUP BY a.k) +SELECT (SELECT COUNT(*) FROM (SELECT * FROM tt_identity_mv EXCEPT ALL SELECT * FROM expected)), + (SELECT COUNT(*) FROM (SELECT * FROM expected EXCEPT ALL SELECT * FROM tt_identity_mv)); +---- +0 0 + +statement ok +SET openivm_input_dialect='duckdb'; + +# With the input dialect back to DuckDB, the native AT (...) spelling still binds +# unchanged for catalogs that support it, and Spark syntax is a parse error again. +statement error +CREATE MATERIALIZED VIEW tt_rejected_again AS + SELECT c_id, SUM(amount) AS total FROM tt_orders VERSION AS OF 366 GROUP BY c_id; +---- +syntax error + +# Earlier sibling CTEs are visible in later bodies, even when they shadow the pinned base. +statement ok +CREATE TABLE tt_scope(k INTEGER, v INTEGER); + +statement ok +INSERT INTO tt_scope VALUES (1, 10), (1, 10), (2, 20); + +statement ok +SET openivm_input_dialect='spark'; + +statement ok +CREATE MATERIALIZED VIEW tt_scoped AS +WITH tt_scope AS (SELECT * FROM tt_scope VERSION AS OF 2), + next_scope AS (SELECT * FROM tt_scope) +SELECT k, SUM(v) AS total FROM next_scope GROUP BY k; + +statement ok +INSERT INTO tt_scope VALUES (1, 5), (2, 30); + +statement ok +UPDATE tt_scope SET v = v + 1 WHERE k = 1; + +statement ok +DELETE FROM tt_scope WHERE k = 2; + +statement ok +INSERT INTO tt_scope VALUES (2, 7), (2, 7); + +# Adaptive planning must reuse the already-peeled query too, without a second pin walk. +statement ok +SET openivm_adaptive_refresh = true; + +query I +SELECT COUNT(*) > 0 AND COUNT(*) FILTER (WHERE sql LIKE '%AT (VERSION%' OR sql LIKE '%VERSION AS OF%') = 0 +FROM openivm_compile_with_facts('tt_scoped', '{"target_dialect":"duckdb","compile_only":true}'); +---- +true + +statement ok +SET openivm_adaptive_refresh = false; + +statement ok +PRAGMA refresh('tt_scoped'); + +query II +SELECT (SELECT COUNT(*) FROM (SELECT * FROM tt_scoped EXCEPT ALL SELECT k, SUM(v) FROM tt_scope GROUP BY k)), + (SELECT COUNT(*) FROM (SELECT k, SUM(v) FROM tt_scope GROUP BY k EXCEPT ALL SELECT * FROM tt_scoped)); +---- +0 0 + +# Escaped identifiers and alias column lists use the parsed table reference, not a second lexer. +statement ok +CREATE TABLE "tt`source"(k INTEGER, v INTEGER); + +statement ok +INSERT INTO "tt`source" VALUES (1, 10), (1, 10), (2, 20); + +statement ok +CREATE MATERIALIZED VIEW tt_escaped AS +SELECT x, SUM(y) AS total FROM `tt``source` VERSION AS OF 7 AS `p``q`(x, y) GROUP BY x; + +statement ok +INSERT INTO "tt`source" VALUES (1, 5), (2, 30); + +statement ok +UPDATE "tt`source" SET v = v + 1 WHERE k = 1; + +statement ok +DELETE FROM "tt`source" WHERE k = 2; + +statement ok +INSERT INTO "tt`source" VALUES (2, 7), (2, 7); + +statement ok +PRAGMA refresh('tt_escaped'); + +query II +SELECT (SELECT COUNT(*) FROM (SELECT * FROM tt_escaped EXCEPT ALL SELECT k, SUM(v) FROM "tt`source" GROUP BY k)), + (SELECT COUNT(*) FROM (SELECT k, SUM(v) FROM "tt`source" GROUP BY k EXCEPT ALL SELECT * FROM tt_escaped)); +---- +0 0 + +statement ok +SET openivm_refresh_mode='full'; + +query I +SELECT string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%`tt``source` VERSION AS OF 7%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%AT (VERSION%' +FROM openivm_compile_with_facts('tt_escaped', '{"target_dialect":"spark","compile_only":true}') +WHERE stmt_kind = 'data'; +---- +true + +statement ok +SET openivm_refresh_mode='incremental'; + +statement ok +SET openivm_input_dialect='duckdb'; diff --git a/test/sql/time_travel_ducklake.test b/test/sql/time_travel_ducklake.test new file mode 100644 index 00000000..1d1daeea --- /dev/null +++ b/test/sql/time_travel_ducklake.test @@ -0,0 +1,147 @@ +# name: test/sql/time_travel_ducklake.test +# description: A pin may never leak onto a same-named relation in a catalog that honours pins. +# group: [sql] + +require openivm + +require parquet + +statement ok +INSTALL ducklake; + +statement ok +LOAD ducklake; + +statement ok +SET openivm_files_path='__TEST_DIR__'; + +statement ok +ATTACH '__TEST_DIR__/time_travel_ducklake.db' AS dl (TYPE ducklake); + +statement ok +CREATE TABLE dl.main.tt_orders(o_id INT, c_id INT, amount INT); + +statement ok +INSERT INTO dl.main.tt_orders VALUES (1, 1, 10), (2, 2, 20); + +statement ok +CREATE TABLE tt_orders(o_id INT, c_id INT, amount INT); + +statement ok +INSERT INTO tt_orders VALUES (1, 1, 10), (2, 2, 20); + +statement ok +SET openivm_input_dialect='spark'; + +# `memory.tt_orders` is pinned, `dl.main.tt_orders` is not. Pins are keyed by relation name, so a +# catalog that honours time travel is exactly where a stray pin would bind and silently read the +# wrong snapshot of a different table. The ambiguity has to be refused instead. +statement error +CREATE MATERIALIZED VIEW tt_mixed AS + SELECT a.c_id AS c_id, SUM(a.amount + b.amount) AS total + FROM tt_orders VERSION AS OF 366 a + JOIN dl.main.tt_orders b ON a.c_id = b.c_id + GROUP BY a.c_id; +---- +scanned both pinned and unpinned + +statement ok +SET openivm_input_dialect='duckdb'; + +# Native snapshots must survive source qualification; foreign pin stripping does not own them. +# Snapshot 2 contains the first INSERT above; later writes must not affect this full-refresh view. +statement ok +CREATE MATERIALIZED VIEW tt_native_snapshot AS +SELECT c_id, SUM(amount) AS total FROM dl.main.tt_orders AT (VERSION => 2) GROUP BY c_id; + +# Two-part catalog.table spelling must resolve to the same native catalog identity. +statement ok +CREATE MATERIALIZED VIEW tt_native_short AS +SELECT c_id, SUM(amount) AS total FROM dl.tt_orders AT (VERSION => 2) GROUP BY c_id; + +query I +SELECT COUNT(*) FROM openivm_views +WHERE view_name IN ('tt_native_snapshot', 'tt_native_short') AND sql_string LIKE '%AT (VERSION => 2)%'; +---- +2 + +# The per-relation resolver cannot distinguish a current pinned scan from an unpinned scan. +statement error +CREATE MATERIALIZED VIEW tt_native_ambiguous AS +SELECT a.o_id FROM dl.tt_orders a AT (VERSION => 2) JOIN dl.tt_orders b ON a.o_id = b.o_id; +---- +scanned both pinned and unpinned + +query I +SELECT COUNT(*) FROM duckdb_indexes() +WHERE table_name IN ('openivm_data_tt_native_snapshot', 'openivm_data_tt_native_short') AND is_unique; +---- +2 + +statement ok +INSERT INTO dl.main.tt_orders VALUES (1, 1, 5), (3, 2, 30); + +statement ok +UPDATE dl.main.tt_orders SET amount = amount + 1 WHERE c_id = 1; + +statement ok +DELETE FROM dl.main.tt_orders WHERE c_id = 2; + +statement ok +INSERT INTO dl.main.tt_orders VALUES (4, 2, 7), (4, 2, 7); + +statement ok +SET openivm_refresh_mode = 'full'; + +query I +SELECT string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%AT (VERSION => 2)%' +FROM openivm_compile_with_facts('tt_native_snapshot', '{"target_dialect":"duckdb","compile_only":true}') +WHERE stmt_kind = 'data'; +---- +true + +statement ok +PRAGMA refresh('tt_native_snapshot'); + +query II +WITH expected AS (SELECT c_id, SUM(amount) AS total FROM dl.main.tt_orders AT (VERSION => 2) GROUP BY c_id) +SELECT (SELECT COUNT(*) FROM (SELECT * FROM tt_native_snapshot EXCEPT ALL SELECT * FROM expected)), + (SELECT COUNT(*) FROM (SELECT * FROM expected EXCEPT ALL SELECT * FROM tt_native_snapshot)); +---- +0 0 + +statement ok +SET openivm_refresh_mode = 'incremental'; + +# The pinned source has an empty delta even after conflicting changes to its latest snapshot. +statement ok +PRAGMA refresh('tt_native_short'); + +query II +WITH expected AS (SELECT c_id, SUM(amount) AS total FROM dl.tt_orders AT (VERSION => 2) GROUP BY c_id) +SELECT (SELECT COUNT(*) FROM (SELECT * FROM tt_native_short EXCEPT ALL SELECT * FROM expected)), + (SELECT COUNT(*) FROM (SELECT * FROM expected EXCEPT ALL SELECT * FROM tt_native_short)); +---- +0 0 + +# Interrupted-refresh recovery must render native pins in the requested output dialect too. +statement ok +UPDATE openivm_views SET refresh_in_progress=true WHERE view_name='tt_native_snapshot'; + +query I +SELECT COUNT(*) > 0 AND COUNT(*) FILTER (WHERE sql LIKE '%`tt_orders` VERSION AS OF 2%') > 0 + AND COUNT(*) FILTER (WHERE sql LIKE '%AT (VERSION%') = 0 +FROM openivm_compile_with_facts('tt_native_snapshot', '{"target_dialect":"spark","compile_only":true}') +WHERE stmt_kind='data'; +---- +true + +statement ok +PRAGMA refresh('tt_native_snapshot'); + +query II +WITH expected AS (SELECT c_id, SUM(amount) AS total FROM dl.tt_orders AT (VERSION => 2) GROUP BY c_id) +SELECT (SELECT COUNT(*) FROM (SELECT * FROM tt_native_snapshot EXCEPT ALL SELECT * FROM expected)), + (SELECT COUNT(*) FROM (SELECT * FROM expected EXCEPT ALL SELECT * FROM tt_native_snapshot)); +---- +0 0 diff --git a/third_party/lpts b/third_party/lpts index b67f84f1..29c606ee 160000 --- a/third_party/lpts +++ b/third_party/lpts @@ -1 +1 @@ -Subproject commit b67f84f173bc8f500ffd8cde799146821fccdc51 +Subproject commit 29c606ee18ad8e4d04cab940f361cbf6fbe5b881