From 7b4f9a33552e8efe3fcbeabc57a3a907507bc3b0 Mon Sep 17 00:00:00 2001 From: skadel Date: Sun, 26 Jul 2026 15:09:33 +0200 Subject: [PATCH 1/2] fix(generator): remove Faker fill and preserve strict parsing --- back/build_query/constraint_simplifier.py | 2 +- back/build_query/examples_generator.py | 218 +----------------- back/poetry.lock | 22 +- back/pyproject.toml | 2 - .../simplifier/test_partition_pinning.py | 14 +- back/tests/test_faker_agg_measure.py | 81 ------- back/tests/test_fix_duck_db_sql.py | 106 +++++---- back/tests/test_generate_type.py | 4 +- back/tests/test_scalar_folder_fix_needed.py | 45 +--- back/tests/test_trino_schema_match.py | 28 --- back/utils/examples.py | 25 +- back/utils/faker_fill.py | 164 ------------- 12 files changed, 109 insertions(+), 602 deletions(-) delete mode 100644 back/tests/test_faker_agg_measure.py delete mode 100644 back/utils/faker_fill.py diff --git a/back/build_query/constraint_simplifier.py b/back/build_query/constraint_simplifier.py index 9925c7e..e64cad5 100644 --- a/back/build_query/constraint_simplifier.py +++ b/back/build_query/constraint_simplifier.py @@ -1353,7 +1353,7 @@ def _extract_from_condition_recursive( """Walk *cond* recursively, collecting constraints from ALL branches (AND + OR). OR branches are not expanded — all constraints from any branch are accumulated - into the same lists. This is conservative for Faker pre-fill: every column + into the same lists. This is conservative for generation: every column mentioned in any OR branch is marked as constrained. """ if isinstance(cond, (exp.And, exp.Or, exp.Paren)): diff --git a/back/build_query/examples_generator.py b/back/build_query/examples_generator.py index 90a5e9b..5f3c246 100644 --- a/back/build_query/examples_generator.py +++ b/back/build_query/examples_generator.py @@ -23,7 +23,6 @@ create_pydantic_models, filter_columns, ) -from utils.faker_fill import generate_faker_rows from utils.llm_factory import make_llm from storage.config import ( get_language, @@ -359,7 +358,8 @@ def _branch_to_dict(result) -> dict: all_constraints = [c for cs in result.source_columns.values() for c in cs] filters = _format_filter_constraints(all_constraints) # Bare columns: referenced in WHERE/JOIN ON/QUALIFY inside complex expressions - # (e.g. UPPER(col) * 2) — no extractable constraint, but must not be Faker-filled. + # (e.g. UPPER(col) * 2) — no extractable constraint, but surfaced to the LLM as a + # bare-column hint so it knows the column participates in the filter. bare = [_col_str_join(col) for col, cs in result.source_columns.items() if not cs] d: dict = {} if joins: @@ -508,50 +508,6 @@ async def _aprepare_generation_constraints( ) -def _strip_unconstrained_from_sql( - sql: str, excluded_col_names: list[str], dialect: str = "bigquery" -) -> str: - """Remove unconstrained columns from SELECT lists in SQL (LLM context only).""" - if not sql or not excluded_col_names: - return sql - - excluded_pairs: set[tuple[str, str]] = set() - for entry in excluded_col_names: - if "." in entry: - tbl, col = entry.rsplit(".", 1) - excluded_pairs.add((tbl.lower(), col.lower())) - - if not excluded_pairs: - return sql - - try: - import sqlglot.expressions as exp - from sqlglot import parse_one - - tree = parse_one(sql, dialect=dialect) - - for select in tree.find_all(exp.Select): - new_exprs = [] - for expr in select.expressions: - col_node = expr.this if isinstance(expr, exp.Alias) else expr - if not isinstance(col_node, exp.Column): - new_exprs.append(expr) - continue - col_name = col_node.name.lower() - table_qualifier = (col_node.table or "").lower().split(".")[-1] - should_exclude = any( - col == col_name and (not table_qualifier or table_qualifier == tbl) - for tbl, col in excluded_pairs - ) - if not should_exclude: - new_exprs.append(expr) - select.set("expressions", new_exprs) - - return tree.sql(dialect=dialect) - except Exception: - return sql - - def _extract_constraints_per_cte(query_decomposed: list, dialect: str) -> dict: """Returns {cte_name: parsed_constraints_dict} for each non-final CTE.""" result_map = {} @@ -1196,35 +1152,6 @@ async def generate_examples_( filtered_schema = filter_columns(schema, used_columns) - # Compute Faker-eligible columns only when constraint extraction fully succeeded - # and all ColumnRefs resolved to known base tables (no silent lineage failure). - # UNNEST queries are skipped: array-of-struct constraints are not reliably captured. - # Faker is also disabled on retry (empty_results) — inconsistent data may have been - # caused by Faker-filled values conflicting with LLM-generated values. - base_tables = {entry["table"].lower() for entry in used_columns} - faker_cols: dict[str, set[str]] = {} - _has_unnest = "unnest" in optimized_sql.lower() - _is_retry = state.get("status") == "empty_results" - logger.debug( - "[generator] _has_unnest=%s _is_retry=%s sim_result=%s", - _has_unnest, - _is_retry, - sim_result is not None, - ) - if ( - sim_result is not None - and not _has_unnest - and not _is_retry - and _all_refs_resolved(sim_result, base_tables) - ): - faker_cols = _compute_faker_columns( - sim_result, used_columns, base_tables, sql=optimized_sql, dialect=dialect - ) - - logger.debug( - "[generator] faker_cols: %s", {k: list(v) for k, v in faker_cols.items()} - ) - # Precompute constraints per column table_to_uc = {} for entry in used_columns: @@ -1281,19 +1208,14 @@ async def generate_examples_( f"Anti-join (NOT IN) avec {other.table}.{other.column}" ) - # Build LLM schema with Faker-eligible columns removed and constraint hints injected + # Build LLM schema with constraint hints injected. Every used column is kept — the + # LLM fills them all, so values stay coherent with the scenario description. llm_filtered_schema = [] - excluded_col_names: list[str] = [] for table_entry in filtered_schema: uc_key = table_entry["table_name"] new_columns = [] for c in table_entry["columns"]: col_name = c["name"].lower() - # Skip if Faker will fill this - if faker_cols and uc_key in faker_cols and col_name in faker_cols[uc_key]: - excluded_col_names.append(f"{uc_key}.{col_name}") - continue - c_copy = dict(c) hints = col_hints.get((uc_key, col_name)) if hints: @@ -1327,14 +1249,6 @@ async def generate_examples_( if new_columns: llm_filtered_schema.append({**table_entry, "columns": new_columns}) - if faker_cols: - logger.debug( - "[generator] Faker pre-fill: %d col(s) across %d table(s) removed from LLM schema — %s", - sum(len(cols) for cols in faker_cols.values()), - len(faker_cols), - excluded_col_names, - ) - # Modèle recommandé (flash/pro) : le thinking natif porte le raisonnement → # le champ in-schema n'est qu'une justification brève. Sinon, fallback sur un # CoT in-schema complet (fonctionnel mais plus lent + risque de troncature) : @@ -1399,7 +1313,6 @@ async def generate_examples_( used_columns, format_instructions, constraints_hint=constraints, - excluded_columns=excluded_col_names, eval_history=eval_history, native_thinking=native_thinking, join_recipes_block=join_recipes_block, @@ -1423,9 +1336,6 @@ async def generate_examples_( "[generator] constraints_hint:\n%s", constraints or "(vide — sous-requêtes corrélées non capturées ?)", ) - logger.diag( - "[generator] faker_cols: %s", {k: list(v) for k, v in faker_cols.items()} - ) try: formatted_msgs = prompt.format_messages() logger.diag( @@ -1462,21 +1372,6 @@ async def generate_examples_( len(rows) if isinstance(rows, list) else "?", ) - # Merge Faker-generated values into LLM output - if faker_cols: - faker_data = generate_faker_rows( - schema, faker_cols, filled_data, profile=state.get("profile") - ) - for uc_key, faker_rows in faker_data.items(): - llm_rows = filled_data.get(uc_key) or [] - if llm_rows: - filled_data[uc_key] = [ - {**(row or {}), **faker_row} - for row, faker_row in zip(llm_rows, faker_rows) - ] - else: - filled_data[uc_key] = faker_rows - # Fix 4 — pin déterministe date→epoch : le LLM ne calcule pas fiablement un epoch # (sf_bq093). Quand le SQL compare une date à une colonne epoch (directive # `epoch_date_eq`), on corrige hors LLM toute valeur qui manque le jour filtré. @@ -1675,7 +1570,6 @@ async def create_appropriate_prompt( used_columns, format_instructions, constraints_hint: str = "", - excluded_columns: list[str] | None = None, eval_history: list | None = None, native_thinking: bool = False, join_recipes_block: str = "", @@ -1685,7 +1579,6 @@ async def create_appropriate_prompt( sql = state.get("optimized_sql", "") dialect = state.get("dialect", "bigquery") profile = state.get("profile") - stripped_sql = _strip_unconstrained_from_sql(sql, excluded_columns or [], dialect) model_context = state.get("model_context") or "" if not existing_tests: return generate_data_prompt( @@ -1694,7 +1587,7 @@ async def create_appropriate_prompt( format_instructions, used_columns, constraints_hint=constraints_hint, - sql=stripped_sql, + sql=sql, profile=profile, model_context=model_context, eval_history=eval_history, @@ -1744,7 +1637,7 @@ async def create_appropriate_prompt( format_instructions, used_columns, constraints_hint=constraints_hint, - sql=stripped_sql, + sql=sql, user_instruction=state["input"], profile=profile, model_context=model_context, @@ -1767,7 +1660,7 @@ async def create_appropriate_prompt( format_instructions, used_columns, constraints_hint=constraints_hint, - sql=stripped_sql, + sql=sql, profile=profile, model_context=model_context, trace_hint=trace_hint, @@ -1790,7 +1683,7 @@ async def create_appropriate_prompt( format_instructions, used_columns, constraints_hint=constraints_hint, - sql=stripped_sql, + sql=sql, profile=profile, model_context=model_context, eval_history=eval_history, @@ -1803,101 +1696,6 @@ async def create_appropriate_prompt( return None -def _all_refs_resolved(sim_result, base_tables: set[str]) -> bool: - """Return True iff every ColumnRef in sim_result maps to a known base table. - - A ColumnRef whose table is NOT in base_tables indicates that lineage resolution - silently fell back to an unresolved CTE alias — in that case Faker must not be - activated because we cannot tell which base-table columns are constrained. - """ - all_refs = ( - list(sim_result.source_columns.keys()) - + list(sim_result.derived_columns.keys()) - + [ref for eq_class in sim_result.equivalence_classes for ref in eq_class] - ) - # is_identity=False refs are DELIBERATELY CTE-qualified (predicate on a derived - # column, never remapped to its base column) — not a silent fallback. Their base - # source columns are excluded from Faker via FilterConstraint.source_columns. - return all(ref.table.lower() in base_tables for ref in all_refs if ref.is_identity) - - -def _compute_faker_columns( - sim_result, - used_columns: list, - base_tables: set[str], - sql: str = "", - dialect: str = "bigquery", -) -> dict[str, set[str]]: - """Return {uc_key: {col_names}} for columns safe to Faker-fill. - - Only called when sim_result is not None and _all_refs_resolved() is True. - uc_key matches the table_name produced by filter_columns() (database_table). - """ - constrained: set[tuple[str, str]] = set() - for ref in sim_result.source_columns: - constrained.add((ref.table.lower(), ref.column.lower())) - for ref in sim_result.derived_columns: - constrained.add((ref.table.lower(), ref.column.lower())) - for eq_class in sim_result.equivalence_classes: - for ref in eq_class: - constrained.add((ref.table.lower(), ref.column.lower())) - # Base columns feeding a constraint kept in CTE form (is_identity=False): - # the constraint key doesn't name them, but Faker must not fill them blindly — - # the LLM has to pick their values so the derived expression satisfies the filter. - for f in sim_result.filters: - for src in f.source_columns: - constrained.add((src.table.lower(), src.column.lower())) - - # Two classes of columns the SQL text reveals but the simplifier's constraint - # extraction misses — both must stay LLM-controlled, never Faker-filled: - # • GROUP BY keys: need repeated values across rows — Faker would assign a - # unique value per row, destroying the aggregation structure (STDDEV=0, - # wrong counts, etc.). - # • Aggregate-argument *measures* (SUM/AVG/COUNT/MIN/MAX/STDDEV…): the test - # scenario pins these to specific values (e.g. "100 then 150 cases"). - # Faker fills them with arbitrary values disconnected from the - # description → description↔data desync (bad_input_description). The - # measure is the point of the test, not an incidental filler column. - if sql: - try: - import sqlglot - import sqlglot.expressions as exp - - pinned_cols: set[str] = set() - for statement in sqlglot.parse(sql, dialect=dialect): - if statement is None: - continue - for node in statement.walk(): - if isinstance(node, (exp.Group, exp.AggFunc)): - for col in node.find_all(exp.Column): - pinned_cols.add(col.name.lower()) - for table in base_tables: - for col in pinned_cols: - constrained.add((table, col)) - except Exception: - pass - - # If the simplifier found no constraints at all (e.g. filters inside an anonymous - # subquery that it can't propagate), don't Faker-fill anything — the LLM sees the - # full SQL and will respect the WHERE clause on its own. - if not constrained: - logger.debug( - "[faker] source_columns empty — skipping Faker fill, delegating to LLM" - ) - return {} - - faker_cols: dict[str, set[str]] = {} - for entry in used_columns: - db = entry.get("database", "") - table = entry["table"] - uc_key = f"{db}_{table}" if db else table - table_lower = table.lower() - for col in entry["used_columns"]: - if (table_lower, col.lower()) not in constrained: - faker_cols.setdefault(uc_key, set()).add(col.lower()) - return faker_cols - - async def create_combined_model(used_columns, schemas): filtered_columns = filter_columns(schemas, used_columns) return create_pydantic_models(filtered_columns) diff --git a/back/poetry.lock b/back/poetry.lock index cf5d020..29a1686 100644 --- a/back/poetry.lock +++ b/back/poetry.lock @@ -715,24 +715,6 @@ files = [ [package.extras] all = ["adbc-driver-manager", "fsspec", "ipython", "numpy", "pandas", "pyarrow"] -[[package]] -name = "faker" -version = "40.23.0" -description = "Faker is a Python package that generates fake data for you." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "faker-40.23.0-py3-none-any.whl", hash = "sha256:775922453e54afa42eaf60eac478fa3a969357f224d09a8022b93e3ad88f18ae"}, - {file = "faker-40.23.0.tar.gz", hash = "sha256:f135e563f1f95f19346bb680bc2e43570bc43b7893e566023746f51f32c69dfc"}, -] - -[package.dependencies] -tzdata = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -tzdata = ["tzdata"] - [[package]] name = "fastapi" version = "0.138.0" @@ -3434,7 +3416,7 @@ description = "Provider of IANA time zone data" optional = false python-versions = ">=2" groups = ["main"] -markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\" or platform_system == \"Windows\"" +markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\" or (extra == \"trino\" or extra == \"all\") and platform_system == \"Windows\"" files = [ {file = "tzdata-2026.1-py2.py3-none-any.whl", hash = "sha256:4b1d2be7ac37ceafd7327b961aa3a54e467efbdb563a23655fbfe0d39cfc42a9"}, {file = "tzdata-2026.1.tar.gz", hash = "sha256:67658a1903c75917309e753fdc349ac0efd8c27db7a0cb406a25be4840f87f98"}, @@ -3891,4 +3873,4 @@ trino = ["trino"] [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.14" -content-hash = "4e166d17acceb2025c3d9423376c67387cf589630965f90925bef8772c0d9cdb" +content-hash = "4cd28bc782e17d2f869a9fa233b0a6dd7d484f0395408430e242f0a94c6edd23" diff --git a/back/pyproject.toml b/back/pyproject.toml index 930a4ec..2285eae 100644 --- a/back/pyproject.toml +++ b/back/pyproject.toml @@ -47,8 +47,6 @@ sqlglot = {version = "^30.11.0", extras = ["c"]} pandas = "^3.0.3" langgraph = "^1.2.6" langchain-google-genai = "^4.2.5" -faker = "^40.23.0" - # sécurité et hashing bcrypt = "^5.0.0" diff --git a/back/tests/simplifier/test_partition_pinning.py b/back/tests/simplifier/test_partition_pinning.py index 33c76a6..58cbfb7 100644 --- a/back/tests/simplifier/test_partition_pinning.py +++ b/back/tests/simplifier/test_partition_pinning.py @@ -7,10 +7,10 @@ WHERE partition_date <= ) Avant le fix, `_dispatch_pred` ignorait silencieusement tout prédicat -`col = (SELECT …)` : la colonne paraissait « non contrainte », sortait du -schéma Pydantic du générateur (le LLM ne pouvait pas la produire) et partait -en remplissage aléatoire via le sparse_filler — la CTE filtrée devenait vide -et la requête retournait 0 ligne (cf. examples/spider_complexified/models/c1.sql). +`col = (SELECT …)` : la colonne paraissait « non contrainte » et ne recevait +aucun indice explicite dans le prompt de génération — la CTE filtrée devenait +vide et la requête retournait 0 ligne +(cf. examples/spider_complexified/models/c1.sql). Comportement attendu : 1. La colonne externe ET la colonne interne du MAX sont marquées contraintes @@ -39,8 +39,10 @@ def _tbl_match(ref, name: str) -> bool: def _constrained(result) -> set[tuple[str, str]]: - """Réplique le calcul du set `constrained` de _compute_faker_columns : - toute colonne absente de ce set est remplie aléatoirement par le sparse_filler.""" + """Colonnes que le simplifier considère contraintes (source_columns / derived / + equivalence / filters). Ces colonnes reçoivent un indice de contrainte dans le + prompt du générateur — le LLM doit produire une valeur qui les satisfait (ex. une + colonne de partition = MAX(...)).""" out: set[tuple[str, str]] = set() for ref in result.source_columns: out.add((ref.table.lower(), ref.column.lower())) diff --git a/back/tests/test_faker_agg_measure.py b/back/tests/test_faker_agg_measure.py deleted file mode 100644 index 3ce72ae..0000000 --- a/back/tests/test_faker_agg_measure.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Régression — une colonne *mesure* (argument d'agrégat) ne doit jamais partir -en Faker-fill. - -Cas réel : bq018 (examples/spider). Le SQL agrège `SUM(cumulative_confirmed)`, -puis applique `LAG` + ranking. `cumulative_confirmed` n'est ni dans un `WHERE`, -ni dans le `GROUP BY` (qui porte sur `date`) → le simplifier ne la voit pas -comme contrainte → elle tombait dans `faker_cols` et était remplie de valeurs -arbitraires (5269→2098, 6603→6238…) déconnectées de la description du scénario -(« 100 puis 150 cas »). Résultat : désync description↔données détectée comme -`bad_input_description`, test invalide. - -Comportement attendu : la colonne argument d'un agrégat (SUM/AVG/COUNT/MIN/MAX/ -STDDEV…) est marquée contrainte → c'est le LLM qui choisit ses valeurs en -cohérence avec la description, jamais Faker. -""" - -from build_query.constraint_simplifier import simplify -from build_query.examples_generator import _compute_faker_columns - - -# ─── 1. La mesure agrégée n'est PAS Faker-fill (régression bq018) ──────────── - - -def test_aggregate_measure_is_not_faker_filled(): - sql = ( - "SELECT date, SUM(cumulative_confirmed) AS cases " - "FROM `bigquery-public-data.covid19_open_data.covid19_open_data` " - "WHERE country_name = 'United States of America' " - "AND date BETWEEN '2020-03-01' AND '2020-04-30' " - "GROUP BY date" - ) - used_columns = [ - { - "project": "bigquery-public-data", - "database": "covid19_open_data", - "table": "covid19_open_data", - "used_columns": ["country_name", "cumulative_confirmed", "date"], - } - ] - base_tables = {"covid19_open_data"} - sim_result = simplify(sql, dialect="bigquery") - - faker = _compute_faker_columns( - sim_result, used_columns, base_tables, sql=sql, dialect="bigquery" - ) - - uc_key = "covid19_open_data_covid19_open_data" - assert "cumulative_confirmed" not in faker.get(uc_key, set()), ( - "cumulative_confirmed est l'argument de SUM(...) — la mesure que la " - "description du test épingle ('100 puis 150 cas'). La Faker-fill avec " - "des valeurs arbitraires crée une désync description↔données " - "(bad_input_description)." - ) - - -# ─── 2. Garde — un dimension de remplissage reste bien Faker-fill ──────────── - - -def test_non_aggregate_passthrough_still_faker_filled(): - """Le fix ne doit pas geler tout : une colonne ni filtrée, ni agrégée, ni - groupée reste éligible au Faker-fill.""" - sql = "SELECT id, label FROM `ds.t` WHERE status = 'active'" - used_columns = [ - { - "database": "ds", - "table": "t", - "used_columns": ["id", "label", "status"], - } - ] - base_tables = {"t"} - sim_result = simplify(sql, dialect="bigquery") - - faker = _compute_faker_columns( - sim_result, used_columns, base_tables, sql=sql, dialect="bigquery" - ) - - filled = faker.get("ds_t", set()) - assert "label" in filled, ( - "label est une dimension de remplissage (ni filtre, ni agrégat, ni " - "GROUP BY) — elle doit rester Faker-fill" - ) diff --git a/back/tests/test_fix_duck_db_sql.py b/back/tests/test_fix_duck_db_sql.py index 282cf23..55dd63d 100644 --- a/back/tests/test_fix_duck_db_sql.py +++ b/back/tests/test_fix_duck_db_sql.py @@ -142,16 +142,46 @@ def test_safe_parse_date_end_to_end(self, con): assert "SAFE.CAST" not in fixed duckdb_ok(con, fixed) - def test_parse_datetime_end_to_end(self, con): - """PARSE_DATETIME → TRY_STRPTIME (retourne NULL si valeur incompatible avec le format).""" + def test_parse_datetime_strict_raises_on_malformed_value(self, con): + """PARSE_DATETIME strict conserve l'erreur que BigQuery lèverait.""" raw = transpile("SELECT PARSE_DATETIME('%Y-%m-%d %H:%M:%S', s) FROM events") duckdb_fails(con, raw) fixed = fix_duck_db_sql(raw) - assert "TRY_STRPTIME" in fixed + assert "STRPTIME" in fixed + assert "TRY_STRPTIME" not in fixed assert "PARSE_DATETIME" not in fixed - duckdb_ok(con, fixed) + duckdb_fails(con, fixed) + + @pytest.mark.parametrize( + "function_name", + ["PARSE_DATE", "PARSE_DATETIME", "PARSE_TIMESTAMP"], + ) + def test_all_strict_parse_variants_raise_on_malformed_value( + self, con, function_name + ): + raw = transpile( + f"SELECT {function_name}('%Y-%m-%d %H:%M:%S', s) FROM events" + ) + fixed = fix_duck_db_sql(raw) + + assert "TRY_STRPTIME" not in fixed.upper() + duckdb_fails(con, fixed) + + @pytest.mark.parametrize( + "function_name", + ["PARSE_DATE", "PARSE_DATETIME", "PARSE_TIMESTAMP"], + ) + def test_all_safe_parse_variants_return_null_on_malformed_value( + self, con, function_name + ): + raw = transpile( + f"SELECT SAFE.{function_name}('%Y-%m-%d %H:%M:%S', s) FROM events" + ) + fixed = fix_duck_db_sql(raw) + + assert con.execute(fixed).fetchone()[0] is None def test_safe_cast_already_translated_by_sqlglot(self, con): """sqlglot traduit SAFE_CAST → TRY_CAST nativement ; fix ne doit pas le casser.""" @@ -691,56 +721,43 @@ class TestParseDatetimeArgOrder: quel que soit l'ordre. """ - def test_canary_sqlglot_30_produces_value_first(self): - """ - CANARY — sqlglot 30+ inverse les args : PARSE_DATETIME(value, '%fmt'). - Si ce test échoue, sqlglot a re-changé l'ordre et le fix doit être adapté. - """ - raw = transpile("PARSE_DATETIME('%Y-%m-%d', col)") - assert raw.startswith("PARSE_DATETIME(col"), ( - f"CANARY : sqlglot ne produit plus value-first pour PARSE_DATETIME. raw={raw!r}. " - "Vérifier si la logique de détection '%' est toujours correcte." - ) - - def test_canary_sqlglot_does_not_translate_parse_datetime(self): - """ - CANARY — sqlglot ne traduit pas PARSE_DATETIME → TRY_STRPTIME nativement. - Si ce test échoue, fix_duck_db_sql n'est plus utile pour ce cas. - """ + def test_canary_sqlglot_translates_parse_datetime_natively(self): + """sqlglot 30.12+ traduit PARSE_DATETIME en STRPTIME strict.""" raw = transpile("PARSE_DATETIME('%Y-%m-%d', col)") - assert "PARSE_DATETIME" in raw.upper(), ( - f"CANARY : sqlglot traduit maintenant PARSE_DATETIME nativement. raw={raw!r}. " - "La correction fix_duck_db_sql est désormais redondante pour ce cas." + assert "STRPTIME" in raw.upper() and "PARSE_DATETIME" not in raw.upper(), ( + f"CANARY : sqlglot ne traduit plus PARSE_DATETIME nativement. raw={raw!r}. " + "Le fallback texte de fix_duck_db_sql doit alors être réévalué." ) + assert "TRY_STRPTIME" not in raw.upper() def test_literal_value_first_correctly_converted(self): """ sqlglot 30+ : PARSE_DATETIME('2024-01-15', '%Y-%m-%d') - fix doit produire : TRY_STRPTIME('2024-01-15', '%Y-%m-%d') + fix doit produire : STRPTIME('2024-01-15', '%Y-%m-%d') résultat attendu : timestamp non-NULL. """ raw_scalar = "PARSE_DATETIME('2024-01-15', '%Y-%m-%d')" fixed = fix_duck_db_sql(f"SELECT {raw_scalar}") fixed_expr = fixed[len("SELECT ") :] - assert "TRY_STRPTIME" in fixed_expr, ( - f"fix n'a pas produit TRY_STRPTIME : {fixed_expr!r}" + assert "STRPTIME" in fixed_expr and "TRY_STRPTIME" not in fixed_expr, ( + f"fix n'a pas produit STRPTIME strict : {fixed_expr!r}" ) result = duckdb.connect().execute(fixed).fetchone()[0] assert result is not None, ( - "TRY_STRPTIME a retourné NULL — args probablement inversés" + "STRPTIME a retourné NULL — args probablement inversés" ) def test_col_value_first_correctly_converted(self, con): - """ - sqlglot 30+ : PARSE_DATETIME(col, '%Y-%m-%d %H:%M:%S') - fix doit produire : TRY_STRPTIME(col, '%Y-%m-%d %H:%M:%S'). - """ - raw = transpile("SELECT PARSE_DATETIME('%Y-%m-%d %H:%M:%S', s) FROM events") - assert "PARSE_DATETIME" in raw - fixed = fix_duck_db_sql(raw) - assert "TRY_STRPTIME" in fixed - assert "PARSE_DATETIME" not in fixed.upper() - con.execute(fixed) + """Le format est appliqué à la bonne colonne, tout en restant strict.""" + matching = fix_duck_db_sql( + transpile("SELECT PARSE_DATETIME('%Y-%m-%d', s) FROM events") + ) + assert con.execute(matching).fetchone()[0] is not None + + mismatching = fix_duck_db_sql( + transpile("SELECT PARSE_DATETIME('%Y-%m-%d %H:%M:%S', s) FROM events") + ) + duckdb_fails(con, mismatching) def test_format_first_legacy_still_converted(self, con): """ @@ -751,8 +768,9 @@ def test_format_first_legacy_still_converted(self, con): "SELECT PARSE_DATETIME('%Y-%m-%d %H:%M:%S', s) FROM events" ) fixed = fix_duck_db_sql(legacy_format_first) - assert "TRY_STRPTIME(s, '%Y-%m-%d %H:%M:%S')" in fixed - con.execute(fixed) + assert "STRPTIME(s, '%Y-%m-%d %H:%M:%S')" in fixed + assert "TRY_STRPTIME" not in fixed + duckdb_fails(con, fixed) # =========================================================================== @@ -772,12 +790,12 @@ class TestSqlglotVersionCanaries: # --- Cas où sqlglot NE corrige PAS (fix encore nécessaire) --- - def test_canary_parse_datetime_not_translated(self): - """sqlglot ne traduit pas PARSE_DATETIME → TRY_STRPTIME.""" + def test_canary_parse_datetime_translated_strictly(self): + """sqlglot traduit PARSE_DATETIME en STRPTIME sans tolérer les erreurs.""" raw = transpile("PARSE_DATETIME('%Y-%m-%d', col)") - assert "PARSE_DATETIME" in raw.upper(), ( - "CANARY ROMPU : sqlglot traduit maintenant PARSE_DATETIME — " - "supprimer la correction dans fix_duck_db_sql." + assert "STRPTIME" in raw.upper() and "TRY_STRPTIME" not in raw.upper(), ( + "CANARY ROMPU : la traduction native de PARSE_DATETIME n'est plus stricte — " + "réévaluer fix_duck_db_sql." ) def test_canary_extract_date_not_translated(self): diff --git a/back/tests/test_generate_type.py b/back/tests/test_generate_type.py index 31bead3..e4ca99a 100644 --- a/back/tests/test_generate_type.py +++ b/back/tests/test_generate_type.py @@ -308,7 +308,7 @@ def test_filter_columns_case_insensitive_dialect_qualification(self): used_columns en minuscules, alors que le schema_cache garde la casse d'origine de l'entrepôt. Le match database/table doit être insensible à la casse — sinon filtered_schema se vide, le modèle de génération n'a - aucune table, et le LLM ne produit aucune donnée (seul Faker survit).""" + aucune table, et le LLM ne produit aucune donnée.""" schema = [ { "table_name": "pipetalk-493612.MONETIQUE_Dataset_Porteur.DS_RCOMP_DASHBOARD_RESEAU", @@ -334,7 +334,7 @@ def test_filter_columns_case_insensitive_dialect_qualification(self): # La table doit être retrouvée malgré la différence de casse… self.assertEqual(len(filtered), 1) # …et la clé émise doit suivre la casse de used_columns (source de vérité du - # pipeline : faker_cols + executor utilisent f"{db}_{table}" issu de used_columns). + # pipeline : le générateur et l'executor utilisent f"{db}_{table}" issu de used_columns). self.assertEqual( filtered[0]["table_name"], "monetique_dataset_porteur_ds_rcomp_dashboard_reseau", diff --git a/back/tests/test_scalar_folder_fix_needed.py b/back/tests/test_scalar_folder_fix_needed.py index 411f515..a918ace 100644 --- a/back/tests/test_scalar_folder_fix_needed.py +++ b/back/tests/test_scalar_folder_fix_needed.py @@ -147,43 +147,10 @@ def test_extract_date_expression_folded_with_fix(self): assert "2024-01-15" in result assert "EXTRACT" not in result.upper() - def test_parse_datetime_fails_in_duckdb(self): - """ - PARSE_DATETIME('%Y-%m-%d', '2024-01-15') : - sqlglot 30 inverse les args → PARSE_DATETIME('2024-01-15', '%Y-%m-%d') - DuckDB ne connaît pas PARSE_DATETIME → erreur → fold raté. - """ - duck_expr = _duck("PARSE_DATETIME('%Y-%m-%d', '2024-01-15')") - - # sqlglot 30+ met la valeur en premier, le format en second - assert duck_expr.startswith("PARSE_DATETIME('2024-01-15'") - - with pytest.raises(duckdb.Error): - _eval_with_duckdb(duck_expr) - - def test_parse_datetime_fix_duck_db_sql_correctly_ordered(self): - """ - Bug corrigé : fix_duck_db_sql détecte maintenant l'arg format via '%'. - entrée : PARSE_DATETIME('2024-01-15', '%Y-%m-%d') ← valeur 1er (sqlglot 30+) - sortie : TRY_STRPTIME('2024-01-15', '%Y-%m-%d') ← ordre correct - résultat DuckDB : timestamp non-NULL - - Intégrer fix_duck_db_sql résout maintenant ce cas. - """ - duck_expr = _duck("PARSE_DATETIME('%Y-%m-%d', '2024-01-15')") - fixed_sql = fix_duck_db_sql(f"SELECT {duck_expr}") - fixed_expr = fixed_sql[len("SELECT ") :] - - assert "TRY_STRPTIME" in fixed_expr - result = _eval_with_duckdb(fixed_expr) - assert result is not None, ( - f"TRY_STRPTIME a retourné NULL — args peut-être encore inversés : {fixed_expr!r}" - ) - def test_parse_datetime_expression_folded_with_fix(self): """ - Avec fix_duck_db_sql intégré, PARSE_DATETIME('%Y-%m-%d', '2024-01-15') - est replié en la valeur timestamp correspondante. + PARSE_DATETIME('%Y-%m-%d', '2024-01-15') est replié en valeur timestamp + avec la traduction STRPTIME native de sqlglot 30.12+. """ sql = "SELECT PARSE_DATETIME('%Y-%m-%d', '2024-01-15') AS dt FROM t" result = _fold(sql) @@ -223,6 +190,14 @@ def test_safe_cast_already_transpiled_to_try_cast(self): result = _eval_with_duckdb(duck_expr) assert result == 123 + def test_parse_datetime_already_transpiled_to_strptime(self): + """sqlglot 30.12+ produit un STRPTIME DuckDB valide et strict.""" + duck_expr = _duck("PARSE_DATETIME('%Y-%m-%d', '2024-01-15')") + assert "STRPTIME" in duck_expr.upper() + assert "PARSE_DATETIME" not in duck_expr.upper() + assert "TRY_STRPTIME" not in duck_expr.upper() + assert _eval_with_duckdb(duck_expr) is not None + def test_date_diff_already_transpiled(self): """ DATE_DIFF(DATE '2024-12-31', DATE '2024-01-01', DAY) : diff --git a/back/tests/test_trino_schema_match.py b/back/tests/test_trino_schema_match.py index aeccf04..54848fb 100644 --- a/back/tests/test_trino_schema_match.py +++ b/back/tests/test_trino_schema_match.py @@ -12,7 +12,6 @@ import unittest from build_query.examples_executor import filter_schemas_by_used_columns -from utils.faker_fill import generate_faker_rows # schema_cache : casse d'origine BigQuery (mixte), noms 3 parties. @@ -83,32 +82,5 @@ def test_bigquery_matching_case_unchanged(self): self.assertTrue(filtered[0]["table_name"].endswith("banques")) -class TestFakerFillCaseInsensitive(unittest.TestCase): - def test_trino_numeric_column_gets_numeric_value(self): - """faker_cols est en minuscules (Trino) mais le nom de table du schéma garde - la casse d'origine. Sans normalisation, la résolution de type échoue → toutes - les colonnes retombent sur STRING → un mot Faker ('help') est injecté dans une - colonne numérique → 'could not convert string to float' au CAST DuckDB.""" - schema = [ - { - "table_name": "pipetalk-493612.MARKETING_GR_source_ref_bpce.coface", - "columns": [ - {"name": "mtcaht", "type": "NUMERIC", "bq_ddl_type": "NUMERIC"}, - {"name": "liensc", "type": "STRING", "bq_ddl_type": "STRING"}, - ], - } - ] - # Clé faker en minuscules (produite par la qualification Trino). - faker_cols = {"marketing_gr_source_ref_bpce_coface": {"mtcaht", "liensc"}} - - rows = generate_faker_rows(schema, faker_cols, filled_data={}, profile=None) - out = rows["marketing_gr_source_ref_bpce_coface"] - self.assertTrue(out, "aucune ligne générée") - # La colonne numérique doit recevoir un nombre, pas un mot. - self.assertIsInstance(out[0]["mtcaht"], (int, float)) - # La colonne texte reste une chaîne. - self.assertIsInstance(out[0]["liensc"], str) - - if __name__ == "__main__": unittest.main() diff --git a/back/utils/examples.py b/back/utils/examples.py index 7752375..93025ff 100644 --- a/back/utils/examples.py +++ b/back/utils/examples.py @@ -45,8 +45,8 @@ def filter_columns(schemas, used_columns): # dialectes (Trino…) met les identifiants de used_columns en minuscules, # alors que le schema_cache conserve la casse d'origine de l'entrepôt # (BigQuery). Sans .lower() des deux côtés, aucune table ne matche → le - # schéma de génération se vide, le LLM ne produit aucune donnée, et seul - # Faker survit (tables de fait manquantes → Catalog Error à l'exécution). + # schéma de génération se vide, le LLM ne produit aucune donnée, puis les + # tables de fait manquent à l'exécution (Catalog Error). used_table_entry = next( ( item @@ -59,8 +59,8 @@ def filter_columns(schemas, used_columns): if used_table_entry: # Nom de table final aligné sur la casse de used_columns (source de - # vérité du pipeline : faker_cols et l'executor construisent tous la - # clé f"{db}_{table}" à partir de used_columns). En BigQuery la casse + # vérité du pipeline : le générateur et l'executor construisent tous + # deux la clé f"{db}_{table}" à partir de used_columns). En BigQuery la casse # coïncide avec le schéma → sortie inchangée ; en Trino elle suit # used_columns (minuscules) → cohérence des clés en aval. db_key = used_table_entry.get("database") or db_name_from_schema @@ -839,10 +839,10 @@ def _fix_date_trunc_week(match): # DATE_TRUNC('WEEK', ...) sans jour spécifié s = re.sub(r"DATE_TRUNC\('WEEK',", "DATE_TRUNC('week',", s, flags=re.IGNORECASE) - # === SAFE.PARSE_DATE / SAFE.PARSE_TIMESTAMP === + # === SAFE.PARSE_DATE / SAFE.PARSE_DATETIME / SAFE.PARSE_TIMESTAMP === # sqlglot <30 : SAFE.PARSE_DATE('%fmt', col) # sqlglot 30+ : SAFE.CAST(STRPTIME(col, '%fmt') AS DATE) - # DuckDB attend : TRY_STRPTIME(col, '%fmt') + # ou SAFE.STRPTIME(col, '%fmt'). DuckDB attend une variante tolérante. s = re.sub( r"SAFE\.CAST\s*\(\s*STRPTIME\s*\(\s*([^,]+?)\s*,\s*'([^']+)'\s*\)\s*AS\s+\w+\s*\)", @@ -865,10 +865,17 @@ def _fix_date_trunc_week(match): flags=re.IGNORECASE, ) + s = re.sub( + r"SAFE\.STRPTIME\s*\(", + "TRY_STRPTIME(", + s, + flags=re.IGNORECASE, + ) + # === PARSE_DATETIME === # sqlglot <30 : PARSE_DATETIME('%fmt', col) — format first # sqlglot 30+ : PARSE_DATETIME(col, '%fmt') — col first (ou littéral first) - # DuckDB attend : TRY_STRPTIME(col, '%fmt') + # DuckDB attend : STRPTIME(col, '%fmt') — la fonction est stricte. # # Stratégie : l'arg format est identifié par son préfixe '%'. # Cela couvre les deux ordres sans dépendre de la version de sqlglot. @@ -876,9 +883,9 @@ def _fix_date_trunc_week(match): def _fix_parse_datetime(m): a1, a2 = m.group(1).strip(), m.group(2).strip() if a1.startswith("'%"): # format en 1er (sqlglot <30) - return f"TRY_STRPTIME({a2}, {a1})" + return f"STRPTIME({a2}, {a1})" elif a2.startswith("'%"): # valeur en 1er (sqlglot 30+) - return f"TRY_STRPTIME({a1}, {a2})" + return f"STRPTIME({a1}, {a2})" return m.group(0) # indéterminable, laisser tel quel s = re.sub( diff --git a/back/utils/faker_fill.py b/back/utils/faker_fill.py deleted file mode 100644 index 89d457f..0000000 --- a/back/utils/faker_fill.py +++ /dev/null @@ -1,164 +0,0 @@ -import random -from faker import Faker - -_faker = Faker() - - -def _table_uc_key(table_name: str) -> str: - """Convert a fully-qualified table name to the uc_key used in filtered_schema.""" - parts = table_name.split(".") - return "_".join(parts[-2:]) if len(parts) >= 2 else parts[-1] - - -def _fake_value_for_type(bq_type: str): - """Return a plausible fake value for the given BigQuery DDL type.""" - upper = bq_type.upper().strip() - if upper.startswith("ARRAY<") or upper.startswith("STRUCT<"): - return None - if any( - t in upper - for t in ("INT64", "INTEGER", "SMALLINT", "BIGINT", "TINYINT", "BYTEINT", "INT") - ): - return random.randint(1, 10_000) - if any( - t in upper for t in ("FLOAT64", "FLOAT", "NUMERIC", "BIGNUMERIC", "DECIMAL") - ): - return round(random.uniform(0.01, 9_999.99), 2) - if "TIMESTAMP" in upper or "DATETIME" in upper: - return _faker.date_time_between("-2y", "now").strftime("%Y-%m-%dT%H:%M:%S") - if "DATE" in upper: - return _faker.date_between("-2y", "today").isoformat() - if "TIME" in upper: - return _faker.time() - if "BOOL" in upper: - return random.choice([True, False]) - if "BYTES" in upper: - return None - return _faker.word() - - -def _build_profile_index(profile: dict | None) -> dict[str, dict[str, dict]]: - """Build {uc_key: {col_name_lower: stats}} from profile.""" - if not profile or not profile.get("tables"): - return {} - index: dict[str, dict[str, dict]] = {} - for tbl_key, tbl_data in profile["tables"].items(): - parts = tbl_key.split(".") - uc_key = ("_".join(parts[-2:]) if len(parts) >= 2 else parts[-1]).lower() - cols = tbl_data.get("columns", {}) - index[uc_key] = {col_name.lower(): stats for col_name, stats in cols.items()} - return index - - -def _value_from_profile(stats: dict, bq_type: str): - """Return a value drawn from profile stats, or None if not applicable.""" - top_values = stats.get("top_values") or [] - if top_values: - return random.choice(top_values) - - min_v = ( - stats.get("min_value") - if stats.get("min_value") is not None - else stats.get("min_val") - ) - max_v = ( - stats.get("max_value") - if stats.get("max_value") is not None - else stats.get("max_val") - ) - if min_v is None or max_v is None: - return None - - upper = bq_type.upper().strip() - try: - if any( - t in upper - for t in ( - "INT64", - "INTEGER", - "SMALLINT", - "BIGINT", - "TINYINT", - "BYTEINT", - "INT", - ) - ): - lo, hi = int(float(min_v)), int(float(max_v)) - return random.randint(lo, hi) if lo <= hi else lo - if any( - t in upper for t in ("FLOAT64", "FLOAT", "NUMERIC", "BIGNUMERIC", "DECIMAL") - ): - lo, hi = float(min_v), float(max_v) - return round(random.uniform(lo, hi), 2) if lo <= hi else lo - except (ValueError, TypeError): - pass - return None - - -def generate_faker_rows( - schema: list, - faker_cols_by_uc_key: dict[str, set[str]], - filled_data: dict[str, list], - profile: dict | None = None, -) -> dict[str, list[dict]]: - """Generate rows for unconstrained columns, preferring profile stats over Faker. - - Priority per column: - 1. top_values from profile → random.choice - 2. min/max from profile (int/float) → random in range - 3. Faker fallback - - Args: - schema: Full project schema (list of table dicts with columns). - faker_cols_by_uc_key: Mapping from uc_key (database_table) to the set of - column names that should be filled. - filled_data: LLM-generated data keyed by uc_key; used to determine row count. - profile: Optional statistical profile dict (same structure as QueryState.profile). - - Returns: - Mapping from uc_key to list of row dicts containing only the filled columns. - """ - # Build a column-type index keyed by uc_key — en MINUSCULES. La qualification - # sqlglot de certains dialectes (Trino…) met les clés de faker_cols en minuscules - # alors que _table_uc_key conserve la casse d'origine du schéma. Sans normalisation, - # aucune table ne matche → type_index vide → toutes les colonnes retombent sur - # STRING → un mot Faker ("help") atterrit dans une colonne numérique → échec du - # CAST DuckDB ("could not convert string to float"). - faker_keys_lower = {k.lower() for k in faker_cols_by_uc_key} - type_index: dict[str, dict[str, str]] = {} - for table_entry in schema: - key = _table_uc_key(table_entry["table_name"]).lower() - if key not in faker_keys_lower: - continue - type_index[key] = { - col["name"].lower(): col.get("bq_ddl_type") or col.get("type", "STRING") - for col in table_entry["columns"] - } - - profile_index = _build_profile_index(profile) - - # Determine row count from LLM data; fall back to 3 - default_n = 3 - if filled_data: - for rows in filled_data.values(): - if isinstance(rows, list) and rows: - default_n = len(rows) - break - - result: dict[str, list[dict]] = {} - for uc_key, col_names in faker_cols_by_uc_key.items(): - col_types = type_index.get(uc_key.lower(), {}) - col_profile = profile_index.get(uc_key.lower(), {}) - llm_rows = filled_data.get(uc_key) or [] - n_rows = len(llm_rows) if llm_rows else default_n - rows = [] - for _ in range(n_rows): - row = {} - for col in col_names: - bq_type = col_types.get(col, "STRING") - stats = col_profile.get(col) - val = _value_from_profile(stats, bq_type) if stats else None - row[col] = val if val is not None else _fake_value_for_type(bq_type) - rows.append(row) - result[uc_key] = rows - return result From 884ea4a0a66be82ab58de4c490448b89aeb793cb Mon Sep 17 00:00:00 2001 From: skadel Date: Sun, 26 Jul 2026 15:22:04 +0200 Subject: [PATCH 2/2] fix(duckdb): support locked sqlglot parse rendering --- back/tests/test_fix_duck_db_sql.py | 27 ++++++++++----------- back/tests/test_scalar_folder_fix_needed.py | 13 +++++----- back/utils/examples.py | 17 +++++++------ 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/back/tests/test_fix_duck_db_sql.py b/back/tests/test_fix_duck_db_sql.py index 55dd63d..08a2e85 100644 --- a/back/tests/test_fix_duck_db_sql.py +++ b/back/tests/test_fix_duck_db_sql.py @@ -161,9 +161,7 @@ def test_parse_datetime_strict_raises_on_malformed_value(self, con): def test_all_strict_parse_variants_raise_on_malformed_value( self, con, function_name ): - raw = transpile( - f"SELECT {function_name}('%Y-%m-%d %H:%M:%S', s) FROM events" - ) + raw = transpile(f"SELECT {function_name}('%Y-%m-%d %H:%M:%S', s) FROM events") fixed = fix_duck_db_sql(raw) assert "TRY_STRPTIME" not in fixed.upper() @@ -721,14 +719,14 @@ class TestParseDatetimeArgOrder: quel que soit l'ordre. """ - def test_canary_sqlglot_translates_parse_datetime_natively(self): - """sqlglot 30.12+ traduit PARSE_DATETIME en STRPTIME strict.""" + def test_parse_datetime_pipeline_is_strict_across_sqlglot_versions(self): + """Le rendu 30.11 ou 30.12+ finit toujours en STRPTIME strict.""" raw = transpile("PARSE_DATETIME('%Y-%m-%d', col)") - assert "STRPTIME" in raw.upper() and "PARSE_DATETIME" not in raw.upper(), ( - f"CANARY : sqlglot ne traduit plus PARSE_DATETIME nativement. raw={raw!r}. " - "Le fallback texte de fix_duck_db_sql doit alors être réévalué." + fixed = fix_duck_db_sql(raw) + assert "STRPTIME" in fixed.upper() and "PARSE_DATETIME" not in fixed.upper(), ( + f"Le pipeline ne traduit plus PARSE_DATETIME. raw={raw!r}, fixed={fixed!r}." ) - assert "TRY_STRPTIME" not in raw.upper() + assert "TRY_STRPTIME" not in fixed.upper() def test_literal_value_first_correctly_converted(self): """ @@ -790,12 +788,13 @@ class TestSqlglotVersionCanaries: # --- Cas où sqlglot NE corrige PAS (fix encore nécessaire) --- - def test_canary_parse_datetime_translated_strictly(self): - """sqlglot traduit PARSE_DATETIME en STRPTIME sans tolérer les erreurs.""" + def test_canary_parse_datetime_pipeline_remains_strict(self): + """Alerte si le pipeline PARSE_DATETIME devient tolérant aux erreurs.""" raw = transpile("PARSE_DATETIME('%Y-%m-%d', col)") - assert "STRPTIME" in raw.upper() and "TRY_STRPTIME" not in raw.upper(), ( - "CANARY ROMPU : la traduction native de PARSE_DATETIME n'est plus stricte — " - "réévaluer fix_duck_db_sql." + fixed = fix_duck_db_sql(raw) + assert "STRPTIME" in fixed.upper() and "TRY_STRPTIME" not in fixed.upper(), ( + "CANARY ROMPU : le pipeline PARSE_DATETIME n'est plus strict — " + f"raw={raw!r}, fixed={fixed!r}." ) def test_canary_extract_date_not_translated(self): diff --git a/back/tests/test_scalar_folder_fix_needed.py b/back/tests/test_scalar_folder_fix_needed.py index a918ace..28e8ed3 100644 --- a/back/tests/test_scalar_folder_fix_needed.py +++ b/back/tests/test_scalar_folder_fix_needed.py @@ -190,13 +190,14 @@ def test_safe_cast_already_transpiled_to_try_cast(self): result = _eval_with_duckdb(duck_expr) assert result == 123 - def test_parse_datetime_already_transpiled_to_strptime(self): - """sqlglot 30.12+ produit un STRPTIME DuckDB valide et strict.""" + def test_parse_datetime_pipeline_produces_strptime(self): + """Le pipeline produit un STRPTIME valide avec sqlglot 30.11 ou 30.12+.""" duck_expr = _duck("PARSE_DATETIME('%Y-%m-%d', '2024-01-15')") - assert "STRPTIME" in duck_expr.upper() - assert "PARSE_DATETIME" not in duck_expr.upper() - assert "TRY_STRPTIME" not in duck_expr.upper() - assert _eval_with_duckdb(duck_expr) is not None + fixed_expr = fix_duck_db_sql(f"SELECT {duck_expr}")[len("SELECT ") :] + assert "STRPTIME" in fixed_expr.upper() + assert "PARSE_DATETIME" not in fixed_expr.upper() + assert "TRY_STRPTIME" not in fixed_expr.upper() + assert _eval_with_duckdb(fixed_expr) is not None def test_date_diff_already_transpiled(self): """ diff --git a/back/utils/examples.py b/back/utils/examples.py index 93025ff..4886c9c 100644 --- a/back/utils/examples.py +++ b/back/utils/examples.py @@ -865,13 +865,6 @@ def _fix_date_trunc_week(match): flags=re.IGNORECASE, ) - s = re.sub( - r"SAFE\.STRPTIME\s*\(", - "TRY_STRPTIME(", - s, - flags=re.IGNORECASE, - ) - # === PARSE_DATETIME === # sqlglot <30 : PARSE_DATETIME('%fmt', col) — format first # sqlglot 30+ : PARSE_DATETIME(col, '%fmt') — col first (ou littéral first) @@ -895,6 +888,16 @@ def _fix_parse_datetime(m): flags=re.IGNORECASE, ) + # À appliquer APRÈS PARSE_DATETIME : avec sqlglot 30.11, le rendu + # SAFE.PARSE_DATETIME est d'abord transformé ci-dessus en SAFE.STRPTIME. + # Avec 30.12+, SAFE.STRPTIME est produit directement. + s = re.sub( + r"SAFE\.STRPTIME\s*\(", + "TRY_STRPTIME(", + s, + flags=re.IGNORECASE, + ) + # === EXTRACT(DATE FROM ...) === # sqlglot 30 enveloppe le littéral timestamp dans un CAST(...AS TIMESTAMPTZ), # ce qui ajoute des parens imbriquées. Le pattern gère un niveau d'imbrication.