From 05ce235a066588e2fa9565940bdcde65867e03dd Mon Sep 17 00:00:00 2001 From: Zach Schuermann Date: Mon, 18 Nov 2024 16:20:31 -0800 Subject: [PATCH 1/3] add col mapping feature --- dat/generated_tables.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dat/generated_tables.py b/dat/generated_tables.py index 4cd6e63b..43f9014b 100644 --- a/dat/generated_tables.py +++ b/dat/generated_tables.py @@ -570,4 +570,5 @@ def create_iceberg_compat_v1(case: TestCaseInfo, spark: SparkSession): .execute() ) delta_table.upgradeTableProtocol(3, 7) + delta_table.addFeatureSupport("columnMapping") df.repartition(1).write.format("delta").mode("append").save(case.delta_root) From 68d6ba5296efcf36182f7e29f1f02443414273b3 Mon Sep 17 00:00:00 2001 From: Zach Schuermann Date: Mon, 18 Nov 2024 16:26:31 -0800 Subject: [PATCH 2/3] update delta --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 02b27425..8bb1159e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ python = "^3.10" pydantic = "^2.6.4" pyspark = "^3.5.1" click = "^8.1.7" -delta-spark = "^3.1.0" +delta-spark = "^3.3.0" rootpath = "^0.1.1" pyarrow = "^15.0.2" From 581b8253d9f268a08f94e2b327d44f9e7c359306 Mon Sep 17 00:00:00 2001 From: Zach Schuermann Date: Mon, 18 Nov 2024 16:31:31 -0800 Subject: [PATCH 3/3] uh try 4.0? --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8bb1159e..0366b111 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ python = "^3.10" pydantic = "^2.6.4" pyspark = "^3.5.1" click = "^8.1.7" -delta-spark = "^3.3.0" +delta-spark = "^4.0.0rc1" rootpath = "^0.1.1" pyarrow = "^15.0.2"