diff --git a/dat/spark_builder.py b/dat/spark_builder.py index 1c4520a3..4eb6743f 100644 --- a/dat/spark_builder.py +++ b/dat/spark_builder.py @@ -18,4 +18,9 @@ def get_spark_session(): 'org.apache.spark.sql.delta.catalog.DeltaCatalog', ) builder = delta.configure_spark_with_delta_pip(builder) - return builder.getOrCreate() + spark = builder.enableHiveSupport().getOrCreate() + hadoop = spark.sparkContext._jvm.org.apache.hadoop # type: ignore + hadoop_conf = spark._jsc.hadoopConfiguration() # type: ignore + fs = hadoop.fs.FileSystem.get(hadoop_conf) # type: ignore + fs.setWriteChecksum(False) + return spark