From cd80d5edfc4a2638ebf4bbaf3e723b32fec00412 Mon Sep 17 00:00:00 2001 From: claudiazi Date: Wed, 18 Jun 2025 17:44:54 +0200 Subject: [PATCH 1/4] feat: convert soda to deferrable --- dagger/dag_creator/airflow/operator_creators/soda_creator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dagger/dag_creator/airflow/operator_creators/soda_creator.py b/dagger/dag_creator/airflow/operator_creators/soda_creator.py index 8b1f68a..fdf9931 100644 --- a/dagger/dag_creator/airflow/operator_creators/soda_creator.py +++ b/dagger/dag_creator/airflow/operator_creators/soda_creator.py @@ -11,6 +11,7 @@ class SodaCreator(BatchCreator): def __init__(self, task, dag): super().__init__(task, dag) + self.deferrable = True self._absolute_job_name = task.absolute_job_name self._table_name = task.table_name self._output_s3_path = task.output_s3_path From f961d1530456ab49b9937101d4254e24d0c15a17 Mon Sep 17 00:00:00 2001 From: claudiazi Date: Thu, 19 Jun 2025 14:05:04 +0200 Subject: [PATCH 2/4] feat: convert soda to deferrable --- dagger/dag_creator/airflow/operator_creators/soda_creator.py | 1 - dagger/dag_creator/airflow/operators/soda_batch.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/dagger/dag_creator/airflow/operator_creators/soda_creator.py b/dagger/dag_creator/airflow/operator_creators/soda_creator.py index fdf9931..8b1f68a 100644 --- a/dagger/dag_creator/airflow/operator_creators/soda_creator.py +++ b/dagger/dag_creator/airflow/operator_creators/soda_creator.py @@ -11,7 +11,6 @@ class SodaCreator(BatchCreator): def __init__(self, task, dag): super().__init__(task, dag) - self.deferrable = True self._absolute_job_name = task.absolute_job_name self._table_name = task.table_name self._output_s3_path = task.output_s3_path diff --git a/dagger/dag_creator/airflow/operators/soda_batch.py b/dagger/dag_creator/airflow/operators/soda_batch.py index d67a26c..f5d492f 100644 --- a/dagger/dag_creator/airflow/operators/soda_batch.py +++ b/dagger/dag_creator/airflow/operators/soda_batch.py @@ -3,3 +3,4 @@ class SodaBatchOperator(AWSBatchOperator): custom_operator_name = 'Soda' ui_color = "#e4f0e7" + deferrable = True From 16493761dd4e8174510c6f719443c27624cd7f9b Mon Sep 17 00:00:00 2001 From: claudiazi Date: Thu, 19 Jun 2025 14:06:50 +0200 Subject: [PATCH 3/4] feat: convert soda to deferrable --- dagger/dag_creator/airflow/operator_creators/soda_creator.py | 1 + dagger/dag_creator/airflow/operators/soda_batch.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/dagger/dag_creator/airflow/operator_creators/soda_creator.py b/dagger/dag_creator/airflow/operator_creators/soda_creator.py index 8b1f68a..9aa2321 100644 --- a/dagger/dag_creator/airflow/operator_creators/soda_creator.py +++ b/dagger/dag_creator/airflow/operator_creators/soda_creator.py @@ -46,6 +46,7 @@ def _create_operator(self, **kwargs): job_queue=self._task.job_queue, container_overrides=overrides, awslogs_enabled=True, + deferrable=True, **kwargs, ) return batch_op diff --git a/dagger/dag_creator/airflow/operators/soda_batch.py b/dagger/dag_creator/airflow/operators/soda_batch.py index f5d492f..d67a26c 100644 --- a/dagger/dag_creator/airflow/operators/soda_batch.py +++ b/dagger/dag_creator/airflow/operators/soda_batch.py @@ -3,4 +3,3 @@ class SodaBatchOperator(AWSBatchOperator): custom_operator_name = 'Soda' ui_color = "#e4f0e7" - deferrable = True From fca31a6ebaa6be2a358a23a07bdac88d677c9eb2 Mon Sep 17 00:00:00 2001 From: claudiazi Date: Thu, 19 Jun 2025 14:24:25 +0200 Subject: [PATCH 4/4] fix: add extra package --- reqs/base.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/reqs/base.txt b/reqs/base.txt index 279ed4b..18ac07e 100644 --- a/reqs/base.txt +++ b/reqs/base.txt @@ -1,3 +1,4 @@ +aiobotocore>=2.5.0 click==8.1.3 croniter==2.0.2 envyaml==1.10.211231