testing airflow and fixed the data type of the day when the data was …

airflow/.astro/config.yaml

@@ -0,0 +1,2 @@
+project:
+    name: airflow

airflow/Dockerfile

@@ -0,0 +1 @@
+FROM astrocrpublic.azurecr.io/runtime:3.1-11

airflow/dags/my_dag.py

@@ -0,0 +1,17 @@
+from airflow.sdk import dag, task
+from pendulum import datetime
+
+@dag(
+    schedule="@daily",
+    start_date=datetime(2026, 1, 31),
+    description="test dag",
+    tags=["first dag"],
+    max_consecutive_failed_dag_runs=3,
+)
+def my_dag():
+
+    @task
+    def _task_a():
+        print("hello")
+
+    _task_a()

worker.py

@@ -15,7 +15,7 @@
 
 
 logger = get_task_logger(__name__)
-todays_date = datetime.now().strftime("%m-%d-%Y")
+todays_date = datetime.now().isoformat()
 S3_BUCKET_NAME = "github-etl-data-bucket"
 
 
@@ -129,12 +129,13 @@ def get_github_data(self, start_in_repo_num: int = 0, batch_size: int = 500, git
             remaining_api_calls = github_instance.rate_limiting
             remaining = remaining_api_calls[0]
 
-            if remaining == 2:
-                print(f"Reached batch size limit of {batch_size}")
+            if counter >= batch_size:
+                print(f"Reached batch size of {batch_size}")
                 break
 
-            #     # start_in_repo_num = counter
-            #     # github_instance = gh_two
+            if remaining < 100:
+                print(f"Rate limit approaching ({remaining}). Stopping worker.")
+                break
 
             #     # raise self.retry(countdown=3600)
             #     break
@@ -191,5 +192,4 @@ def build_repo_chord(total: int = 5000, batch_size: int = 500):
 #         for start in range(0, 5000, 500)
 #     ])
 
-#     return chord(jobs)()
-
+#     return chord(jobs)()