59 error handling for wonky characters in lid

.github/workflows/main.yml

@@ -12,7 +12,7 @@ jobs:
     - name: Set up Python 3.9
       uses: actions/setup-python@v1
       with:
-        python-version: 3.9
+        python-version: 3.12
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/pythonapp.yml

@@ -9,10 +9,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python 3.9
+    - name: Set up Python 3.12
       uses: actions/setup-python@v1
       with:
-        python-version: 3.9
+        python-version: 3.12
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.gitignore

@@ -0,0 +1 @@
+.idea/*

.pre-commit-config.yaml

@@ -27,7 +27,7 @@ repos:
     -   id: black
         language_version: python3
 -   repo: https://github.com/PyCQA/flake8
-    rev: 4.0.1
+    rev: 7.3.0
     hooks:
     -   id: flake8
 -   repo: https://github.com/sqlfluff/sqlfluff

linkage_dag.py

@@ -641,6 +641,11 @@
         trigger_dag_id="affiliations",
     )
 
+    trigger_almanac_classifiers = TriggerDagRunOperator(
+        task_id="trigger_almanac_classifiers",
+        trigger_dag_id="almanac_classifiers",
+    )
+
     curr_date = datetime.now().strftime("%Y%m%d")
     with open(
         f"{os.environ.get('DAGS_FOLDER')}/schemas/{gcs_folder}/table_descriptions.json"
@@ -679,6 +684,7 @@
             >> update_archive
             >> success_alert
             >> trigger_affiliations
+            >> trigger_almanac_classifiers
         )
 
     # We don't show the "all metadata" table in the production dataset, but we do need to
@@ -732,6 +738,7 @@
     gce_instance_start >> gce_instance_stop
 
     gce_instance_create >> run_lid >> gce_instance_delete
+    # run_lid >> gce_instance_stop
 
     (
         gce_instance_delete

utils/run_lid.py

@@ -42,8 +42,17 @@ def add_cld2_outputs(self, record: dict) -> None:
                     if encoding is None:
                         encoding = "latin-1"  # last-ditch effort...
                     is_reliable, text_bytes_found, details = cld2.detect(
-                        record[field].encode("utf-8").decode(encoding)
+                        record[field].encode(encoding, "surrogatepass").decode(encoding)
                     )
+                except UnicodeDecodeError as e:
+                    logging.warning(
+                        "utf-8 failed, attempting to get rid of surrogate characters"
+                    )
+                    logging.warning(e)
+                    is_reliable, text_bytes_found, details = cld2.detect(
+                        record[field].encode("utf-8", "surrogatepass").decode("utf-8")
+                    )
+
                 record[field + "_cld2_lid_success"] = True
                 record[field + "_cld2_lid_is_reliable"] = is_reliable
                 # details looks like: (('RUSSIAN', 'ru', 98, 404.0), ('Unknown', 'un', 0, 0.0), ('Unknown', 'un', 0, 0.0))