From 1cc57e0e327577a0e46c6c779d1eeeb80cbe82c5 Mon Sep 17 00:00:00 2001 From: Katherine Quinn Date: Thu, 14 May 2026 09:25:11 -0400 Subject: [PATCH 1/5] adds handling of surrogate characters, triggers almanac classifier --- .gitignore | 1 + linkage_dag.py | 7 +++++++ utils/run_lid.py | 9 ++++++++- 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bc8a670 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea/* \ No newline at end of file diff --git a/linkage_dag.py b/linkage_dag.py index 27fc15f..d34d302 100644 --- a/linkage_dag.py +++ b/linkage_dag.py @@ -641,6 +641,11 @@ trigger_dag_id="affiliations", ) + trigger_almanac_classifiers = TriggerDagRunOperator( + task_id="trigger_almanac_classifiers", + trigger_dag_id="almanac_classifiers", + ) + curr_date = datetime.now().strftime("%Y%m%d") with open( f"{os.environ.get('DAGS_FOLDER')}/schemas/{gcs_folder}/table_descriptions.json" @@ -679,6 +684,7 @@ >> update_archive >> success_alert >> trigger_affiliations + >> trigger_almanac_classifiers ) # We don't show the "all metadata" table in the production dataset, but we do need to @@ -732,6 +738,7 @@ gce_instance_start >> gce_instance_stop gce_instance_create >> run_lid >> gce_instance_delete + #run_lid >> gce_instance_stop ( gce_instance_delete diff --git a/utils/run_lid.py b/utils/run_lid.py index 1ed7f35..b1b7fdf 100644 --- a/utils/run_lid.py +++ b/utils/run_lid.py @@ -42,8 +42,15 @@ def add_cld2_outputs(self, record: dict) -> None: if encoding is None: encoding = "latin-1" # last-ditch effort... is_reliable, text_bytes_found, details = cld2.detect( - record[field].encode("utf-8").decode(encoding) + record[field].encode(encoding, 'surrogatepass').decode(encoding) ) + except UnicodeDecodeError as e: + logging.warning("utf-8 failed, attempting to get rid of surrogate characters") + logging.warning(e) + is_reliable, text_bytes_found, details = cld2.detect( + record[field].encode('utf-8', 'surrogatepass').decode('utf-8') + ) + record[field + "_cld2_lid_success"] = True record[field + "_cld2_lid_is_reliable"] = is_reliable # details looks like: (('RUSSIAN', 'ru', 98, 404.0), ('Unknown', 'un', 0, 0.0), ('Unknown', 'un', 0, 0.0)) From 0722a61fc1ee633919b920f3ae1b1792a98bf415 Mon Sep 17 00:00:00 2001 From: Katherine Quinn Date: Thu, 14 May 2026 09:29:50 -0400 Subject: [PATCH 2/5] updates to python 3.12 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1f9c048..2132172 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,7 +12,7 @@ jobs: - name: Set up Python 3.9 uses: actions/setup-python@v1 with: - python-version: 3.9 + python-version: 3.12 - name: Install dependencies run: | python -m pip install --upgrade pip From b89a172c6a9e62529b95b361fb38c2f11b981683 Mon Sep 17 00:00:00 2001 From: Katherine Quinn Date: Thu, 14 May 2026 09:31:36 -0400 Subject: [PATCH 3/5] updates to python 3.12 for realz --- .github/workflows/pythonapp.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 75c648b..82f4846 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -9,10 +9,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python 3.12 uses: actions/setup-python@v1 with: - python-version: 3.9 + python-version: 3.12 - name: Install dependencies run: | python -m pip install --upgrade pip From c187dba76cf29cc4c7c0da5631d3338e9ed13267 Mon Sep 17 00:00:00 2001 From: Katherine Quinn Date: Thu, 14 May 2026 09:35:27 -0400 Subject: [PATCH 4/5] linting --- .gitignore | 2 +- linkage_dag.py | 2 +- utils/run_lid.py | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index bc8a670..1c2d52b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -.idea/* \ No newline at end of file +.idea/* diff --git a/linkage_dag.py b/linkage_dag.py index d34d302..20b23c0 100644 --- a/linkage_dag.py +++ b/linkage_dag.py @@ -738,7 +738,7 @@ gce_instance_start >> gce_instance_stop gce_instance_create >> run_lid >> gce_instance_delete - #run_lid >> gce_instance_stop + # run_lid >> gce_instance_stop ( gce_instance_delete diff --git a/utils/run_lid.py b/utils/run_lid.py index b1b7fdf..4c9b69b 100644 --- a/utils/run_lid.py +++ b/utils/run_lid.py @@ -42,13 +42,15 @@ def add_cld2_outputs(self, record: dict) -> None: if encoding is None: encoding = "latin-1" # last-ditch effort... is_reliable, text_bytes_found, details = cld2.detect( - record[field].encode(encoding, 'surrogatepass').decode(encoding) + record[field].encode(encoding, "surrogatepass").decode(encoding) ) except UnicodeDecodeError as e: - logging.warning("utf-8 failed, attempting to get rid of surrogate characters") + logging.warning( + "utf-8 failed, attempting to get rid of surrogate characters" + ) logging.warning(e) is_reliable, text_bytes_found, details = cld2.detect( - record[field].encode('utf-8', 'surrogatepass').decode('utf-8') + record[field].encode("utf-8", "surrogatepass").decode("utf-8") ) record[field + "_cld2_lid_success"] = True From 19c552ba841763379ff6410d19335cf32ab014e4 Mon Sep 17 00:00:00 2001 From: Katherine Quinn Date: Thu, 14 May 2026 09:42:53 -0400 Subject: [PATCH 5/5] updating flake8 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 92379f6..86c0552 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: - id: black language_version: python3 - repo: https://github.com/PyCQA/flake8 - rev: 4.0.1 + rev: 7.3.0 hooks: - id: flake8 - repo: https://github.com/sqlfluff/sqlfluff