diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1f9c048..2132172 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,7 +12,7 @@ jobs: - name: Set up Python 3.9 uses: actions/setup-python@v1 with: - python-version: 3.9 + python-version: 3.12 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 75c648b..82f4846 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -9,10 +9,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python 3.12 uses: actions/setup-python@v1 with: - python-version: 3.9 + python-version: 3.12 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1c2d52b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea/* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 92379f6..86c0552 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: - id: black language_version: python3 - repo: https://github.com/PyCQA/flake8 - rev: 4.0.1 + rev: 7.3.0 hooks: - id: flake8 - repo: https://github.com/sqlfluff/sqlfluff diff --git a/linkage_dag.py b/linkage_dag.py index 27fc15f..20b23c0 100644 --- a/linkage_dag.py +++ b/linkage_dag.py @@ -641,6 +641,11 @@ trigger_dag_id="affiliations", ) + trigger_almanac_classifiers = TriggerDagRunOperator( + task_id="trigger_almanac_classifiers", + trigger_dag_id="almanac_classifiers", + ) + curr_date = datetime.now().strftime("%Y%m%d") with open( f"{os.environ.get('DAGS_FOLDER')}/schemas/{gcs_folder}/table_descriptions.json" @@ -679,6 +684,7 @@ >> update_archive >> success_alert >> trigger_affiliations + >> trigger_almanac_classifiers ) # We don't show the "all metadata" table in the production dataset, but we do need to @@ -732,6 +738,7 @@ gce_instance_start >> gce_instance_stop gce_instance_create >> run_lid >> gce_instance_delete + # run_lid >> gce_instance_stop ( gce_instance_delete diff --git a/utils/run_lid.py b/utils/run_lid.py index 1ed7f35..4c9b69b 100644 --- a/utils/run_lid.py +++ b/utils/run_lid.py @@ -42,8 +42,17 @@ def add_cld2_outputs(self, record: dict) -> None: if encoding is None: encoding = "latin-1" # last-ditch effort... is_reliable, text_bytes_found, details = cld2.detect( - record[field].encode("utf-8").decode(encoding) + record[field].encode(encoding, "surrogatepass").decode(encoding) ) + except UnicodeDecodeError as e: + logging.warning( + "utf-8 failed, attempting to get rid of surrogate characters" + ) + logging.warning(e) + is_reliable, text_bytes_found, details = cld2.detect( + record[field].encode("utf-8", "surrogatepass").decode("utf-8") + ) + record[field + "_cld2_lid_success"] = True record[field + "_cld2_lid_is_reliable"] = is_reliable # details looks like: (('RUSSIAN', 'ru', 98, 404.0), ('Unknown', 'un', 0, 0.0), ('Unknown', 'un', 0, 0.0))