From f2b8644030c1d1f6b348eca413eee3200af60b96 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 6 Mar 2023 12:34:59 -0600 Subject: [PATCH 1/8] fix: packaging migration scripts Running `./pants package ::` results in the following error. We included the v3.5 migration script in `setup.py` and in the `pants_distribution`. But, that script does not end in `.py`, so the `python_sources` target did not "own" it. In order to correct this, we explicitly include the extension-less binaries. > ValueError: The explicit dependency > `st2common/bin/migrations/v3.5/st2-migrate-db-dict-field-values` of > the target at `st2common:st2common` does not provide enough address > parameters to identify which parametrization of the dependency target > should be used. > Target `st2common/bin/migrations/v3.5:v3.5` can be addressed as: > * st2common/bin/migrations/v3.5:v3.5 > * st2common/bin/migrations/v3.5/__init__.py > * st2common/bin/migrations/v3.5/st2_migrate_db_dict_field_values.py In fixing this, I also noticed that we did not include the v3.8 migration script, so I marked that executable and included it as well. --- st2common/BUILD | 1 + st2common/bin/migrations/v3.5/BUILD | 4 +++- st2common/bin/migrations/v3.8/BUILD | 3 +++ .../migrations/v3.8/st2-drop-st2exporter-marker-collections | 0 4 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 st2common/bin/migrations/v3.8/BUILD mode change 100644 => 100755 st2common/bin/migrations/v3.8/st2-drop-st2exporter-marker-collections diff --git a/st2common/BUILD b/st2common/BUILD index 75e35c9fc5..8622bceddd 100644 --- a/st2common/BUILD +++ b/st2common/BUILD @@ -20,6 +20,7 @@ st2_component_python_distribution( "bin/st2-pack-download", "bin/st2-pack-setup-virtualenv", "bin/migrations/v3.5/st2-migrate-db-dict-field-values", + "bin/migrations/v3.8/st2-drop-st2exporter-marker-collections", "bin/st2-run-pack-tests:shell", "bin/st2ctl:shell", "bin/st2-self-check:shell", diff --git a/st2common/bin/migrations/v3.5/BUILD b/st2common/bin/migrations/v3.5/BUILD index e574adca51..66fcf26bb9 100644 --- a/st2common/bin/migrations/v3.5/BUILD +++ b/st2common/bin/migrations/v3.5/BUILD @@ -1,3 +1,5 @@ # TODO: what to do about st2-migrate-db-dict-field-values ? # st2_migrate_db_dict_field_values.py is a symlink to st2-migrate-db-dict-field-values -python_sources() +python_sources( + sources=["*.py", "st2*"], +) diff --git a/st2common/bin/migrations/v3.8/BUILD b/st2common/bin/migrations/v3.8/BUILD new file mode 100644 index 0000000000..05411bee10 --- /dev/null +++ b/st2common/bin/migrations/v3.8/BUILD @@ -0,0 +1,3 @@ +python_sources( + sources=["st2*"], +) diff --git a/st2common/bin/migrations/v3.8/st2-drop-st2exporter-marker-collections b/st2common/bin/migrations/v3.8/st2-drop-st2exporter-marker-collections old mode 100644 new mode 100755 From c10ca30fb3289523d653997279cf4d5ede1bb928 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 9 Mar 2023 00:11:13 -0600 Subject: [PATCH 2/8] chore: add explicit deps for shell scripts in st2common/bin --- st2common/bin/BUILD | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/st2common/bin/BUILD b/st2common/bin/BUILD index a05ce529eb..69f1ac4102 100644 --- a/st2common/bin/BUILD +++ b/st2common/bin/BUILD @@ -9,4 +9,22 @@ st2_shell_sources_and_resources( name="shell", sources=["st2ctl", "st2-self-check", "st2-run-pack-tests"], skip_shellcheck=True, + overrides={ + "st2ctl": { + "dependencies": [ + "./st2-register-content", + "./st2-cleanup-db", + ], + }, + "st2-self-check": { + "dependencies": [ + "./st2ctl:shell", + # TODO: dep on st2client cli? + ], + }, + # st2-run-pack-tests creates its own virtualenv on the fly and + # installs its dependencies, so they don't need to be listed here. + # It can optionally use the deps installed with st2tests package. + # "st2-run-pack-tests": {}, + }, ) From 3913bdebe430f7ea36a13b195f67f7bbe8251544 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 7 Mar 2023 20:35:40 -0600 Subject: [PATCH 3/8] add note about commands to use when searching for missing deps --- st2common/BUILD | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/st2common/BUILD b/st2common/BUILD index 8622bceddd..ed852b135d 100644 --- a/st2common/BUILD +++ b/st2common/BUILD @@ -37,4 +37,12 @@ st2_component_python_distribution( "noop": "st2common.rbac.backends.noop:NoOpRBACBackend", }, }, + dependencies=[ + ], + # commands helpful in inspecting the dependencies (the "=(...)" is zsh syntax) + # python files under st2common that will not be included in the wheel + # comm -13 =(./pants dependencies --transitive st2common:st2common | grep -v -e : -e __init__.py | grep st2common/st2common | sort) =(find st2common/st2common -name '*.py' -and -not -name '__init__.py' | sort) + # + # python files required by other wheels that are missing from st2common + # comm -13 =(./pants dependencies --transitive st2common:st2common | grep st2common/st2common | sort) =(./pants list --filter-target-type=python_distribution --filter-address-regex=-st2common:st2common :: | xargs ./pants dependencies --transitive | grep st2common/st2common | sort) ) From 7fbfa5fe7023c22896b703c4f531d696a10f990b Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 7 Mar 2023 01:16:59 -0600 Subject: [PATCH 4/8] chore: note known dead code in st2common/BUILD This should probably be deleted. But for now, we just leave a note about why it is not included in the st2common python_distribution. --- st2common/BUILD | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/st2common/BUILD b/st2common/BUILD index ed852b135d..2144b1f5f8 100644 --- a/st2common/BUILD +++ b/st2common/BUILD @@ -38,6 +38,22 @@ st2_component_python_distribution( }, }, dependencies=[ + # Added gunicorn bug workaround for SyncWorker users in #2571. No known active users. To use: + # `gunicorn -k st2common.util.gunicorn_workers.EventletSyncWorker ...` + # "./st2common/util/gunicorn_workers.py", + # + # Known dead code that should be deleted. Do not add these: + # ./st2common/callback/* (was for the old mistral_v2 runner. see #4038) + # ./st2common/constants/scheduler.py (unused since #4397) + # ./st2common/content/validators.py (unused since #939) + # ./st2common/exceptions/api.py (unused since #1840) + # ./st2common/exceptions/connection.py (unused since #1794) + # ./st2common/exceptions/resultstracker.py (unused since #5011) + # ./st2common/models/api/actionrunner.py (unused since #442) + # ./st2common/models/db/reactor.py (unused since #5922) + # ./st2common/persistence/reactor.py (unused since #5922) + # ./st2common/util/argument_parser.py (never used since added in e9ae7e31e1eb47e49f0fdc414ed6d2b8eccf4748) + # ./st2common/validators/workflow (unused since #5011) ], # commands helpful in inspecting the dependencies (the "=(...)" is zsh syntax) # python files under st2common that will not be included in the wheel From ae693580cd8490070bb1c572db559935380dac78 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 7 Mar 2023 16:47:55 -0600 Subject: [PATCH 5/8] Define API dependencies on code that should be in st2common wheel Since this code is not imported by one of our scripts or entry points, we need to add an explicit dependency. This starts with several things that form part of the official "API" of the st2common library. --- st2common/BUILD | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/st2common/BUILD b/st2common/BUILD index 2144b1f5f8..19cb0c0844 100644 --- a/st2common/BUILD +++ b/st2common/BUILD @@ -38,6 +38,45 @@ st2_component_python_distribution( }, }, dependencies=[ + # no entry-point or script yet + "./st2common/garbage_collection/inquiries.py", # missing cmd + ./bin/st2-purge-inquiries + # + # Things that need to be included as part of the st2common library's API: + # + # ### Public API ### + # + "./st2common/logging", # used by all of our logging conf files + "./st2common/models/system", # used by runners + "./st2common/policies", # used by policies (see st2actions.policies) + "./st2common/runners", # used by runners and python actions + # + # ### Mixed Public+Internal API ### + # + "./st2common/services", # used by runners, python actions, st2api, ... + # + # ### Internal API ### + # + "./st2common/constants/garbage_collection.py", # used by garbage collector + "./st2common/constants/policy.py", # used by st2scheduler (in st2actions) + "./st2common/constants/timer.py", # used by st2timersengine (in st2reactor) + "./st2common/middleware", # used by st2auth, st2api, st2stream + "./st2common/models/api", # used by st2auth, st2api, st2stream + "./st2common/models/system", # used by st2auth, st2api, st2stream + "./st2common/models/db/timer.py", # used by st2api + "./st2common/models/db/webhook.py", # used by st2api + "./st2common/persistence/execution_queue.py", # used by st2scheduler (in st2actions) + "./st2common/stream", # used by st2stream + "./st2common/transport/consumers.py", # used by st2actions- and st2reactor-related services + "./st2common/util/actionalias_helpstring.py", # used by st2api + "./st2common/util/auth.py", # used by st2api, st2auth + "./st2common/util/keyvalue.py", # used by st2api + "./st2common/util/sandboxing.py", # used by python runner and sensor container + "./st2common/util/service.py", # used by st2scheduler (in st2actions) + "./st2common/util/wsgi.py", # used by st2stream + "./st2common/validators/api/misc.py", # used by st2api + # + # ### Dead Code (?) ### + # # Added gunicorn bug workaround for SyncWorker users in #2571. No known active users. To use: # `gunicorn -k st2common.util.gunicorn_workers.EventletSyncWorker ...` # "./st2common/util/gunicorn_workers.py", From 57031951324a828f67efeb4d9fde517cd3eb6534 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 9 Mar 2023 14:24:56 -0600 Subject: [PATCH 6/8] bugfix: remove bad arg in st2common/bin/migrations/v3.8/st2-drop-st2exporter-marker-collections --- .../bin/migrations/v3.8/st2-drop-st2exporter-marker-collections | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/bin/migrations/v3.8/st2-drop-st2exporter-marker-collections b/st2common/bin/migrations/v3.8/st2-drop-st2exporter-marker-collections index 569cb1037a..5769db6672 100755 --- a/st2common/bin/migrations/v3.8/st2-drop-st2exporter-marker-collections +++ b/st2common/bin/migrations/v3.8/st2-drop-st2exporter-marker-collections @@ -56,7 +56,7 @@ def main(): db_setup() try: - delete_marker_collections(display_prompt=not cfg.CONF.yes) + delete_marker_collections() exit_code = 0 except Exception as e: print("ABORTED: Collection deletion aborted on first failure: %s" % (str(e))) From 40db4a1c651089a0062a992e431002e3534c16e3 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 9 Mar 2023 14:25:43 -0600 Subject: [PATCH 7/8] fix: drop unused imports --- .../migrations/v3.8/st2-drop-st2exporter-marker-collections | 3 --- 1 file changed, 3 deletions(-) diff --git a/st2common/bin/migrations/v3.8/st2-drop-st2exporter-marker-collections b/st2common/bin/migrations/v3.8/st2-drop-st2exporter-marker-collections index 5769db6672..10fa7e1913 100755 --- a/st2common/bin/migrations/v3.8/st2-drop-st2exporter-marker-collections +++ b/st2common/bin/migrations/v3.8/st2-drop-st2exporter-marker-collections @@ -24,10 +24,7 @@ and the collections were not configured to be created automatically. import sys import traceback -import mongoengine as me - from mongoengine.connection import get_db -from oslo_config import cfg from st2common import config from st2common.service_setup import db_setup From 2fa16d82c1080e7c0b3b7427edf4a42102f30566 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 9 Mar 2023 13:54:59 -0600 Subject: [PATCH 8/8] update changelog entry --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bb13e1ba5a..3e5777133d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,7 +16,7 @@ Added to pants' use of PEX lockfiles. This is not a user-facing addition. #5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850 #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5864 #5874 #5884 #5893 #5891 - #5890 #5898 #5901 #5906 #5899 #5907 #5909 #5922 #5926 #5927 + #5890 #5898 #5901 #5906 #5899 #5907 #5909 #5922 #5926 #5927 #5925 Contributed by @cognifloyd * Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805