From 05067fafb071e374340dda0c9c33675d0a81cae1 Mon Sep 17 00:00:00 2001 From: rod-glover Date: Wed, 24 Sep 2025 16:42:56 -0700 Subject: [PATCH 1/2] Fix Python comments --- .../trigger_functions/version_7ab87f8fbcf4.py | 74 ++++++++++++------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/pycds/orm/trigger_functions/version_7ab87f8fbcf4.py b/pycds/orm/trigger_functions/version_7ab87f8fbcf4.py index 2d80c920..74a6159c 100644 --- a/pycds/orm/trigger_functions/version_7ab87f8fbcf4.py +++ b/pycds/orm/trigger_functions/version_7ab87f8fbcf4.py @@ -2,7 +2,7 @@ Define trigger functions for change history tracking. These trigger functions maintain the contents of the history table that tracks each -primary table. Triggers calling these functions are established on the primary and +main table. Triggers calling these functions are established on the main and history table. To understand the trigger functions, it is necessary to understand the overall setup of @@ -11,30 +11,30 @@ Table setup =========== -For each meta/data table (called the primary table) that has history tracking, the +For each meta/data table (called the main table) that has history tracking, the following things are true: -* The primary table is a slightly modified version of the existing table. Its name is +* The main table is a slightly modified version of the existing table. Its name is the same, its existing columns are in the same order and position, and it continues - to provide the main user-facing interface to the meta/data. It also contains a couple - of supplementary columns that expose history information. + to provide the main user-facing interface to the meta/data (hence the term "main"). + It also contains two supplementary columns that expose history information. -* The history table contains all the columns of the primary table, plus several history +* The history table contains all the columns of the main table, plus several history -specific columns. It is append-only, and contains records of every past and present state of the meta/data. * Each row of the history table represents a single state (in time) of a single metadata item. - * The primary table and history table both store the contents of the most recent - items. That is, the history table duplicates contents of the primary table. + * The main table and history table both store the contents of the most recent + items. That is, the history table duplicates contents of the main table. -* Triggers attached to the primary intercept each INSERT, UPDATE and DELETE operation +* Triggers attached to the main intercept each INSERT, UPDATE and DELETE operation and append appropriate record(s) to the history table. Trigger functions ================= -As noted above, triggers are defined on each primary table to convert insert, update, and -delete operations on the primary to inserts (only) on the history table. +As noted above, triggers are defined on each main table to convert insert, update, and +delete operations on the main to inserts (only) on the history table. We define generic trigger functions that work for all the different meta/data tables, rather defining a separate trigger function for each table. Parametrization of trigger @@ -45,26 +45,26 @@ * The *history* id, which is a unique identifier for the history record. It is provided by a corresponding sequence. -* The *primary* or *metadata* id, which identifies a single item in the collection, - but which can have many different history records for it, each with a different - timestamp. - The combination of history id and timestamp is unique. (That pair could be used as - the primary key, but implementation is simpler if we tolerate a slight - lack of normalization and use an independent primary key column.) +* The *item* id, which identifies a single item in the collection (and is the primary + key in the main table). A given item (with a given item id) can have many different + history records for it, each with a different timestamp, corresponding to successive + updates to that item. -Naming conventions for tables, history id columns, and sequences enable us to write +* NOTE: The combination of item id and timestamp is unique. (That pair could be used as + the history table primary key, but implementation is much simpler if we tolerate a + slight lack of normalization and use an independent primary key column.) + +Naming conventions for tables and history-related columns enable us to write simpler, more self-configuring trigger functions. These conventions are: -* The primary table, history table, history id column, and history id sequence +* The main table, history table, history id column, and history id sequence must be named as follows: - * Primary table: ```` (the original table name, e.g., ``meta_network``) + * Main table: ```` (the original table name, e.g., ``meta_network``) * History table: ``_hx`` * History id column: ``_hx_id`` - * History id sequence: ``__seq`` - (i.e., the default name for automatically created primary key sequences) -* The primary table is extended with the following columns (mainly for the convenience +* The main table is extended with the following columns (mainly for the convenience of the user): * ``mod_time``: most recent modification time of this record @@ -72,20 +72,20 @@ * The history table columns must be defined as follows, in this order: - * Primary table columns (including ``mod_time`` and ``mod_user``). + * main table columns (including ``mod_time`` and ``mod_user``). * History maintenance columns * ``deleted``: flag indicating if this record was deleted * history id - * For each foreign key in the primary table to another primary (history-tracked) + * For each foreign key in the main table to another main (history-tracked) table: - * A foreign key to the corresponding history table + * A foreign key to the corresponding history table using the history id Other notes: * One of the trigger functions sets ``mod_time`` and ``mod_user`` in operations on the - primary table so that they cannot be set inaccurately by a user. + main table so that they cannot be set inaccurately by a user. * To do some of the manipulations we require on the ``NEW``/``OLD`` records, we must access their contents by attribute name. By far the easiest way to do this in ``pgplsql`` is to use the ``hstore`` extension. Its syntax and usage are slightly @@ -94,6 +94,26 @@ The performance of these triggers / functions may be able to be improved by converting them to statement-level. This will be somewhat more complicated, particularly for the code that fills in the history table foreign keys. + +NOTE: Terminology has changed a little over time: We used to use the term "primary table" +instead of "main table". "Primary" is a little overloaded, so we adopted "main". +Comments have been updated, but database code has not, because that actually calls for +a migration, which we may or may not want to do. Below is a list of suggested renamings +for identifiers in code. + +Existing | New +--------------------------------|---------------------------------------- +hxtk_primary_control_hx_cols | hxtk_main_control_hx_cols +hxtk_primary_ops_to_hx | hxtk_main_ops_to_hx +new_metadata_hx_id | new_hx_id +fk_metadata_collection_name | fk_collection_name +fk_metadata_id_name | fk_item_id_name -- Name of primary key in foreign table +fk_metadata_history_id_name | fk_history_id_name +fk_metadata_history_table_name | fk_history_table_name +fk_metadata_id | fk_item_id +fk_metadata_history_id | fk_history_id + +And of course "main" for "primary" in comments. """ from pycds.alembic.extensions.replaceable_objects import ReplaceableFunction From d9fb03ea1514376777b0983a269666c3af41c4b4 Mon Sep 17 00:00:00 2001 From: rod-glover Date: Wed, 24 Sep 2025 16:54:14 -0700 Subject: [PATCH 2/2] Rename some Python functions --- pycds/alembic/change_history_utils.py | 12 ++++++------ .../versions/8c05da87cb79_add_hx_tkg_to_obs_raw.py | 12 ++++++------ ...59d64cf16ca_add_hx_tkg_to_main_metadata_tables.py | 12 ++++++------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/pycds/alembic/change_history_utils.py b/pycds/alembic/change_history_utils.py index a9cc2571..c0d7ea21 100644 --- a/pycds/alembic/change_history_utils.py +++ b/pycds/alembic/change_history_utils.py @@ -42,7 +42,7 @@ def sql_array(a: Iterable[Any]) -> str: return f"{{{', '.join(a)}}}" -def add_history_cols_to_primary( +def add_history_cols_to_main( collection_name: str, columns: tuple[str] = ( "mod_time timestamp without time zone NOT NULL DEFAULT NOW()", @@ -56,7 +56,7 @@ def add_history_cols_to_primary( op.execute(f"ALTER TABLE {main_table_name(collection_name)} {add_columns}") -def drop_history_cols_from_primary( +def drop_history_cols_from_main( collection_name: str, columns: tuple[str] = ("mod_time", "mod_user") ): drop_columns = ", ".join(f"DROP COLUMN {c}" for c in columns) @@ -99,7 +99,7 @@ def create_history_table_indexes( """ for columns in ( - # Index on primary table primary key, mod_time, mod_user + # Index on main table primary key, mod_time, mod_user ([pri_id_name], ["mod_time"], ["mod_user"]) # Index on all foreign main table primary keys + tuple([ft_pk_name] for _, ft_pk_name in (foreign_tables or tuple())) @@ -188,8 +188,8 @@ def populate_history_table( op.execute(stmt) -def create_primary_table_triggers(collection_name: str, prefix: str = "t100_"): - # Trigger: Enforce mod_time and mod_user values on primary table. +def create_main_table_triggers(collection_name: str, prefix: str = "t100_"): + # Trigger: Enforce mod_time and mod_user values on main table. op.execute( f"CREATE TRIGGER {prefix}primary_control_hx_cols " f" BEFORE INSERT OR DELETE OR UPDATE " @@ -198,7 +198,7 @@ def create_primary_table_triggers(collection_name: str, prefix: str = "t100_"): f" EXECUTE FUNCTION {qualified_name('hxtk_primary_control_hx_cols')}()" ) - # Trigger: Append history records to history table when primary updated. + # Trigger: Append history records to history table when main updated. op.execute( f"CREATE TRIGGER {prefix}primary_ops_to_hx " f" AFTER INSERT OR DELETE OR UPDATE " diff --git a/pycds/alembic/versions/8c05da87cb79_add_hx_tkg_to_obs_raw.py b/pycds/alembic/versions/8c05da87cb79_add_hx_tkg_to_obs_raw.py index 25be4d04..324a1dec 100644 --- a/pycds/alembic/versions/8c05da87cb79_add_hx_tkg_to_obs_raw.py +++ b/pycds/alembic/versions/8c05da87cb79_add_hx_tkg_to_obs_raw.py @@ -12,14 +12,14 @@ from pycds import get_schema_name from pycds.alembic.change_history_utils import ( - add_history_cols_to_primary, + add_history_cols_to_main, create_history_table, populate_history_table, drop_history_triggers, drop_history_table, - drop_history_cols_from_primary, + drop_history_cols_from_main, create_history_table_triggers, - create_primary_table_triggers, + create_main_table_triggers, create_history_table_indexes, hx_table_name, main_table_name, @@ -52,7 +52,7 @@ def upgrade(): #### # Add missing history col - add_history_cols_to_primary( + add_history_cols_to_main( table_name, columns=( 'mod_user character varying(64) COLLATE pg_catalog."default" ' @@ -63,7 +63,7 @@ def upgrade(): op.execute( text(f"DROP TRIGGER IF EXISTS update_mod_time ON {main_table_name(table_name)}") ) - create_primary_table_triggers(table_name) + create_main_table_triggers(table_name) # History table #### @@ -85,7 +85,7 @@ def upgrade(): def downgrade(): drop_history_triggers(table_name) drop_history_table(table_name) - drop_history_cols_from_primary(table_name, columns=("mod_user",)) + drop_history_cols_from_main(table_name, columns=("mod_user",)) # Restore original mod_time trigger op.execute( text( diff --git a/pycds/alembic/versions/a59d64cf16ca_add_hx_tkg_to_main_metadata_tables.py b/pycds/alembic/versions/a59d64cf16ca_add_hx_tkg_to_main_metadata_tables.py index 522e4a10..90dafb0c 100644 --- a/pycds/alembic/versions/a59d64cf16ca_add_hx_tkg_to_main_metadata_tables.py +++ b/pycds/alembic/versions/a59d64cf16ca_add_hx_tkg_to_main_metadata_tables.py @@ -11,14 +11,14 @@ from pycds import get_schema_name from pycds.alembic.change_history_utils import ( - add_history_cols_to_primary, + add_history_cols_to_main, create_history_table, populate_history_table, drop_history_triggers, drop_history_table, - drop_history_cols_from_primary, + drop_history_cols_from_main, create_history_table_triggers, - create_primary_table_triggers, + create_main_table_triggers, create_history_table_indexes, hx_table_name, ) @@ -50,8 +50,8 @@ def upgrade(): for table_name, primary_key_name, foreign_tables, extra_indexes in table_info: # Primary table - add_history_cols_to_primary(table_name) - create_primary_table_triggers(table_name) + add_history_cols_to_main(table_name) + create_main_table_triggers(table_name) # History table create_history_table(table_name, foreign_tables) @@ -68,4 +68,4 @@ def downgrade(): for table_name, _, _, _ in reversed(table_info): drop_history_triggers(table_name) drop_history_table(table_name) - drop_history_cols_from_primary(table_name) + drop_history_cols_from_main(table_name)