Spr 939 test runner improvement

.github/workflows/nightly.overrides.json

@@ -0,0 +1,10 @@
+{
+    "storage": {
+        "vacuum_config": {
+            "enabled": true
+        }
+    },
+    "otel": {
+        "enabled": false
+    }
+}

.github/workflows/nightly.yml

@@ -67,19 +67,19 @@ jobs:
       - name: Prepare Tests
         run: |
           git config --global --add safe.directory ${GITHUB_WORKSPACE}
-          # Turns off the OTEL in the system.json.settings
-          jq '.otel.enabled = false' system.json.settings > tmp.json && mv tmp.json system.json.settings
+          # Override some settings in the config file
+          .github/workflows/scripts/merge_system_settings.py system.json.settings .github/workflows/nightly.overrides.json system.json.test
+
           # Replace the correct `springtail` code directory
-          sed -i -e "s|/home/dev/springtail|${GITHUB_WORKSPACE}|g" system.json.settings
-          cp system.json.settings system.json.test
+          sed -i -e "s|/home/dev/springtail|${GITHUB_WORKSPACE}|g" system.json.test
           mkdir -p /opt/springtail/pids
           service redis-server start
 
-          # Copies over the fdw_config file to the designated location
+          # Copy over the fdw_config file to the designated location
           conf_file=/etc/postgresql/16/main/postgresql.conf
           fdw_config_file_path=$(grep -E "^springtail_fdw.config_file_path" $conf_file | awk '{print $3}' | tr -d "'")
           mkdir -p $(dirname $fdw_config_file_path)
-          cp system.json.settings $fdw_config_file_path
+          cp system.json.test $fdw_config_file_path
 
       - name: Build
         run: |
@@ -208,7 +208,7 @@ jobs:
 
             _LOG_FLAG="failed"
             echo "*** Starting iteration $i/${INTEGRATION_TEST_ITERATIONS} of integration tests"
-            python3 test_runner.py -a -c config.yaml -j ${GITHUB_WORKSPACE}/integration-test-report.xml 2>&1 | tee tmplog
+            python3 test_runner.py -c nightly -j ${GITHUB_WORKSPACE}/integration-test-report.xml 2>&1 | tee tmplog
             # Grab the success/failure status of the integration tests
             if [ $? -eq 0 ] && ! grep -qE "Tests failed: [1-9][0-9]*" tmplog
             then

.github/workflows/scripts/merge_system_settings.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+# JSON config override script
+# This script merges two JSON files, where the second file overrides the first.
+# Usage:
+#   ./merge_system_settings.py <base_file> <override_file> <output_file>
+
+
+import json
+
+
+def merge_dict(base, override):
+    """
+    Merges two dictionaries, where the second dictionary overrides the first.
+    If a key is missing from the second dictionary, the value from the first is used.
+    If a key exists in both dictionaries and both values are dictionaries, we merge them recursively.
+    If a key exists in both dictionaries and the values are not dictionaries, the value from the second dictionary is used.
+
+    :param base: The base dictionary.
+    :param override: The dictionary with overrides.
+    :return: A new dictionary with merged values.
+    """
+    merged = base.copy()  # Start with a copy of the base dictionary
+    for key, value in override.items():
+        if key not in merged:
+            raise KeyError(f"Key '{key}' in overrides not found in base.")
+        # Assert types are the same if both exist
+        base_value = merged[key]
+        bvt, ovt = type(base_value), type(value)
+        if bvt != ovt:
+            raise TypeError(f"Type mismatch for key '{key}': base type {bvt} does not match override type {ovt}.")
+        if bvt is dict:
+            # If both are dictionaries, merge them recursively
+            merged[key] = merge_dict(base_value, value)
+        else:
+            # Otherwise, override the value
+            merged[key] = value
+    return merged
+
+
+def merge_system_settings(base_file, override_file, output_file):
+    """
+    Merges two YAML files containing system settings, where the second file overrides the first.
+
+    :param base_file: Path to the base system settings YAML file.
+    :param override_file: Path to the override system settings YAML file.
+    :param output_file: Path to save the merged output YAML file.
+    """
+    with open(base_file, 'r') as f:
+        base_settings = json.load(f)
+
+    with open(override_file, 'r') as f:
+        override_settings = json.load(f)
+
+    merged_settings = merge_dict(base_settings, override_settings)
+
+    with open(output_file, 'w') as f:
+        json.dump(merged_settings, f, indent=4)
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Merge system settings JSON files.")
+    parser.add_argument("base_file", help="Path to the base system settings file.")
+    parser.add_argument("override_file", help="Path to the override system settings file.")
+    parser.add_argument("output_file", help="Path to save the merged output file.")
+
+    args = parser.parse_args()
+
+    merge_system_settings(args.base_file, args.override_file, args.output_file)

.github/workflows/test.yml

@@ -179,7 +179,7 @@ jobs:
 
           set +e
           _LOG_FLAG="failed"          
-          python3 test_runner.py -c config.yaml -j /__w/springtail/springtail/integration-test-report.xml --skip-downloads 2>&1 | tee tmplog
+          python3 test_runner.py -c github_ci -j /__w/springtail/springtail/integration-test-report.xml --skip-downloads 2>&1 | tee tmplog
           # Grab the success/failure status of the integration tests
           if [ $? -eq 0 ] && ! grep -qE "Tests failed: [1-9][0-9]*" tmplog
           then

include/pg_fdw/pg_fdw_ddl_common.hh

@@ -55,8 +55,7 @@ namespace springtail::pg_fdw {
         std::string _partition_bound;
     };
 
-    class PgFdwCommon : public Singleton<PgFdwCommon> {
-        friend class Singleton<PgFdwCommon>;
+    class PgFdwCommon {
     public:
         struct TableEntry {
             uint64_t table_id;
@@ -347,8 +346,12 @@ namespace springtail::pg_fdw {
             auto fields = table->extent_schema()->get_fields();
             for (auto row : (*table)) {
                 uint64_t tid = fields->at(sys_tbl::Schemas::Data::TABLE_ID)->get_uint64(&row);
+                uint64_t xid = fields->at(sys_tbl::Schemas::Data::XID)->get_uint64(&row);
+                if (xid > schema_xid) {
+                    continue;
+                }
 
-                LOG_DEBUG(LOG_FDW, "Found table in schemas table: {}", tid);
+                LOG_DEBUG(LOG_FDW, "Found table in schemas table: {}, xid: {}, schema_xid: {}", tid, xid, schema_xid);
 
                 // check if we have moved to next tid
                 if (tid != current_tid) {

python/shared/sysutils.py

@@ -18,7 +18,8 @@ def stop_daemons(pid_path : str, daemons : List[tuple] = []) -> None:
     """Stop all daemons."""
     # Stop the daemons
     if not os.path.exists(pid_path):
-        raise Exception(f"PID path not found: {pid_path}")
+        print(f"PID path: {pid_path} does not exist")
+        return
 
     found_pids = []
 

python/testing/overlays/small_log_rotate.json

@@ -1,6 +1,11 @@
 {
     "log_mgr": {
-	"log_size_rollover_threshold": 32768,
-	"archive_logs": true
-    }
+        "log_size_rollover_threshold": 32768,
+        "archive_logs": true
+    },
+    "storage": {
+        "vacuum_config": {
+            "enabled": false
+        }   
+    }   
 }

python/testing/overlays/small_log_rotate_with_streaming.json

@@ -0,0 +1,11 @@
+{
+    "log_mgr": {
+        "log_size_rollover_threshold": 32768,
+        "archive_logs": true
+    },
+    "storage": {
+        "vacuum_config": {
+            "enabled": false
+        }
+    }
+}

python/testing/overlays/streaming_postgres_config.json

@@ -1 +1,7 @@
-{}
+{
+    "storage": {
+        "vacuum_config": {
+            "enabled": false
+        }   
+    }   
+}