From b9cd61e71594fbe55f0892adb4a9d1b49b93b91d Mon Sep 17 00:00:00 2001 From: Grigorii Heifetz Date: Tue, 26 May 2026 11:33:52 +0300 Subject: [PATCH 01/20] Fix issue with SET LOCAL inside PL functions (6.x) What happened? Local GUCs (SET LOCAL) get lost inside PLPG functions (see test case for more details), thus writing commands may have unexpected results or fail. Why it happens? It happens because SET LOCAL is executed in a separate transaction, because no DTX (2PC) is set up, so its effect is popped. How do we fix this? Let's make all SET LOCAL commands on segments be just SET, so that they will last until master synchronizes them using previously saved value (gp_guc_restore_list). Synchronization happens on next master's transaction or when transaction control statement is met (COMMIT, ROLLBACK). Specifically,syncing is handled inside AtEOXact_SPI(). Note - on master, SET LOCAL is still executed as SET LOCAL. Alternate solutions? DTX transaction context could be set up on the SET step, so that all subsequent writing commands get inside this new transaction and thus, get required GUC. Tests? Test case to check consistency and correct passage of local GUCs inside DO was developed. Changes from original commit? As 6.x does not support control statements inside DO - removed in between SPI GUC synchronization code from AtEOXact_SPI(), connected test. GUC syncing happens on the next master's transaction using mechanism inside PostgresMain(). (cherry picked from commit 762fb85) Ticket: GG-479 --------- Co-authored-by: Georgy Shelkovy Co-authored-by: Viktor Kurilko --- src/backend/utils/misc/guc.c | 9 ++++++- src/test/regress/expected/plpgsql.out | 21 ++++++++++++++++ .../regress/expected/plpgsql_optimizer.out | 21 ++++++++++++++++ src/test/regress/sql/plpgsql.sql | 25 +++++++++++++++++++ 4 files changed, 75 insertions(+), 1 deletion(-) diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 35315b2a0514..78053eea4dfb 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -5241,7 +5241,6 @@ AtEOXact_GUC(bool isCommit, int nestLevel) * record it and restore QE before next query start */ if (Gp_role == GP_ROLE_DISPATCH - && !IsTransactionBlock() && changed && ((isCommit) || (!isCommit && gp_guc_need_restore)) && (gconf->flags & GUC_GPDB_NEED_SYNC)) @@ -5921,6 +5920,14 @@ set_config_option(const char *name, const char *value, errmsg("unrecognized configuration parameter \"%s\"", name))); return 0; } + /* + * Make SET LOCAL just SET when executing on segments. It is needed for + * correct passage of local GUCs as they might be discarded. Anyway, + * master executes SET as LOCAL, so on next transaction GUC will + * resynchronize and everything will be back in place. + */ + if (Gp_role == GP_ROLE_EXECUTE && action == GUC_ACTION_LOCAL) + action = GUC_ACTION_SET; /* * Check if option can be set by the user. diff --git a/src/test/regress/expected/plpgsql.out b/src/test/regress/expected/plpgsql.out index e909d02e8a44..aadd00ab92e3 100755 --- a/src/test/regress/expected/plpgsql.out +++ b/src/test/regress/expected/plpgsql.out @@ -5555,3 +5555,24 @@ NOTICE: outer_func() done drop function outer_outer_func(int); drop function outer_func(int); drop function inner_func(int); +-- Test consistency and passage of SET LOCAL GUC to writing commands +create schema s; +do $$ +begin + set local search_path to s; + create table test_table(a int) distributed by (a); + drop table test_table; +end $$; +-- check existence on master and segments +with c as ( + select gp_segment_id, relname, relkind, relnamespace from pg_class + union all + select gp_segment_id, relname, relkind, relnamespace from gp_dist_random('pg_class') +) select gp_segment_id from c +join pg_namespace n on n.oid = c.relnamespace +where c.relname = 'test_table' and c.relkind = 'r' and n.nspname = 's'; + gp_segment_id +--------------- +(0 rows) + +drop schema s cascade; diff --git a/src/test/regress/expected/plpgsql_optimizer.out b/src/test/regress/expected/plpgsql_optimizer.out index c9c9347b8f15..6af96c227cbe 100755 --- a/src/test/regress/expected/plpgsql_optimizer.out +++ b/src/test/regress/expected/plpgsql_optimizer.out @@ -5533,3 +5533,24 @@ NOTICE: outer_func() done drop function outer_outer_func(int); drop function outer_func(int); drop function inner_func(int); +-- Test consistency and passage of SET LOCAL GUC to writing commands +create schema s; +do $$ +begin + set local search_path to s; + create table test_table(a int) distributed by (a); + drop table test_table; +end $$; +-- check existence on master and segments +with c as ( + select gp_segment_id, relname, relkind, relnamespace from pg_class + union all + select gp_segment_id, relname, relkind, relnamespace from gp_dist_random('pg_class') +) select gp_segment_id from c +join pg_namespace n on n.oid = c.relnamespace +where c.relname = 'test_table' and c.relkind = 'r' and n.nspname = 's'; + gp_segment_id +--------------- +(0 rows) + +drop schema s cascade; diff --git a/src/test/regress/sql/plpgsql.sql b/src/test/regress/sql/plpgsql.sql index 62dc38e92e90..f3a10de5d3ba 100644 --- a/src/test/regress/sql/plpgsql.sql +++ b/src/test/regress/sql/plpgsql.sql @@ -4232,3 +4232,28 @@ select outer_outer_func(20); drop function outer_outer_func(int); drop function outer_func(int); drop function inner_func(int); + +-- Test consistency and passage of SET LOCAL GUC to writing commands +--start_ignore +drop schema if exists s cascade; +--end_ignore + +create schema s; + +do $$ +begin + set local search_path to s; + create table test_table(a int) distributed by (a); + drop table test_table; +end $$; + +-- check existence on master and segments +with c as ( + select gp_segment_id, relname, relkind, relnamespace from pg_class + union all + select gp_segment_id, relname, relkind, relnamespace from gp_dist_random('pg_class') +) select gp_segment_id from c +join pg_namespace n on n.oid = c.relnamespace +where c.relname = 'test_table' and c.relkind = 'r' and n.nspname = 's'; + +drop schema s cascade; From bfec70051dff028dcce5d12e308d4e9dceb7d096 Mon Sep 17 00:00:00 2001 From: Maxim Gajdaj Date: Tue, 2 Jun 2026 13:32:04 +0700 Subject: [PATCH 02/20] CI behave gpexpand @skip, SQL dump changes 6.x (#453) Bump CI behave tests v28 to v35 changes: - Detect `@skip` tag in Behave CI matrix generation - Replace `ls | grep` with glob to handle non-alphanumeric filenames - Split matrix into `run_matrix` / `skip_matrix` based on `@skip` tag detection on the `Feature:` line - Skipped features excluded from the run matrix and listed in the Job Summary of `generate-matrix` step - `continue-on-error: true` on SQL dump fetch step for `gpexpand` - failure surfaces in the test run itself - Add `set -e` to Allure report generation step Task: CI-5599 - Look for ubuntu24.04 SQL dump artifact on 6.x. The `gpexpand` behave test was looking for `sqldump_ggdb6_ubuntu` but the SQL dump workflow now generates `sqldump_ggdb6_ubuntu24.04` after regression tests were fully switched to `ubuntu24`. - Fix `artifact_name` and `artifact_archive_name` to include `ubuntu24.04` suffix for `6.x`, keeping legacy empty suffix for `7.x`. Task: CI-5678 --------- Co-authored-by: Vladislav Pavlov --- .github/workflows/greengage-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/greengage-ci.yml b/.github/workflows/greengage-ci.yml index 5f15e6292f85..ea453602000c 100644 --- a/.github/workflows/greengage-ci.yml +++ b/.github/workflows/greengage-ci.yml @@ -48,7 +48,7 @@ jobs: contents: read # Explicit for default behavior packages: read # Explicit for GHCR access clarity actions: write # Required for artifact upload - uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-tests-behave.yml@v28 + uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-tests-behave.yml@v35 with: version: 6 target_os: ${{ matrix.target_os }} From 0c064bca90bd0ad51b7a9b06402c38602e1c5698 Mon Sep 17 00:00:00 2001 From: Vasiliy Ivanov Date: Tue, 9 Jun 2026 12:51:02 +0200 Subject: [PATCH 03/20] Rework unpickled exception test This test checks that the coordinator gets exception info even in case of troubles with python serialization. But the check relies on the outher bug, caused by PyGreSQL wrapping to sub-directory. Replace exception with explicitly unpickled one to rework PyGreSQL later. --- .../gppylib/operations/test/unit/test_unit_utils.py | 10 +++------- gpMgmt/bin/gppylib/operations/test_utils_helper.py | 8 ++++++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/gpMgmt/bin/gppylib/operations/test/unit/test_unit_utils.py b/gpMgmt/bin/gppylib/operations/test/unit/test_unit_utils.py index a1f61fa8bf2a..e0a9fcf7ff67 100755 --- a/gpMgmt/bin/gppylib/operations/test/unit/test_unit_utils.py +++ b/gpMgmt/bin/gppylib/operations/test/unit/test_unit_utils.py @@ -81,17 +81,13 @@ def test_proper_exceptions_with_args(self): # It is crucial that the RMI is debuggable! def test_Remote_harden(self): """ Ensure that some logging occurs in event of error. """ - # One case encountered thus far is the raising of a pygresql DatabaseError, - # which due to the import from a shared object (I think), does not behave - # nicely in terms of imports and namespacing. """ try: RemoteOperation(RaiseOperation_Unpicklable(), "localhost").run() except ExecutionError as e: - self.assertTrue(e.cmd.get_results().stderr.strip().endswith("raise pg.DatabaseError()")) + self.assertTrue(e.cmd.get_results().stderr.strip().endswith("raise Unpicklable()")) else: - self.fail("""A pg.DatabaseError should have been raised remotely, and because it cannot - be pickled cleanly (due to a strange import in pickle.py), - an ExecutionError should have ultimately been caused.""") + self.fail("""A Unpicklable should have been raised remotely, and because it cannot + be pickled cleanly, an ExecutionError should have ultimately been caused.""") # TODO: Check logs on disk. With gplogfilter? def test_ParallelOperation_succeeds(self): diff --git a/gpMgmt/bin/gppylib/operations/test_utils_helper.py b/gpMgmt/bin/gppylib/operations/test_utils_helper.py index 31a001b735bd..a1e168ded11c 100755 --- a/gpMgmt/bin/gppylib/operations/test_utils_helper.py +++ b/gpMgmt/bin/gppylib/operations/test_utils_helper.py @@ -1,3 +1,5 @@ +import pickle + from gppylib.operations import Operation """ @@ -40,7 +42,9 @@ class ExceptionWithArgsUnsafe(Exception): def __init__(self, x, y): self.x, self.y = x, y +class Unpicklable(Exception): + def __reduce__(self): + raise pickle.PicklingError("intentionally unpicklable") class RaiseOperation_Unpicklable(Operation): def execute(self): - from pygresql import pg - raise pg.DatabaseError() + raise Unpicklable() From 3a5a00f8a675e7bfea43ae146e4b17c6c7ed8469 Mon Sep 17 00:00:00 2001 From: Vasiliy Ivanov Date: Tue, 9 Jun 2026 23:40:52 +0200 Subject: [PATCH 04/20] Stop modifying PyGreSQL layout Greengage has custom installation scripts for PyGreSQL to place module to the dedicated direcory. There already was an issue with moving shared library of the module to the proper directory level during migration to python 3. Also, this approach makes impossible to use common installation with pip. Further more, there is 10 years old issue with pickling PyGreSQL exceptions: ``` >>> import pickle >>> from pygresql import pg >>> pickle.dumps(pg.DatabaseError()) Traceback (most recent call last): File "", line 1, in _pickle.PicklingError: Can't pickle : import of module 'pg' failed ``` This patch throw out additional directory layer for PyGreSQL and implements a small shim python module to preserve compatibility with existing Greengage codebase. So that `from pygresql.pg import DB` works as expected. --- gpMgmt/Makefile | 6 +----- gpMgmt/bin/Makefile | 3 +++ gpMgmt/bin/ext/pygresql/__init__.py | 0 gpMgmt/bin/pythonSrc/PyGreSQL-compat/.gitignore | 1 + .../bin/pythonSrc/PyGreSQL-compat/pygresql/__init__.py | 7 +++++++ gpMgmt/bin/pythonSrc/PyGreSQL-compat/setup.cfg | 3 +++ gpMgmt/bin/pythonSrc/PyGreSQL-compat/setup.py | 9 +++++++++ 7 files changed, 24 insertions(+), 5 deletions(-) delete mode 100644 gpMgmt/bin/ext/pygresql/__init__.py create mode 100644 gpMgmt/bin/pythonSrc/PyGreSQL-compat/.gitignore create mode 100644 gpMgmt/bin/pythonSrc/PyGreSQL-compat/pygresql/__init__.py create mode 100644 gpMgmt/bin/pythonSrc/PyGreSQL-compat/setup.cfg create mode 100644 gpMgmt/bin/pythonSrc/PyGreSQL-compat/setup.py diff --git a/gpMgmt/Makefile b/gpMgmt/Makefile index e95ff97f068b..a002c52cc2e0 100644 --- a/gpMgmt/Makefile +++ b/gpMgmt/Makefile @@ -23,11 +23,7 @@ install: generate_greengage_path_file fi # Move _pg extension to sys.path root for Python 3 import (from _pg import *) if [ -e bin/ext/pygresql ]; then \ - cp -rp bin/ext/pygresql $(DESTDIR)$(prefix)/lib/python ; \ - sofile=`find "$(DESTDIR)$(prefix)/lib/python/pygresql" -maxdepth 1 -type f -name "_pg*.so" | head -n 1`; \ - if [ -n "$$sofile" ]; then \ - mv -f "$$sofile" "$(DESTDIR)$(prefix)/lib/python/"; \ - fi; \ + cp -rp bin/ext/pygresql/* $(DESTDIR)$(prefix)/lib/python; \ fi if [ -e bin/ext/yaml ]; then \ cp -rp bin/ext/yaml $(DESTDIR)$(prefix)/lib/python ; \ diff --git a/gpMgmt/bin/Makefile b/gpMgmt/bin/Makefile index b1cdf7fc060d..be10a8d32976 100644 --- a/gpMgmt/bin/Makefile +++ b/gpMgmt/bin/Makefile @@ -86,6 +86,7 @@ pygresql: else \ cd $(PYLIB_SRC)/$(PYGRESQL_DIR) && DESTDIR="$(DESTDIR)" CC="$(CC)" LDFLAGS='$(LDFLAGS) $(PYGRESQL_LDFLAGS)' python setup.py build; \ fi + cd $(PYLIB_SRC)/PyGreSQL-compat && python setup.py build; mkdir -p $(PYLIB_DIR)/pygresql if [ `uname -s` = 'Darwin' ]; then \ cp -r $(PYLIB_SRC)/$(PYGRESQL_DIR)/build/lib.macosx*/* $(PYLIB_DIR)/pygresql; \ @@ -94,6 +95,7 @@ pygresql: else \ cp -r $(PYLIB_SRC)/$(PYGRESQL_DIR)/build/lib.linux*/* $(PYLIB_DIR)/pygresql; \ fi + cp -r $(PYLIB_SRC)/PyGreSQL-compat/build/lib*/* $(PYLIB_DIR)/pygresql; touch $(PYLIB_DIR)/__init__.py # @@ -225,6 +227,7 @@ clean distclean: rm -rf $(PYLIB_SRC_EXT)/$(LOGILAB_ASTNG_DIR) rm -rf $(PYLIB_SRC_EXT)/$(PYGRESQL_DIR)/build rm -rf $(PYLIB_SRC)/$(PYGRESQL_DIR)/build + rm -rf $(PYLIB_SRC)/PyGreSQL-compat/build rm -rf $(PYLIB_SRC)/subprocess32/build rm -rf *.pyc rm -f analyzedbc gpactivatestandbyc gpaddmirrorsc gpcheckcatc \ diff --git a/gpMgmt/bin/ext/pygresql/__init__.py b/gpMgmt/bin/ext/pygresql/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/gpMgmt/bin/pythonSrc/PyGreSQL-compat/.gitignore b/gpMgmt/bin/pythonSrc/PyGreSQL-compat/.gitignore new file mode 100644 index 000000000000..567609b1234a --- /dev/null +++ b/gpMgmt/bin/pythonSrc/PyGreSQL-compat/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/gpMgmt/bin/pythonSrc/PyGreSQL-compat/pygresql/__init__.py b/gpMgmt/bin/pythonSrc/PyGreSQL-compat/pygresql/__init__.py new file mode 100644 index 000000000000..b08cfb3e1217 --- /dev/null +++ b/gpMgmt/bin/pythonSrc/PyGreSQL-compat/pygresql/__init__.py @@ -0,0 +1,7 @@ +# pygresql/__init__.py +from __future__ import absolute_import +import sys +import pg, pgdb +sys.modules.setdefault('pygresql.pg', pg) +sys.modules.setdefault('pygresql.pgdb', pgdb) + diff --git a/gpMgmt/bin/pythonSrc/PyGreSQL-compat/setup.cfg b/gpMgmt/bin/pythonSrc/PyGreSQL-compat/setup.cfg new file mode 100644 index 000000000000..b8a1655fa4e9 --- /dev/null +++ b/gpMgmt/bin/pythonSrc/PyGreSQL-compat/setup.cfg @@ -0,0 +1,3 @@ +[bdist_wheel] +universal = 1 + diff --git a/gpMgmt/bin/pythonSrc/PyGreSQL-compat/setup.py b/gpMgmt/bin/pythonSrc/PyGreSQL-compat/setup.py new file mode 100644 index 000000000000..1983ed79788a --- /dev/null +++ b/gpMgmt/bin/pythonSrc/PyGreSQL-compat/setup.py @@ -0,0 +1,9 @@ +from setuptools import setup + +setup( + name="PyGreSQL-compat", + version="1.0.0", + packages=["pygresql"], + install_requires=["PyGreSQL>=5.2,<6"], +) + From 475213b2ba547d235bc8157d3289db510bc67d18 Mon Sep 17 00:00:00 2001 From: Vasiliy Ivanov Date: Tue, 9 Jun 2026 23:54:14 +0200 Subject: [PATCH 05/20] Add lsof as dependency for Greengage At least this tools is used in the greengage_path.sh. Add this one to the Docker containers and as the deb package dependency. --- README.CentOS.bash | 1 + README.Rhel-Rocky.bash | 1 + README.ubuntu.bash | 1 + gpAux/debian/control | 1 + 4 files changed, 4 insertions(+) diff --git a/README.CentOS.bash b/README.CentOS.bash index bee249e73f57..fb51d730b3b1 100755 --- a/README.CentOS.bash +++ b/README.CentOS.bash @@ -22,6 +22,7 @@ sudo yum install -y \ libyaml-devel \ libzstd-devel \ libzstd-static \ + lsof \ net-tools \ openldap-devel \ openssl \ diff --git a/README.Rhel-Rocky.bash b/README.Rhel-Rocky.bash index 866c646e08fb..702f38911871 100755 --- a/README.Rhel-Rocky.bash +++ b/README.Rhel-Rocky.bash @@ -26,6 +26,7 @@ sudo dnf -y install\ libxml2-devel \ libxslt-devel \ libyaml-devel \ + lsof \ net-tools \ openldap-devel \ openssl \ diff --git a/README.ubuntu.bash b/README.ubuntu.bash index 4404a848e7d9..a968f6e8c58a 100755 --- a/README.ubuntu.bash +++ b/README.ubuntu.bash @@ -41,6 +41,7 @@ apt-get install -y \ libyaml-dev \ libzstd-dev \ locales \ + lsof \ net-tools \ openssh-client \ openssh-server \ diff --git a/gpAux/debian/control b/gpAux/debian/control index 6a14f16350aa..6a1a25886e82 100644 --- a/gpAux/debian/control +++ b/gpAux/debian/control @@ -13,6 +13,7 @@ Depends: ${shlibs:Depends}, iproute2, iputils-ping, less, + lsof, openssh-client, openssh-server, openssl, From ff4c3c53dfdb5a6a58c84e0c7728dd036d201267 Mon Sep 17 00:00:00 2001 From: Vasiliy Ivanov Date: Tue, 9 Jun 2026 23:57:49 +0200 Subject: [PATCH 06/20] Install isolation2 tests deps for plpython3 8f9aa03 switches all regression and isolation2 tests to use plpython3. But some tests require additional python modules. Moreover, part of them require PyGreSQL that depends on Greengage libPQ. And currently, this module is built only for default python interpreter as a part of Greengage build process. 8f9aa03 tries to pass Greengage PYTHONPATH to plpython3. As a result, tests work correctly only on python 3 builds (e.g. Ubuntu 24). Otherwise, plpython3 environment is polluted by python 2 modules. To run isolation2 tests in the python 2 env we should provide tests dependencies for python 3 additionally. The biggest problem is PyGreSQL that depends on Greengage artifacts. To solve this problem I suggest: 1. build PyGreSQL wheels on build Dockerfile stage if needed; 2. install them on test stage. Also, source Greengage python environment explicitly to run python 2 cluster utilities in the plpython3 environment. Make it as local as possible to avoid leaking. os.system has an issue with sourcing greengage_path.sh when called inside plpython3u function. So I've replaced this one with appropriate subprocess call. --- README.ubuntu.bash | 2 ++ ci/Dockerfile.ubuntu | 14 ++++++++++++++ src/test/isolation2/expected/packcore.out | 2 +- .../expected/segwalrep/dtm_recovery_on_standby.out | 2 +- .../input/resgroup/enable_resgroup.source | 4 ++-- .../output/resgroup/enable_resgroup.source | 4 ++-- src/test/isolation2/sql/packcore.sql | 4 +++- .../sql/segwalrep/dtm_recovery_on_standby.sql | 4 ++-- src/test/isolation2/sql/setup.sql | 4 ---- 9 files changed, 27 insertions(+), 13 deletions(-) diff --git a/README.ubuntu.bash b/README.ubuntu.bash index a968f6e8c58a..f65ab5dcec1f 100755 --- a/README.ubuntu.bash +++ b/README.ubuntu.bash @@ -48,6 +48,8 @@ apt-get install -y \ pkg-config \ protobuf-compiler \ python3-dev \ + python3-installer \ + python3-psutil \ rsync \ sudo \ zlib1g-dev diff --git a/ci/Dockerfile.ubuntu b/ci/Dockerfile.ubuntu index 788ab25f4a20..c6c9e1cfe5f4 100644 --- a/ci/Dockerfile.ubuntu +++ b/ci/Dockerfile.ubuntu @@ -49,6 +49,16 @@ ENV TARGET_OS=ubuntu \ RUN test "$(lsb_release -sr)" == "22.04" && export PYTHON3=python3; \ gpdb_src/concourse/scripts/compile_gpdb.bash +# Build python modules for isolation2 tests +# PyGresSQL requires libpq, so build wheel here to install later +RUN set -eux; \ + source /usr/local/greengage-db-devel/greengage_path.sh; \ + unset PYTHONPATH; \ + export PYTHON_SRC="/home/gpadmin/gpdb_src/gpMgmt/bin/pythonSrc/"; \ + apt update && apt install -y python3-pip; \ + python3 -m pip wheel $PYTHON_SRC/PyGreSQL-5.2.5 -w bin_gpdb/; \ + python3 -m pip wheel $PYTHON_SRC/PyGreSQL-compat -w bin_gpdb/; + FROM base AS code # Use --exclude, when it will be available in stable syntax. COPY . gpdb_src @@ -60,6 +70,10 @@ COPY --from=code /home/gpadmin/gpdb_src gpdb_src COPY --from=build /home/gpadmin/bin_gpdb bin_gpdb COPY --from=build /home/gpadmin/gpdb_src/VERSION gpdb_src +# Install pygresql for python3 +RUN find bin_gpdb -name "PyGreSQL*.whl" -exec python3 \ + -m installer {} \; + # Install entab used by pgindent utility. # This should be done using gpdb sources. RUN make -C gpdb_src/src/tools/entab install clean diff --git a/src/test/isolation2/expected/packcore.out b/src/test/isolation2/expected/packcore.out index 74721727de3c..9d228d294567 100644 --- a/src/test/isolation2/expected/packcore.out +++ b/src/test/isolation2/expected/packcore.out @@ -5,7 +5,7 @@ CREATE DO LANGUAGE plpython3u $$ import os import sys import glob import shutil import subprocess if sys.platform not in ('linux', 'linux2'): # packcore only works on linux return -def check_call(cmds): ret = subprocess.Popen(cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out = ret.communicate() if ret.returncode != 0: raise SystemError('''\ Command {cmds} returned non-zero exit status {retcode} stdout: {stdout} stderr: {stderr} '''.format(cmds=cmds, retcode=ret.returncode, stdout=out[0].decode('utf-8'), stderr=out[1].decode('utf-8'))) +def check_call(cmds): py_path = os.path.join(os.getenv("GPHOME"), 'lib/python') ret = subprocess.Popen(cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=dict(os.environ, PYTHONPATH=py_path)) out = ret.communicate() if ret.returncode != 0: raise SystemError('''\ Command {cmds} returned non-zero exit status {retcode} stdout: {stdout} stderr: {stderr} '''.format(cmds=cmds, retcode=ret.returncode, stdout=out[0].decode('utf-8'), stderr=out[1].decode('utf-8'))) # generate and verify a packcore tarball # # TODO: packcore can list shared libraries with gdb, ldd, or ld-linux.so, # we should verify all of them, but so far there is no cmdline option to # specify it. although we could rename the commands to fallback to others, # we should not do it, it requires root permission and might corrupt the # developer system. on concourse, gdb is not installed by default, so the # gdb way is not covered by the pipelines. def test_packcore(cmds): # cleanup old files and dirs shutil.rmtree(tarball, ignore_errors=True) shutil.rmtree(dirname, ignore_errors=True) # generate the tarball, the packcore command should return 0 check_call(cmds) assert os.path.isfile(tarball) # extract the tarball check_call(['tar', '-zxf', tarball]) assert os.path.isdir(dirname) diff --git a/src/test/isolation2/expected/segwalrep/dtm_recovery_on_standby.out b/src/test/isolation2/expected/segwalrep/dtm_recovery_on_standby.out index 8715b6f61267..767da1e0ab3c 100644 --- a/src/test/isolation2/expected/segwalrep/dtm_recovery_on_standby.out +++ b/src/test/isolation2/expected/segwalrep/dtm_recovery_on_standby.out @@ -120,7 +120,7 @@ ERROR: terminating connection due to administrator command (seg0 slice1 192.16 create table standby_config as (select hostname, datadir, port, role from gp_segment_configuration where content = -1) distributed by (hostname); CREATE 2 -create or replace function reinitialize_standby() returns text as $$ import subprocess rv = plpy.execute("select hostname, datadir, port from standby_config order by role", 2) standby = rv[0] # role = 'm' master = rv[1] # role = 'p' try: cmd = 'rm -rf %s.dtm_recovery && cp -R %s %s.dtm_recovery' % (standby['datadir'], standby['datadir'], standby['datadir']) remove_output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT).decode('utf-8') cmd = 'gpinitstandby -ar -P %d' % master['port'] remove_output += subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT).decode('utf-8') cmd = 'export PGPORT=%d; gpinitstandby -a -s %s -S %s -P %d' % (master['port'], standby['hostname'], standby['datadir'], standby['port']) init_output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT).decode('utf-8') except subprocess.CalledProcessError as e: plpy.info(e.output) raise +create or replace function reinitialize_standby() returns text as $$ import subprocess rv = plpy.execute("select hostname, datadir, port from standby_config order by role", 2) standby = rv[0] # role = 'm' master = rv[1] # role = 'p' try: cmd = 'rm -rf %s.dtm_recovery && cp -R %s %s.dtm_recovery' % (standby['datadir'], standby['datadir'], standby['datadir']) remove_output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT).decode('utf-8') cmd = '. $GPHOME/greengage_path.sh; gpinitstandby -ar -P %d' % master['port'] remove_output += subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT).decode('utf-8') cmd = '. $GPHOME/greengage_path.sh; export PGPORT=%d; gpinitstandby -a -s %s -S %s -P %d' % (master['port'], standby['hostname'], standby['datadir'], standby['port']) init_output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT).decode('utf-8') except subprocess.CalledProcessError as e: plpy.info(e.output) raise return remove_output + "\n" + init_output $$ language plpython3u; CREATE diff --git a/src/test/isolation2/input/resgroup/enable_resgroup.source b/src/test/isolation2/input/resgroup/enable_resgroup.source index de8fd8c05355..d67b6cdb5eee 100644 --- a/src/test/isolation2/input/resgroup/enable_resgroup.source +++ b/src/test/isolation2/input/resgroup/enable_resgroup.source @@ -19,8 +19,8 @@ -- -- with the simulation each primary segment should manage 682MB memory. DO LANGUAGE plpython3u $$ - import os import psutil + import subprocess mem = psutil.virtual_memory().total swap = psutil.swap_memory().total @@ -37,7 +37,7 @@ DO LANGUAGE plpython3u $$ ''')[0]['nsegs']) limit = (2 << 30) * 1.0 * nsegs / 3 / total - os.system('gpconfig -c gp_resource_group_memory_limit -v {:f}'.format(limit)) + subprocess.check_call('. $GPHOME/greengage_path.sh; gpconfig -c gp_resource_group_memory_limit -v {:f}'.format(limit), shell=True) $$; -- enable resource group and restart cluster. diff --git a/src/test/isolation2/output/resgroup/enable_resgroup.source b/src/test/isolation2/output/resgroup/enable_resgroup.source index 4f8de1e5bf1e..3a554bfff19a 100644 --- a/src/test/isolation2/output/resgroup/enable_resgroup.source +++ b/src/test/isolation2/output/resgroup/enable_resgroup.source @@ -24,10 +24,10 @@ -- so: limit = 2GB * 1.0 / 3 * nsegs / total -- -- with the simulation each primary segment should manage 682MB memory. -DO LANGUAGE plpython3u $$ import os import psutil +DO LANGUAGE plpython3u $$ import psutil import subprocess mem = psutil.virtual_memory().total swap = psutil.swap_memory().total overcommit = int(open('/proc/sys/vm/overcommit_ratio').readline()) total = swap + mem * overcommit / 100. nsegs = int(plpy.execute(''' SELECT count(hostname) as nsegs FROM gp_segment_configuration WHERE preferred_role = 'p' GROUP BY hostname ORDER BY count(hostname) DESC LIMIT 1 ''')[0]['nsegs']) -limit = (2 << 30) * 1.0 * nsegs / 3 / total os.system('gpconfig -c gp_resource_group_memory_limit -v {:f}'.format(limit)) $$; +limit = (2 << 30) * 1.0 * nsegs / 3 / total subprocess.check_call('. $GPHOME/greengage_path.sh; gpconfig -c gp_resource_group_memory_limit -v {:f}'.format(limit), shell=True) $$; DO -- enable resource group and restart cluster. diff --git a/src/test/isolation2/sql/packcore.sql b/src/test/isolation2/sql/packcore.sql index 74a722a6dd71..21a400033440 100644 --- a/src/test/isolation2/sql/packcore.sql +++ b/src/test/isolation2/sql/packcore.sql @@ -14,9 +14,11 @@ DO LANGUAGE plpython3u $$ return def check_call(cmds): + py_path = os.path.join(os.getenv("GPHOME"), 'lib/python') ret = subprocess.Popen(cmds, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + env=dict(os.environ, PYTHONPATH=py_path)) out = ret.communicate() if ret.returncode != 0: raise SystemError('''\ diff --git a/src/test/isolation2/sql/segwalrep/dtm_recovery_on_standby.sql b/src/test/isolation2/sql/segwalrep/dtm_recovery_on_standby.sql index e1a18edbfa78..8e45d99e69ef 100644 --- a/src/test/isolation2/sql/segwalrep/dtm_recovery_on_standby.sql +++ b/src/test/isolation2/sql/segwalrep/dtm_recovery_on_standby.sql @@ -89,9 +89,9 @@ returns text as $$ try: cmd = 'rm -rf %s.dtm_recovery && cp -R %s %s.dtm_recovery' % (standby['datadir'], standby['datadir'], standby['datadir']) remove_output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT).decode('utf-8') - cmd = 'gpinitstandby -ar -P %d' % master['port'] + cmd = '. $GPHOME/greengage_path.sh; gpinitstandby -ar -P %d' % master['port'] remove_output += subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT).decode('utf-8') - cmd = 'export PGPORT=%d; gpinitstandby -a -s %s -S %s -P %d' % (master['port'], standby['hostname'], standby['datadir'], standby['port']) + cmd = '. $GPHOME/greengage_path.sh; export PGPORT=%d; gpinitstandby -a -s %s -S %s -P %d' % (master['port'], standby['hostname'], standby['datadir'], standby['port']) init_output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT).decode('utf-8') except subprocess.CalledProcessError as e: plpy.info(e.output) diff --git a/src/test/isolation2/sql/setup.sql b/src/test/isolation2/sql/setup.sql index 73cb88683ba1..22356b9c6f8d 100644 --- a/src/test/isolation2/sql/setup.sql +++ b/src/test/isolation2/sql/setup.sql @@ -1,7 +1,3 @@ --- start_ignore -! gpconfig -c plpython3.python_path -v "'$GPHOME/lib/python'" --skipvalidation; -! gpstop -u; --- end_ignore CREATE OR REPLACE LANGUAGE plpython3u; -- Helper function, to call either __gp_aoseg, or gp_aocsseg, depending From 106ad51d0becc50822ca0b271fe43b1b64cc0cf0 Mon Sep 17 00:00:00 2001 From: Artem Shapatin Date: Mon, 22 Jun 2026 16:40:37 +0700 Subject: [PATCH 07/20] Fix python coverage configs (#489) This commit fixes minor faults at coverage configs and collection process. Changes in current commit: 1. Update behave tests coverage collection with GPHOME variable, so remappings are done to each test. 2. Update regression tests script so it will collect and report coverage (otherwise we forced to do it in CI). 3. Add new remappings to configs for gpperfmon test and gp_replica_check.py file. 4. Add to exceptions new(moved) pg.py and pgdb.py files. 5. Update $GPHOME hardcode in coveragerc_unit file. --- ci/scripts/run_behave_tests.bash | 1 + concourse/scripts/ic_gpdb.bash | 12 ++++++++++++ gpMgmt/test/coveragerc_behave | 8 ++++++++ gpMgmt/test/coveragerc_combine_report | 14 +++++++++++--- gpMgmt/test/coveragerc_unit | 28 ++++++++++++++------------- 5 files changed, 47 insertions(+), 16 deletions(-) diff --git a/ci/scripts/run_behave_tests.bash b/ci/scripts/run_behave_tests.bash index 31cc9ebabded..74f89f9bc4b1 100755 --- a/ci/scripts/run_behave_tests.bash +++ b/ci/scripts/run_behave_tests.bash @@ -74,6 +74,7 @@ run_feature() { -e FEATURE="$feature" -e PROJECT="$project" \ cdw bash -eux <<'EOF' set -ex + source /usr/local/greengage-db-devel/greengage_path.sh cd /tmp/coverage-data if [ "$(ls "$PROJECT"-coverage-data/ | wc -l)" -gt 0 ]; then diff --git a/concourse/scripts/ic_gpdb.bash b/concourse/scripts/ic_gpdb.bash index 7f4fc654ba6a..93b5f8c5209d 100755 --- a/concourse/scripts/ic_gpdb.bash +++ b/concourse/scripts/ic_gpdb.bash @@ -34,6 +34,18 @@ function gen_env(){ export TEST_PGFDW=1 export COVERAGE_PROCESS_START="\${1}/gpdb_src/gpMgmt/test/coveragerc_unit" make -s ${MAKE_TEST_COMMAND} + + cd /tmp/coverage-data + if [ "\$(ls coverage-data* 2>/dev/null | wc -l)" -gt 0 ]; then + coverage combine --append --keep \ + --rcfile=/home/gpadmin/gpdb_src/gpMgmt/test/coveragerc_combine_report \ + coverage-data* + coverage html \ + --rcfile=/home/gpadmin/gpdb_src/gpMgmt/test/coveragerc_combine_report \ + --show-contexts -d ./coverage-html + else + echo "No coverage-data files found, skipping coverage report" + fi EOF chmod a+x /opt/run_test.sh diff --git a/gpMgmt/test/coveragerc_behave b/gpMgmt/test/coveragerc_behave index 699be39093ea..f4863d1eb17e 100644 --- a/gpMgmt/test/coveragerc_behave +++ b/gpMgmt/test/coveragerc_behave @@ -11,6 +11,8 @@ omit = */gpload_test/* $GPHOME/lib/python/pygresql/* $GPHOME/lib/python/yaml/* + $GPHOME/lib/python/pg.py + $GPHOME/lib/python/pgdb.py $GPHOME/lib/python/sitecustomize.py $GPHOME/lib/python/subprocess32.py $GPHOME/lib/python/subprocess32.pyc @@ -36,3 +38,9 @@ source3 = /home/gpadmin/gpdb_src/gpMgmt/bin/gppylib /home/gpadmin/gpdb_src/gpMgmt/bin/gppylib $GPHOME/lib/python/gppylib + +source4 = + /home/gpadmin/gpdb_src/gpAux/gpperfmon/src/gpmon + /home/gpadmin/gpdb_src/gpAux/gpperfmon/src/gpmon + $GPHOME/bin + $GPHOME/sbin diff --git a/gpMgmt/test/coveragerc_combine_report b/gpMgmt/test/coveragerc_combine_report index 23729e3b43cc..7e1b8e185365 100644 --- a/gpMgmt/test/coveragerc_combine_report +++ b/gpMgmt/test/coveragerc_combine_report @@ -19,15 +19,23 @@ source3 = $GPHOME/lib/python/gppylib source4 = - /home/gpadmin/gpdb_src/gpcontrib/gp_replica_check/gp_replica_check.py - /home/gpadmin/gpdb_src/gpcontrib/gp_replica_check/gp_replica_check.py - $GPHOME/bin/gp_replica_check.py + /home/gpadmin/gpdb_src/gpcontrib/gp_replica_check + /home/gpadmin/gpdb_src/gpcontrib/gp_replica_check + $GPHOME/bin + +source5 = + /home/gpadmin/gpdb_src/gpAux/gpperfmon/src/gpmon + /home/gpadmin/gpdb_src/gpAux/gpperfmon/src/gpmon + $GPHOME/bin + $GPHOME/sbin [report] omit = */__init__.py* */test/* $GPHOME/lib/python/pygresql/* + $GPHOME/lib/python/pg.py + $GPHOME/lib/python/pgdb.py $GPHOME/lib/python/yaml/* $GPHOME/lib/python/sitecustomize.py $GPHOME/lib/python/subprocess32.py diff --git a/gpMgmt/test/coveragerc_unit b/gpMgmt/test/coveragerc_unit index 6fa5b11654b6..75ca49e5fbdb 100644 --- a/gpMgmt/test/coveragerc_unit +++ b/gpMgmt/test/coveragerc_unit @@ -9,13 +9,15 @@ omit = */sitecustomize.py* */test/* */gpload_test/* - /usr/local/greengage-db-devel/lib/python/pygresql/* - /usr/local/greengage-db-devel/lib/python/yaml/* - /usr/local/greengage-db-devel/lib/python/sitecustomize.py - /usr/local/greengage-db-devel/lib/python/subprocess32.py - /usr/local/greengage-db-devel/lib/python/subprocess32.pyc - /usr/local/greengage-db-devel/bin/lib/pexpect/* - /usr/local/greengage-db-devel/lib/python/psutil/* + $GPHOME/lib/python/pygresql/* + $GPHOME/lib/python/pg.py + $GPHOME/lib/python/pgdb.py + $GPHOME/lib/python/yaml/* + $GPHOME/lib/python/sitecustomize.py + $GPHOME/lib/python/subprocess32.py + $GPHOME/lib/python/subprocess32.pyc + $GPHOME/bin/lib/pexpect/* + $GPHOME/lib/python/psutil/* */dist-packages/* /usr/local/bin/behave /usr/local/bin/coverage @@ -25,19 +27,19 @@ omit = source = /home/gpadmin/gpdb_src/gpMgmt/bin /home/gpadmin/gpdb_src/gpMgmt/bin - /usr/local/greengage-db-devel/bin + $GPHOME/bin source2 = /home/gpadmin/gpdb_src/gpMgmt/sbin /home/gpadmin/gpdb_src/gpMgmt/sbin - /usr/local/greengage-db-devel/sbin + $GPHOME/sbin source3 = /home/gpadmin/gpdb_src/gpMgmt/bin/gppylib /home/gpadmin/gpdb_src/gpMgmt/bin/gppylib - /usr/local/greengage-db-devel/lib/python/gppylib + /$GPHOME/lib/python/gppylib source4 = - /home/gpadmin/gpdb_src/gpcontrib/gp_replica_check/gp_replica_check.py - /home/gpadmin/gpdb_src/gpcontrib/gp_replica_check/gp_replica_check.py - /usr/local/greengage-db-devel/bin/gp_replica_check.py + /home/gpadmin/gpdb_src/gpcontrib/gp_replica_check + /home/gpadmin/gpdb_src/gpcontrib/gp_replica_check + $GPHOME/bin From 14cf0b71a003f0552fc9845ea143db612974a968 Mon Sep 17 00:00:00 2001 From: Denis Kovalev Date: Mon, 22 Jun 2026 18:50:13 +0300 Subject: [PATCH 08/20] Add pg_event_trigger to the list of master only tables (#483) Event triggers are expected to work only at the master side. However gpexpand copies content of pg_event_trigger table to the newly created segment(s). This commit adds pg_event_trigger to the list of master only table and adds behave test to verify it. GG-526 Co-authored-by: Viktor Kurilko --- .../gpcheckcat_modules/foreign_key_check.py | 3 ++- gpMgmt/bin/gppylib/gpcatalog.py | 1 + .../test/behave/mgmt_utils/gpexpand.feature | 17 +++++++++++++ .../behave/mgmt_utils/steps/mgmt_utils.py | 25 +++++++++++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py b/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py index a0e1702d3396..f612f706d449 100644 --- a/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py +++ b/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py @@ -67,7 +67,8 @@ def checkTableForeignKey(self, cat): # skip these master-only tables skipped_masteronly = ['gp_relation_node', 'pg_description', 'pg_shdescription', 'pg_stat_last_operation', - 'pg_stat_last_shoperation', 'pg_statistic'] + 'pg_stat_last_shoperation', 'pg_statistic', + 'pg_event_trigger'] if catname in skipped_masteronly: return diff --git a/gpMgmt/bin/gppylib/gpcatalog.py b/gpMgmt/bin/gppylib/gpcatalog.py index a237b299f2e0..1429cf723961 100644 --- a/gpMgmt/bin/gppylib/gpcatalog.py +++ b/gpMgmt/bin/gppylib/gpcatalog.py @@ -40,6 +40,7 @@ class GPCatalogException(Exception): 'pg_stat_last_shoperation', 'pg_statistic', 'pg_partition_encoding', + 'pg_event_trigger' ] # Hard coded tables that have different values on every segment diff --git a/gpMgmt/test/behave/mgmt_utils/gpexpand.feature b/gpMgmt/test/behave/mgmt_utils/gpexpand.feature index 1bf817c00be4..0e89444bf9fe 100644 --- a/gpMgmt/test/behave/mgmt_utils/gpexpand.feature +++ b/gpMgmt/test/behave/mgmt_utils/gpexpand.feature @@ -820,3 +820,20 @@ Feature: expand the cluster by adding more segments When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--verbose" Then gpexpand should print "[DEBUG]:-Skipping tar file (gpexpand_schema.tar) copy to cdw" escaped to stdout And verify that the cluster has 1 new segments + + @gpexpand_verify_master_only + Scenario: Verify master only tables are not copied to segment + Given the database is not running + And a working directory of the test as '/data/gpdata/gpexpand' + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into + And a cluster is created with no mirrors on "cdw" and "sdw1" + And database "gptest" exists + And the user creates an event trigger test_trigger + And verify that event trigger test_trigger exists + Then verify that the query "SELECT count(*) FROM gp_dist_random('pg_event_trigger');" in database "gptest" returns "0" + When the user runs gpexpand interview to add 1 new segment and 0 new host "ignored.host" + Then the number of segments have been saved + When the user runs gpexpand with the latest gpexpand_inputfile without ret code check + Then gpexpand should return a return code of 0 + And verify that the cluster has 1 new segments + Then verify that the query "SELECT count(*) FROM gp_dist_random('pg_event_trigger');" in database "gptest" returns "0" diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index 0f4866327ada..aa2ee38bf507 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -4676,3 +4676,28 @@ def impl(context): def impl(context, second): cmd = Command(name='psql', cmdStr="-c 'SELECT * from generate_series(1, %s) a where pg_sleep(1) is not null;'" % second) cmd.runNoWait() + +@given('the user creates an event trigger {trigger_name}') +def impl(context, trigger_name): + func_name = "%s_fn" % trigger_name + with dbconn.connect(dbconn.DbURL(dbname=context.dbname), unsetSearchPath=False) as conn: + sql = """CREATE OR REPLACE FUNCTION %s() +RETURNS event_trigger +LANGUAGE plpgsql AS $$ +BEGIN + RAISE NOTICE 'Event trigger %s_fn fired after DDL command'; +END; +$$; +""" % (func_name, trigger_name) + + dbconn.execSQL(conn, sql) + sql = "CREATE EVENT TRIGGER %s ON ddl_command_start EXECUTE PROCEDURE %s();" % (trigger_name, func_name) + dbconn.execSQL(conn, sql) + conn.commit() + +@given('verify that event trigger {trigger_name} exists') +def impl(context, trigger_name): + with dbconn.connect(dbconn.DbURL(dbname=context.dbname), unsetSearchPath=False) as conn: + sql = "SELECT evtname FROM pg_event_trigger WHERE evtname = '%s';" % trigger_name + cursor = dbconn.execSQL(conn, sql) + assert cursor.rowcount == 1 From 729cec8d93f74cc95df7bf801a1e52e3ca6c0167 Mon Sep 17 00:00:00 2001 From: Artem Shapatin Date: Tue, 23 Jun 2026 19:00:24 +0700 Subject: [PATCH 09/20] Fix handling path with trailing slashes (#498) Previously gpstart utility couldn't start standby on directory specified with trailing slash, like that: /path/to/standby/. It was enough to add normalization on path, to fix the issue. --- gpMgmt/bin/gppylib/commands/gp.py | 2 ++ .../behave/mgmt_utils/gpactivatestandby.feature | 14 ++++++++++++++ .../behave/mgmt_utils/gpinitstandby.feature | 10 ++++++++++ gpMgmt/test/behave/mgmt_utils/gpstart.feature | 17 +++++++++++++++++ 4 files changed, 43 insertions(+) diff --git a/gpMgmt/bin/gppylib/commands/gp.py b/gpMgmt/bin/gppylib/commands/gp.py index cebc46538495..a973663bc872 100644 --- a/gpMgmt/bin/gppylib/commands/gp.py +++ b/gpMgmt/bin/gppylib/commands/gp.py @@ -1380,6 +1380,8 @@ def start_standbymaster(host, datadir, port, era=None, wrapper=None, wrapper_args=None): logger.info("Starting standby master") + datadir = os.path.normpath(datadir) + logger.info("Checking if standby master is running on host: %s in directory: %s" % (host,datadir)) cmd = Command("recovery_startup", ("python -c " diff --git a/gpMgmt/test/behave/mgmt_utils/gpactivatestandby.feature b/gpMgmt/test/behave/mgmt_utils/gpactivatestandby.feature index db4f2df47bce..0041b5ee7711 100644 --- a/gpMgmt/test/behave/mgmt_utils/gpactivatestandby.feature +++ b/gpMgmt/test/behave/mgmt_utils/gpactivatestandby.feature @@ -76,6 +76,20 @@ Feature: gpactivatestandby And the tablespace is valid on the standby master And clean up and revert back to original master + Scenario: master can be made on dir with trailing slash + Given the database is running + And the standby is not initialized + + When the user runs gpinitstandby with options "-S /tmp/standby_data/" + Then gpinitstandby should return a return code of 0 + And verify the standby master entries in catalog + + When the master goes down + And the user runs gpactivatestandby with options "-d /tmp/standby_data/" + Then gpactivatestandby should return a return code of 0 + And verify the standby master is now acting as master + And clean up and revert back to original master + ########################### @concourse_cluster tests ########################### # The @concourse_cluster tag denotes the scenario that requires a remote cluster diff --git a/gpMgmt/test/behave/mgmt_utils/gpinitstandby.feature b/gpMgmt/test/behave/mgmt_utils/gpinitstandby.feature index 8e4e37cdb631..bf9c5557775c 100644 --- a/gpMgmt/test/behave/mgmt_utils/gpinitstandby.feature +++ b/gpMgmt/test/behave/mgmt_utils/gpinitstandby.feature @@ -101,6 +101,16 @@ Feature: Tests for gpinitstandby feature Then gpinitstandby should return a return code of 0 And verify that the file "pg_hba.conf" in the master data directory has "no" line starting with "host.*replication.*(127.0.0.1|::1).*trust" + Scenario: gpinitstandby on dir with trailing slash + Given the database is running + And the standby is not initialized + + When the user runs gpinitstandby with options "-S /tmp/standby_data/" + Then gpinitstandby should return a return code of 0 + And verify the standby master entries in catalog + When execute sql "select datadir from gp_segment_configuration where content = -1 and role = 'm'" in db "postgres" and store result in the context + Then validate that "/tmp/standby_data/" is in the stored rows + @backup_restore_bashrc Scenario: gpinitstandby should not throw error when banner exists on the hsot Given the database is running diff --git a/gpMgmt/test/behave/mgmt_utils/gpstart.feature b/gpMgmt/test/behave/mgmt_utils/gpstart.feature index 4c7c21c74a8d..37ada7c76350 100644 --- a/gpMgmt/test/behave/mgmt_utils/gpstart.feature +++ b/gpMgmt/test/behave/mgmt_utils/gpstart.feature @@ -172,3 +172,20 @@ Feature: gpstart behave tests When the user runs psql with "-c 'drop user foouser;'" against database "postgres" Then psql should return a return code of 0 + + Scenario: gpstart on dir with trailing slash + Given the database is running + And the catalog has a standby master entry + And the standby is not initialized + + When the user runs gpinitstandby with options "-S /tmp/standby_data/" + Then gpinitstandby should return a return code of 0 + And verify the standby master entries in catalog + + When the master goes down + And the user runs "gpstart -a" + Then gpstart should return a return code of 0 + And verify the standby master entries in catalog + + When execute sql "select datadir from gp_segment_configuration where content = -1 and role = 'm'" in db "postgres" and store result in the context + Then validate that "/tmp/standby_data/" is in the stored rows From 9b97b1fb58ff06b3c740147e94896edd11318d4d Mon Sep 17 00:00:00 2001 From: Maxim Gajdaj Date: Wed, 24 Jun 2026 02:08:48 +0700 Subject: [PATCH 10/20] Build rpm-distro docker image 6.x (#436) Adds Docker-based build environment for Rocky Linux 8 and 9, on par with the existing Ubuntu support. - add `ci/Dockerfile.rockylinux` - multi-stage build (base/build/code/test) - upd `README.Rhel-Rocky.bash` - OS-version-aware dependency installation: - Python 2 + 3 for RHEL/Rocky 8, Python 3.11 for Rocky 9 - Perl packages extended for RHEL/Rocky 9 (Opcode, Test-Simple, Thread-Queue) - zstd built from source (unavailable as a package on RHEL/Rocky) - `concourse/scripts/common.bash` - fix `os_id()` to correctly detect Rocky Linux; previously any RHEL-like system was returned as `centos`, making `rocky8`/`rocky9` CONFIGFLAGS unreachable dead code - `ci/readme.md` - add Rocky Linux 8/9 build image instructions - `README.linux.md` - add Version 9 to RHEL/Rocky description - CI matrix - Rocky Linux 8/9 added for `build` workflow - Bump build workflow to `v42`; changes: Add Rocky Linux support to reusable build workflow - Expose `TARGET_OS` as a job-level env var for use in shell scripts - Replace unconditional Ubuntu mirror resolution with a `case`: Ubuntu keeps the existing Azure mirror optimization, Rocky Linux has a stub for a future mirror when available - Use `$TARGET_OS` instead of expression interpolation in `docker build` Task: CI-5657 --- .github/workflows/greengage-ci.yml | 6 ++- README.Rhel-Rocky.bash | 76 ++++++++++++++++++++------- README.linux.md | 12 +---- ci/Dockerfile.rockylinux | 82 ++++++++++++++++++++++++++++++ ci/readme.md | 12 +++++ concourse/scripts/common.bash | 12 ++--- 6 files changed, 164 insertions(+), 36 deletions(-) create mode 100644 ci/Dockerfile.rockylinux diff --git a/.github/workflows/greengage-ci.yml b/.github/workflows/greengage-ci.yml index ea453602000c..a0ecbd5ff9ab 100644 --- a/.github/workflows/greengage-ci.yml +++ b/.github/workflows/greengage-ci.yml @@ -22,11 +22,15 @@ jobs: - target_os: ubuntu - target_os: ubuntu target_os_version: "24.04" + - target_os: rockylinux + target_os_version: "8" + - target_os: rockylinux + target_os_version: "9" permissions: contents: read # Explicit for default behavior packages: write # Required for GHCR access actions: write # Required for artifact upload - uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-build.yml@v33 + uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-build.yml@v42 with: version: 6 target_os: ${{ matrix.target_os }} diff --git a/README.Rhel-Rocky.bash b/README.Rhel-Rocky.bash index 702f38911871..1b55d5f95bc1 100755 --- a/README.Rhel-Rocky.bash +++ b/README.Rhel-Rocky.bash @@ -1,21 +1,57 @@ #!/bin/bash +# FILE: README.Rhel-Rocky.bash +# CONTEXT: Called from ci/Dockerfile.rockylinux for Greengage build +# PURPOSE: Install build dependencies, compile zstd static library, +# Install Python based on OS version -sudo dnf -y update -sudo dnf -y install epel-release -sudo dnf -y install 'dnf-command(config-manager)' -sudo dnf config-manager --set-enabled devel -sudo dnf makecache +set -eux -sudo dnf -y install\ +dnf -y install epel-release + +# Detect OS version if not already set +export OS_VERSION="${OS_VERSION:-$(grep -oP '(?<= release )\d+' /etc/redhat-release)}" + +case "$OS_VERSION" in + 8) + dnf config-manager --set-enabled powertools + python_packages="python2 python2-devel python2-setuptools \ + python3 python3-devel python3-setuptools" + perl_packages="perl-Env perl-ExtUtils-Embed \ + perl-IPC-Run perl-JSON perl-Test-Base" + ;; + 9) + dnf config-manager --set-enabled crb + python_packages="python3.11 python3.11-devel python3.11-setuptools" + perl_packages="perl-Env perl-ExtUtils-Embed \ + perl-IPC-Run perl-JSON perl-Test-Base \ + perl-Opcode perl-Test-Simple perl-Thread-Queue perl-devel" + ;; + *) + echo "Unsupported Rocky Linux version: $OS_VERSION" + exit 1 + ;; +esac +# shellcheck disable=SC2086 # intentional: word splitting for package lists +dnf -y install \ apr-devel \ + apr-util-devel \ + autoconf \ bison \ bzip2-devel \ - cmake3 \ + cmake \ + expat-devel \ flex \ - gcc \ gcc-c++ \ + git \ + glibc-langpack-en \ + gperf \ + indent \ iproute \ + java-11-openjdk-devel \ + jq \ krb5-devel \ + krb5-server \ + krb5-workstation \ libcurl-devel \ libevent-devel \ libicu \ @@ -29,18 +65,24 @@ sudo dnf -y install\ lsof \ net-tools \ openldap-devel \ - openssl \ + openssh-server \ openssl-devel \ pam-devel \ - perl-Env \ - perl-ExtUtils-Embed \ - perl-IPC-Run \ - perl-JSON \ - perl-Test-Base \ procps-ng \ - python2-devel \ - python2-pip \ readline-devel \ + rsync \ snappy-devel \ + sudo \ + time \ + unzip \ + vim \ + wget \ xerces-c-devel \ - zlib-devel + zlib-devel \ + $python_packages $perl_packages + +# Build zstd with static library (not available as a package on Rocky) +curl -Ls https://github.com/facebook/zstd/releases/download/v1.4.4/zstd-1.4.4.tar.gz | tar -xzf - +make -j"$(nproc)" -C zstd-1.4.4 +make install PREFIX=/usr/local -C zstd-1.4.4 +rm -rf zstd-1.4.4 diff --git a/README.linux.md b/README.linux.md index c73dfd9c4d96..1e646b26166d 100644 --- a/README.linux.md +++ b/README.linux.md @@ -6,23 +6,13 @@ ``` Note: CentOS 7 is EOL — configure `yum` to use a valid repo (e.g., `vault.centos.org`) before installing dependencies. -## For RHEL/Rocky 8: +## For RHEL/Rocky (versions 8 or 9): - Install dependencies using README.Rhel-Rocky.bash script: ```bash ./README.Rhel-Rocky.bash ``` -- Build and install zstd with static library, e.g.: - ```bash - cd /tmp - curl -LO https://github.com/facebook/zstd/releases/download/v1.4.4/zstd-1.4.4.tar.gz - tar -xf zstd-1.4.4.tar.gz - cd zstd-1.4.4 - make -j$(nproc) - sudo make install PREFIX=/usr/local - ``` - - Create symbolic link to Python 2 in `/usr/bin`: ```bash diff --git a/ci/Dockerfile.rockylinux b/ci/Dockerfile.rockylinux new file mode 100644 index 000000000000..6e066c0b8b4f --- /dev/null +++ b/ci/Dockerfile.rockylinux @@ -0,0 +1,82 @@ +# FILE: ci/Dockerfile.rockylinux +# CONTEXT: Build Greengage on Rocky Linux +# PURPOSE: Multi-stage Docker build for Greengage database + +ARG OS_VERSION=8 + +FROM rockylinux:${OS_VERSION} AS base +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +WORKDIR /home/gpadmin + +ARG OS_VERSION +ENV OS_VERSION=${OS_VERSION} + +COPY README.Rhel-Rocky.bash ./ +COPY gpMgmt/bin/gpload_test/pytest_requirement.txt ./ +RUN set -eux; \ + ./README.Rhel-Rocky.bash; \ + rm README.Rhel-Rocky.bash; \ +# Set Python as default based on OS version + case "$OS_VERSION" in \ + 8) \ + ln -sf /usr/bin/python2 /usr/bin/python; \ + PIP_PACKAGES='allure-behave==2.4.0 mock==3.0.5 psi==0.3b3 future==0.16'; \ + ;; \ + 9) \ + ln -sf /usr/bin/python3.11 /usr/bin/python; \ + python -m ensurepip --upgrade; \ + PIP_PACKAGES='allure-behave==2.4.0 future==1.0.0'; \ + ;; \ + *) \ + echo "Unsupported Rocky Linux version: $OS_VERSION"; \ + exit 1; \ + ;; \ + esac; \ +# Install pg_bsd_indent used by pgindent utility + wget --progress=dot:giga --no-hsts https://ftp.postgresql.org/pub/dev/pg_bsd_indent-1.3.tar.gz -O - | tar -xzf -; \ + make install -C pg_bsd_indent; \ + rm -r pg_bsd_indent; \ +# To run sshd directly, but not using service + mkdir -p /run/sshd; \ +# Alter precedence in favor of IPv4 during resolving + echo 'precedence ::ffff:0:0/96 100' >> /etc/gai.conf; \ +# Upgrade pip to support current package versions + python -m pip install --no-cache-dir --upgrade pip; \ +# Install allure-behave for behave tests + python -m pip install --no-cache-dir $PIP_PACKAGES; \ +# Install pytest for gpload test + python -m pip install --no-cache-dir --requirement pytest_requirement.txt; \ + rm pytest_requirement.txt; \ +# Cleanup to reduce image size + dnf clean all; \ + rm -rf /tmp/* /var/tmp/* /var/cache/man/ + +ENV LANG=en_US.UTF-8 +ENV CONFIGURE_FLAGS="--enable-debug-extensions --with-gssapi --enable-cassert --enable-debug --enable-depend" + +FROM base AS build + +COPY . gpdb_src + +RUN mkdir bin_gpdb + +ENV TARGET_OS=rockylinux \ + OUTPUT_ARTIFACT_DIR=bin_gpdb + +# Use python3 to compile into GPDB's plpython3u +# Set PYTHON3 only on Rocky 8 to avoid compiling plpython3 twice +# On Rocky 9, Python 3 is the default +RUN test "$OS_VERSION" -eq 8 && export PYTHON3=python3; \ + gpdb_src/concourse/scripts/compile_gpdb.bash + +FROM base AS code +COPY . gpdb_src +RUN rm -rf gpdb_src/.git/ + +FROM base AS test + +COPY --from=code /home/gpadmin/gpdb_src gpdb_src +COPY --from=build /home/gpadmin/bin_gpdb bin_gpdb +COPY --from=build /home/gpadmin/gpdb_src/VERSION gpdb_src + +RUN make -C gpdb_src/src/tools/entab install clean diff --git a/ci/readme.md b/ci/readme.md index 2aa31f3c5a52..09bc4eecc3ff 100644 --- a/ci/readme.md +++ b/ci/readme.md @@ -14,6 +14,18 @@ To build an image based on Ubuntu 24.04, specify the version in build args: docker build -t gpdb6_regress:latest --build-arg OS_VERSION=24.04 -f ci/Dockerfile.ubuntu . ``` +To build a Rocky Linux 8 image: + +```bash +docker build -t gpdb6_rockylinux8:latest -f ci/Dockerfile.rockylinux . +``` + +To build a Rocky Linux 9 image: + +```bash +docker build -t gpdb6_rockylinux9:latest --build-arg OS_VERSION=9 -f ci/Dockerfile.rockylinux . +``` + ## Full regression tests suite run diff --git a/concourse/scripts/common.bash b/concourse/scripts/common.bash index 873a4d130a53..020eeec46198 100644 --- a/concourse/scripts/common.bash +++ b/concourse/scripts/common.bash @@ -10,14 +10,12 @@ function set_env() { } function os_id() { - if [[ ! -f "/etc/altlinux-release" ]] && [[ -f "/etc/redhat-release" ]]; then - echo "centos" - else - echo "$( - . /etc/os-release - echo "${ID}" - )" + if [[ -f "/etc/os-release" ]]; then + . /etc/os-release + elif [[ -f "/etc/redhat-release" ]]; then + ID="centos" fi + echo "${ID}" } function os_version() { From 81440f0a9c6f9ac346515e612d3053bd613400c6 Mon Sep 17 00:00:00 2001 From: Artem Shapatin Date: Tue, 30 Jun 2026 15:20:18 +0700 Subject: [PATCH 11/20] Fix "Host key verification failed" error in tests (#511) Some behave tests had problem with host key verification. They threw Host key verification failed on attempt to ssh to some host. It was happening because of lack of host keys in file known_hosts, which population was happening in init_containers.sh. ssh-keyscan was responsible for this task. It was successfully opening sockets on hosts for each of 5 default algorithms for key authentication (see ssh-keyscan code to understand details, get_keytypes holds these default algorithms), but still was failing to gather required information out of these sockets: because out of 5 opened sockets only 3 could provide keys by the chosen algorithm (KT_RSA, KT_ECDSA, KT_ED25519). And these 3 sockets could be closed after beginning of ssh-protocol, because server was always having default sshd utility configuration, which included default settings of MaxStartups (10:30:100, it means that if number of unauthorized connections were bigger then 10, it could close the connection with 30% chance). To fix this issue, patch suggests to change MaxStartups default setting. Considering that this is infrastructural issue only and doesn't affect anything in core, tests are not provided. --- concourse/scripts/setup_gpadmin_user.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/concourse/scripts/setup_gpadmin_user.bash b/concourse/scripts/setup_gpadmin_user.bash index abaadae69db7..7b051e242a59 100755 --- a/concourse/scripts/setup_gpadmin_user.bash +++ b/concourse/scripts/setup_gpadmin_user.bash @@ -92,6 +92,7 @@ setup_sshd() { # Disable password authentication so builds never hang given bad keys sed -ri 's/PasswordAuthentication yes/PasswordAuthentication no/g' /etc/ssh/sshd_config + echo "MaxStartups 100:30:200" >> /etc/ssh/sshd_config case "$TEST_OS" in centos6 | sles*) From e42c8346ea79016a6cfa40eb2ba0a75110532523 Mon Sep 17 00:00:00 2001 From: Vladislav Pavlov Date: Wed, 1 Jul 2026 16:01:32 +0300 Subject: [PATCH 12/20] Add code coverage workflow (#464) - updated the behave tests workflow to upload coverage artifacts; - updated the regression tests workflow to upload coverage artifacts. - added a coverage job that runs on PR's and depends on succesfull completion of both behave and regression tests. Task: CI-5692 --- .github/workflows/greengage-ci.yml | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/greengage-ci.yml b/.github/workflows/greengage-ci.yml index a0ecbd5ff9ab..dab38d5d199b 100644 --- a/.github/workflows/greengage-ci.yml +++ b/.github/workflows/greengage-ci.yml @@ -52,7 +52,7 @@ jobs: contents: read # Explicit for default behavior packages: read # Explicit for GHCR access clarity actions: write # Required for artifact upload - uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-tests-behave.yml@v35 + uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-tests-behave.yml@v46 with: version: 6 target_os: ${{ matrix.target_os }} @@ -73,7 +73,7 @@ jobs: contents: read # Explicit for default behavior packages: read # Explicit for GHCR access clarity actions: write # Required for artifact upload - uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-tests-regression.yml@v28 + uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-tests-regression.yml@v46 with: version: 6 target_os: ${{ matrix.target_os }} @@ -81,6 +81,28 @@ jobs: secrets: ghcr_token: ${{ secrets.GITHUB_TOKEN }} + coverage: + needs: [behave-tests, regression-tests] + if: github.event_name == 'pull_request' && needs.behave-tests.result == 'success' && needs.regression-tests.result == 'success' + strategy: + fail-fast: false + matrix: + include: + - target_os: ubuntu + target_os_version: "24.04" + permissions: + contents: read + packages: read + actions: write + uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-coverage.yml@v46 + with: + version: 6 + target_os: ${{ matrix.target_os }} + target_os_version: ${{ matrix.target_os_version }} + coverage_threshold: 75 + secrets: + ghcr_token: ${{ secrets.GITHUB_TOKEN }} + orca-tests: needs: build if: github.event_name == 'pull_request' # Only for PR From 9c86b3287ee93679a9d30b19680e0b1088f48716 Mon Sep 17 00:00:00 2001 From: Maxim Gajdaj Date: Fri, 3 Jul 2026 13:56:11 +0700 Subject: [PATCH 13/20] Bump Package CI to v44 for 6.x (#486) Replace inline test-docker bash job with composite action tests/install/deb: - OS version auto-detection for Greengage apt repository - per-package `dpkg -s` report written to GitHub Actions job summary - full installation report uploaded as workflow artifact - remove unused `test_lima` input Task: CI-5835 --- .github/workflows/greengage-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/greengage-ci.yml b/.github/workflows/greengage-ci.yml index dab38d5d199b..72c94510e7bf 100644 --- a/.github/workflows/greengage-ci.yml +++ b/.github/workflows/greengage-ci.yml @@ -180,7 +180,7 @@ jobs: contents: read # Explicit for default behavior packages: write # Required for GHCR access actions: write # Required for artifact upload - uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-package.yml@v38 + uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-package.yml@v44 with: version: 6 target_os: ${{ matrix.target_os }} From 45abc189575e10932672c089402eb9428c8a5fd5 Mon Sep 17 00:00:00 2001 From: Maxim Gajdaj Date: Sun, 5 Jul 2026 21:41:01 +0700 Subject: [PATCH 14/20] Create build rpm package 6.x (#439) Adds RPM packaging support for Greengage 6.x. - .gitignore - add /RPM/ - README.Rhel-Rocky.bash: - add 'rpm-build' and 'iputils' dependencies - simplify Python package installation for Rocky 8/9 - install python{2,3}-pip where appropriate - enable pipefail - README.linux.md: - run dependency installation script with 'sudo' - document Python symlink setup for Rocky 8/9 - document 'future==0.16' installation for Rocky 8 - ci/Dockerfile.rockylinux: - remove 'future' installation - stop installing 'pytest' requirements - concourse/scripts/compile_gpdb.bash - add rocky8/rocky9 to vendored libraries conditions - concourse/scripts/common.bash - update BLD_ARCH comment for rocky9 - gpAux/Makefile: - add 'pkg-rpm' target and RPM_TOPDIR variable - replace 'lsb_release' with /etc/os-release for codename detection - rename DISTRO_CODENAME to VERSION_CODENAME - add rocky9_x86_64 to SERVER_PLATFORMS and MPP_ARCH - gpAux/Makefile.global - add BLD_CFLAGS for rocky8 and rocky9 - gpAux/releng/set_bld_arch.sh - detect Rocky Linux via /etc/rocky-release instead of misclassifying it as 'rhel' - gpAux/README.package.md: - document RPM packaging alongside Debian packaging - describe 'pkg-rpm' workflow, RPM build process, dependencies, and maintenance - gpAux/rpm/greengage6.spec - RPM spec file for 'greengage6' package - gpAux/rpm/greengage6.rpmlintrc - rpmlint overrides analogous to debian/lintian-overrides - .github/workflows/greengage-ci.yml: - add Rocky Linux 8 and 9 to package job matrix - replace Docker install test with test_install flag - bump reusable workflow to @v47 Task: CI-5452 --- .github/workflows/greengage-ci.yml | 14 ++- .gitignore | 1 + README.Rhel-Rocky.bash | 17 ++-- README.linux.md | 17 ++-- ci/Dockerfile.rockylinux | 7 +- concourse/scripts/common.bash | 2 +- concourse/scripts/compile_gpdb.bash | 4 +- gpAux/Makefile | 30 +++++-- gpAux/Makefile.global | 2 + gpAux/README.package.md | 129 ++++++++++++++++++++++++++-- gpAux/releng/set_bld_arch.sh | 7 +- gpAux/rpm/greengage6.rpmlintrc | 20 +++++ gpAux/rpm/greengage6.spec | 78 +++++++++++++++++ 13 files changed, 287 insertions(+), 41 deletions(-) create mode 100644 gpAux/rpm/greengage6.rpmlintrc create mode 100644 gpAux/rpm/greengage6.spec diff --git a/.github/workflows/greengage-ci.yml b/.github/workflows/greengage-ci.yml index 72c94510e7bf..81ec73d45d0c 100644 --- a/.github/workflows/greengage-ci.yml +++ b/.github/workflows/greengage-ci.yml @@ -166,7 +166,7 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - # Rebuild prod-redy version without debug extensions and pack it to deb + # Rebuild prod-redy version without debug extensions and pack it to deb and rpm package: needs: build strategy: @@ -174,17 +174,25 @@ jobs: matrix: include: - target_os: ubuntu + test_install: true - target_os: ubuntu target_os_version: "24.04" + test_install: true + - target_os: rockylinux + target_os_version: "8" + test_install: false + - target_os: rockylinux + target_os_version: "9" + test_install: false permissions: contents: read # Explicit for default behavior packages: write # Required for GHCR access actions: write # Required for artifact upload - uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-package.yml@v44 + uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-package.yml@v47 with: version: 6 target_os: ${{ matrix.target_os }} target_os_version: ${{ matrix.target_os_version }} - test_docker: ${{ matrix.target_os }}:${{ matrix.target_os_version || '22.04' }} + test_install: ${{ matrix.test_install }} secrets: ghcr_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 799206a138da..2e6df3a72aed 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,4 @@ compile_commands.json /Release/ /CMakeLists.txt /Package/ +/RPM/ diff --git a/README.Rhel-Rocky.bash b/README.Rhel-Rocky.bash index 1b55d5f95bc1..78c22192c3e7 100755 --- a/README.Rhel-Rocky.bash +++ b/README.Rhel-Rocky.bash @@ -4,27 +4,24 @@ # PURPOSE: Install build dependencies, compile zstd static library, # Install Python based on OS version -set -eux +set -euxo pipefail dnf -y install epel-release # Detect OS version if not already set export OS_VERSION="${OS_VERSION:-$(grep -oP '(?<= release )\d+' /etc/redhat-release)}" +perl_packages="perl-Env perl-ExtUtils-Embed perl-IPC-Run perl-JSON perl-Test-Base" +python_packages="python3 python3-devel python3-setuptools python3-pip python3-future" + case "$OS_VERSION" in 8) dnf config-manager --set-enabled powertools - python_packages="python2 python2-devel python2-setuptools \ - python3 python3-devel python3-setuptools" - perl_packages="perl-Env perl-ExtUtils-Embed \ - perl-IPC-Run perl-JSON perl-Test-Base" + python_packages="python2 python2-devel python2-setuptools python2-pip $python_packages" ;; 9) dnf config-manager --set-enabled crb - python_packages="python3.11 python3.11-devel python3.11-setuptools" - perl_packages="perl-Env perl-ExtUtils-Embed \ - perl-IPC-Run perl-JSON perl-Test-Base \ - perl-Opcode perl-Test-Simple perl-Thread-Queue perl-devel" + perl_packages="$perl_packages perl-Opcode perl-Test-Simple perl-Thread-Queue perl-devel" ;; *) echo "Unsupported Rocky Linux version: $OS_VERSION" @@ -47,6 +44,7 @@ dnf -y install \ gperf \ indent \ iproute \ + iputils \ java-11-openjdk-devel \ jq \ krb5-devel \ @@ -70,6 +68,7 @@ dnf -y install \ pam-devel \ procps-ng \ readline-devel \ + rpm-build \ rsync \ snappy-devel \ sudo \ diff --git a/README.linux.md b/README.linux.md index 1e646b26166d..6d95d1850946 100644 --- a/README.linux.md +++ b/README.linux.md @@ -10,14 +10,21 @@ - Install dependencies using README.Rhel-Rocky.bash script: ```bash - ./README.Rhel-Rocky.bash + sudo ./README.Rhel-Rocky.bash ``` -- Create symbolic link to Python 2 in `/usr/bin`: +- Create a symbolic link to Python in `/usr/bin` and install `future` v0.16: - ```bash - sudo ln -s python2 /usr/bin/python - ``` + - Rocky 8: + ```bash + sudo ln -s python2 /usr/bin/python + sudo python -m pip install --no-cache-dir future==0.16 + ``` + + - Rocky 9: + ```bash + sudo ln -s python3 /usr/bin/python + ``` ## For Ubuntu (versions 22.04 or 24.04): diff --git a/ci/Dockerfile.rockylinux b/ci/Dockerfile.rockylinux index 6e066c0b8b4f..1d958bec133f 100644 --- a/ci/Dockerfile.rockylinux +++ b/ci/Dockerfile.rockylinux @@ -23,9 +23,9 @@ RUN set -eux; \ PIP_PACKAGES='allure-behave==2.4.0 mock==3.0.5 psi==0.3b3 future==0.16'; \ ;; \ 9) \ - ln -sf /usr/bin/python3.11 /usr/bin/python; \ + ln -sf /usr/bin/python3 /usr/bin/python; \ python -m ensurepip --upgrade; \ - PIP_PACKAGES='allure-behave==2.4.0 future==1.0.0'; \ + PIP_PACKAGES='allure-behave==2.4.0'; \ ;; \ *) \ echo "Unsupported Rocky Linux version: $OS_VERSION"; \ @@ -44,9 +44,6 @@ RUN set -eux; \ python -m pip install --no-cache-dir --upgrade pip; \ # Install allure-behave for behave tests python -m pip install --no-cache-dir $PIP_PACKAGES; \ -# Install pytest for gpload test - python -m pip install --no-cache-dir --requirement pytest_requirement.txt; \ - rm pytest_requirement.txt; \ # Cleanup to reduce image size dnf clean all; \ rm -rf /tmp/* /var/tmp/* /var/cache/man/ diff --git a/concourse/scripts/common.bash b/concourse/scripts/common.bash index 020eeec46198..633baa74a392 100644 --- a/concourse/scripts/common.bash +++ b/concourse/scripts/common.bash @@ -37,7 +37,7 @@ function os_version() { function build_arch() { local id=$(os_id) local version=$(os_version) - # BLD_ARCH expects rhel{6,7,8}_x86_64 || rocky8_x86_64 || sles12_x86_64 || ubuntu20.04_x86_64 + # BLD_ARCH expects rhel{6,7,8}_x86_64 || rocky8_x86_64 || rocky9_x86_64 || sles12_x86_64 || ubuntu20.04_x86_64 # for oel7 and oel8 platform, the id will return centos case ${id} in sles | rocky) version=$(os_version | cut -d. -f1) ;; diff --git a/concourse/scripts/compile_gpdb.bash b/concourse/scripts/compile_gpdb.bash index 9092a83855f5..60574fc05990 100755 --- a/concourse/scripts/compile_gpdb.bash +++ b/concourse/scripts/compile_gpdb.bash @@ -105,10 +105,10 @@ function include_dependencies() { vendored_headers=(zstd*.h uv.h uv ) pkgconfigs=(libzstd.pc libuv.pc quicklz.pc) # rocky9/oel9/rhel9 won't vendor zstd because of rsync on these platform does not work libzstd 1.3.7 - if [[ ${BLD_ARCH} == "rhel9"* ]]; then + if [[ ${BLD_ARCH} == "rhel9"* || ${BLD_ARCH} == "rocky9"* ]]; then vendored_libs=(libquicklz.so{,.1,.1.5.0} libuv.so{,.1,.1.0.0} libxerces-c.so) # rocky8/oel8/rhel8 needs zstd 1.4.4 to be vendor because these platform support system libzstd 1.4.4 - elif [[ ${BLD_ARCH} == "rhel8"* ]]; then + elif [[ ${BLD_ARCH} == "rhel8"* || ${BLD_ARCH} == "rocky8"* ]]; then vendored_libs=(libquicklz.so{,.1,.1.5.0} libzstd.so{,.1,.1.4.4} libuv.so{,.1,.1.0.0} libxerces-c.so) else vendored_libs=(libquicklz.so{,.1,.1.5.0} libzstd.so{,.1,.1.3.7} libuv.so{,.1,.1.0.0} libxerces-c.so) diff --git a/gpAux/Makefile b/gpAux/Makefile index f4624ae82845..15c9c690f58f 100644 --- a/gpAux/Makefile +++ b/gpAux/Makefile @@ -93,7 +93,7 @@ ISCONFIG=$(GPPGDIR)/GNUmakefile ## On these platforms, we do the full build including the server. On other ## platforms, we do a client-only build. ## -SERVER_PLATFORMS=rhel7_x86_64 rhel6_x86_64 rhel8_x86_64 rocky8_x86_64 linux_x86_64 +SERVER_PLATFORMS=rhel7_x86_64 rhel6_x86_64 rhel8_x86_64 rocky8_x86_64 rocky9_x86_64 linux_x86_64 #--------------------------------------------------------------------- # Compiler options @@ -354,6 +354,7 @@ PACKAGE_NAME := $(shell grep '^Package:' debian/control | head -1 | awk '{print MAINTAINER := $(shell grep '^Maintainer:' debian/control | sed 's/Maintainer: //') DATE_RFC := $(shell date -R) ARTIFACTS_DIR := $(CURDIR)/../Package +RPM_TOPDIR := $(CURDIR)/../RPM ../VERSION : @echo "Update $@" @@ -363,21 +364,21 @@ ARTIFACTS_DIR := $(CURDIR)/../Package version-vars : ../VERSION $(eval FULL_VERSION := $(shell [ -f ../VERSION ] && perl -pe 's, ,-,g' ../VERSION)) $(eval PACKAGE_VERSION := $(shell [ -f ../VERSION ] && perl -pe 's, .*,,g' ../VERSION)) - $(eval DISTRO_CODENAME := $(shell lsb_release -sc)) + $(eval VERSION_CODENAME := $(shell . /etc/os-release && echo $$VERSION_CODENAME)) $(eval IS_RELEASE := $(if $(findstring +dev,$(PACKAGE_VERSION)),no,yes)) $(eval BUILD_TYPE := $(if $(filter yes,$(IS_RELEASE)),Release build,Development build)) version-info : version-vars @echo "PACKAGE_VERSION: $(PACKAGE_VERSION)" @echo "FULL_VERSION: $(FULL_VERSION)" - @echo "DISTRO_CODENAME: $(DISTRO_CODENAME)" + @echo "VERSION_CODENAME: $(VERSION_CODENAME)" @echo "IS_RELEASE: $(IS_RELEASE)" @echo "BUILD_TYPE: $(BUILD_TYPE)" # Generate package control files changelog : debian/changelog debian/changelog : version-vars - @echo "$(PACKAGE_NAME) ($(PACKAGE_VERSION)) $(DISTRO_CODENAME); urgency=low" > $@ + @echo "$(PACKAGE_NAME) ($(PACKAGE_VERSION)) $(VERSION_CODENAME); urgency=low" > $@ @echo "" >> $@ @echo " * $(BUILD_TYPE)" >> $@ @echo "" >> $@ @@ -404,7 +405,25 @@ pkg-deb : debian/changelog debian/install -o -name "*.changes" \) \ -exec mv -f {} $(ARTIFACTS_DIR)/ \; -.PHONY: pkg pkg-deb changelog version-vars version-info +# Build RPM package +pkg-rpm : GPROOT = /opt/greengagedb +pkg-rpm : GPDIR = $(PACKAGE_NAME) +pkg-rpm : version-vars + @echo "Building RPM with GPROOT=$(GPROOT), GPDIR=$(GPDIR), PACKAGE_NAME=$(PACKAGE_NAME)" + @mkdir -p $(RPM_TOPDIR)/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} + rpmbuild -bb \ + --define "_topdir $(RPM_TOPDIR)" \ + --define "gpdb_version $(subst +,.,$(subst -,.,$(PACKAGE_VERSION)))" \ + --define "gpdb_release 1" \ + --define "gproot $(GPROOT)" \ + --define "gpdir $(GPDIR)" \ + --define "sourcedir $(CURDIR)/.." \ + rpm/greengage6.spec + @mkdir -p $(ARTIFACTS_DIR) + @find $(RPM_TOPDIR)/RPMS -name "*.rpm" \ + -exec mv -f {} $(ARTIFACTS_DIR)/ \; + +.PHONY: pkg pkg-deb pkg-rpm changelog version-vars version-info #--------------------------------------------------------------------- # clientTools @@ -468,6 +487,7 @@ rhel6_x86_64_MPP_ARCH=RHEL6-x86_64 rhel7_x86_64_MPP_ARCH=RHEL7-x86_64 rhel8_x86_64_MPP_ARCH=RHEL8-x86_64 rocky8_x86_64_MPP_ARCH=ROCKY8-x86_64 +rocky9_x86_64_MPP_ARCH=ROCKY9-x86_64 ifneq "$($(BLD_ARCH)_MPP_ARCH)" "" export MPP_ARCH=$($(BLD_ARCH)_MPP_ARCH) else diff --git a/gpAux/Makefile.global b/gpAux/Makefile.global index 88c3560d5199..d99581cc4574 100644 --- a/gpAux/Makefile.global +++ b/gpAux/Makefile.global @@ -47,6 +47,8 @@ rhel6_x86_64_BLD_CFLAGS=-m64 -gdwarf-2 -gstrict-dwarf rhel7_x86_64_BLD_CFLAGS=-m64 rhel8_x86_64_BLD_CFLAGS=-m64 rhel7_ppc64le_BLD_CFLAGS=-m64 -fasynchronous-unwind-tables -fsigned-char +rocky8_x86_64_BLD_CFLAGS=-m64 +rocky9_x86_64_BLD_CFLAGS=-m64 altlinux8.4_x86_64_BLD_CFLAGS=-m64 altlinux10.2_x86_64_BLD_CFLAGS=-m64 astra1.7_x86_64_BLD_CFLAGS=-m64 diff --git a/gpAux/README.package.md b/gpAux/README.package.md index b71adaae2ee5..f37c617c3a39 100644 --- a/gpAux/README.package.md +++ b/gpAux/README.package.md @@ -2,9 +2,9 @@ ## Overview -This documentation describes the Debian packaging system for Greengage Database -located in the `gpAux/` subdirectory. The system builds Debian packages using a -custom Makefile and `debian/rules` file. +This documentation describes the packaging system for Greengage Database +located in the `gpAux/` subdirectory: Debian packages (`pkg-deb`, using +`debian/rules`) and RPM packages (`pkg-rpm`, using `rpm/greengage6.spec`). ## Location and Structure @@ -22,6 +22,9 @@ The main components are: - `debian/control` - Package metadata and dependencies - `debian/copyright` - Copyright information - `debian/lintian-overrides` - Lintian warning overrides +- `rpm/greengage6.spec` - RPM spec file defining package metadata and build steps +- `rpm/greengage6.rpmlintrc` - rpmlint warning overrides, analogous to + `debian/lintian-overrides` ## Key Components @@ -30,13 +33,15 @@ The main components are: 1. **Version Management**: - `../VERSION`: Generates version file using `../getversion` - `version-vars`: Sets build variables (`FULL_VERSION`, `PACKAGE_VERSION`, - `DISTRO_CODENAME`, `IS_RELEASE`, `BUILD_TYPE`) from `../VERSION` file + `VERSION_CODENAME`, `IS_RELEASE`, `BUILD_TYPE`) from `../VERSION` file - `version-info`: Displays version information for debugging 2. **Packaging Targets**: - `pkg`: Default target (aliases to `pkg-deb`) - `pkg-deb`: Builds Debian package, preserves environment variables, and collects artifacts (`.deb`, `.ddeb`, `.build`, `.buildinfo`, `.changes`) + - `pkg-rpm`: Builds RPM package via `rpmbuild` and collects the + resulting `.rpm` artifact - `changelog`: Generates `debian/changelog` (not stored in repo) - `debian/install`: Creates installation manifest (not stored in repo) @@ -60,9 +65,31 @@ The `debian/rules` file uses debhelper (dh) with custom overrides: - Injects Python dependencies via `-VpythonRequires`, `-VpythonConflicts` options in `dh_gencontrol` +### RPM Spec File + +The `rpm/greengage6.spec` file defines the RPM build: + +1. **Distribution-specific Dependencies**: + - `python3.11` and `python3.11-pip` required on RHEL/Rocky 9 and newer + - `python2`, `python2-pip`, `python3`, `python3-pip` required on + RHEL/Rocky 8 and older + +2. **Build Process**: + - `%install` invokes the project's `make dist` target with `DESTDIR` + set to `%{buildroot}` + - Excludes the entire install prefix from `brp-mangle-shebangs` via + `__brp_mangle_shebangs_exclude_from` — scripts intentionally use + `#!/usr/bin/env python` and `#!/usr/bin/env perl`; rewriting shebangs + would diverge package contents from the source and mask issues + - Strips debug info from `*.so` files via `strip --strip-debug` to + remove BUILDROOT paths embedded by the compiler (e.g., PyGreSQL's + `_pg.so`), which would otherwise fail `check-buildroot` validation + - Removes the executable bit from Python/Perl/Bash files without a + shebang and from `*.md` files, mirroring `debian/rules` `dh_fixperms` + ## Usage -### Building the Package +### Building the Debian Package From the project root directory, run: @@ -70,12 +97,21 @@ From the project root directory, run: make -C ./gpAux pkg-deb ``` +### Building the RPM Package + +From the project root directory, run: + +```bash +make -C ./gpAux pkg-rpm +``` + ### Custom Installation Paths To customize installation paths, set environment variables: ```bash make -C ./gpAux pkg-deb GPROOT=/custom/path GPDIR=custom_dir +make -C ./gpAux pkg-rpm GPROOT=/custom/path GPDIR=custom_dir ``` ### Environment Variables @@ -84,10 +120,15 @@ make -C ./gpAux pkg-deb GPROOT=/custom/path GPDIR=custom_dir - `GPDIR`: Subdirectory under `GPROOT` (default: same as `PACKAGE_NAME`) - `PACKAGE_NAME`: Package name (default: from `Package:` in `debian/control`, e.g., `greengage6`) -- `ARTIFACTS_DIR`: Directory for artifacts (default: `$(CURDIR)/../Package`) +- `ARTIFACTS_DIR`: Directory for artifacts (default: `$(CURDIR)/../Package`, + shared between `.deb` and `.rpm` outputs) +- `RPM_TOPDIR`: `rpmbuild` working directory used only by `pkg-rpm` + default: `$(CURDIR)/../RPM`, kept after the build for inspection ## Build Process Details +### Debian Package + 1. **Version Generation**: - Runs `../getversion` to create `../VERSION` - Processes version string into `FULL_VERSION` and `PACKAGE_VERSION` @@ -104,11 +145,50 @@ make -C ./gpAux pkg-deb GPROOT=/custom/path GPDIR=custom_dir - Uses `make dist` for installation into `debian/tmp/$(PACKAGE_NAME)` - Generates file manifest in `debian/install` +### RPM Package + +1. **Version Generation**: + - Reuses `version-vars` (same as Debian) to derive `PACKAGE_VERSION`, + normalized for RPM (`+`/`-` replaced with `.`) and passed to + `rpmbuild` via `--define gpdb_version` + +2. **Package Building**: + - Sets up `rpmbuild` tree under `RPM_TOPDIR` + - Runs `rpmbuild -bb rpm/greengage6.spec` with `gproot`, `gpdir`, and + `sourcedir` passed via `--define` + - Collects the resulting `.rpm` into `ARTIFACTS_DIR` + +3. **Installation**: + - Uses `make dist` inside `%install`, installing into `%{buildroot}` + +## Build Dependencies + +Both `pkg-deb` and `pkg-rpm` assume that all build dependencies and the +GPDB build itself (`make dist`) are already satisfied/buildable on the +host — neither target builds GPDB from scratch; they package an already +configured and built source tree. + +To get a complete environment with all dependencies installed, use the +provided Docker images instead of installing dependencies manually: + +- `ci/Dockerfile.ubuntu` - Ubuntu 22.04/24.04 +- `ci/Dockerfile.rockylinux` - Rocky Linux 8/9 + +See [ci/readme.md](../ci/readme.md) for instructions on building and +using these images. To install dependencies directly on a host without +Docker, see `README.linux.md` together with `README.ubuntu.bash` (Ubuntu) +or `README.Rhel-Rocky.bash` (RHEL/Rocky Linux). + ## Dependencies -Package dependencies and conflicts are defined in `debian/control`. -Python dependencies and conflicts are dynamically detected in `debian/rules` and -injected via `${pythonRequires}` and `${pythonConflicts}` substitution variables. +Package dependencies and conflicts for the Debian package are defined in +`debian/control`. Python dependencies and conflicts are dynamically +detected in `debian/rules` and injected via `${pythonRequires}` and +`${pythonConflicts}` substitution variables. + +Package dependencies for the RPM package are defined directly in +`rpm/greengage6.spec` via `Requires:`/`Conflicts:` tags, including a +conditional on `%{?rhel}` for the Python version requirement. ## Maintenance @@ -120,6 +200,10 @@ Edit `debian/control` to update: - Maintainer information - General dependencies +Edit `rpm/greengage6.spec` to update the equivalent metadata for the +RPM package (`Summary`, `License`, `URL`, `Requires`, `Conflicts`, +`%description`). + ### Adding Distribution Support Modify distribution detection in `debian/rules`: @@ -138,8 +222,25 @@ else endif ``` +For RPM, add distribution-specific conditionals in `rpm/greengage6.spec`, +for example: + +```spec +%if 0%{?rhel} >= 9 +Requires: python3.11 +Requires: python3.11-pip +%else +Requires: python2 +Requires: python3 +Requires: python2-pip +Requires: python3-pip +%endif +``` + ## Notes +### Debian + - Skips tests (`DEB_BUILD_OPTIONS=nocheck`) for faster builds - Unsets compiler flags to avoid conflicts with the project's build system - Enables parallel builds using all available CPU cores @@ -150,3 +251,13 @@ endif - Custom `dh_fixperms` removes executable bit from Python/Perl/Bash files without shebang and `*.md` files - Generated `debian/{changelog,install}` is not committed to the repository + +### RPM + +- Builds without signing for development convenience +- Strips debug info from `*.so` files to remove BUILDROOT paths embedded + by the compiler, which would otherwise fail `check-buildroot` validation +- Excludes the entire install prefix from shebang mangling via + `__brp_mangle_shebangs_exclude_from` +- Collects only the resulting `.rpm` into `$(CURDIR)/../Package` +- `RPM_TOPDIR` is kept after the build for inspection diff --git a/gpAux/releng/set_bld_arch.sh b/gpAux/releng/set_bld_arch.sh index 4eab100dd9fc..c45f0c31fc6e 100755 --- a/gpAux/releng/set_bld_arch.sh +++ b/gpAux/releng/set_bld_arch.sh @@ -4,10 +4,10 @@ case "`uname -s`" in Linux) - if [ -f /etc/redhat-release -a ! -f /etc/altlinux-release -a ! -f /etc/redos-release ]; then + if [ -f /etc/redhat-release -a ! -f /etc/altlinux-release -a ! -f /etc/redos-release -a ! -f /etc/rocky-release ]; then case "`cat /etc/redhat-release`" in *) - BLD_ARCH_HOST="rhel`cat /etc/redhat-release | sed -e 's/CentOS Linux/RedHat/' -e 's/Red Hat Enterprise Linux/RedHat/' -e 's/Rocky Linux/RedHat/' -e 's/WS//' -e 's/Server//' -e 's/Client//' | awk '{print $3}' | awk -F. '{print $1}'`_`uname -m | sed -e s/i686/x86_32/`" + BLD_ARCH_HOST="rhel`cat /etc/redhat-release | sed -e 's/CentOS Linux/RedHat/' -e 's/Red Hat Enterprise Linux/RedHat/' -e 's/WS//' -e 's/Server//' -e 's/Client//' | awk '{print $3}' | awk -F. '{print $1}'`_`uname -m | sed -e s/i686/x86_32/`" ;; esac fi @@ -18,6 +18,9 @@ case "`uname -s`" in ;; esac fi + if [ -f /etc/rocky-release ]; then + BLD_ARCH_HOST="$(. /etc/os-release; echo ${ID}$(echo ${VERSION_ID} | cut -d. -f1)_$(uname -m))" + fi if [ -f /etc/astra_version ]; then BLD_ARCH_HOST="$(. /etc/os-release; echo ${ID}${VERSION_ID} | sed 's/-/_/')" fi diff --git a/gpAux/rpm/greengage6.rpmlintrc b/gpAux/rpm/greengage6.rpmlintrc new file mode 100644 index 000000000000..7dea542a62b0 --- /dev/null +++ b/gpAux/rpm/greengage6.rpmlintrc @@ -0,0 +1,20 @@ +# 3rd-party package installs to /opt — intentional for database software +addFilter("dir-or-file-in-opt") + +# lib/python/ contains gppylib modules imported via Python's import system, +# not executed directly. Shebangs are for developer convenience only. +addFilter("non-executable-script") +addFilter("script-without-shebang") + +# bin/, sbin/, lib/python/ use bare 'python' interpreter intentionally. +# On Rocky 8, python -> python2; on Rocky 9, python -> python3.11. +addFilter("python-shebang-ambiguous") +addFilter("env-script-interpreter") + +# Timezone data files are hardlinked by upstream PostgreSQL build system +# to save disk space — standard PostgreSQL behavior, not a packaging error. +addFilter("hardlink") + +# plperl.so RUNPATH points to the exact Perl version this package depends on. +addFilter("library-without-ldconfig-postin") +addFilter("library-without-ldconfig-postun") diff --git a/gpAux/rpm/greengage6.spec b/gpAux/rpm/greengage6.spec new file mode 100644 index 000000000000..3e336a0078f1 --- /dev/null +++ b/gpAux/rpm/greengage6.spec @@ -0,0 +1,78 @@ +# Run command: `make -C ./gpdb_src/gpAux pkg-rpm` + +%{!?gproot: %global gproot /opt/greengagedb} +%{!?gpdir: %global gpdir greengage6} +%global prefix %{gproot}/%{gpdir} + +# bin/, sbin/, lib/python/ use bare 'python' interpreter intentionally. +# On Rocky 8, python -> python2; on Rocky 9, python -> python3.11. +# Perl test scripts use /usr/bin/env perl for portability across environments +# where Perl is not at /usr/bin/perl — see commit:eab7246a65e3c7834f8d4763d73972e4a6c0fcbd +%global __brp_mangle_shebangs_exclude_from ^%{prefix}/.*$ + +Name: greengage6 +Version: %{gpdb_version} +Release: %{gpdb_release}%{?dist} +Summary: Greengage MPP database engine +License: ASL 2.0 +URL: https://greengagedb.org + +Requires: findutils +Requires: glibc-langpack-en +Requires: iproute +Requires: iputils +Requires: less +Requires: net-tools +Requires: openssh-clients +Requires: openssh-server +Requires: openssl +Requires: procps-ng +Requires: rsync +Requires: zip + +%if 0%{?rhel} >= 9 +Requires: python3 +Requires: python3-pip +%else +Requires: python2 +Requires: python3 +Requires: python2-pip +Requires: python3-pip +%endif + +Conflicts: greengage-loaders + +%description +Greengage Database (GPDB) is an advanced, fully featured, open +source data warehouse, based on PostgreSQL. It provides powerful and +rapid analytics on petabyte scale data volumes. Uniquely geared toward +big data analytics, Greengage Database is powered by the world's most +advanced cost-based query optimizer delivering high analytical query +performance on large data volumes. + +%install +rm -rf %{buildroot} +env -u CFLAGS -u CPPFLAGS -u CXXFLAGS -u LDFLAGS \ + make dist \ + DESTDIR=%{buildroot} \ + GPROOT=%{gproot} \ + GPDIR=%{gpdir} \ + PARALLEL_MAKE_OPTS=%{?_smp_mflags} \ + -C %{sourcedir}/gpAux + +# Remove executable bit from scripts without shebang +find %{buildroot}%{prefix} -type f \ + \( -name "*.py" -o -name "*.pm" -o -name "*.sh" \) \ + -executable | while read f; do \ + head -1 "$f" | grep -q '^#!' || chmod -x "$f"; \ + done +find %{buildroot}%{prefix} -name "*.md" -executable \ + -exec chmod -x {} + + +# Remove debug info from *.so +find %{buildroot}%{prefix} -type f -name "*.so" -exec strip --strip-debug {} \; + +%files +%{prefix} + +%changelog From c45d3fa4364dcbee8653d006df6bfd80030d7a13 Mon Sep 17 00:00:00 2001 From: Ivan Sergeenko Date: Fri, 10 Jul 2026 16:50:00 +0300 Subject: [PATCH 15/20] Report tables and columns removed from Greengage 7 (6.x) (#514) This patch makes several adjustments to pg_upgrade_support functions to report more objects that would make upgrade fail. These changes are as follows: - backport pg_upgrade_support functions that check for removed tables and columns. - report removed objects in materialized views, in addition to regular views. - also, adapt static/dynamic OID logic for schemas other than pg_catalog, as it was done in 5.x. Don't add tests because they belong to the ggupgrade repo and will be added there after this patch. Ticket: GG-551 --- .../pg_upgrade_support/pg_upgrade_support.c | 826 +++++++++++++++++- 1 file changed, 814 insertions(+), 12 deletions(-) diff --git a/contrib/pg_upgrade_support/pg_upgrade_support.c b/contrib/pg_upgrade_support/pg_upgrade_support.c index 5beef9490412..d45e41d6ac1a 100644 --- a/contrib/pg_upgrade_support/pg_upgrade_support.c +++ b/contrib/pg_upgrade_support/pg_upgrade_support.c @@ -18,6 +18,15 @@ #include "catalog/oid_dispatch.h" #include "catalog/pg_authid.h" #include "catalog/pg_class.h" +#include "catalog/pg_constraint.h" +#include "catalog/pg_attrdef.h" +#include "catalog/pg_appendonly.h" +#include "catalog/pg_am.h" +#include "catalog/pg_proc.h" +#include "catalog/pg_exttable.h" +#include "catalog/pg_compression.h" +#include "catalog/pg_resqueuecapability.h" +#include "catalog/gp_configuration_history.h" #include "catalog/pg_enum.h" #include "catalog/pg_namespace.h" #include "catalog/pg_tablespace.h" @@ -29,6 +38,8 @@ #include "rewrite/rewriteHandler.h" #include "utils/array.h" #include "utils/builtins.h" +#include "utils/syscache.h" +#include "utils/lsyscache.h" /* THIS IS USED ONLY FOR PG >= 9.0 */ @@ -66,6 +77,20 @@ PG_FUNCTION_INFO_V1(view_has_removed_operators); PG_FUNCTION_INFO_V1(view_has_removed_functions); PG_FUNCTION_INFO_V1(view_has_removed_types); PG_FUNCTION_INFO_V1(view_has_changed_function_signatures); +PG_FUNCTION_INFO_V1(get_removed_tables); +PG_FUNCTION_INFO_V1(get_removed_columns); + +typedef struct RemovedTablesWalkerContext RemovedTablesWalkerContext; +typedef struct RemovedColumnsWalkerContext RemovedColumnsWalkerContext; +typedef struct RemovedColumnStatic RemovedColumnStatic; +typedef struct RemovedColumnDynamic RemovedColumnDynamic; +typedef struct RemovedFunctionDynamic RemovedFunctionDynamic; +typedef struct ReportedColumn ReportedColumn; + +static Oid get_function(const char *name, const Oid *args, int args_count, Oid namespace); +static Oid get_type(const char *name, Oid namespace); + +static Query *get_matview_query(Relation matview); static bool check_node_anyarray_walker(Node *node, void *context); static bool check_node_unknown_walker(Node *node, void *context); @@ -73,6 +98,277 @@ static bool check_node_removed_operators_walker(Node *node, void *context); static bool check_node_removed_functions_walker(Node *node, void *context); static bool check_node_removed_types_walker(Node *node, void *context); static bool check_node_changed_function_signatures_walker(Node *node, void *context); +static void report_removed_table(RemovedTablesWalkerContext *context, Oid reloid); +static void check_and_report_removed_table(RemovedTablesWalkerContext *context, Oid reloid); +static bool check_node_removed_tables_walker(Node *node, void *context); +static void report_removed_column(RemovedColumnsWalkerContext *context, Oid reloid, int attnum); +static bool check_and_report_removed_columns(RemovedColumnsWalkerContext *context, Oid reloid, int attnum); +static bool check_node_removed_columns_walker(Node *node, RemovedColumnsWalkerContext *context); + +/* + * Some objects are treated as 'dynamic', because they are present in the database + * by default, but live outside of 'pg_catalog', inside schemas like + * 'gp_toolkit' and 'information_schema', which can be dropped using + * 'DROP SCHEMA ... CASCADE'. This means that we should check for their absence. + * Moreover, they can be recreated from respective sql scripts after that, + * changing OIDs of the objects. So, we need to ask the database for their OIDs first. + */ + +struct RemovedTablesWalkerContext +{ + List *removedTables; +}; + +struct RemovedColumnsWalkerContext +{ + List *rtableStack; + List *removedColumns; + bool inside_whole_row_reference; +}; + +struct RemovedColumnStatic +{ + Oid reloid; + int attnum; +}; + +struct RemovedColumnDynamic +{ + const char *relnamespace; + const char *relname; + int attnum; +}; + +struct RemovedFunctionDynamic +{ + const char *pronamespace; + const char *name; + const Oid *args; + const int args_count; +}; + +struct ReportedColumn +{ + Oid attrelid; + int attnum; + bool comes_from_whole_row_reference; +}; + +/* Lists of objects removed from Greenage 7 */ +static Oid pg_resgroup_check_move_query_oids[] = {23, 26}; +static Oid __gp_remove_ao_entry_from_cache_oids[] = {26}; +static Oid __gp_get_ao_entry_from_cache_oids[] = {26}; + +static RemovedFunctionDynamic removed_functions_dynamic[] = +{ + {"gp_toolkit", "pg_resgroup_check_move_query", pg_resgroup_check_move_query_oids, 2}, + {"gp_toolkit", "__gp_remove_ao_entry_from_cache", __gp_remove_ao_entry_from_cache_oids, 1}, + {"gp_toolkit", "__gp_get_ao_entry_from_cache", __gp_get_ao_entry_from_cache_oids, 1}, + +}; +static const int num_removed_functions_dynamic = sizeof(removed_functions_dynamic) / sizeof(RemovedFunctionDynamic); + +static Oid __gp_aocsseg_oids[] = {2205}; +static Oid __gp_aocsseg_history_oids[] = {2205}; +static Oid __gp_aoseg_oids[] = {2205}; +static Oid __gp_aoseg_history_oids[] = {2205}; + +static RemovedFunctionDynamic functions_with_changed_signatures_dynamic[] = +{ + {"gp_toolkit", "__gp_aocsseg", __gp_aocsseg_oids, 1}, + {"gp_toolkit", "__gp_aocsseg_history", __gp_aocsseg_history_oids, 1}, + {"gp_toolkit", "__gp_aoseg", __gp_aoseg_oids, 1}, + {"gp_toolkit", "__gp_aoseg_history", __gp_aoseg_history_oids, 1} +}; +static const int num_functions_with_changed_signatures_dynamic = sizeof(functions_with_changed_signatures_dynamic) / sizeof(RemovedFunctionDynamic); + + +static const Oid removed_tables_static[] = +{ + 5010, /* pg_catalog.pg_partition */ + 11786, /* pg_catalog.pg_partition_columns */ + 9903, /* pg_catalog.pg_partition_encoding */ + 5011, /* pg_catalog.pg_partition_rule */ + 11782, /* pg_catalog.pg_partitions */ + 11789, /* pg_catalog.pg_partition_templates */ + 11796 /* pg_catalog.pg_stat_partition_operations */ +}; +static const int num_removed_tables_static = sizeof(removed_tables_static) / sizeof(Oid); + +/* Assuming that all of these tables live inside gp_toolkit schema */ +static char *removed_tables_dynamic[] = +{ + "gp_size_of_partition_and_indexes_disk", + "__gp_user_data_tables" +}; +static const int num_removed_tables_dynamic = sizeof(removed_tables_dynamic) / sizeof(char*); + + +static const RemovedColumnStatic removed_columns_static[] = +{ + {11636, 6}, /* pg_catalog.pg_roles.rolcatupdate */ + {11639, 5}, /* pg_catalog.pg_shadow.usecatupd */ + {11645, 5}, /* pg_catalog.pg_user.usecatupd */ + {11758, 11}, /* pg_catalog.pg_stat_replication.sent_location */ + {11758, 12}, /* pg_catalog.pg_stat_replication.write_location */ + {11758, 13}, /* pg_catalog.pg_stat_replication.flush_location */ + {11758, 14}, /* pg_catalog.pg_stat_replication.replay_location */ + {11755, 15}, /* pg_catalog.pg_stat_activity.waiting */ + {11755, 20}, /* pg_catalog.pg_stat_activity.waiting_reason */ + {11755, 23}, /* pg_catalog.pg_stat_activity.rsgqueueduration */ + {12345, 4}, /* pg_catalog.gp_distributed_log.distributed_id */ + {12339, 2}, /* pg_catalog.gp_distributed_xacts.distributed_id */ + {11764, 14}, /* pg_catalog.gp_stat_replication.flush_location */ + {11764, 15}, /* pg_catalog.gp_stat_replication.replay_location */ + {11764, 12}, /* pg_catalog.gp_stat_replication.sent_location */ + {11764, 13}, /* pg_catalog.gp_stat_replication.write_location */ + {6439, -2}, /* pg_catalog.pg_resgroupcapability.oid */ + {GpConfigHistoryRelationId, Anum_gp_configuration_history_desc}, + {ProcedureRelationId, Anum_pg_proc_protransform}, + {ProcedureRelationId, Anum_pg_proc_proisagg}, + {ProcedureRelationId, Anum_pg_proc_proiswindow}, + {ProcedureRelationId, Anum_pg_proc_prodataaccess}, + {AccessMethodRelationId, Anum_pg_am_ambeginscan}, + {AccessMethodRelationId, Anum_pg_am_ambuild}, + {AccessMethodRelationId, Anum_pg_am_ambuildempty}, + {AccessMethodRelationId, Anum_pg_am_ambulkdelete}, + {AccessMethodRelationId, Anum_pg_am_amcanbackward}, + {AccessMethodRelationId, Anum_pg_am_amcanmulticol}, + {AccessMethodRelationId, Anum_pg_am_amcanorder}, + {AccessMethodRelationId, Anum_pg_am_amcanorderbyop}, + {AccessMethodRelationId, Anum_pg_am_amcanreturn}, + {AccessMethodRelationId, Anum_pg_am_amcanunique}, + {AccessMethodRelationId, Anum_pg_am_amclusterable}, + {AccessMethodRelationId, Anum_pg_am_amcostestimate}, + {AccessMethodRelationId, Anum_pg_am_amendscan}, + {AccessMethodRelationId, Anum_pg_am_amgetbitmap}, + {AccessMethodRelationId, Anum_pg_am_amgettuple}, + {AccessMethodRelationId, Anum_pg_am_aminsert}, + {AccessMethodRelationId, Anum_pg_am_amkeytype}, + {AccessMethodRelationId, Anum_pg_am_ammarkpos}, + {AccessMethodRelationId, Anum_pg_am_amoptionalkey}, + {AccessMethodRelationId, Anum_pg_am_amoptions}, + {AccessMethodRelationId, Anum_pg_am_ampredlocks}, + {AccessMethodRelationId, Anum_pg_am_amrescan}, + {AccessMethodRelationId, Anum_pg_am_amrestrpos}, + {AccessMethodRelationId, Anum_pg_am_amsearcharray}, + {AccessMethodRelationId, Anum_pg_am_amsearchnulls}, + {AccessMethodRelationId, Anum_pg_am_amstorage}, + {AccessMethodRelationId, Anum_pg_am_amstrategies}, + {AccessMethodRelationId, Anum_pg_am_amsupport}, + {AccessMethodRelationId, Anum_pg_am_amvacuumcleanup}, + {AppendOnlyRelationId, Anum_pg_appendonly_blkdiridxid}, + {AppendOnlyRelationId, Anum_pg_appendonly_blocksize}, + {AppendOnlyRelationId, Anum_pg_appendonly_checksum}, + {AppendOnlyRelationId, Anum_pg_appendonly_columnstore}, + {AppendOnlyRelationId, Anum_pg_appendonly_compresslevel}, + {AppendOnlyRelationId, Anum_pg_appendonly_compresstype}, + {AppendOnlyRelationId, Anum_pg_appendonly_safefswritesize}, + {AppendOnlyRelationId, Anum_pg_appendonly_visimapidxid}, + {AttrDefaultRelationId, Anum_pg_attrdef_adsrc}, + {AuthIdRelationId, Anum_pg_authid_rolcatupdate}, + {RelationRelationId, Anum_pg_class_relhasoids}, + {RelationRelationId, Anum_pg_class_relhaspkey}, + {RelationRelationId, Anum_pg_class_relstorage}, + {ConstraintRelationId, Anum_pg_constraint_consrc}, + {CompressionRelationId, -2 /* oid */}, + {ExtTableRelationId, -8 /* gp_segment_id */}, + {ExtTableRelationId, -7 /* tableoid */}, + {ExtTableRelationId, -6 /* cmax */}, + {ExtTableRelationId, -5 /* xmax */}, + {ExtTableRelationId, -4 /* cmin */}, + {ExtTableRelationId, -3 /* xmin */}, + {ExtTableRelationId, -1 /* ctid*/}, + {ResQueueCapabilityRelationId, -2 /* oid */} +}; +static const int num_removed_columns_static = sizeof(removed_columns_static) / sizeof(RemovedColumnStatic); + +static const RemovedColumnDynamic removed_columns_dynamic[] = +{ + {"gp_toolkit", "gp_locks_on_resqueue", 9 /* lorwaiting */}, + {"gp_toolkit", "gp_resgroup_config", 4 /* cpu_rate_limit */}, + {"gp_toolkit", "gp_resgroup_config", 8 /* memory_auditor */}, + {"gp_toolkit", "gp_resgroup_config", 6 /* memory_shared_quota */}, + {"gp_toolkit", "gp_resgroup_config", 7 /* memory_spill_ratio */}, + {"gp_toolkit", "gp_resgroup_status", 8 /* cpu_usage */}, + {"gp_toolkit", "gp_resgroup_status", 9 /* memory_usage */}, + {"gp_toolkit", "gp_resgroup_status", 1 /* rsgname */ }, + {"gp_toolkit", "gp_resgroup_status_per_host", 4 /* cpu */ }, + {"gp_toolkit", "gp_resgroup_status_per_host", 6 /* memory_available */ }, + {"gp_toolkit", "gp_resgroup_status_per_host", 8 /* memory_quota_available */}, + {"gp_toolkit", "gp_resgroup_status_per_host", 7 /* memory_quota_used */}, + {"gp_toolkit", "gp_resgroup_status_per_host", 10 /* memory_shared_available */}, + {"gp_toolkit", "gp_resgroup_status_per_host", 9 /* memory_shared_used */}, + {"gp_toolkit", "gp_resgroup_status_per_host", 5 /* memory_used */}, + {"gp_toolkit", "gp_resgroup_status_per_host", 1 /* rsgname */}, + {"gp_toolkit", "gp_resgroup_status_per_segment", 5 /* cpu */}, + {"gp_toolkit", "gp_resgroup_status_per_segment", 3 /* hostname */}, + {"gp_toolkit", "gp_resgroup_status_per_segment", 7 /* memory_available */}, + {"gp_toolkit", "gp_resgroup_status_per_segment", 9 /* memory_quota_available */}, + {"gp_toolkit", "gp_resgroup_status_per_segment", 8 /* memory_quota_used */}, + {"gp_toolkit", "gp_resgroup_status_per_segment", 11 /* memory_shared_available */}, + {"gp_toolkit", "gp_resgroup_status_per_segment", 10 /* memory_shared_used */}, + {"gp_toolkit", "gp_resgroup_status_per_segment", 6 /* memory_used */}, + {"gp_toolkit", "gp_resgroup_status_per_segment", 1 /* rsgname */}, + {"gp_toolkit", "__gp_user_tables", 9 /* autrelstorage */ }, + {"gp_toolkit", "__gp_user_data_tables_readable", 9 /* autrelstorage */ }, + {"information_schema", "routines", 65 /* result_cast_character_set_name */}, + {"information_schema", "routines", 43 /* sql_data_access */}, + {"gp_toolkit", "__gp_log_master_ext", -8 /* gp_segment_id */}, + {"gp_toolkit", "__gp_log_master_ext", -7 /* tableoid */}, + {"gp_toolkit", "__gp_log_master_ext", -6 /* cmax */}, + {"gp_toolkit", "__gp_log_master_ext", -5 /* xmax */}, + {"gp_toolkit", "__gp_log_master_ext", -4 /* cmin */}, + {"gp_toolkit", "__gp_log_master_ext", -3 /* xmin */}, + {"gp_toolkit", "__gp_log_master_ext", -1 /* ctid */} +}; +static const int num_removed_columns_dynamic = sizeof(removed_columns_dynamic) / sizeof(RemovedColumnDynamic); + + +static const char* removed_types_gp_toolkit[] = +{ + "gp_size_of_partition_and_indexes_disk", + "__gp_user_data_tables" +}; +static const int num_removed_types_gp_toolkit = sizeof(removed_types_gp_toolkit) / sizeof (char*); + +/* + * Helper function like get_view_query, but for materialized views. + * It works similarly to ExecRefreshMatView. + */ +static Query * +get_matview_query(Relation matview) +{ + RewriteRule *rule; + List *actions; + + Assert(matview->rd_rel->relkind == RELKIND_MATVIEW); + + if (matview->rd_rel->relhasrules == false || + matview->rd_rules->numLocks < 1) + elog(ERROR, + "materialized view \"%s\" is missing rewrite information", + RelationGetRelationName(matview)); + + if (matview->rd_rules->numLocks > 1) + elog(ERROR, + "materialized view \"%s\" has too many rules", + RelationGetRelationName(matview)); + + rule = matview->rd_rules->rules[0]; + if (rule->event != CMD_SELECT || !(rule->isInstead)) + elog(ERROR, + "the rule for materialized view \"%s\" is not a SELECT INSTEAD OF rule", + RelationGetRelationName(matview)); + + actions = rule->actions; + if (list_length(actions) != 1) + elog(ERROR, + "the rule for materialized view \"%s\" is not a single action", + RelationGetRelationName(matview)); + + return (Query *) linitial(rule->actions); +} Datum set_next_pg_type_oid(PG_FUNCTION_ARGS) @@ -311,6 +607,11 @@ view_has_anyarray_casts(PG_FUNCTION_ARGS) viewquery = get_view_query(rel); found = query_tree_walker(viewquery, check_node_anyarray_walker, NULL, 0); } + else if (rel->rd_rel->relkind == RELKIND_MATVIEW) + { + viewquery = get_matview_query(rel); + found = query_tree_walker(viewquery, check_node_anyarray_walker, NULL, 0); + } else found = false; @@ -368,6 +669,11 @@ view_has_unknown_casts(PG_FUNCTION_ARGS) viewquery = get_view_query(rel); found = query_tree_walker(viewquery, check_node_unknown_walker, NULL, 0); } + else if(rel->rd_rel->relkind == RELKIND_MATVIEW) + { + viewquery = get_matview_query(rel); + found = query_tree_walker(viewquery, check_node_unknown_walker, NULL, 0); + } else found = false; @@ -432,6 +738,12 @@ view_has_removed_operators(PG_FUNCTION_ARGS) viewquery = get_view_query(rel); found = query_tree_walker(viewquery, check_node_removed_operators_walker, NULL, 0); } + else if(rel->rd_rel->relkind == RELKIND_MATVIEW) + { + viewquery = get_matview_query(rel); + found = query_tree_walker(viewquery, check_node_removed_operators_walker, NULL, 0); + } + else found = false; @@ -482,6 +794,12 @@ view_has_removed_functions(PG_FUNCTION_ARGS) viewquery = get_view_query(rel); found = query_tree_walker(viewquery, check_node_removed_functions_walker, NULL, 0); } + else if(rel->rd_rel->relkind == RELKIND_MATVIEW) + { + viewquery = get_matview_query(rel); + found = query_tree_walker(viewquery, check_node_removed_functions_walker, NULL, 0); + } + else found = false; @@ -490,6 +808,24 @@ view_has_removed_functions(PG_FUNCTION_ARGS) PG_RETURN_BOOL(found); } +/* Helper functions to get object OIDs by their signatures */ +Oid +get_function(const char *name, const Oid *args, int args_count, Oid namespace) +{ + return GetSysCacheOid3(PROCNAMEARGSNSP, + PointerGetDatum(name), + PointerGetDatum(buildoidvector(args, args_count)), + ObjectIdGetDatum(namespace)); +} + +Oid +get_type(const char *name, Oid namespace) +{ + return GetSysCacheOid2(TYPENAMENSP, + PointerGetDatum(name), + ObjectIdGetDatum(namespace)); +} + static bool check_node_removed_functions_walker(Node *node, void *context) { @@ -500,11 +836,9 @@ check_node_removed_functions_walker(Node *node, void *context) if (IsA(node, FuncExpr)) { + Oid schema_oid; Oid func_oid = ((FuncExpr *)node)->funcid; - if (func_oid == 12512 || // gp_toolkit.__gp_get_ao_entry_from_cache - func_oid == 12511 || // gp_toolkit.__gp_remove_ao_entry_from_cache - func_oid == 12498 || // gp_toolkit.pg_resgroup_check_move_query - func_oid == 7188 || // pg_catalog.bmbeginscan + if (func_oid == 7188 || // pg_catalog.bmbeginscan func_oid == 7193 || // pg_catalog.bmbuild func_oid == 7011 || // pg_catalog.bmbuildempty func_oid == 7194 || // pg_catalog.bmbulkdelete @@ -633,6 +967,22 @@ check_node_removed_functions_walker(Node *node, void *context) func_oid == 3097) // pg_catalog.varchar_transform return true; + for (int i = 0; i < num_removed_functions_dynamic; ++i) + { + schema_oid = GetSysCacheOid(NAMESPACENAME, + CStringGetDatum(removed_functions_dynamic[i].pronamespace), + 0, 0, 0); + + if (OidIsValid(schema_oid)) + { + if (func_oid == get_function(removed_functions_dynamic[i].name, + removed_functions_dynamic[i].args, + removed_functions_dynamic[i].args_count, + schema_oid)) + return true; + } + } + return false; } else if (IsA(node, Query)) @@ -662,6 +1012,11 @@ view_has_removed_types(PG_FUNCTION_ARGS) viewquery = get_view_query(rel); found = query_tree_walker(viewquery, check_node_removed_types_walker, NULL, 0); } + else if(rel->rd_rel->relkind == RELKIND_MATVIEW) + { + viewquery = get_matview_query(rel); + found = query_tree_walker(viewquery, check_node_removed_types_walker, NULL, 0); + } else found = false; @@ -680,15 +1035,14 @@ check_node_removed_types_walker(Node *node, void *context) if (IsA(node, Var) || IsA(node, Const)) { + Oid gp_toolkit_oid; Oid type_oid; if IsA(node, Var) type_oid = ((Var *)node)->vartype; else type_oid = ((Const *)node)->consttype; - if (type_oid == 12475 || // gp_toolkit.gp_size_of_partition_and_indexes_disk - type_oid == 12366 || // gp_toolkit.__gp_user_data_tables - type_oid == 1023 || // pg_catalog._abstime + if (type_oid == 1023 || // pg_catalog._abstime type_oid == 702 || // pg_catalog.abstime type_oid == 11612 || // pg_catalog.pg_partition type_oid == 11787 || // pg_catalog.pg_partition_columns @@ -704,6 +1058,20 @@ check_node_removed_types_walker(Node *node, void *context) type_oid == 704) // pg_catalog.tinterval return true; + gp_toolkit_oid = GetSysCacheOid(NAMESPACENAME, + CStringGetDatum("gp_toolkit"), + 0, 0, 0); + + if (OidIsValid(gp_toolkit_oid)) + { + for (int i = 0; i < num_removed_types_gp_toolkit; i++) + { + if (type_oid == get_type(removed_types_gp_toolkit[i], + gp_toolkit_oid)) + return true; + } + } + return false; } else if (IsA(node, Query)) @@ -732,6 +1100,11 @@ view_has_changed_function_signatures(PG_FUNCTION_ARGS) viewquery = get_view_query(rel); found = query_tree_walker(viewquery, check_node_changed_function_signatures_walker, NULL, 0); } + else if(rel->rd_rel->relkind == RELKIND_MATVIEW) + { + viewquery = get_matview_query(rel); + found = query_tree_walker(viewquery, check_node_changed_function_signatures_walker, NULL, 0); + } else found = false; @@ -750,12 +1123,9 @@ check_node_changed_function_signatures_walker(Node *node, void *context) if (IsA(node, FuncExpr)) { + Oid schema_oid; Oid func_oid = ((FuncExpr *)node)->funcid; - if (func_oid == 12501 || // gp_toolkit.__gp_aocsseg - func_oid == 12502 || // gp_toolkit.__gp_aocsseg_history - func_oid == 12506 || // gp_toolkit.__gp_aoseg - func_oid == 12500 || // gp_toolkit.__gp_aoseg_history - func_oid == 2335 || // pg_catalog.array_agg + if (func_oid == 2335 || // pg_catalog.array_agg func_oid == 2334 || // pg_catalog.array_agg_finalfn func_oid == 2333 || // pg_catalog.array_agg_transfn func_oid == 3484 || // pg_catalog.gin_consistent_jsonb @@ -812,6 +1182,22 @@ check_node_changed_function_signatures_walker(Node *node, void *context) func_oid == 3493) // pg_catalog.to_regtype return true; + for (int i = 0; i < num_functions_with_changed_signatures_dynamic; i++) + { + schema_oid = GetSysCacheOid(NAMESPACENAME, + CStringGetDatum(functions_with_changed_signatures_dynamic[i].pronamespace), + 0, 0, 0); + + if (OidIsValid(schema_oid)) + { + if (func_oid == get_function(functions_with_changed_signatures_dynamic[i].name, + functions_with_changed_signatures_dynamic[i].args, + functions_with_changed_signatures_dynamic[i].args_count, + schema_oid)) + return true; + } + } + return false; } else if (IsA(node, Query)) @@ -823,3 +1209,419 @@ check_node_changed_function_signatures_walker(Node *node, void *context) return expression_tree_walker(node, check_node_changed_function_signatures_walker, context); } + +static void +report_removed_table(RemovedTablesWalkerContext *context, Oid reloid) +{ + Oid already_reported_table; + ListCell *lc; + + /* + * Go thorogh already reported tables to remove + * duplicates + */ + foreach (lc, context->removedTables) + { + already_reported_table = lfirst_oid(lc); + if (reloid == already_reported_table) + return; + } + + context->removedTables = lappend_oid(context->removedTables, reloid); +} + +static void +check_and_report_removed_table(RemovedTablesWalkerContext *context, Oid reloid) +{ + int i; + Oid gp_toolkit_oid; + + for (i = 0; i < num_removed_tables_static; i++) + { + if (reloid == removed_tables_static[i]) + report_removed_table(context, reloid); + } + + gp_toolkit_oid = GetSysCacheOid(NAMESPACENAME, + CStringGetDatum("gp_toolkit"), + 0, 0, 0); + + if (OidIsValid(gp_toolkit_oid)) + { + for (i = 0; i < num_removed_tables_dynamic; i++) + { + if (reloid == get_relname_relid(removed_tables_dynamic[i], gp_toolkit_oid)) + report_removed_table(context, reloid); + + } + } +} + +static bool +check_node_removed_tables_walker(Node *node, void *context) +{ + Assert(context != NULL); + + if (node == NULL) + return false; + + if (IsA(node, RangeTblEntry)) + { + RangeTblEntry *rte = (RangeTblEntry *) node; + if (rte->rtekind == RTE_RELATION) + check_and_report_removed_table(context, rte->relid); + return false; + } + else if(IsA(node, Query)) + { + /* + * Recurse into (sub)queries to look for removed tables. + */ + return query_tree_walker((Query *) node, + check_node_removed_tables_walker, + context, + QTW_EXAMINE_RTES); + } + + /* + * This ensures that we look for removed tables embedded inside + * expressions (e.g. CTEs, sublinks etc.) which can contain range tables. + */ + return expression_tree_walker(node, check_node_removed_tables_walker, context); +} + +Datum +get_removed_tables(PG_FUNCTION_ARGS) +{ + Oid view_oid = PG_GETARG_OID(0); + Relation rel; + StringInfoData buf; + Oid reported_table; + Oid relnamespace; + ListCell *lc; + char *nspname; + char *relname; + Query *viewquery; + RemovedTablesWalkerContext context; + + rel = try_relation_open(view_oid, AccessShareLock, false); + if (!RelationIsValid(rel)) + elog(ERROR, "Could not open relation file for relation oid %u", view_oid); + + context.removedTables = NIL; + if (rel->rd_rel->relkind == RELKIND_VIEW) + { + viewquery = get_view_query(rel); + check_node_removed_tables_walker((Node *) viewquery, &context); + } + else if (rel->rd_rel->relkind == RELKIND_MATVIEW) + { + viewquery = get_matview_query(rel); + check_node_removed_tables_walker((Node *) viewquery, &context); + } + + relation_close(rel, AccessShareLock); + + /* + * Make a single formatted string, listing all unique removed tables. + * It will be displayed to the user. + */ + initStringInfo(&buf); + foreach (lc, context.removedTables) + { + reported_table = lfirst_oid(lc); + relname = get_rel_name(reported_table); + if (!relname) + elog(ERROR, "cache lookup failed for relation %u", reported_table); + + relnamespace = get_rel_namespace(reported_table); + if (!OidIsValid(relnamespace)) + elog(ERROR, "cache lookup failed for relation %u", reported_table); + + nspname = get_namespace_name(relnamespace); + if (!nspname) + elog(ERROR, "cache lookup failed for namespace %u", relnamespace); + + appendStringInfo(&buf, "\t%s.%s\n", nspname, relname); + } + + PG_RETURN_TEXT_P(cstring_to_text(buf.data)); +} + + +static void +report_removed_column(RemovedColumnsWalkerContext *context, Oid reloid, int attnum) +{ + ListCell *lc; + ReportedColumn *already_reported_column; + ReportedColumn *column; + + /* + * Go through already reported columns in a nested loop manner + * to remove duplicates. This should be fast enough, because the + * number of removed columns is not that large + * (currently, 110), and most view wouldn't have all of them. + * But it is hard to tell without user data. + */ + foreach (lc, context->removedColumns) + { + already_reported_column = lfirst(lc); + if (reloid == already_reported_column->attrelid && + attnum == already_reported_column->attnum && + context->inside_whole_row_reference == already_reported_column->comes_from_whole_row_reference) + return; + } + + column = palloc(sizeof(ReportedColumn)); + column->attrelid = reloid; + column->attnum = attnum; + column->comes_from_whole_row_reference = context->inside_whole_row_reference; + + context->removedColumns = lappend(context->removedColumns, column); +} + +static bool +check_and_report_removed_columns(RemovedColumnsWalkerContext *context, Oid reloid, int attnum) +{ + int i; + Oid schema_oid; + int removed_column_attnum; + + for (i = 0; i < num_removed_columns_static; i++) + { + removed_column_attnum = removed_columns_static[i].attnum; + if (reloid == removed_columns_static[i].reloid && + (attnum == removed_column_attnum || attnum == InvalidAttrNumber)) + report_removed_column(context, reloid, removed_column_attnum); + } + + for (i = 0; i < num_removed_columns_dynamic; i++) + { + schema_oid = GetSysCacheOid(NAMESPACENAME, + CStringGetDatum(removed_columns_dynamic[i].relnamespace), + 0, 0, 0); + + if (OidIsValid(schema_oid)) + { + removed_column_attnum = removed_columns_dynamic[i].attnum; + if (reloid == get_relname_relid(removed_columns_dynamic[i].relname, schema_oid) && + (attnum == removed_column_attnum || attnum == InvalidAttrNumber)) + report_removed_column(context, reloid, removed_column_attnum); + } + } + + return false; +} + +/* + * Check whether a query contains a reference to a removed column, or a whole + * row reference to a table with removed columns. + * + * The first case will always cause pg_upgrade to fail, while the second is more + * complicated. Whole row references by themselves won't cause upgrade to fail, + * because row type will be taken from the target cluster. For example, + * the following view won't cause any troubles: + * + * CREATE VIEW view1 AS SELECT pg_class FROM pg_class; + * + * However, there could be another view that references specific columns from the + * previous one: + * + * CREATE VIEW view2 AS SELECT (pg_class).relhasoids FROM view1; + * + * and this columns may be indeed absent in the new version. Because of that, + * conservatively report any whole row reference to any table with removed + * columns. + */ +bool +check_node_removed_columns_walker(Node *node, RemovedColumnsWalkerContext *context) +{ + Assert(context != NULL); + + if (node == NULL) + { + return false; + } + + if (IsA(node, Var)) + { + Var *var; + List *rtable; + RangeTblEntry *rte; + bool save_inside_whole_row_reference; + + var = (Var *) node; + if (var->varlevelsup >= list_length(context->rtableStack)) + elog(ERROR, "invalid varlevelsup %d", var->varlevelsup); + + rtable = (List *) list_nth(context->rtableStack, var->varlevelsup); + if (var->varno <= 0 || var->varno > list_length(rtable)) + elog(ERROR, "invalid varno %d", var->varno); + + save_inside_whole_row_reference = context->inside_whole_row_reference; + if (var->varattno == InvalidAttrNumber) + context->inside_whole_row_reference = true; + + rte = (RangeTblEntry *) list_nth(rtable, var->varno - 1); + if (rte->rtekind == RTE_RELATION) + { + /* + * It's a plain relation, simply check that Var doesn't reference + * removed column(s) + */ + check_and_report_removed_columns(context, rte->relid, var->varattno); + } + else if (rte->rtekind == RTE_JOIN) + { + /* + * It's a join entry, we need to recursively go through + * the RTE tree to get to the source entry for this attribute. + */ + int i; + List *save_rtables = context->rtableStack; + + context->rtableStack = list_copy_tail(context->rtableStack, + var->varlevelsup); + + if (var->varattno == InvalidAttrNumber) + { + /* + * For a whole table reference, check every column of the RTE + */ + for (i = 0; i < list_length(rte->joinaliasvars); i++) + check_node_removed_columns_walker((Node *) list_nth(rte->joinaliasvars, i), + context); + } + else + { + /* Regular attribute */ + if (var->varattno <= 0 || + var->varattno > list_length(rte->joinaliasvars)) + elog(ERROR, "invalid varattno %d", var->varattno); + + check_node_removed_columns_walker((Node *) list_nth(rte->joinaliasvars, + var->varattno - 1), + context); + } + list_free(context->rtableStack); + context->rtableStack = save_rtables; + } + + /* + * Don't do anything special for other RTE kinds. Most notably, RTE_SUBQUERY, + * because subqueries will be handled when we recurse into them. + */ + context->inside_whole_row_reference = save_inside_whole_row_reference; + return false; + } + else if (IsA(node, Query)) + { + /* + * Recurse into (sub)queries to search for removed columns. + * + * Pass QTW_IGNORE_JOINALIASES to avoid recursing into a join RTE's + * joinaliasvars, as they always contain every unique column from + * the joined tables. Meaning that without this flag, each join with + * a table with removed columns would trigger this check. + * For example: + * + * CREATE VIEW err AS SELECT jn.relname FROM (pg_class JOIN pg_namespace ON true) jn; + * + * will be erroneously reported as referencing removed columns. Legit cases like: + * + * CREATE VIEW rte_join AS SELECT jn.relhasoids FROM (pg_class JOIN pg_namespace ON true) jn; + * + * are handled when processing Var nodes, for them (rte->rtekind == RTE_JOIN) + */ + Query *query = (Query *) node; + context->rtableStack = lcons(query->rtable, context->rtableStack); + query_tree_walker(query, + check_node_removed_columns_walker, + context, + QTW_IGNORE_JOINALIASES); + context->rtableStack = list_delete_first(context->rtableStack); + return false; + } + + /* + * This ensures we look at all expressions, including entities that contain + * subqueries (such as CTEs and sublinks) + */ + return expression_tree_walker(node, check_node_removed_columns_walker, context); +} + +Datum +get_removed_columns(PG_FUNCTION_ARGS) +{ + Oid view_oid = PG_GETARG_OID(0); + Relation rel; + StringInfoData buf; + Oid relnamespace; + ListCell *lc; + char *nspname; + char *relname; + char *attname; + char *comes_from_whole_row_reference_string; + ReportedColumn *removed_column; + Query *viewquery; + RemovedColumnsWalkerContext context; + + rel = try_relation_open(view_oid, AccessShareLock, false); + if (!RelationIsValid(rel)) + elog(ERROR, "Could not open relation file for relation oid %u", view_oid); + + context.rtableStack = NIL; + context.removedColumns = NIL; + context.inside_whole_row_reference = false; + if (rel->rd_rel->relkind == RELKIND_VIEW) + { + viewquery = get_view_query(rel); + check_node_removed_columns_walker((Node *) viewquery, &context); + } + else if (rel->rd_rel->relkind == RELKIND_MATVIEW) + { + viewquery = get_matview_query(rel); + check_node_removed_columns_walker((Node *) viewquery, &context); + } + + relation_close(rel, AccessShareLock); + + /* + * Make a single formatted string, listing all unique removed columns. + * It will be displayed to the user. + */ + initStringInfo(&buf); + foreach (lc, context.removedColumns) + { + removed_column = lfirst(lc); + relname = get_rel_name(removed_column->attrelid); + if (!relname) + elog(ERROR, "cache lookup failed for relation %u", removed_column->attrelid); + + relnamespace = get_rel_namespace(removed_column->attrelid); + if (!OidIsValid(relnamespace)) + elog(ERROR, "cache lookup failed for relation %u", removed_column->attrelid); + + nspname = get_namespace_name(relnamespace); + if (!nspname) + elog(ERROR, "cache lookup failed for namespace %u", relnamespace); + + attname = get_attname(removed_column->attrelid, removed_column->attnum); + if (!attname) + elog(ERROR, "cache lookup failed for attribute %d for relation %u", + removed_column->attnum, removed_column->attrelid); + + comes_from_whole_row_reference_string = ""; + if (removed_column->comes_from_whole_row_reference) + comes_from_whole_row_reference_string = "(comes from a whole row reference)"; + + appendStringInfo(&buf, "\t%s.%s.%s %s\n", nspname, relname, attname, + comes_from_whole_row_reference_string); + + pfree(relname); + pfree(nspname); + pfree(attname); + } + + PG_RETURN_TEXT_P(cstring_to_text(buf.data)); +} From d0d75651335eb0c766ac952b4d67297c836b90af Mon Sep 17 00:00:00 2001 From: Vladislav Pavlov Date: Tue, 14 Jul 2026 20:44:27 +0300 Subject: [PATCH 16/20] Bump regression workflow 6.x (#527) Regression workflow was bumped. Heredoc script removed from regression workflow due to silent failures. Task: CI-5943 --- .github/workflows/greengage-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/greengage-ci.yml b/.github/workflows/greengage-ci.yml index 81ec73d45d0c..f337b3618e3c 100644 --- a/.github/workflows/greengage-ci.yml +++ b/.github/workflows/greengage-ci.yml @@ -73,7 +73,7 @@ jobs: contents: read # Explicit for default behavior packages: read # Explicit for GHCR access clarity actions: write # Required for artifact upload - uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-tests-regression.yml@v46 + uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-tests-regression.yml@v49 with: version: 6 target_os: ${{ matrix.target_os }} From 9472a254d1bc2b766600026ebbeed39c4d0c0702 Mon Sep 17 00:00:00 2001 From: Evgeniy Ratkov Date: Wed, 15 Jul 2026 10:48:02 +0300 Subject: [PATCH 17/20] Fix initialization of unlogged tables at recovery (#520) Function heap_create_with_catalog creates file for new relation. It always creates "main" fork file and creates "init" fork files for unlogged relation. If segment was down, we can recovery it by gprecoverseg, when, simultaneously, unlogged table is being created by parallel process. Basebackup copies all files at time, when "main" fork was created by the heap_create_with_catalog function, but "init" fork wasn't, "main" fork will not be filtered and will be added to basebackup (without "init" fork). Next, "Init" fork will be created little later by parallel process, and new record will be added to the WAL. Basebackup with WAL will be sent to segment to start recovery process. At recovery, segment will apply WAL to basebackup, create "init" fork, try to copy from "init" to "main" fork and get error at the ResetUnloggedRelations function, because main fork was created earlier. Patch adds removing "main" fork file if it exists when "init" fork is copied to main at recovery at the ResetUnloggedRelations function. Test checks recovery of segment, if it contains just "main" fork. "Init" fork should be created correctly at recovery. --- src/backend/storage/file/reinit.c | 20 +++++++++++ src/test/recovery/t/014_unlogged_reinit.pl | 41 +++++++++++++++++++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/src/backend/storage/file/reinit.c b/src/backend/storage/file/reinit.c index debdd37cd6c2..111ddf882ad9 100644 --- a/src/backend/storage/file/reinit.c +++ b/src/backend/storage/file/reinit.c @@ -15,6 +15,7 @@ #include "postgres.h" #include +#include #include "catalog/catalog.h" #include "catalog/pg_tablespace.h" @@ -335,6 +336,25 @@ ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op) dbspacedirname, oidbuf, de->d_name + oidchars + 1 + strlen(forkNames[INIT_FORKNUM])); + /* + * Sometimes, before recovery there is no init fork, but there is + * main fork. So at cleanup it was not removed. Next, at + * recovery init fork will be created and if we will not remove main + * fork here, function copy_file will return error. Let's check main + * fork here and remove it if presented. + * Such situations may happen when pg_basebackup checks main fork + * file at time when init fork is not created yet. pg_basebackup + * does not filter such file and adds it to the backup. Next, init + * fork will be created by reading xlog at recovery. + */ + struct stat statbuf; + if (lstat(dstpath, &statbuf) == 0) + { + if (unlink(dstpath) != 0) + elog(ERROR, "unlink extra main fork %s error: %m", dstpath); + elog(DEBUG2, "unlink extra main fork %s", dstpath); + } + /* OK, we're ready to perform the actual copy. */ elog(DEBUG2, "copying %s to %s", srcpath, dstpath); copy_file(srcpath, dstpath); diff --git a/src/test/recovery/t/014_unlogged_reinit.pl b/src/test/recovery/t/014_unlogged_reinit.pl index 0af1d5848047..f973d0fc635a 100644 --- a/src/test/recovery/t/014_unlogged_reinit.pl +++ b/src/test/recovery/t/014_unlogged_reinit.pl @@ -7,7 +7,7 @@ use warnings; use PostgresNode; use TestLib; -use Test::More tests => 12; +use Test::More tests => 16; my $node = get_new_node('main'); @@ -79,3 +79,42 @@ 'vm fork in tablespace removed at startup'); ok( !-f "$pgdata/${ts1UnloggedPath}_fsm", 'fsm fork in tablespace removed at startup'); + + +# Create new unlogged tables to check recovery when DB contains main fork only. + +# Make it impossible to get a checkpoint after table creation before a crash. +$node->append_conf('postgresql.conf', <restart; + +$node->safe_psql('postgres', 'CREATE UNLOGGED TABLE base_unlogged2 (id int)'); +$node->safe_psql('postgres', + 'CREATE UNLOGGED TABLE ts1_unlogged2 (id int) TABLESPACE ts1'); + +$baseUnloggedPath = $node->safe_psql('postgres', + q{select pg_relation_filepath('base_unlogged2')}); +$ts1UnloggedPath = $node->safe_psql('postgres', + q{select pg_relation_filepath('ts1_unlogged2')}); + +# Crash the postmaster. +$node->stop('immediate'); + +# Remove init fork to test that it is recreated from init. +unlink("$pgdata/${baseUnloggedPath}_init") + or BAIL_OUT("could not remove \"${baseUnloggedPath}_init\": $!"); +unlink("$pgdata/${ts1UnloggedPath}_init") + or BAIL_OUT("could not remove \"${ts1UnloggedPath}_init\": $!"); + +$node->start; + +# check unlogged table in tablespace +ok( -f "$pgdata/${ts1UnloggedPath}_init", + 'init fork recreated at startup in tablespace'); +ok(-f "$pgdata/$ts1UnloggedPath", + 'main fork in tablespace recreated at startup'); + +# check unlogged table in base +ok(-f "$pgdata/${baseUnloggedPath}_init", 'init fork in base recreated at startup'); +ok(-f "$pgdata/${baseUnloggedPath}", 'main fork in base recreated at startup'); From 08c5ec573121ecf51ef27624f249a86f69535a7b Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Fri, 17 Jul 2026 12:27:26 +1000 Subject: [PATCH 18/20] Fix wrong table data distribution after gpexpand (#532) Problem description: In case 'synchronous_commit' was "off", after a table had been expanded and a primary segment had gone down, the data distribution could be wrong (tuples with the same distribution hash could be duplicated over several segments). Root cause: If 'synchronous_commit' is off, the PREPARE of the transaction doesn't wait for the mirror to receive all WAL records. The issue happened if the corresponding mirror didn't receive all WAL records by the moment the primary had gone down. The mirror was promoted, but it had the table's data prior to the expand operation. Fix: Emit an error at ALTER TABLE EXPAND TABLE and ALTER TABLE SET DISTRIBUTED BY if 'synchronous_commit' is off. --- src/backend/commands/tablecmds.c | 26 +++++ src/test/regress/expected/expand_table.out | 119 +++++++++++++++++++++ src/test/regress/greengage_schedule | 4 +- src/test/regress/sql/expand_table.sql | 78 ++++++++++++++ 4 files changed, 226 insertions(+), 1 deletion(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 3f9b85c279b9..c063355b91cb 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -118,6 +118,7 @@ #include "utils/memutils.h" #include "utils/metrics_utils.h" #include "utils/relcache.h" +#include "utils/guc.h" #include "utils/snapmgr.h" #include "utils/syscache.h" #include "utils/tqual.h" @@ -15051,6 +15052,20 @@ ATExecExpandTable(List **wqueue, Relation rel, AlterTableCmd *cmd) (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied: \"%s\" is a system catalog", RelationGetRelationName(rel)))); + /* + * synchronous_commit can be "off" at the session or cluster level for + * reasons unrelated to this command. With it off, a segment's commit never + * waits for its mirror to acknowledge the redistributed data, so a primary + * crash immediately followed by an automatic mirror promotion can silently + * lose/duplicate the rows this command just moved, if not all WAL was sent + * to the mirror. + * Therefore, forbid operation if synchronous_commit is not fully enabled. + */ + if (synchronous_commit != SYNCHRONOUS_COMMIT_ON) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("synchronous_commit should be enabled during EXPAND"))); + oldContext = MemoryContextSwitchTo(GetMemoryChunkContext(rel)); newPolicy = GpPolicyCopy(policy); MemoryContextSwitchTo(oldContext); @@ -15394,6 +15409,17 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied: \"%s\" is a system catalog", RelationGetRelationName(rel)))); + /* + * Like ATExecExpandTable, this command can rewrite and redistribute the + * table's data via an internal CTAS. See the comment there. + * And, forbid operation if synchronous_commit is not fully enabled here + * as well. + */ + if (synchronous_commit != SYNCHRONOUS_COMMIT_ON) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("synchronous_commit should be enabled during SET DISTRIBUTED BY"))); + Assert(PointerIsValid(node)); Assert(IsA(node, List)); diff --git a/src/test/regress/expected/expand_table.out b/src/test/regress/expected/expand_table.out index 56d181d6222c..91a4eac052bd 100644 --- a/src/test/regress/expected/expand_table.out +++ b/src/test/regress/expected/expand_table.out @@ -1079,3 +1079,122 @@ select numsegments from gp_distribution_policy where localoid='expand_domain_tab reset search_path; drop schema test_reshuffle cascade; -- end_ignore +-- Check that table expand doesn't lead to invalid data distribution +-- if WAL wasn't replicated to a mirror in time, the respective Primary failed, +-- and the mirror with obsolete data is promoted. That could happen if the user +-- disabled 'synchronous_commit'. +create extension if not exists gp_inject_fault; +select gp_debug_set_create_table_default_numsegments(2); + gp_debug_set_create_table_default_numsegments +----------------------------------------------- + 2 +(1 row) + +drop table if exists test; +create table test(a int) distributed by (a); +insert into test select generate_series(1, 100); +set synchronous_commit = off; +select gp_inject_fault('wal_sender_loop', 'suspend', dbid) from gp_segment_configuration where content=0 and role='p'; + gp_inject_fault +----------------- + Success: +(1 row) + +alter table test expand table; +ERROR: synchronous_commit should be enabled during EXPAND +select '\! pg_ctl -D ' || datadir || ' stop -m immediate' as cmd_stop_primary_0 from gp_segment_configuration where content = 0 and role = 'p' +\gset +:cmd_stop_primary_0 +waiting for server to shut down.... done +server stopped +select gp_request_fts_probe_scan(); + gp_request_fts_probe_scan +--------------------------- + t +(1 row) + +select role, preferred_role, status from gp_segment_configuration where content = 0 order by dbid; + role | preferred_role | status +------+----------------+-------- + m | p | d + p | m | u +(2 rows) + +select count(*) from test; + count +------- + 100 +(1 row) + +select count(*), gp_segment_id from test group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 53 | 0 + 47 | 1 +(2 rows) + +reset synchronous_commit; +drop table test; +-- Check the same for SET WITH (REORGANIZE=TRUE), which is used for partitioned table. +select gp_debug_set_create_table_default_numsegments(2); + gp_debug_set_create_table_default_numsegments +----------------------------------------------- + 2 +(1 row) + +create table test (a int) distributed by (a) +partition by range (a) +( + start (0) end (1000) every (1000) +); +NOTICE: CREATE TABLE will create partition "test_1_prt_1" for table "test" +insert into test select generate_series(1, 100); +alter table test expand partition prepare; +set synchronous_commit = off; +select gp_inject_fault('wal_sender_loop', 'suspend', dbid) from gp_segment_configuration where content=0 and role='p'; + gp_inject_fault +----------------- + Success: +(1 row) + +alter table test_1_prt_1 set with (reorganize=true) distributed by (a); +ERROR: synchronous_commit should be enabled during SET DISTRIBUTED BY +select '\! pg_ctl -D ' || datadir || ' stop -m immediate' as cmd_stop_primary_0 from gp_segment_configuration where content = 0 and role = 'p' +\gset +:cmd_stop_primary_0 +waiting for server to shut down.... done +server stopped +select gp_request_fts_probe_scan(); + gp_request_fts_probe_scan +--------------------------- + t +(1 row) + +select role, preferred_role, status from gp_segment_configuration where content = 0 order by dbid; + role | preferred_role | status +------+----------------+-------- + m | p | d + p | m | u +(2 rows) + +select count(*) from test; + count +------- + 100 +(1 row) + +select count(*), gp_segment_id from test group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 53 | 0 + 47 | 1 +(2 rows) + +reset synchronous_commit; +drop table test; +select gp_debug_reset_create_table_default_numsegments(); + gp_debug_reset_create_table_default_numsegments +------------------------------------------------- + +(1 row) + diff --git a/src/test/regress/greengage_schedule b/src/test/regress/greengage_schedule index ad3f9cb5d81b..7bb10b9961aa 100755 --- a/src/test/regress/greengage_schedule +++ b/src/test/regress/greengage_schedule @@ -159,7 +159,9 @@ test: resource_group_gucs test: wrkloadadmin # expand_table tests may affect the result of 'gp_explain', keep them below that -test: gp_toolkit_ao_funcs trig auth_constraint role portals_updatable plpgsql_cache timeseries pg_stat_last_operation pg_stat_last_shoperation gp_numeric_agg partindex_test partition_pruning runtime_stats expand_table expand_table_ao expand_table_aoco expand_table_regression +test: gp_toolkit_ao_funcs trig auth_constraint role portals_updatable plpgsql_cache timeseries pg_stat_last_operation pg_stat_last_shoperation gp_numeric_agg partindex_test partition_pruning runtime_stats expand_table_ao expand_table_aoco expand_table_regression +# expand_table contains segment stop and recover, so keep it separate from other tests +test: expand_table test: rle rle_delta dsp not_out_of_shmem_exit_slots # direct dispatch tests diff --git a/src/test/regress/sql/expand_table.sql b/src/test/regress/sql/expand_table.sql index 393b2d2b4778..ddfe18c045c3 100644 --- a/src/test/regress/sql/expand_table.sql +++ b/src/test/regress/sql/expand_table.sql @@ -455,3 +455,81 @@ select numsegments from gp_distribution_policy where localoid='expand_domain_tab reset search_path; drop schema test_reshuffle cascade; -- end_ignore + +-- Check that table expand doesn't lead to invalid data distribution +-- if WAL wasn't replicated to a mirror in time, the respective Primary failed, +-- and the mirror with obsolete data is promoted. That could happen if the user +-- disabled 'synchronous_commit'. + +create extension if not exists gp_inject_fault; + +select gp_debug_set_create_table_default_numsegments(2); +drop table if exists test; +create table test(a int) distributed by (a); +insert into test select generate_series(1, 100); + +set synchronous_commit = off; + +select gp_inject_fault('wal_sender_loop', 'suspend', dbid) from gp_segment_configuration where content=0 and role='p'; + +alter table test expand table; + +select '\! pg_ctl -D ' || datadir || ' stop -m immediate' as cmd_stop_primary_0 from gp_segment_configuration where content = 0 and role = 'p' +\gset +:cmd_stop_primary_0 + +select gp_request_fts_probe_scan(); +select role, preferred_role, status from gp_segment_configuration where content = 0 order by dbid; + +select count(*) from test; + +select count(*), gp_segment_id from test group by gp_segment_id order by gp_segment_id; + +reset synchronous_commit; + +-- start_ignore +\! gprecoverseg -aF --no-progress; +\! gprecoverseg -ar; +-- end_ignore + +drop table test; + +-- Check the same for SET WITH (REORGANIZE=TRUE), which is used for partitioned table. +select gp_debug_set_create_table_default_numsegments(2); +create table test (a int) distributed by (a) +partition by range (a) +( + start (0) end (1000) every (1000) +); +insert into test select generate_series(1, 100); + +alter table test expand partition prepare; + +set synchronous_commit = off; + +select gp_inject_fault('wal_sender_loop', 'suspend', dbid) from gp_segment_configuration where content=0 and role='p'; + +alter table test_1_prt_1 set with (reorganize=true) distributed by (a); + +select '\! pg_ctl -D ' || datadir || ' stop -m immediate' as cmd_stop_primary_0 from gp_segment_configuration where content = 0 and role = 'p' +\gset +:cmd_stop_primary_0 + +select gp_request_fts_probe_scan(); +select role, preferred_role, status from gp_segment_configuration where content = 0 order by dbid; + + +select count(*) from test; + +select count(*), gp_segment_id from test group by gp_segment_id order by gp_segment_id; + +reset synchronous_commit; + +-- start_ignore +\! gprecoverseg -aF --no-progress; +\! gprecoverseg -ar; +-- end_ignore + +drop table test; + +select gp_debug_reset_create_table_default_numsegments(); From 88351ee392923b8198624c7aa0c3d0cc4d41e2f6 Mon Sep 17 00:00:00 2001 From: Vladimir Sarmin Date: Mon, 13 Jul 2026 18:33:26 +0300 Subject: [PATCH 19/20] Implement 'rows out' in EXPLAIN ANALYZE (#524) Backport the gp_enable_explain_rows_out GUC, off by default. When it is on, EXPLAIN ANALYZE reports the spread of rows produced across segments for each plan node. The figures make uneven data distribution visible per node. Changes: 1. Add the gp_enable_explain_rows_out boolean GUC in the experimental feature group, exposed as USERSET and hidden from SHOW ALL. 2. Emit a "Rows out" line in cdbexplain_showExecStats when the GUC is on, reporting the average rows across workers with the maximum and minimum rows and the segments that produced them, in both text and structured output formats. 3. Compute the figures as per-loop rows (ntuples/nloops) to match the row count already shown on the node. 4. Skip non-participating (T_Invalid) segment slots, taking the average, the worker count, and the minimum only over the segments that ran the node. 5. Cover the feature in the gp_explain regression test on a uniform and a skewed table, and assert the line is emitted. The 6.x branch is PostgreSQL 9.4 based, so ExplainPropertyInteger and ExplainPropertyFloat take no unit argument, and the per-instance segment id is ns->segindex0 + i. EXPLAIN ANALYZE now reports per-node row-count skew across segments. Backported from GreengageDB/greengage@cc5a070, originally taken from open-gpdb/gpdb@1d41230. Co-authored-by: Vladimir Rachkin <33569237+robozmey@users.noreply.github.com> --- .abi-check/6.31.0/postgres.symbols.ignore | 1 + src/backend/cdb/cdbvars.c | 1 + src/backend/commands/explain_gp.c | 69 +++++++++++++++++++ src/backend/utils/misc/guc_gp.c | 11 +++ src/include/cdb/cdbvars.h | 6 ++ src/include/utils/unsync_guc_name.h | 1 + src/test/regress/expected/gp_explain.out | 63 +++++++++++++---- .../regress/expected/gp_explain_optimizer.out | 63 +++++++++++++---- src/test/regress/sql/gp_explain.sql | 68 ++++++++++++++---- 9 files changed, 246 insertions(+), 37 deletions(-) create mode 100644 .abi-check/6.31.0/postgres.symbols.ignore diff --git a/.abi-check/6.31.0/postgres.symbols.ignore b/.abi-check/6.31.0/postgres.symbols.ignore new file mode 100644 index 000000000000..aa4c33e2be25 --- /dev/null +++ b/.abi-check/6.31.0/postgres.symbols.ignore @@ -0,0 +1 @@ +ConfigureNamesBool_gp diff --git a/src/backend/cdb/cdbvars.c b/src/backend/cdb/cdbvars.c index 63ea1bcac9bb..f324871761ae 100644 --- a/src/backend/cdb/cdbvars.c +++ b/src/backend/cdb/cdbvars.c @@ -278,6 +278,7 @@ int gp_motion_slice_noop = 0; /* Greengage Database Experimental Feature GUCs */ int gp_distinct_grouping_sets_threshold = 32; +bool gp_enable_explain_rows_out = FALSE; bool gp_enable_explain_allstat = FALSE; bool gp_enable_motion_deadlock_sanity = FALSE; /* planning time sanity * check */ diff --git a/src/backend/commands/explain_gp.c b/src/backend/commands/explain_gp.c index 8765f4c28de7..c69dec48f607 100644 --- a/src/backend/commands/explain_gp.c +++ b/src/backend/commands/explain_gp.c @@ -1857,6 +1857,75 @@ cdbexplain_showExecStats(struct PlanState *planstate, ExplainState *es) } pfree(extraData.data); + /* + * Print "Rows out" + */ + if (gp_enable_explain_rows_out && es->analyze && ns->ninst > 0) + { + double alltuples = 0; + double maxtuples = 0; + int maxseg = -1; + double mintuples = 0; + int minseg = -1; + int workercount = 0; + double avgtuples; + + for (i = 0; i < ns->ninst; i++) + { + CdbExplain_StatInst *nsi = &ns->insts[i]; + double rows; + + if (nsi->pstype == T_Invalid) + continue; + + rows = nsi->nloops > 0 ? nsi->ntuples / nsi->nloops : 0; + + if (workercount == 0 || rows > maxtuples) + { + maxtuples = rows; + maxseg = ns->segindex0 + i; + } + + if (workercount == 0 || rows < mintuples) + { + mintuples = rows; + minseg = ns->segindex0 + i; + } + + alltuples += rows; + workercount++; + } + + if (workercount > 0) + { + avgtuples = alltuples / workercount; + + if (es->format == EXPLAIN_FORMAT_TEXT) + { + appendStringInfoSpaces(es->str, es->indent * 2); + appendStringInfoString(es->str, "Rows out: "); + + appendStringInfo(es->str, + "%.2f rows avg x %d workers, %.0f rows max (seg%d), %.0f rows min (seg%d).\n", + avgtuples, + workercount, + maxtuples, + maxseg, + mintuples, + minseg); + } + else + { + ExplainPropertyInteger("Workers", workercount, es); + ExplainPropertyFloat("Average Rows", avgtuples, 2, es); + ExplainPropertyFloat("Max Rows", maxtuples, 0, es); + ExplainPropertyInteger("Max Rows Segment", maxseg, es); + ExplainPropertyFloat("Min Rows", mintuples, 0, es); + ExplainPropertyInteger("Min Rows Segment", minseg, es); + } + } + } + /* * Dump stats for all workers. */ diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index 2d5a687d2208..1ccc7c31698a 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -890,6 +890,17 @@ struct config_bool ConfigureNamesBool_gp[] = NULL, NULL, NULL }, + { + {"gp_enable_explain_rows_out", PGC_USERSET, CLIENT_CONN_OTHER, + gettext_noop("Experimental feature: print avg, min and max rows out in segments in EXPLAIN ANALYZE."), + NULL, + GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE + }, + &gp_enable_explain_rows_out, + false, + NULL, NULL, NULL + }, + { {"gp_enable_sort_limit", PGC_USERSET, QUERY_TUNING_METHOD, gettext_noop("Enable LIMIT operation to be performed while sorting."), diff --git a/src/include/cdb/cdbvars.h b/src/include/cdb/cdbvars.h index 4823798085d4..95ad3cbd0ec9 100644 --- a/src/include/cdb/cdbvars.h +++ b/src/include/cdb/cdbvars.h @@ -744,6 +744,12 @@ extern bool gp_enable_preunique; */ extern bool gp_eager_preunique; +/* + * May Greengage print statistics as average, minimum and maximum rows out + * during EXPLAIN ANALYZE? + */ +extern bool gp_enable_explain_rows_out; + /* May Greengage dump statistics for all segments as a huge ugly string * during EXPLAIN ANALYZE? * diff --git a/src/include/utils/unsync_guc_name.h b/src/include/utils/unsync_guc_name.h index 979ecb2d6365..faef133e92cb 100644 --- a/src/include/utils/unsync_guc_name.h +++ b/src/include/utils/unsync_guc_name.h @@ -176,6 +176,7 @@ "gp_enable_direct_dispatch", "gp_enable_exchange_default_partition", "gp_enable_explain_allstat", + "gp_enable_explain_rows_out", "gp_enable_fast_sri", "gp_enable_global_deadlock_detector", "gp_enable_gpperfmon", diff --git a/src/test/regress/expected/gp_explain.out b/src/test/regress/expected/gp_explain.out index 77010f95787a..984c2dbe31b6 100644 --- a/src/test/regress/expected/gp_explain.out +++ b/src/test/regress/expected/gp_explain.out @@ -29,13 +29,27 @@ begin end loop; end; $$ language plpgsql; +-- Return EXPLAIN ANALYZE result as xml to manipulate it further. +create or replace function get_explain_analyze_xml_output(explain_query text) +returns xml as +$$ +declare + x xml; +begin + execute 'EXPLAIN (ANALYZE, VERBOSE, FORMAT XML) ' || explain_query + into x; + return x; +end; +$$ language plpgsql; -- -- Test explain_memory_verbosity option -- +set gp_use_legacy_hashops=off; CREATE TABLE explaintest (id int4); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Greengage Database data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. INSERT INTO explaintest SELECT generate_series(1, 10); +reset gp_use_legacy_hashops; EXPLAIN ANALYZE SELECT * FROM explaintest; QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------- @@ -337,21 +351,46 @@ explain analyze SELECT * FROM explaintest; (8 rows) set gp_enable_explain_allstat=DEFAULT; +-- Test explain rows out. +set gp_enable_explain_rows_out=on; +\pset format unaligned +\pset tuples_only on +WITH query_plan (et) AS +( + select get_explain_analyze_output($$ + SELECT * FROM explaintest; + $$) +) +SELECT trim(et) FROM query_plan WHERE et like '%Rows out:%' AND et not like '%(seg-1)%'; +Rows out: 3.33 rows avg x 3 workers, 5 rows max (seg0), 1 rows min (seg1). +-- Rows out on a skewed distribution, so max and min land on different segments. +set gp_use_legacy_hashops=off; +CREATE TABLE explain_rows_skew (id int) DISTRIBUTED BY (id); +INSERT INTO explain_rows_skew SELECT 2 FROM generate_series(1, 100); +INSERT INTO explain_rows_skew SELECT 1 FROM generate_series(1, 10); +INSERT INTO explain_rows_skew VALUES (5); +reset gp_use_legacy_hashops; +ANALYZE explain_rows_skew; +SELECT xpath( + '//*[local-name()="Relation-Name" and text()="explain_rows_skew"]/.. + /*[local-name()="Workers" + or local-name()="Average-Rows" + or local-name()="Max-Rows" + or local-name()="Max-Rows-Segment" + or local-name()="Min-Rows" + or local-name()="Min-Rows-Segment"]/text()', + x) +FROM get_explain_analyze_xml_output($$ + SELECT * FROM explain_rows_skew; + $$) AS query_plan(x); +{3,37.00,100,0,1,2} +\pset tuples_only off +\pset format aligned +reset gp_enable_explain_rows_out; +DROP TABLE explain_rows_skew; -- -- Test output of EXPLAIN ANALYZE for Bitmap index scan's actual rows. -- --- Return EXPLAIN ANALYZE result as xml to manipulate it further. -create or replace function get_explain_analyze_xml_output(explain_query text) -returns xml as -$$ -declare - x xml; -begin - execute 'EXPLAIN (ANALYZE, VERBOSE, FORMAT XML) ' || explain_query - into x; - return x; -end; -$$ language plpgsql; -- force (Dynamic) Bitmap Index Scan set optimizer_enable_dynamictablescan=off; set enable_seqscan=off; diff --git a/src/test/regress/expected/gp_explain_optimizer.out b/src/test/regress/expected/gp_explain_optimizer.out index afc7272c09ad..3bf7a5b76afc 100644 --- a/src/test/regress/expected/gp_explain_optimizer.out +++ b/src/test/regress/expected/gp_explain_optimizer.out @@ -29,13 +29,27 @@ begin end loop; end; $$ language plpgsql; +-- Return EXPLAIN ANALYZE result as xml to manipulate it further. +create or replace function get_explain_analyze_xml_output(explain_query text) +returns xml as +$$ +declare + x xml; +begin + execute 'EXPLAIN (ANALYZE, VERBOSE, FORMAT XML) ' || explain_query + into x; + return x; +end; +$$ language plpgsql; -- -- Test explain_memory_verbosity option -- +set gp_use_legacy_hashops=off; CREATE TABLE explaintest (id int4); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Greengage Database data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. INSERT INTO explaintest SELECT generate_series(1, 10); +reset gp_use_legacy_hashops; EXPLAIN ANALYZE SELECT * FROM explaintest; QUERY PLAN -------------------------------------------------------------------------------------------------------------------------- @@ -368,21 +382,46 @@ explain analyze SELECT * FROM explaintest; (8 rows) set gp_enable_explain_allstat=DEFAULT; +-- Test explain rows out. +set gp_enable_explain_rows_out=on; +\pset format unaligned +\pset tuples_only on +WITH query_plan (et) AS +( + select get_explain_analyze_output($$ + SELECT * FROM explaintest; + $$) +) +SELECT trim(et) FROM query_plan WHERE et like '%Rows out:%' AND et not like '%(seg-1)%'; +Rows out: 3.33 rows avg x 3 workers, 5 rows max (seg0), 1 rows min (seg1). +-- Rows out on a skewed distribution, so max and min land on different segments. +set gp_use_legacy_hashops=off; +CREATE TABLE explain_rows_skew (id int) DISTRIBUTED BY (id); +INSERT INTO explain_rows_skew SELECT 2 FROM generate_series(1, 100); +INSERT INTO explain_rows_skew SELECT 1 FROM generate_series(1, 10); +INSERT INTO explain_rows_skew VALUES (5); +reset gp_use_legacy_hashops; +ANALYZE explain_rows_skew; +SELECT xpath( + '//*[local-name()="Relation-Name" and text()="explain_rows_skew"]/.. + /*[local-name()="Workers" + or local-name()="Average-Rows" + or local-name()="Max-Rows" + or local-name()="Max-Rows-Segment" + or local-name()="Min-Rows" + or local-name()="Min-Rows-Segment"]/text()', + x) +FROM get_explain_analyze_xml_output($$ + SELECT * FROM explain_rows_skew; + $$) AS query_plan(x); +{3,37.00,100,0,1,2} +\pset tuples_only off +\pset format aligned +reset gp_enable_explain_rows_out; +DROP TABLE explain_rows_skew; -- -- Test output of EXPLAIN ANALYZE for Bitmap index scan's actual rows. -- --- Return EXPLAIN ANALYZE result as xml to manipulate it further. -create or replace function get_explain_analyze_xml_output(explain_query text) -returns xml as -$$ -declare - x xml; -begin - execute 'EXPLAIN (ANALYZE, VERBOSE, FORMAT XML) ' || explain_query - into x; - return x; -end; -$$ language plpgsql; -- force (Dynamic) Bitmap Index Scan set optimizer_enable_dynamictablescan=off; set enable_seqscan=off; diff --git a/src/test/regress/sql/gp_explain.sql b/src/test/regress/sql/gp_explain.sql index 3deba9eb8beb..281460a45d57 100644 --- a/src/test/regress/sql/gp_explain.sql +++ b/src/test/regress/sql/gp_explain.sql @@ -34,12 +34,27 @@ begin end; $$ language plpgsql; +-- Return EXPLAIN ANALYZE result as xml to manipulate it further. +create or replace function get_explain_analyze_xml_output(explain_query text) +returns xml as +$$ +declare + x xml; +begin + execute 'EXPLAIN (ANALYZE, VERBOSE, FORMAT XML) ' || explain_query + into x; + return x; +end; +$$ language plpgsql; + -- -- Test explain_memory_verbosity option -- +set gp_use_legacy_hashops=off; CREATE TABLE explaintest (id int4); INSERT INTO explaintest SELECT generate_series(1, 10); +reset gp_use_legacy_hashops; EXPLAIN ANALYZE SELECT * FROM explaintest; @@ -162,23 +177,50 @@ set gp_enable_explain_allstat=on; explain analyze SELECT * FROM explaintest; set gp_enable_explain_allstat=DEFAULT; +-- Test explain rows out. +set gp_enable_explain_rows_out=on; + +\pset format unaligned +\pset tuples_only on +WITH query_plan (et) AS +( + select get_explain_analyze_output($$ + SELECT * FROM explaintest; + $$) +) +SELECT trim(et) FROM query_plan WHERE et like '%Rows out:%' AND et not like '%(seg-1)%'; + +-- Rows out on a skewed distribution, so max and min land on different segments. +set gp_use_legacy_hashops=off; +CREATE TABLE explain_rows_skew (id int) DISTRIBUTED BY (id); +INSERT INTO explain_rows_skew SELECT 2 FROM generate_series(1, 100); +INSERT INTO explain_rows_skew SELECT 1 FROM generate_series(1, 10); +INSERT INTO explain_rows_skew VALUES (5); +reset gp_use_legacy_hashops; +ANALYZE explain_rows_skew; + +SELECT xpath( + '//*[local-name()="Relation-Name" and text()="explain_rows_skew"]/.. + /*[local-name()="Workers" + or local-name()="Average-Rows" + or local-name()="Max-Rows" + or local-name()="Max-Rows-Segment" + or local-name()="Min-Rows" + or local-name()="Min-Rows-Segment"]/text()', + x) +FROM get_explain_analyze_xml_output($$ + SELECT * FROM explain_rows_skew; + $$) AS query_plan(x); + +\pset tuples_only off +\pset format aligned +reset gp_enable_explain_rows_out; +DROP TABLE explain_rows_skew; + -- -- Test output of EXPLAIN ANALYZE for Bitmap index scan's actual rows. -- --- Return EXPLAIN ANALYZE result as xml to manipulate it further. -create or replace function get_explain_analyze_xml_output(explain_query text) -returns xml as -$$ -declare - x xml; -begin - execute 'EXPLAIN (ANALYZE, VERBOSE, FORMAT XML) ' || explain_query - into x; - return x; -end; -$$ language plpgsql; - -- force (Dynamic) Bitmap Index Scan set optimizer_enable_dynamictablescan=off; set enable_seqscan=off; From 64dd42f3363077a008b4e4a2ec2580b2867e4eb8 Mon Sep 17 00:00:00 2001 From: Matvei Nelasov Date: Wed, 22 Jul 2026 15:34:26 +0300 Subject: [PATCH 20/20] Add RHEL9 and RedOS8.0 support (#540) Add support for several distros - RedHat 9 support - RedOS 8.0 support Ticket: CI-6025 --- gpAux/Makefile | 2 ++ gpAux/Makefile.global | 2 ++ 2 files changed, 4 insertions(+) diff --git a/gpAux/Makefile b/gpAux/Makefile index 15c9c690f58f..a690e5f41d27 100644 --- a/gpAux/Makefile +++ b/gpAux/Makefile @@ -132,6 +132,7 @@ astra1.8_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi --enable-mapreduce rhel7_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs rhel7_ppc64le_CONFIGFLAGS=--enable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs rhel8_x86_64_CONFIGFLAGS=--disable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs +rhel9_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs rocky8_x86_64_CONFIGFLAGS=--with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-pythonsrc-ext --with-uuid=e2fs rocky9_x86_64_CONFIGFLAGS=--disable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs linux_x86_64_CONFIGFLAGS=${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs @@ -141,6 +142,7 @@ ubuntu22.04_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi --enable-mapredu ubuntu24.04_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs sles12_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs redos7.3_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs +redos8.0_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs BLD_CONFIGFLAGS=$($(BLD_ARCH)_CONFIGFLAGS) CONFIGFLAGS=$(strip $(BLD_CONFIGFLAGS) --with-pgport=$(DEFPORT) $(BLD_DEPLOYMENT_SETTING)) diff --git a/gpAux/Makefile.global b/gpAux/Makefile.global index d99581cc4574..59ab99da360e 100644 --- a/gpAux/Makefile.global +++ b/gpAux/Makefile.global @@ -46,6 +46,7 @@ export NO_M64=1 rhel6_x86_64_BLD_CFLAGS=-m64 -gdwarf-2 -gstrict-dwarf rhel7_x86_64_BLD_CFLAGS=-m64 rhel8_x86_64_BLD_CFLAGS=-m64 +rhel9_x86_64_BLD_CFLAGS=-m64 rhel7_ppc64le_BLD_CFLAGS=-m64 -fasynchronous-unwind-tables -fsigned-char rocky8_x86_64_BLD_CFLAGS=-m64 rocky9_x86_64_BLD_CFLAGS=-m64 @@ -56,6 +57,7 @@ astra1.8_x86_64_BLD_CFLAGS=-m64 ubuntu22.04_x86_64_BLD_CFLAGS=-m64 ubuntu24.04_x86_64_BLD_CFLAGS=-m64 redos7.3_x86_64_BLD_CFLAGS=-m64 +redos8.0_x86_64_BLD_CFLAGS=-m64 BLD_CFLAGS=$($(BLD_ARCH)_BLD_CFLAGS) BLD_LDFLAGS=$($(BLD_ARCH)_BLD_LDFLAGS)