diff --git a/.abi-check/6.31.0/postgres.symbols.ignore b/.abi-check/6.31.0/postgres.symbols.ignore new file mode 100644 index 000000000000..aa4c33e2be25 --- /dev/null +++ b/.abi-check/6.31.0/postgres.symbols.ignore @@ -0,0 +1 @@ +ConfigureNamesBool_gp diff --git a/.github/workflows/greengage-ci.yml b/.github/workflows/greengage-ci.yml index 5f15e6292f85..f337b3618e3c 100644 --- a/.github/workflows/greengage-ci.yml +++ b/.github/workflows/greengage-ci.yml @@ -22,11 +22,15 @@ jobs: - target_os: ubuntu - target_os: ubuntu target_os_version: "24.04" + - target_os: rockylinux + target_os_version: "8" + - target_os: rockylinux + target_os_version: "9" permissions: contents: read # Explicit for default behavior packages: write # Required for GHCR access actions: write # Required for artifact upload - uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-build.yml@v33 + uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-build.yml@v42 with: version: 6 target_os: ${{ matrix.target_os }} @@ -48,7 +52,7 @@ jobs: contents: read # Explicit for default behavior packages: read # Explicit for GHCR access clarity actions: write # Required for artifact upload - uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-tests-behave.yml@v28 + uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-tests-behave.yml@v46 with: version: 6 target_os: ${{ matrix.target_os }} @@ -69,7 +73,7 @@ jobs: contents: read # Explicit for default behavior packages: read # Explicit for GHCR access clarity actions: write # Required for artifact upload - uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-tests-regression.yml@v28 + uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-tests-regression.yml@v49 with: version: 6 target_os: ${{ matrix.target_os }} @@ -77,6 +81,28 @@ jobs: secrets: ghcr_token: ${{ secrets.GITHUB_TOKEN }} + coverage: + needs: [behave-tests, regression-tests] + if: github.event_name == 'pull_request' && needs.behave-tests.result == 'success' && needs.regression-tests.result == 'success' + strategy: + fail-fast: false + matrix: + include: + - target_os: ubuntu + target_os_version: "24.04" + permissions: + contents: read + packages: read + actions: write + uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-coverage.yml@v46 + with: + version: 6 + target_os: ${{ matrix.target_os }} + target_os_version: ${{ matrix.target_os_version }} + coverage_threshold: 75 + secrets: + ghcr_token: ${{ secrets.GITHUB_TOKEN }} + orca-tests: needs: build if: github.event_name == 'pull_request' # Only for PR @@ -140,7 +166,7 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - # Rebuild prod-redy version without debug extensions and pack it to deb + # Rebuild prod-redy version without debug extensions and pack it to deb and rpm package: needs: build strategy: @@ -148,17 +174,25 @@ jobs: matrix: include: - target_os: ubuntu + test_install: true - target_os: ubuntu target_os_version: "24.04" + test_install: true + - target_os: rockylinux + target_os_version: "8" + test_install: false + - target_os: rockylinux + target_os_version: "9" + test_install: false permissions: contents: read # Explicit for default behavior packages: write # Required for GHCR access actions: write # Required for artifact upload - uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-package.yml@v38 + uses: greengagedb/greengage-ci/.github/workflows/greengage-reusable-package.yml@v47 with: version: 6 target_os: ${{ matrix.target_os }} target_os_version: ${{ matrix.target_os_version }} - test_docker: ${{ matrix.target_os }}:${{ matrix.target_os_version || '22.04' }} + test_install: ${{ matrix.test_install }} secrets: ghcr_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 799206a138da..2e6df3a72aed 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,4 @@ compile_commands.json /Release/ /CMakeLists.txt /Package/ +/RPM/ diff --git a/README.Rhel-Rocky.bash b/README.Rhel-Rocky.bash index 702f38911871..78c22192c3e7 100755 --- a/README.Rhel-Rocky.bash +++ b/README.Rhel-Rocky.bash @@ -1,21 +1,55 @@ #!/bin/bash +# FILE: README.Rhel-Rocky.bash +# CONTEXT: Called from ci/Dockerfile.rockylinux for Greengage build +# PURPOSE: Install build dependencies, compile zstd static library, +# Install Python based on OS version -sudo dnf -y update -sudo dnf -y install epel-release -sudo dnf -y install 'dnf-command(config-manager)' -sudo dnf config-manager --set-enabled devel -sudo dnf makecache +set -euxo pipefail -sudo dnf -y install\ +dnf -y install epel-release + +# Detect OS version if not already set +export OS_VERSION="${OS_VERSION:-$(grep -oP '(?<= release )\d+' /etc/redhat-release)}" + +perl_packages="perl-Env perl-ExtUtils-Embed perl-IPC-Run perl-JSON perl-Test-Base" +python_packages="python3 python3-devel python3-setuptools python3-pip python3-future" + +case "$OS_VERSION" in + 8) + dnf config-manager --set-enabled powertools + python_packages="python2 python2-devel python2-setuptools python2-pip $python_packages" + ;; + 9) + dnf config-manager --set-enabled crb + perl_packages="$perl_packages perl-Opcode perl-Test-Simple perl-Thread-Queue perl-devel" + ;; + *) + echo "Unsupported Rocky Linux version: $OS_VERSION" + exit 1 + ;; +esac +# shellcheck disable=SC2086 # intentional: word splitting for package lists +dnf -y install \ apr-devel \ + apr-util-devel \ + autoconf \ bison \ bzip2-devel \ - cmake3 \ + cmake \ + expat-devel \ flex \ - gcc \ gcc-c++ \ + git \ + glibc-langpack-en \ + gperf \ + indent \ iproute \ + iputils \ + java-11-openjdk-devel \ + jq \ krb5-devel \ + krb5-server \ + krb5-workstation \ libcurl-devel \ libevent-devel \ libicu \ @@ -29,18 +63,25 @@ sudo dnf -y install\ lsof \ net-tools \ openldap-devel \ - openssl \ + openssh-server \ openssl-devel \ pam-devel \ - perl-Env \ - perl-ExtUtils-Embed \ - perl-IPC-Run \ - perl-JSON \ - perl-Test-Base \ procps-ng \ - python2-devel \ - python2-pip \ readline-devel \ + rpm-build \ + rsync \ snappy-devel \ + sudo \ + time \ + unzip \ + vim \ + wget \ xerces-c-devel \ - zlib-devel + zlib-devel \ + $python_packages $perl_packages + +# Build zstd with static library (not available as a package on Rocky) +curl -Ls https://github.com/facebook/zstd/releases/download/v1.4.4/zstd-1.4.4.tar.gz | tar -xzf - +make -j"$(nproc)" -C zstd-1.4.4 +make install PREFIX=/usr/local -C zstd-1.4.4 +rm -rf zstd-1.4.4 diff --git a/README.linux.md b/README.linux.md index c73dfd9c4d96..6d95d1850946 100644 --- a/README.linux.md +++ b/README.linux.md @@ -6,28 +6,25 @@ ``` Note: CentOS 7 is EOL — configure `yum` to use a valid repo (e.g., `vault.centos.org`) before installing dependencies. -## For RHEL/Rocky 8: +## For RHEL/Rocky (versions 8 or 9): - Install dependencies using README.Rhel-Rocky.bash script: ```bash - ./README.Rhel-Rocky.bash + sudo ./README.Rhel-Rocky.bash ``` -- Build and install zstd with static library, e.g.: - ```bash - cd /tmp - curl -LO https://github.com/facebook/zstd/releases/download/v1.4.4/zstd-1.4.4.tar.gz - tar -xf zstd-1.4.4.tar.gz - cd zstd-1.4.4 - make -j$(nproc) - sudo make install PREFIX=/usr/local - ``` +- Create a symbolic link to Python in `/usr/bin` and install `future` v0.16: -- Create symbolic link to Python 2 in `/usr/bin`: + - Rocky 8: + ```bash + sudo ln -s python2 /usr/bin/python + sudo python -m pip install --no-cache-dir future==0.16 + ``` - ```bash - sudo ln -s python2 /usr/bin/python - ``` + - Rocky 9: + ```bash + sudo ln -s python3 /usr/bin/python + ``` ## For Ubuntu (versions 22.04 or 24.04): diff --git a/ci/Dockerfile.rockylinux b/ci/Dockerfile.rockylinux new file mode 100644 index 000000000000..1d958bec133f --- /dev/null +++ b/ci/Dockerfile.rockylinux @@ -0,0 +1,79 @@ +# FILE: ci/Dockerfile.rockylinux +# CONTEXT: Build Greengage on Rocky Linux +# PURPOSE: Multi-stage Docker build for Greengage database + +ARG OS_VERSION=8 + +FROM rockylinux:${OS_VERSION} AS base +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +WORKDIR /home/gpadmin + +ARG OS_VERSION +ENV OS_VERSION=${OS_VERSION} + +COPY README.Rhel-Rocky.bash ./ +COPY gpMgmt/bin/gpload_test/pytest_requirement.txt ./ +RUN set -eux; \ + ./README.Rhel-Rocky.bash; \ + rm README.Rhel-Rocky.bash; \ +# Set Python as default based on OS version + case "$OS_VERSION" in \ + 8) \ + ln -sf /usr/bin/python2 /usr/bin/python; \ + PIP_PACKAGES='allure-behave==2.4.0 mock==3.0.5 psi==0.3b3 future==0.16'; \ + ;; \ + 9) \ + ln -sf /usr/bin/python3 /usr/bin/python; \ + python -m ensurepip --upgrade; \ + PIP_PACKAGES='allure-behave==2.4.0'; \ + ;; \ + *) \ + echo "Unsupported Rocky Linux version: $OS_VERSION"; \ + exit 1; \ + ;; \ + esac; \ +# Install pg_bsd_indent used by pgindent utility + wget --progress=dot:giga --no-hsts https://ftp.postgresql.org/pub/dev/pg_bsd_indent-1.3.tar.gz -O - | tar -xzf -; \ + make install -C pg_bsd_indent; \ + rm -r pg_bsd_indent; \ +# To run sshd directly, but not using service + mkdir -p /run/sshd; \ +# Alter precedence in favor of IPv4 during resolving + echo 'precedence ::ffff:0:0/96 100' >> /etc/gai.conf; \ +# Upgrade pip to support current package versions + python -m pip install --no-cache-dir --upgrade pip; \ +# Install allure-behave for behave tests + python -m pip install --no-cache-dir $PIP_PACKAGES; \ +# Cleanup to reduce image size + dnf clean all; \ + rm -rf /tmp/* /var/tmp/* /var/cache/man/ + +ENV LANG=en_US.UTF-8 +ENV CONFIGURE_FLAGS="--enable-debug-extensions --with-gssapi --enable-cassert --enable-debug --enable-depend" + +FROM base AS build + +COPY . gpdb_src + +RUN mkdir bin_gpdb + +ENV TARGET_OS=rockylinux \ + OUTPUT_ARTIFACT_DIR=bin_gpdb + +# Use python3 to compile into GPDB's plpython3u +# Set PYTHON3 only on Rocky 8 to avoid compiling plpython3 twice +# On Rocky 9, Python 3 is the default +RUN test "$OS_VERSION" -eq 8 && export PYTHON3=python3; \ + gpdb_src/concourse/scripts/compile_gpdb.bash + +FROM base AS code +COPY . gpdb_src +RUN rm -rf gpdb_src/.git/ + +FROM base AS test + +COPY --from=code /home/gpadmin/gpdb_src gpdb_src +COPY --from=build /home/gpadmin/bin_gpdb bin_gpdb +COPY --from=build /home/gpadmin/gpdb_src/VERSION gpdb_src + +RUN make -C gpdb_src/src/tools/entab install clean diff --git a/ci/readme.md b/ci/readme.md index 788480991f0a..3f233cf88d36 100644 --- a/ci/readme.md +++ b/ci/readme.md @@ -14,12 +14,17 @@ To build an image based on Ubuntu 24.04, specify the version in build args: docker build -t gpdb6_regress:latest --build-arg OS_VERSION=24.04 -f ci/Dockerfile.ubuntu . ``` -There are two additional options in [Dockerfile](./Dockerfile) to passthrough urls for [sigar](https://github.com/hyperic/sigar) packages: +To build a Rocky Linux 8 image: -* `--build-arg sigar=https://path_to_sigar.rpm` for package with sigar library -* `--build-arg sigar_headers=https://path_to_sigar_headers.rpm` for package with sigar headers files +```bash +docker build -t gpdb6_rockylinux8:latest -f ci/Dockerfile.rockylinux . +``` + +To build a Rocky Linux 9 image: -These options may be useful to build image on different platforms. +```bash +docker build -t gpdb6_rockylinux9:latest --build-arg OS_VERSION=9 -f ci/Dockerfile.rockylinux . +``` CI pushes docker images to the internal registry for each branch. We can pull it with usage of: diff --git a/ci/scripts/run_behave_tests.bash b/ci/scripts/run_behave_tests.bash index f5930c7a9879..f9c0c3b3fb3e 100755 --- a/ci/scripts/run_behave_tests.bash +++ b/ci/scripts/run_behave_tests.bash @@ -72,6 +72,7 @@ run_feature() { -e FEATURE="$feature" -e PROJECT="$project" \ cdw bash -eux <<'EOF' set -ex + source /usr/local/greengage-db-devel/greengage_path.sh cd /tmp/coverage-data if [ "$(ls "$PROJECT"-coverage-data/ | wc -l)" -gt 0 ]; then diff --git a/concourse/scripts/common.bash b/concourse/scripts/common.bash index 01b18be4caf5..b4142cc368f1 100644 --- a/concourse/scripts/common.bash +++ b/concourse/scripts/common.bash @@ -10,14 +10,12 @@ function set_env() { } function os_id() { - if [[ ! -f "/etc/altlinux-release" ]] && [[ -f "/etc/redhat-release" ]]; then - echo "centos" - else - echo "$( - . /etc/os-release - echo "${ID}" - )" + if [[ -f "/etc/os-release" ]]; then + . /etc/os-release + elif [[ -f "/etc/redhat-release" ]]; then + ID="centos" fi + echo "${ID}" } function os_version() { @@ -39,7 +37,7 @@ function os_version() { function build_arch() { local id=$(os_id) local version=$(os_version) - # BLD_ARCH expects rhel{6,7,8}_x86_64 || rocky8_x86_64 || sles12_x86_64 || ubuntu20.04_x86_64 + # BLD_ARCH expects rhel{6,7,8}_x86_64 || rocky8_x86_64 || rocky9_x86_64 || sles12_x86_64 || ubuntu20.04_x86_64 # for oel7 and oel8 platform, the id will return centos case ${id} in sles | rocky) version=$(os_version | cut -d. -f1) ;; diff --git a/concourse/scripts/compile_gpdb.bash b/concourse/scripts/compile_gpdb.bash index 9092a83855f5..60574fc05990 100755 --- a/concourse/scripts/compile_gpdb.bash +++ b/concourse/scripts/compile_gpdb.bash @@ -105,10 +105,10 @@ function include_dependencies() { vendored_headers=(zstd*.h uv.h uv ) pkgconfigs=(libzstd.pc libuv.pc quicklz.pc) # rocky9/oel9/rhel9 won't vendor zstd because of rsync on these platform does not work libzstd 1.3.7 - if [[ ${BLD_ARCH} == "rhel9"* ]]; then + if [[ ${BLD_ARCH} == "rhel9"* || ${BLD_ARCH} == "rocky9"* ]]; then vendored_libs=(libquicklz.so{,.1,.1.5.0} libuv.so{,.1,.1.0.0} libxerces-c.so) # rocky8/oel8/rhel8 needs zstd 1.4.4 to be vendor because these platform support system libzstd 1.4.4 - elif [[ ${BLD_ARCH} == "rhel8"* ]]; then + elif [[ ${BLD_ARCH} == "rhel8"* || ${BLD_ARCH} == "rocky8"* ]]; then vendored_libs=(libquicklz.so{,.1,.1.5.0} libzstd.so{,.1,.1.4.4} libuv.so{,.1,.1.0.0} libxerces-c.so) else vendored_libs=(libquicklz.so{,.1,.1.5.0} libzstd.so{,.1,.1.3.7} libuv.so{,.1,.1.0.0} libxerces-c.so) diff --git a/concourse/scripts/ic_gpdb.bash b/concourse/scripts/ic_gpdb.bash index 7f4fc654ba6a..93b5f8c5209d 100755 --- a/concourse/scripts/ic_gpdb.bash +++ b/concourse/scripts/ic_gpdb.bash @@ -34,6 +34,18 @@ function gen_env(){ export TEST_PGFDW=1 export COVERAGE_PROCESS_START="\${1}/gpdb_src/gpMgmt/test/coveragerc_unit" make -s ${MAKE_TEST_COMMAND} + + cd /tmp/coverage-data + if [ "\$(ls coverage-data* 2>/dev/null | wc -l)" -gt 0 ]; then + coverage combine --append --keep \ + --rcfile=/home/gpadmin/gpdb_src/gpMgmt/test/coveragerc_combine_report \ + coverage-data* + coverage html \ + --rcfile=/home/gpadmin/gpdb_src/gpMgmt/test/coveragerc_combine_report \ + --show-contexts -d ./coverage-html + else + echo "No coverage-data files found, skipping coverage report" + fi EOF chmod a+x /opt/run_test.sh diff --git a/concourse/scripts/setup_gpadmin_user.bash b/concourse/scripts/setup_gpadmin_user.bash index abaadae69db7..7b051e242a59 100755 --- a/concourse/scripts/setup_gpadmin_user.bash +++ b/concourse/scripts/setup_gpadmin_user.bash @@ -92,6 +92,7 @@ setup_sshd() { # Disable password authentication so builds never hang given bad keys sed -ri 's/PasswordAuthentication yes/PasswordAuthentication no/g' /etc/ssh/sshd_config + echo "MaxStartups 100:30:200" >> /etc/ssh/sshd_config case "$TEST_OS" in centos6 | sles*) diff --git a/contrib/pg_upgrade_support/pg_upgrade_support.c b/contrib/pg_upgrade_support/pg_upgrade_support.c index 5beef9490412..d45e41d6ac1a 100644 --- a/contrib/pg_upgrade_support/pg_upgrade_support.c +++ b/contrib/pg_upgrade_support/pg_upgrade_support.c @@ -18,6 +18,15 @@ #include "catalog/oid_dispatch.h" #include "catalog/pg_authid.h" #include "catalog/pg_class.h" +#include "catalog/pg_constraint.h" +#include "catalog/pg_attrdef.h" +#include "catalog/pg_appendonly.h" +#include "catalog/pg_am.h" +#include "catalog/pg_proc.h" +#include "catalog/pg_exttable.h" +#include "catalog/pg_compression.h" +#include "catalog/pg_resqueuecapability.h" +#include "catalog/gp_configuration_history.h" #include "catalog/pg_enum.h" #include "catalog/pg_namespace.h" #include "catalog/pg_tablespace.h" @@ -29,6 +38,8 @@ #include "rewrite/rewriteHandler.h" #include "utils/array.h" #include "utils/builtins.h" +#include "utils/syscache.h" +#include "utils/lsyscache.h" /* THIS IS USED ONLY FOR PG >= 9.0 */ @@ -66,6 +77,20 @@ PG_FUNCTION_INFO_V1(view_has_removed_operators); PG_FUNCTION_INFO_V1(view_has_removed_functions); PG_FUNCTION_INFO_V1(view_has_removed_types); PG_FUNCTION_INFO_V1(view_has_changed_function_signatures); +PG_FUNCTION_INFO_V1(get_removed_tables); +PG_FUNCTION_INFO_V1(get_removed_columns); + +typedef struct RemovedTablesWalkerContext RemovedTablesWalkerContext; +typedef struct RemovedColumnsWalkerContext RemovedColumnsWalkerContext; +typedef struct RemovedColumnStatic RemovedColumnStatic; +typedef struct RemovedColumnDynamic RemovedColumnDynamic; +typedef struct RemovedFunctionDynamic RemovedFunctionDynamic; +typedef struct ReportedColumn ReportedColumn; + +static Oid get_function(const char *name, const Oid *args, int args_count, Oid namespace); +static Oid get_type(const char *name, Oid namespace); + +static Query *get_matview_query(Relation matview); static bool check_node_anyarray_walker(Node *node, void *context); static bool check_node_unknown_walker(Node *node, void *context); @@ -73,6 +98,277 @@ static bool check_node_removed_operators_walker(Node *node, void *context); static bool check_node_removed_functions_walker(Node *node, void *context); static bool check_node_removed_types_walker(Node *node, void *context); static bool check_node_changed_function_signatures_walker(Node *node, void *context); +static void report_removed_table(RemovedTablesWalkerContext *context, Oid reloid); +static void check_and_report_removed_table(RemovedTablesWalkerContext *context, Oid reloid); +static bool check_node_removed_tables_walker(Node *node, void *context); +static void report_removed_column(RemovedColumnsWalkerContext *context, Oid reloid, int attnum); +static bool check_and_report_removed_columns(RemovedColumnsWalkerContext *context, Oid reloid, int attnum); +static bool check_node_removed_columns_walker(Node *node, RemovedColumnsWalkerContext *context); + +/* + * Some objects are treated as 'dynamic', because they are present in the database + * by default, but live outside of 'pg_catalog', inside schemas like + * 'gp_toolkit' and 'information_schema', which can be dropped using + * 'DROP SCHEMA ... CASCADE'. This means that we should check for their absence. + * Moreover, they can be recreated from respective sql scripts after that, + * changing OIDs of the objects. So, we need to ask the database for their OIDs first. + */ + +struct RemovedTablesWalkerContext +{ + List *removedTables; +}; + +struct RemovedColumnsWalkerContext +{ + List *rtableStack; + List *removedColumns; + bool inside_whole_row_reference; +}; + +struct RemovedColumnStatic +{ + Oid reloid; + int attnum; +}; + +struct RemovedColumnDynamic +{ + const char *relnamespace; + const char *relname; + int attnum; +}; + +struct RemovedFunctionDynamic +{ + const char *pronamespace; + const char *name; + const Oid *args; + const int args_count; +}; + +struct ReportedColumn +{ + Oid attrelid; + int attnum; + bool comes_from_whole_row_reference; +}; + +/* Lists of objects removed from Greenage 7 */ +static Oid pg_resgroup_check_move_query_oids[] = {23, 26}; +static Oid __gp_remove_ao_entry_from_cache_oids[] = {26}; +static Oid __gp_get_ao_entry_from_cache_oids[] = {26}; + +static RemovedFunctionDynamic removed_functions_dynamic[] = +{ + {"gp_toolkit", "pg_resgroup_check_move_query", pg_resgroup_check_move_query_oids, 2}, + {"gp_toolkit", "__gp_remove_ao_entry_from_cache", __gp_remove_ao_entry_from_cache_oids, 1}, + {"gp_toolkit", "__gp_get_ao_entry_from_cache", __gp_get_ao_entry_from_cache_oids, 1}, + +}; +static const int num_removed_functions_dynamic = sizeof(removed_functions_dynamic) / sizeof(RemovedFunctionDynamic); + +static Oid __gp_aocsseg_oids[] = {2205}; +static Oid __gp_aocsseg_history_oids[] = {2205}; +static Oid __gp_aoseg_oids[] = {2205}; +static Oid __gp_aoseg_history_oids[] = {2205}; + +static RemovedFunctionDynamic functions_with_changed_signatures_dynamic[] = +{ + {"gp_toolkit", "__gp_aocsseg", __gp_aocsseg_oids, 1}, + {"gp_toolkit", "__gp_aocsseg_history", __gp_aocsseg_history_oids, 1}, + {"gp_toolkit", "__gp_aoseg", __gp_aoseg_oids, 1}, + {"gp_toolkit", "__gp_aoseg_history", __gp_aoseg_history_oids, 1} +}; +static const int num_functions_with_changed_signatures_dynamic = sizeof(functions_with_changed_signatures_dynamic) / sizeof(RemovedFunctionDynamic); + + +static const Oid removed_tables_static[] = +{ + 5010, /* pg_catalog.pg_partition */ + 11786, /* pg_catalog.pg_partition_columns */ + 9903, /* pg_catalog.pg_partition_encoding */ + 5011, /* pg_catalog.pg_partition_rule */ + 11782, /* pg_catalog.pg_partitions */ + 11789, /* pg_catalog.pg_partition_templates */ + 11796 /* pg_catalog.pg_stat_partition_operations */ +}; +static const int num_removed_tables_static = sizeof(removed_tables_static) / sizeof(Oid); + +/* Assuming that all of these tables live inside gp_toolkit schema */ +static char *removed_tables_dynamic[] = +{ + "gp_size_of_partition_and_indexes_disk", + "__gp_user_data_tables" +}; +static const int num_removed_tables_dynamic = sizeof(removed_tables_dynamic) / sizeof(char*); + + +static const RemovedColumnStatic removed_columns_static[] = +{ + {11636, 6}, /* pg_catalog.pg_roles.rolcatupdate */ + {11639, 5}, /* pg_catalog.pg_shadow.usecatupd */ + {11645, 5}, /* pg_catalog.pg_user.usecatupd */ + {11758, 11}, /* pg_catalog.pg_stat_replication.sent_location */ + {11758, 12}, /* pg_catalog.pg_stat_replication.write_location */ + {11758, 13}, /* pg_catalog.pg_stat_replication.flush_location */ + {11758, 14}, /* pg_catalog.pg_stat_replication.replay_location */ + {11755, 15}, /* pg_catalog.pg_stat_activity.waiting */ + {11755, 20}, /* pg_catalog.pg_stat_activity.waiting_reason */ + {11755, 23}, /* pg_catalog.pg_stat_activity.rsgqueueduration */ + {12345, 4}, /* pg_catalog.gp_distributed_log.distributed_id */ + {12339, 2}, /* pg_catalog.gp_distributed_xacts.distributed_id */ + {11764, 14}, /* pg_catalog.gp_stat_replication.flush_location */ + {11764, 15}, /* pg_catalog.gp_stat_replication.replay_location */ + {11764, 12}, /* pg_catalog.gp_stat_replication.sent_location */ + {11764, 13}, /* pg_catalog.gp_stat_replication.write_location */ + {6439, -2}, /* pg_catalog.pg_resgroupcapability.oid */ + {GpConfigHistoryRelationId, Anum_gp_configuration_history_desc}, + {ProcedureRelationId, Anum_pg_proc_protransform}, + {ProcedureRelationId, Anum_pg_proc_proisagg}, + {ProcedureRelationId, Anum_pg_proc_proiswindow}, + {ProcedureRelationId, Anum_pg_proc_prodataaccess}, + {AccessMethodRelationId, Anum_pg_am_ambeginscan}, + {AccessMethodRelationId, Anum_pg_am_ambuild}, + {AccessMethodRelationId, Anum_pg_am_ambuildempty}, + {AccessMethodRelationId, Anum_pg_am_ambulkdelete}, + {AccessMethodRelationId, Anum_pg_am_amcanbackward}, + {AccessMethodRelationId, Anum_pg_am_amcanmulticol}, + {AccessMethodRelationId, Anum_pg_am_amcanorder}, + {AccessMethodRelationId, Anum_pg_am_amcanorderbyop}, + {AccessMethodRelationId, Anum_pg_am_amcanreturn}, + {AccessMethodRelationId, Anum_pg_am_amcanunique}, + {AccessMethodRelationId, Anum_pg_am_amclusterable}, + {AccessMethodRelationId, Anum_pg_am_amcostestimate}, + {AccessMethodRelationId, Anum_pg_am_amendscan}, + {AccessMethodRelationId, Anum_pg_am_amgetbitmap}, + {AccessMethodRelationId, Anum_pg_am_amgettuple}, + {AccessMethodRelationId, Anum_pg_am_aminsert}, + {AccessMethodRelationId, Anum_pg_am_amkeytype}, + {AccessMethodRelationId, Anum_pg_am_ammarkpos}, + {AccessMethodRelationId, Anum_pg_am_amoptionalkey}, + {AccessMethodRelationId, Anum_pg_am_amoptions}, + {AccessMethodRelationId, Anum_pg_am_ampredlocks}, + {AccessMethodRelationId, Anum_pg_am_amrescan}, + {AccessMethodRelationId, Anum_pg_am_amrestrpos}, + {AccessMethodRelationId, Anum_pg_am_amsearcharray}, + {AccessMethodRelationId, Anum_pg_am_amsearchnulls}, + {AccessMethodRelationId, Anum_pg_am_amstorage}, + {AccessMethodRelationId, Anum_pg_am_amstrategies}, + {AccessMethodRelationId, Anum_pg_am_amsupport}, + {AccessMethodRelationId, Anum_pg_am_amvacuumcleanup}, + {AppendOnlyRelationId, Anum_pg_appendonly_blkdiridxid}, + {AppendOnlyRelationId, Anum_pg_appendonly_blocksize}, + {AppendOnlyRelationId, Anum_pg_appendonly_checksum}, + {AppendOnlyRelationId, Anum_pg_appendonly_columnstore}, + {AppendOnlyRelationId, Anum_pg_appendonly_compresslevel}, + {AppendOnlyRelationId, Anum_pg_appendonly_compresstype}, + {AppendOnlyRelationId, Anum_pg_appendonly_safefswritesize}, + {AppendOnlyRelationId, Anum_pg_appendonly_visimapidxid}, + {AttrDefaultRelationId, Anum_pg_attrdef_adsrc}, + {AuthIdRelationId, Anum_pg_authid_rolcatupdate}, + {RelationRelationId, Anum_pg_class_relhasoids}, + {RelationRelationId, Anum_pg_class_relhaspkey}, + {RelationRelationId, Anum_pg_class_relstorage}, + {ConstraintRelationId, Anum_pg_constraint_consrc}, + {CompressionRelationId, -2 /* oid */}, + {ExtTableRelationId, -8 /* gp_segment_id */}, + {ExtTableRelationId, -7 /* tableoid */}, + {ExtTableRelationId, -6 /* cmax */}, + {ExtTableRelationId, -5 /* xmax */}, + {ExtTableRelationId, -4 /* cmin */}, + {ExtTableRelationId, -3 /* xmin */}, + {ExtTableRelationId, -1 /* ctid*/}, + {ResQueueCapabilityRelationId, -2 /* oid */} +}; +static const int num_removed_columns_static = sizeof(removed_columns_static) / sizeof(RemovedColumnStatic); + +static const RemovedColumnDynamic removed_columns_dynamic[] = +{ + {"gp_toolkit", "gp_locks_on_resqueue", 9 /* lorwaiting */}, + {"gp_toolkit", "gp_resgroup_config", 4 /* cpu_rate_limit */}, + {"gp_toolkit", "gp_resgroup_config", 8 /* memory_auditor */}, + {"gp_toolkit", "gp_resgroup_config", 6 /* memory_shared_quota */}, + {"gp_toolkit", "gp_resgroup_config", 7 /* memory_spill_ratio */}, + {"gp_toolkit", "gp_resgroup_status", 8 /* cpu_usage */}, + {"gp_toolkit", "gp_resgroup_status", 9 /* memory_usage */}, + {"gp_toolkit", "gp_resgroup_status", 1 /* rsgname */ }, + {"gp_toolkit", "gp_resgroup_status_per_host", 4 /* cpu */ }, + {"gp_toolkit", "gp_resgroup_status_per_host", 6 /* memory_available */ }, + {"gp_toolkit", "gp_resgroup_status_per_host", 8 /* memory_quota_available */}, + {"gp_toolkit", "gp_resgroup_status_per_host", 7 /* memory_quota_used */}, + {"gp_toolkit", "gp_resgroup_status_per_host", 10 /* memory_shared_available */}, + {"gp_toolkit", "gp_resgroup_status_per_host", 9 /* memory_shared_used */}, + {"gp_toolkit", "gp_resgroup_status_per_host", 5 /* memory_used */}, + {"gp_toolkit", "gp_resgroup_status_per_host", 1 /* rsgname */}, + {"gp_toolkit", "gp_resgroup_status_per_segment", 5 /* cpu */}, + {"gp_toolkit", "gp_resgroup_status_per_segment", 3 /* hostname */}, + {"gp_toolkit", "gp_resgroup_status_per_segment", 7 /* memory_available */}, + {"gp_toolkit", "gp_resgroup_status_per_segment", 9 /* memory_quota_available */}, + {"gp_toolkit", "gp_resgroup_status_per_segment", 8 /* memory_quota_used */}, + {"gp_toolkit", "gp_resgroup_status_per_segment", 11 /* memory_shared_available */}, + {"gp_toolkit", "gp_resgroup_status_per_segment", 10 /* memory_shared_used */}, + {"gp_toolkit", "gp_resgroup_status_per_segment", 6 /* memory_used */}, + {"gp_toolkit", "gp_resgroup_status_per_segment", 1 /* rsgname */}, + {"gp_toolkit", "__gp_user_tables", 9 /* autrelstorage */ }, + {"gp_toolkit", "__gp_user_data_tables_readable", 9 /* autrelstorage */ }, + {"information_schema", "routines", 65 /* result_cast_character_set_name */}, + {"information_schema", "routines", 43 /* sql_data_access */}, + {"gp_toolkit", "__gp_log_master_ext", -8 /* gp_segment_id */}, + {"gp_toolkit", "__gp_log_master_ext", -7 /* tableoid */}, + {"gp_toolkit", "__gp_log_master_ext", -6 /* cmax */}, + {"gp_toolkit", "__gp_log_master_ext", -5 /* xmax */}, + {"gp_toolkit", "__gp_log_master_ext", -4 /* cmin */}, + {"gp_toolkit", "__gp_log_master_ext", -3 /* xmin */}, + {"gp_toolkit", "__gp_log_master_ext", -1 /* ctid */} +}; +static const int num_removed_columns_dynamic = sizeof(removed_columns_dynamic) / sizeof(RemovedColumnDynamic); + + +static const char* removed_types_gp_toolkit[] = +{ + "gp_size_of_partition_and_indexes_disk", + "__gp_user_data_tables" +}; +static const int num_removed_types_gp_toolkit = sizeof(removed_types_gp_toolkit) / sizeof (char*); + +/* + * Helper function like get_view_query, but for materialized views. + * It works similarly to ExecRefreshMatView. + */ +static Query * +get_matview_query(Relation matview) +{ + RewriteRule *rule; + List *actions; + + Assert(matview->rd_rel->relkind == RELKIND_MATVIEW); + + if (matview->rd_rel->relhasrules == false || + matview->rd_rules->numLocks < 1) + elog(ERROR, + "materialized view \"%s\" is missing rewrite information", + RelationGetRelationName(matview)); + + if (matview->rd_rules->numLocks > 1) + elog(ERROR, + "materialized view \"%s\" has too many rules", + RelationGetRelationName(matview)); + + rule = matview->rd_rules->rules[0]; + if (rule->event != CMD_SELECT || !(rule->isInstead)) + elog(ERROR, + "the rule for materialized view \"%s\" is not a SELECT INSTEAD OF rule", + RelationGetRelationName(matview)); + + actions = rule->actions; + if (list_length(actions) != 1) + elog(ERROR, + "the rule for materialized view \"%s\" is not a single action", + RelationGetRelationName(matview)); + + return (Query *) linitial(rule->actions); +} Datum set_next_pg_type_oid(PG_FUNCTION_ARGS) @@ -311,6 +607,11 @@ view_has_anyarray_casts(PG_FUNCTION_ARGS) viewquery = get_view_query(rel); found = query_tree_walker(viewquery, check_node_anyarray_walker, NULL, 0); } + else if (rel->rd_rel->relkind == RELKIND_MATVIEW) + { + viewquery = get_matview_query(rel); + found = query_tree_walker(viewquery, check_node_anyarray_walker, NULL, 0); + } else found = false; @@ -368,6 +669,11 @@ view_has_unknown_casts(PG_FUNCTION_ARGS) viewquery = get_view_query(rel); found = query_tree_walker(viewquery, check_node_unknown_walker, NULL, 0); } + else if(rel->rd_rel->relkind == RELKIND_MATVIEW) + { + viewquery = get_matview_query(rel); + found = query_tree_walker(viewquery, check_node_unknown_walker, NULL, 0); + } else found = false; @@ -432,6 +738,12 @@ view_has_removed_operators(PG_FUNCTION_ARGS) viewquery = get_view_query(rel); found = query_tree_walker(viewquery, check_node_removed_operators_walker, NULL, 0); } + else if(rel->rd_rel->relkind == RELKIND_MATVIEW) + { + viewquery = get_matview_query(rel); + found = query_tree_walker(viewquery, check_node_removed_operators_walker, NULL, 0); + } + else found = false; @@ -482,6 +794,12 @@ view_has_removed_functions(PG_FUNCTION_ARGS) viewquery = get_view_query(rel); found = query_tree_walker(viewquery, check_node_removed_functions_walker, NULL, 0); } + else if(rel->rd_rel->relkind == RELKIND_MATVIEW) + { + viewquery = get_matview_query(rel); + found = query_tree_walker(viewquery, check_node_removed_functions_walker, NULL, 0); + } + else found = false; @@ -490,6 +808,24 @@ view_has_removed_functions(PG_FUNCTION_ARGS) PG_RETURN_BOOL(found); } +/* Helper functions to get object OIDs by their signatures */ +Oid +get_function(const char *name, const Oid *args, int args_count, Oid namespace) +{ + return GetSysCacheOid3(PROCNAMEARGSNSP, + PointerGetDatum(name), + PointerGetDatum(buildoidvector(args, args_count)), + ObjectIdGetDatum(namespace)); +} + +Oid +get_type(const char *name, Oid namespace) +{ + return GetSysCacheOid2(TYPENAMENSP, + PointerGetDatum(name), + ObjectIdGetDatum(namespace)); +} + static bool check_node_removed_functions_walker(Node *node, void *context) { @@ -500,11 +836,9 @@ check_node_removed_functions_walker(Node *node, void *context) if (IsA(node, FuncExpr)) { + Oid schema_oid; Oid func_oid = ((FuncExpr *)node)->funcid; - if (func_oid == 12512 || // gp_toolkit.__gp_get_ao_entry_from_cache - func_oid == 12511 || // gp_toolkit.__gp_remove_ao_entry_from_cache - func_oid == 12498 || // gp_toolkit.pg_resgroup_check_move_query - func_oid == 7188 || // pg_catalog.bmbeginscan + if (func_oid == 7188 || // pg_catalog.bmbeginscan func_oid == 7193 || // pg_catalog.bmbuild func_oid == 7011 || // pg_catalog.bmbuildempty func_oid == 7194 || // pg_catalog.bmbulkdelete @@ -633,6 +967,22 @@ check_node_removed_functions_walker(Node *node, void *context) func_oid == 3097) // pg_catalog.varchar_transform return true; + for (int i = 0; i < num_removed_functions_dynamic; ++i) + { + schema_oid = GetSysCacheOid(NAMESPACENAME, + CStringGetDatum(removed_functions_dynamic[i].pronamespace), + 0, 0, 0); + + if (OidIsValid(schema_oid)) + { + if (func_oid == get_function(removed_functions_dynamic[i].name, + removed_functions_dynamic[i].args, + removed_functions_dynamic[i].args_count, + schema_oid)) + return true; + } + } + return false; } else if (IsA(node, Query)) @@ -662,6 +1012,11 @@ view_has_removed_types(PG_FUNCTION_ARGS) viewquery = get_view_query(rel); found = query_tree_walker(viewquery, check_node_removed_types_walker, NULL, 0); } + else if(rel->rd_rel->relkind == RELKIND_MATVIEW) + { + viewquery = get_matview_query(rel); + found = query_tree_walker(viewquery, check_node_removed_types_walker, NULL, 0); + } else found = false; @@ -680,15 +1035,14 @@ check_node_removed_types_walker(Node *node, void *context) if (IsA(node, Var) || IsA(node, Const)) { + Oid gp_toolkit_oid; Oid type_oid; if IsA(node, Var) type_oid = ((Var *)node)->vartype; else type_oid = ((Const *)node)->consttype; - if (type_oid == 12475 || // gp_toolkit.gp_size_of_partition_and_indexes_disk - type_oid == 12366 || // gp_toolkit.__gp_user_data_tables - type_oid == 1023 || // pg_catalog._abstime + if (type_oid == 1023 || // pg_catalog._abstime type_oid == 702 || // pg_catalog.abstime type_oid == 11612 || // pg_catalog.pg_partition type_oid == 11787 || // pg_catalog.pg_partition_columns @@ -704,6 +1058,20 @@ check_node_removed_types_walker(Node *node, void *context) type_oid == 704) // pg_catalog.tinterval return true; + gp_toolkit_oid = GetSysCacheOid(NAMESPACENAME, + CStringGetDatum("gp_toolkit"), + 0, 0, 0); + + if (OidIsValid(gp_toolkit_oid)) + { + for (int i = 0; i < num_removed_types_gp_toolkit; i++) + { + if (type_oid == get_type(removed_types_gp_toolkit[i], + gp_toolkit_oid)) + return true; + } + } + return false; } else if (IsA(node, Query)) @@ -732,6 +1100,11 @@ view_has_changed_function_signatures(PG_FUNCTION_ARGS) viewquery = get_view_query(rel); found = query_tree_walker(viewquery, check_node_changed_function_signatures_walker, NULL, 0); } + else if(rel->rd_rel->relkind == RELKIND_MATVIEW) + { + viewquery = get_matview_query(rel); + found = query_tree_walker(viewquery, check_node_changed_function_signatures_walker, NULL, 0); + } else found = false; @@ -750,12 +1123,9 @@ check_node_changed_function_signatures_walker(Node *node, void *context) if (IsA(node, FuncExpr)) { + Oid schema_oid; Oid func_oid = ((FuncExpr *)node)->funcid; - if (func_oid == 12501 || // gp_toolkit.__gp_aocsseg - func_oid == 12502 || // gp_toolkit.__gp_aocsseg_history - func_oid == 12506 || // gp_toolkit.__gp_aoseg - func_oid == 12500 || // gp_toolkit.__gp_aoseg_history - func_oid == 2335 || // pg_catalog.array_agg + if (func_oid == 2335 || // pg_catalog.array_agg func_oid == 2334 || // pg_catalog.array_agg_finalfn func_oid == 2333 || // pg_catalog.array_agg_transfn func_oid == 3484 || // pg_catalog.gin_consistent_jsonb @@ -812,6 +1182,22 @@ check_node_changed_function_signatures_walker(Node *node, void *context) func_oid == 3493) // pg_catalog.to_regtype return true; + for (int i = 0; i < num_functions_with_changed_signatures_dynamic; i++) + { + schema_oid = GetSysCacheOid(NAMESPACENAME, + CStringGetDatum(functions_with_changed_signatures_dynamic[i].pronamespace), + 0, 0, 0); + + if (OidIsValid(schema_oid)) + { + if (func_oid == get_function(functions_with_changed_signatures_dynamic[i].name, + functions_with_changed_signatures_dynamic[i].args, + functions_with_changed_signatures_dynamic[i].args_count, + schema_oid)) + return true; + } + } + return false; } else if (IsA(node, Query)) @@ -823,3 +1209,419 @@ check_node_changed_function_signatures_walker(Node *node, void *context) return expression_tree_walker(node, check_node_changed_function_signatures_walker, context); } + +static void +report_removed_table(RemovedTablesWalkerContext *context, Oid reloid) +{ + Oid already_reported_table; + ListCell *lc; + + /* + * Go thorogh already reported tables to remove + * duplicates + */ + foreach (lc, context->removedTables) + { + already_reported_table = lfirst_oid(lc); + if (reloid == already_reported_table) + return; + } + + context->removedTables = lappend_oid(context->removedTables, reloid); +} + +static void +check_and_report_removed_table(RemovedTablesWalkerContext *context, Oid reloid) +{ + int i; + Oid gp_toolkit_oid; + + for (i = 0; i < num_removed_tables_static; i++) + { + if (reloid == removed_tables_static[i]) + report_removed_table(context, reloid); + } + + gp_toolkit_oid = GetSysCacheOid(NAMESPACENAME, + CStringGetDatum("gp_toolkit"), + 0, 0, 0); + + if (OidIsValid(gp_toolkit_oid)) + { + for (i = 0; i < num_removed_tables_dynamic; i++) + { + if (reloid == get_relname_relid(removed_tables_dynamic[i], gp_toolkit_oid)) + report_removed_table(context, reloid); + + } + } +} + +static bool +check_node_removed_tables_walker(Node *node, void *context) +{ + Assert(context != NULL); + + if (node == NULL) + return false; + + if (IsA(node, RangeTblEntry)) + { + RangeTblEntry *rte = (RangeTblEntry *) node; + if (rte->rtekind == RTE_RELATION) + check_and_report_removed_table(context, rte->relid); + return false; + } + else if(IsA(node, Query)) + { + /* + * Recurse into (sub)queries to look for removed tables. + */ + return query_tree_walker((Query *) node, + check_node_removed_tables_walker, + context, + QTW_EXAMINE_RTES); + } + + /* + * This ensures that we look for removed tables embedded inside + * expressions (e.g. CTEs, sublinks etc.) which can contain range tables. + */ + return expression_tree_walker(node, check_node_removed_tables_walker, context); +} + +Datum +get_removed_tables(PG_FUNCTION_ARGS) +{ + Oid view_oid = PG_GETARG_OID(0); + Relation rel; + StringInfoData buf; + Oid reported_table; + Oid relnamespace; + ListCell *lc; + char *nspname; + char *relname; + Query *viewquery; + RemovedTablesWalkerContext context; + + rel = try_relation_open(view_oid, AccessShareLock, false); + if (!RelationIsValid(rel)) + elog(ERROR, "Could not open relation file for relation oid %u", view_oid); + + context.removedTables = NIL; + if (rel->rd_rel->relkind == RELKIND_VIEW) + { + viewquery = get_view_query(rel); + check_node_removed_tables_walker((Node *) viewquery, &context); + } + else if (rel->rd_rel->relkind == RELKIND_MATVIEW) + { + viewquery = get_matview_query(rel); + check_node_removed_tables_walker((Node *) viewquery, &context); + } + + relation_close(rel, AccessShareLock); + + /* + * Make a single formatted string, listing all unique removed tables. + * It will be displayed to the user. + */ + initStringInfo(&buf); + foreach (lc, context.removedTables) + { + reported_table = lfirst_oid(lc); + relname = get_rel_name(reported_table); + if (!relname) + elog(ERROR, "cache lookup failed for relation %u", reported_table); + + relnamespace = get_rel_namespace(reported_table); + if (!OidIsValid(relnamespace)) + elog(ERROR, "cache lookup failed for relation %u", reported_table); + + nspname = get_namespace_name(relnamespace); + if (!nspname) + elog(ERROR, "cache lookup failed for namespace %u", relnamespace); + + appendStringInfo(&buf, "\t%s.%s\n", nspname, relname); + } + + PG_RETURN_TEXT_P(cstring_to_text(buf.data)); +} + + +static void +report_removed_column(RemovedColumnsWalkerContext *context, Oid reloid, int attnum) +{ + ListCell *lc; + ReportedColumn *already_reported_column; + ReportedColumn *column; + + /* + * Go through already reported columns in a nested loop manner + * to remove duplicates. This should be fast enough, because the + * number of removed columns is not that large + * (currently, 110), and most view wouldn't have all of them. + * But it is hard to tell without user data. + */ + foreach (lc, context->removedColumns) + { + already_reported_column = lfirst(lc); + if (reloid == already_reported_column->attrelid && + attnum == already_reported_column->attnum && + context->inside_whole_row_reference == already_reported_column->comes_from_whole_row_reference) + return; + } + + column = palloc(sizeof(ReportedColumn)); + column->attrelid = reloid; + column->attnum = attnum; + column->comes_from_whole_row_reference = context->inside_whole_row_reference; + + context->removedColumns = lappend(context->removedColumns, column); +} + +static bool +check_and_report_removed_columns(RemovedColumnsWalkerContext *context, Oid reloid, int attnum) +{ + int i; + Oid schema_oid; + int removed_column_attnum; + + for (i = 0; i < num_removed_columns_static; i++) + { + removed_column_attnum = removed_columns_static[i].attnum; + if (reloid == removed_columns_static[i].reloid && + (attnum == removed_column_attnum || attnum == InvalidAttrNumber)) + report_removed_column(context, reloid, removed_column_attnum); + } + + for (i = 0; i < num_removed_columns_dynamic; i++) + { + schema_oid = GetSysCacheOid(NAMESPACENAME, + CStringGetDatum(removed_columns_dynamic[i].relnamespace), + 0, 0, 0); + + if (OidIsValid(schema_oid)) + { + removed_column_attnum = removed_columns_dynamic[i].attnum; + if (reloid == get_relname_relid(removed_columns_dynamic[i].relname, schema_oid) && + (attnum == removed_column_attnum || attnum == InvalidAttrNumber)) + report_removed_column(context, reloid, removed_column_attnum); + } + } + + return false; +} + +/* + * Check whether a query contains a reference to a removed column, or a whole + * row reference to a table with removed columns. + * + * The first case will always cause pg_upgrade to fail, while the second is more + * complicated. Whole row references by themselves won't cause upgrade to fail, + * because row type will be taken from the target cluster. For example, + * the following view won't cause any troubles: + * + * CREATE VIEW view1 AS SELECT pg_class FROM pg_class; + * + * However, there could be another view that references specific columns from the + * previous one: + * + * CREATE VIEW view2 AS SELECT (pg_class).relhasoids FROM view1; + * + * and this columns may be indeed absent in the new version. Because of that, + * conservatively report any whole row reference to any table with removed + * columns. + */ +bool +check_node_removed_columns_walker(Node *node, RemovedColumnsWalkerContext *context) +{ + Assert(context != NULL); + + if (node == NULL) + { + return false; + } + + if (IsA(node, Var)) + { + Var *var; + List *rtable; + RangeTblEntry *rte; + bool save_inside_whole_row_reference; + + var = (Var *) node; + if (var->varlevelsup >= list_length(context->rtableStack)) + elog(ERROR, "invalid varlevelsup %d", var->varlevelsup); + + rtable = (List *) list_nth(context->rtableStack, var->varlevelsup); + if (var->varno <= 0 || var->varno > list_length(rtable)) + elog(ERROR, "invalid varno %d", var->varno); + + save_inside_whole_row_reference = context->inside_whole_row_reference; + if (var->varattno == InvalidAttrNumber) + context->inside_whole_row_reference = true; + + rte = (RangeTblEntry *) list_nth(rtable, var->varno - 1); + if (rte->rtekind == RTE_RELATION) + { + /* + * It's a plain relation, simply check that Var doesn't reference + * removed column(s) + */ + check_and_report_removed_columns(context, rte->relid, var->varattno); + } + else if (rte->rtekind == RTE_JOIN) + { + /* + * It's a join entry, we need to recursively go through + * the RTE tree to get to the source entry for this attribute. + */ + int i; + List *save_rtables = context->rtableStack; + + context->rtableStack = list_copy_tail(context->rtableStack, + var->varlevelsup); + + if (var->varattno == InvalidAttrNumber) + { + /* + * For a whole table reference, check every column of the RTE + */ + for (i = 0; i < list_length(rte->joinaliasvars); i++) + check_node_removed_columns_walker((Node *) list_nth(rte->joinaliasvars, i), + context); + } + else + { + /* Regular attribute */ + if (var->varattno <= 0 || + var->varattno > list_length(rte->joinaliasvars)) + elog(ERROR, "invalid varattno %d", var->varattno); + + check_node_removed_columns_walker((Node *) list_nth(rte->joinaliasvars, + var->varattno - 1), + context); + } + list_free(context->rtableStack); + context->rtableStack = save_rtables; + } + + /* + * Don't do anything special for other RTE kinds. Most notably, RTE_SUBQUERY, + * because subqueries will be handled when we recurse into them. + */ + context->inside_whole_row_reference = save_inside_whole_row_reference; + return false; + } + else if (IsA(node, Query)) + { + /* + * Recurse into (sub)queries to search for removed columns. + * + * Pass QTW_IGNORE_JOINALIASES to avoid recursing into a join RTE's + * joinaliasvars, as they always contain every unique column from + * the joined tables. Meaning that without this flag, each join with + * a table with removed columns would trigger this check. + * For example: + * + * CREATE VIEW err AS SELECT jn.relname FROM (pg_class JOIN pg_namespace ON true) jn; + * + * will be erroneously reported as referencing removed columns. Legit cases like: + * + * CREATE VIEW rte_join AS SELECT jn.relhasoids FROM (pg_class JOIN pg_namespace ON true) jn; + * + * are handled when processing Var nodes, for them (rte->rtekind == RTE_JOIN) + */ + Query *query = (Query *) node; + context->rtableStack = lcons(query->rtable, context->rtableStack); + query_tree_walker(query, + check_node_removed_columns_walker, + context, + QTW_IGNORE_JOINALIASES); + context->rtableStack = list_delete_first(context->rtableStack); + return false; + } + + /* + * This ensures we look at all expressions, including entities that contain + * subqueries (such as CTEs and sublinks) + */ + return expression_tree_walker(node, check_node_removed_columns_walker, context); +} + +Datum +get_removed_columns(PG_FUNCTION_ARGS) +{ + Oid view_oid = PG_GETARG_OID(0); + Relation rel; + StringInfoData buf; + Oid relnamespace; + ListCell *lc; + char *nspname; + char *relname; + char *attname; + char *comes_from_whole_row_reference_string; + ReportedColumn *removed_column; + Query *viewquery; + RemovedColumnsWalkerContext context; + + rel = try_relation_open(view_oid, AccessShareLock, false); + if (!RelationIsValid(rel)) + elog(ERROR, "Could not open relation file for relation oid %u", view_oid); + + context.rtableStack = NIL; + context.removedColumns = NIL; + context.inside_whole_row_reference = false; + if (rel->rd_rel->relkind == RELKIND_VIEW) + { + viewquery = get_view_query(rel); + check_node_removed_columns_walker((Node *) viewquery, &context); + } + else if (rel->rd_rel->relkind == RELKIND_MATVIEW) + { + viewquery = get_matview_query(rel); + check_node_removed_columns_walker((Node *) viewquery, &context); + } + + relation_close(rel, AccessShareLock); + + /* + * Make a single formatted string, listing all unique removed columns. + * It will be displayed to the user. + */ + initStringInfo(&buf); + foreach (lc, context.removedColumns) + { + removed_column = lfirst(lc); + relname = get_rel_name(removed_column->attrelid); + if (!relname) + elog(ERROR, "cache lookup failed for relation %u", removed_column->attrelid); + + relnamespace = get_rel_namespace(removed_column->attrelid); + if (!OidIsValid(relnamespace)) + elog(ERROR, "cache lookup failed for relation %u", removed_column->attrelid); + + nspname = get_namespace_name(relnamespace); + if (!nspname) + elog(ERROR, "cache lookup failed for namespace %u", relnamespace); + + attname = get_attname(removed_column->attrelid, removed_column->attnum); + if (!attname) + elog(ERROR, "cache lookup failed for attribute %d for relation %u", + removed_column->attnum, removed_column->attrelid); + + comes_from_whole_row_reference_string = ""; + if (removed_column->comes_from_whole_row_reference) + comes_from_whole_row_reference_string = "(comes from a whole row reference)"; + + appendStringInfo(&buf, "\t%s.%s.%s %s\n", nspname, relname, attname, + comes_from_whole_row_reference_string); + + pfree(relname); + pfree(nspname); + pfree(attname); + } + + PG_RETURN_TEXT_P(cstring_to_text(buf.data)); +} diff --git a/gpAux/Makefile b/gpAux/Makefile index f4624ae82845..a690e5f41d27 100644 --- a/gpAux/Makefile +++ b/gpAux/Makefile @@ -93,7 +93,7 @@ ISCONFIG=$(GPPGDIR)/GNUmakefile ## On these platforms, we do the full build including the server. On other ## platforms, we do a client-only build. ## -SERVER_PLATFORMS=rhel7_x86_64 rhel6_x86_64 rhel8_x86_64 rocky8_x86_64 linux_x86_64 +SERVER_PLATFORMS=rhel7_x86_64 rhel6_x86_64 rhel8_x86_64 rocky8_x86_64 rocky9_x86_64 linux_x86_64 #--------------------------------------------------------------------- # Compiler options @@ -132,6 +132,7 @@ astra1.8_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi --enable-mapreduce rhel7_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs rhel7_ppc64le_CONFIGFLAGS=--enable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs rhel8_x86_64_CONFIGFLAGS=--disable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs +rhel9_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs rocky8_x86_64_CONFIGFLAGS=--with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-pythonsrc-ext --with-uuid=e2fs rocky9_x86_64_CONFIGFLAGS=--disable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs linux_x86_64_CONFIGFLAGS=${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs @@ -141,6 +142,7 @@ ubuntu22.04_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi --enable-mapredu ubuntu24.04_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs sles12_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs redos7.3_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs +redos8.0_x86_64_CONFIGFLAGS=--enable-gpperfmon --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --with-libxml --with-pythonsrc-ext --with-uuid=e2fs BLD_CONFIGFLAGS=$($(BLD_ARCH)_CONFIGFLAGS) CONFIGFLAGS=$(strip $(BLD_CONFIGFLAGS) --with-pgport=$(DEFPORT) $(BLD_DEPLOYMENT_SETTING)) @@ -354,6 +356,7 @@ PACKAGE_NAME := $(shell grep '^Package:' debian/control | head -1 | awk '{print MAINTAINER := $(shell grep '^Maintainer:' debian/control | sed 's/Maintainer: //') DATE_RFC := $(shell date -R) ARTIFACTS_DIR := $(CURDIR)/../Package +RPM_TOPDIR := $(CURDIR)/../RPM ../VERSION : @echo "Update $@" @@ -363,21 +366,21 @@ ARTIFACTS_DIR := $(CURDIR)/../Package version-vars : ../VERSION $(eval FULL_VERSION := $(shell [ -f ../VERSION ] && perl -pe 's, ,-,g' ../VERSION)) $(eval PACKAGE_VERSION := $(shell [ -f ../VERSION ] && perl -pe 's, .*,,g' ../VERSION)) - $(eval DISTRO_CODENAME := $(shell lsb_release -sc)) + $(eval VERSION_CODENAME := $(shell . /etc/os-release && echo $$VERSION_CODENAME)) $(eval IS_RELEASE := $(if $(findstring +dev,$(PACKAGE_VERSION)),no,yes)) $(eval BUILD_TYPE := $(if $(filter yes,$(IS_RELEASE)),Release build,Development build)) version-info : version-vars @echo "PACKAGE_VERSION: $(PACKAGE_VERSION)" @echo "FULL_VERSION: $(FULL_VERSION)" - @echo "DISTRO_CODENAME: $(DISTRO_CODENAME)" + @echo "VERSION_CODENAME: $(VERSION_CODENAME)" @echo "IS_RELEASE: $(IS_RELEASE)" @echo "BUILD_TYPE: $(BUILD_TYPE)" # Generate package control files changelog : debian/changelog debian/changelog : version-vars - @echo "$(PACKAGE_NAME) ($(PACKAGE_VERSION)) $(DISTRO_CODENAME); urgency=low" > $@ + @echo "$(PACKAGE_NAME) ($(PACKAGE_VERSION)) $(VERSION_CODENAME); urgency=low" > $@ @echo "" >> $@ @echo " * $(BUILD_TYPE)" >> $@ @echo "" >> $@ @@ -404,7 +407,25 @@ pkg-deb : debian/changelog debian/install -o -name "*.changes" \) \ -exec mv -f {} $(ARTIFACTS_DIR)/ \; -.PHONY: pkg pkg-deb changelog version-vars version-info +# Build RPM package +pkg-rpm : GPROOT = /opt/greengagedb +pkg-rpm : GPDIR = $(PACKAGE_NAME) +pkg-rpm : version-vars + @echo "Building RPM with GPROOT=$(GPROOT), GPDIR=$(GPDIR), PACKAGE_NAME=$(PACKAGE_NAME)" + @mkdir -p $(RPM_TOPDIR)/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} + rpmbuild -bb \ + --define "_topdir $(RPM_TOPDIR)" \ + --define "gpdb_version $(subst +,.,$(subst -,.,$(PACKAGE_VERSION)))" \ + --define "gpdb_release 1" \ + --define "gproot $(GPROOT)" \ + --define "gpdir $(GPDIR)" \ + --define "sourcedir $(CURDIR)/.." \ + rpm/greengage6.spec + @mkdir -p $(ARTIFACTS_DIR) + @find $(RPM_TOPDIR)/RPMS -name "*.rpm" \ + -exec mv -f {} $(ARTIFACTS_DIR)/ \; + +.PHONY: pkg pkg-deb pkg-rpm changelog version-vars version-info #--------------------------------------------------------------------- # clientTools @@ -468,6 +489,7 @@ rhel6_x86_64_MPP_ARCH=RHEL6-x86_64 rhel7_x86_64_MPP_ARCH=RHEL7-x86_64 rhel8_x86_64_MPP_ARCH=RHEL8-x86_64 rocky8_x86_64_MPP_ARCH=ROCKY8-x86_64 +rocky9_x86_64_MPP_ARCH=ROCKY9-x86_64 ifneq "$($(BLD_ARCH)_MPP_ARCH)" "" export MPP_ARCH=$($(BLD_ARCH)_MPP_ARCH) else diff --git a/gpAux/Makefile.global b/gpAux/Makefile.global index 88c3560d5199..59ab99da360e 100644 --- a/gpAux/Makefile.global +++ b/gpAux/Makefile.global @@ -46,7 +46,10 @@ export NO_M64=1 rhel6_x86_64_BLD_CFLAGS=-m64 -gdwarf-2 -gstrict-dwarf rhel7_x86_64_BLD_CFLAGS=-m64 rhel8_x86_64_BLD_CFLAGS=-m64 +rhel9_x86_64_BLD_CFLAGS=-m64 rhel7_ppc64le_BLD_CFLAGS=-m64 -fasynchronous-unwind-tables -fsigned-char +rocky8_x86_64_BLD_CFLAGS=-m64 +rocky9_x86_64_BLD_CFLAGS=-m64 altlinux8.4_x86_64_BLD_CFLAGS=-m64 altlinux10.2_x86_64_BLD_CFLAGS=-m64 astra1.7_x86_64_BLD_CFLAGS=-m64 @@ -54,6 +57,7 @@ astra1.8_x86_64_BLD_CFLAGS=-m64 ubuntu22.04_x86_64_BLD_CFLAGS=-m64 ubuntu24.04_x86_64_BLD_CFLAGS=-m64 redos7.3_x86_64_BLD_CFLAGS=-m64 +redos8.0_x86_64_BLD_CFLAGS=-m64 BLD_CFLAGS=$($(BLD_ARCH)_BLD_CFLAGS) BLD_LDFLAGS=$($(BLD_ARCH)_BLD_LDFLAGS) diff --git a/gpAux/README.package.md b/gpAux/README.package.md index b71adaae2ee5..f37c617c3a39 100644 --- a/gpAux/README.package.md +++ b/gpAux/README.package.md @@ -2,9 +2,9 @@ ## Overview -This documentation describes the Debian packaging system for Greengage Database -located in the `gpAux/` subdirectory. The system builds Debian packages using a -custom Makefile and `debian/rules` file. +This documentation describes the packaging system for Greengage Database +located in the `gpAux/` subdirectory: Debian packages (`pkg-deb`, using +`debian/rules`) and RPM packages (`pkg-rpm`, using `rpm/greengage6.spec`). ## Location and Structure @@ -22,6 +22,9 @@ The main components are: - `debian/control` - Package metadata and dependencies - `debian/copyright` - Copyright information - `debian/lintian-overrides` - Lintian warning overrides +- `rpm/greengage6.spec` - RPM spec file defining package metadata and build steps +- `rpm/greengage6.rpmlintrc` - rpmlint warning overrides, analogous to + `debian/lintian-overrides` ## Key Components @@ -30,13 +33,15 @@ The main components are: 1. **Version Management**: - `../VERSION`: Generates version file using `../getversion` - `version-vars`: Sets build variables (`FULL_VERSION`, `PACKAGE_VERSION`, - `DISTRO_CODENAME`, `IS_RELEASE`, `BUILD_TYPE`) from `../VERSION` file + `VERSION_CODENAME`, `IS_RELEASE`, `BUILD_TYPE`) from `../VERSION` file - `version-info`: Displays version information for debugging 2. **Packaging Targets**: - `pkg`: Default target (aliases to `pkg-deb`) - `pkg-deb`: Builds Debian package, preserves environment variables, and collects artifacts (`.deb`, `.ddeb`, `.build`, `.buildinfo`, `.changes`) + - `pkg-rpm`: Builds RPM package via `rpmbuild` and collects the + resulting `.rpm` artifact - `changelog`: Generates `debian/changelog` (not stored in repo) - `debian/install`: Creates installation manifest (not stored in repo) @@ -60,9 +65,31 @@ The `debian/rules` file uses debhelper (dh) with custom overrides: - Injects Python dependencies via `-VpythonRequires`, `-VpythonConflicts` options in `dh_gencontrol` +### RPM Spec File + +The `rpm/greengage6.spec` file defines the RPM build: + +1. **Distribution-specific Dependencies**: + - `python3.11` and `python3.11-pip` required on RHEL/Rocky 9 and newer + - `python2`, `python2-pip`, `python3`, `python3-pip` required on + RHEL/Rocky 8 and older + +2. **Build Process**: + - `%install` invokes the project's `make dist` target with `DESTDIR` + set to `%{buildroot}` + - Excludes the entire install prefix from `brp-mangle-shebangs` via + `__brp_mangle_shebangs_exclude_from` — scripts intentionally use + `#!/usr/bin/env python` and `#!/usr/bin/env perl`; rewriting shebangs + would diverge package contents from the source and mask issues + - Strips debug info from `*.so` files via `strip --strip-debug` to + remove BUILDROOT paths embedded by the compiler (e.g., PyGreSQL's + `_pg.so`), which would otherwise fail `check-buildroot` validation + - Removes the executable bit from Python/Perl/Bash files without a + shebang and from `*.md` files, mirroring `debian/rules` `dh_fixperms` + ## Usage -### Building the Package +### Building the Debian Package From the project root directory, run: @@ -70,12 +97,21 @@ From the project root directory, run: make -C ./gpAux pkg-deb ``` +### Building the RPM Package + +From the project root directory, run: + +```bash +make -C ./gpAux pkg-rpm +``` + ### Custom Installation Paths To customize installation paths, set environment variables: ```bash make -C ./gpAux pkg-deb GPROOT=/custom/path GPDIR=custom_dir +make -C ./gpAux pkg-rpm GPROOT=/custom/path GPDIR=custom_dir ``` ### Environment Variables @@ -84,10 +120,15 @@ make -C ./gpAux pkg-deb GPROOT=/custom/path GPDIR=custom_dir - `GPDIR`: Subdirectory under `GPROOT` (default: same as `PACKAGE_NAME`) - `PACKAGE_NAME`: Package name (default: from `Package:` in `debian/control`, e.g., `greengage6`) -- `ARTIFACTS_DIR`: Directory for artifacts (default: `$(CURDIR)/../Package`) +- `ARTIFACTS_DIR`: Directory for artifacts (default: `$(CURDIR)/../Package`, + shared between `.deb` and `.rpm` outputs) +- `RPM_TOPDIR`: `rpmbuild` working directory used only by `pkg-rpm` + default: `$(CURDIR)/../RPM`, kept after the build for inspection ## Build Process Details +### Debian Package + 1. **Version Generation**: - Runs `../getversion` to create `../VERSION` - Processes version string into `FULL_VERSION` and `PACKAGE_VERSION` @@ -104,11 +145,50 @@ make -C ./gpAux pkg-deb GPROOT=/custom/path GPDIR=custom_dir - Uses `make dist` for installation into `debian/tmp/$(PACKAGE_NAME)` - Generates file manifest in `debian/install` +### RPM Package + +1. **Version Generation**: + - Reuses `version-vars` (same as Debian) to derive `PACKAGE_VERSION`, + normalized for RPM (`+`/`-` replaced with `.`) and passed to + `rpmbuild` via `--define gpdb_version` + +2. **Package Building**: + - Sets up `rpmbuild` tree under `RPM_TOPDIR` + - Runs `rpmbuild -bb rpm/greengage6.spec` with `gproot`, `gpdir`, and + `sourcedir` passed via `--define` + - Collects the resulting `.rpm` into `ARTIFACTS_DIR` + +3. **Installation**: + - Uses `make dist` inside `%install`, installing into `%{buildroot}` + +## Build Dependencies + +Both `pkg-deb` and `pkg-rpm` assume that all build dependencies and the +GPDB build itself (`make dist`) are already satisfied/buildable on the +host — neither target builds GPDB from scratch; they package an already +configured and built source tree. + +To get a complete environment with all dependencies installed, use the +provided Docker images instead of installing dependencies manually: + +- `ci/Dockerfile.ubuntu` - Ubuntu 22.04/24.04 +- `ci/Dockerfile.rockylinux` - Rocky Linux 8/9 + +See [ci/readme.md](../ci/readme.md) for instructions on building and +using these images. To install dependencies directly on a host without +Docker, see `README.linux.md` together with `README.ubuntu.bash` (Ubuntu) +or `README.Rhel-Rocky.bash` (RHEL/Rocky Linux). + ## Dependencies -Package dependencies and conflicts are defined in `debian/control`. -Python dependencies and conflicts are dynamically detected in `debian/rules` and -injected via `${pythonRequires}` and `${pythonConflicts}` substitution variables. +Package dependencies and conflicts for the Debian package are defined in +`debian/control`. Python dependencies and conflicts are dynamically +detected in `debian/rules` and injected via `${pythonRequires}` and +`${pythonConflicts}` substitution variables. + +Package dependencies for the RPM package are defined directly in +`rpm/greengage6.spec` via `Requires:`/`Conflicts:` tags, including a +conditional on `%{?rhel}` for the Python version requirement. ## Maintenance @@ -120,6 +200,10 @@ Edit `debian/control` to update: - Maintainer information - General dependencies +Edit `rpm/greengage6.spec` to update the equivalent metadata for the +RPM package (`Summary`, `License`, `URL`, `Requires`, `Conflicts`, +`%description`). + ### Adding Distribution Support Modify distribution detection in `debian/rules`: @@ -138,8 +222,25 @@ else endif ``` +For RPM, add distribution-specific conditionals in `rpm/greengage6.spec`, +for example: + +```spec +%if 0%{?rhel} >= 9 +Requires: python3.11 +Requires: python3.11-pip +%else +Requires: python2 +Requires: python3 +Requires: python2-pip +Requires: python3-pip +%endif +``` + ## Notes +### Debian + - Skips tests (`DEB_BUILD_OPTIONS=nocheck`) for faster builds - Unsets compiler flags to avoid conflicts with the project's build system - Enables parallel builds using all available CPU cores @@ -150,3 +251,13 @@ endif - Custom `dh_fixperms` removes executable bit from Python/Perl/Bash files without shebang and `*.md` files - Generated `debian/{changelog,install}` is not committed to the repository + +### RPM + +- Builds without signing for development convenience +- Strips debug info from `*.so` files to remove BUILDROOT paths embedded + by the compiler, which would otherwise fail `check-buildroot` validation +- Excludes the entire install prefix from shebang mangling via + `__brp_mangle_shebangs_exclude_from` +- Collects only the resulting `.rpm` into `$(CURDIR)/../Package` +- `RPM_TOPDIR` is kept after the build for inspection diff --git a/gpAux/releng/set_bld_arch.sh b/gpAux/releng/set_bld_arch.sh index 4eab100dd9fc..c45f0c31fc6e 100755 --- a/gpAux/releng/set_bld_arch.sh +++ b/gpAux/releng/set_bld_arch.sh @@ -4,10 +4,10 @@ case "`uname -s`" in Linux) - if [ -f /etc/redhat-release -a ! -f /etc/altlinux-release -a ! -f /etc/redos-release ]; then + if [ -f /etc/redhat-release -a ! -f /etc/altlinux-release -a ! -f /etc/redos-release -a ! -f /etc/rocky-release ]; then case "`cat /etc/redhat-release`" in *) - BLD_ARCH_HOST="rhel`cat /etc/redhat-release | sed -e 's/CentOS Linux/RedHat/' -e 's/Red Hat Enterprise Linux/RedHat/' -e 's/Rocky Linux/RedHat/' -e 's/WS//' -e 's/Server//' -e 's/Client//' | awk '{print $3}' | awk -F. '{print $1}'`_`uname -m | sed -e s/i686/x86_32/`" + BLD_ARCH_HOST="rhel`cat /etc/redhat-release | sed -e 's/CentOS Linux/RedHat/' -e 's/Red Hat Enterprise Linux/RedHat/' -e 's/WS//' -e 's/Server//' -e 's/Client//' | awk '{print $3}' | awk -F. '{print $1}'`_`uname -m | sed -e s/i686/x86_32/`" ;; esac fi @@ -18,6 +18,9 @@ case "`uname -s`" in ;; esac fi + if [ -f /etc/rocky-release ]; then + BLD_ARCH_HOST="$(. /etc/os-release; echo ${ID}$(echo ${VERSION_ID} | cut -d. -f1)_$(uname -m))" + fi if [ -f /etc/astra_version ]; then BLD_ARCH_HOST="$(. /etc/os-release; echo ${ID}${VERSION_ID} | sed 's/-/_/')" fi diff --git a/gpAux/rpm/greengage6.rpmlintrc b/gpAux/rpm/greengage6.rpmlintrc new file mode 100644 index 000000000000..7dea542a62b0 --- /dev/null +++ b/gpAux/rpm/greengage6.rpmlintrc @@ -0,0 +1,20 @@ +# 3rd-party package installs to /opt — intentional for database software +addFilter("dir-or-file-in-opt") + +# lib/python/ contains gppylib modules imported via Python's import system, +# not executed directly. Shebangs are for developer convenience only. +addFilter("non-executable-script") +addFilter("script-without-shebang") + +# bin/, sbin/, lib/python/ use bare 'python' interpreter intentionally. +# On Rocky 8, python -> python2; on Rocky 9, python -> python3.11. +addFilter("python-shebang-ambiguous") +addFilter("env-script-interpreter") + +# Timezone data files are hardlinked by upstream PostgreSQL build system +# to save disk space — standard PostgreSQL behavior, not a packaging error. +addFilter("hardlink") + +# plperl.so RUNPATH points to the exact Perl version this package depends on. +addFilter("library-without-ldconfig-postin") +addFilter("library-without-ldconfig-postun") diff --git a/gpAux/rpm/greengage6.spec b/gpAux/rpm/greengage6.spec new file mode 100644 index 000000000000..3e336a0078f1 --- /dev/null +++ b/gpAux/rpm/greengage6.spec @@ -0,0 +1,78 @@ +# Run command: `make -C ./gpdb_src/gpAux pkg-rpm` + +%{!?gproot: %global gproot /opt/greengagedb} +%{!?gpdir: %global gpdir greengage6} +%global prefix %{gproot}/%{gpdir} + +# bin/, sbin/, lib/python/ use bare 'python' interpreter intentionally. +# On Rocky 8, python -> python2; on Rocky 9, python -> python3.11. +# Perl test scripts use /usr/bin/env perl for portability across environments +# where Perl is not at /usr/bin/perl — see commit:eab7246a65e3c7834f8d4763d73972e4a6c0fcbd +%global __brp_mangle_shebangs_exclude_from ^%{prefix}/.*$ + +Name: greengage6 +Version: %{gpdb_version} +Release: %{gpdb_release}%{?dist} +Summary: Greengage MPP database engine +License: ASL 2.0 +URL: https://greengagedb.org + +Requires: findutils +Requires: glibc-langpack-en +Requires: iproute +Requires: iputils +Requires: less +Requires: net-tools +Requires: openssh-clients +Requires: openssh-server +Requires: openssl +Requires: procps-ng +Requires: rsync +Requires: zip + +%if 0%{?rhel} >= 9 +Requires: python3 +Requires: python3-pip +%else +Requires: python2 +Requires: python3 +Requires: python2-pip +Requires: python3-pip +%endif + +Conflicts: greengage-loaders + +%description +Greengage Database (GPDB) is an advanced, fully featured, open +source data warehouse, based on PostgreSQL. It provides powerful and +rapid analytics on petabyte scale data volumes. Uniquely geared toward +big data analytics, Greengage Database is powered by the world's most +advanced cost-based query optimizer delivering high analytical query +performance on large data volumes. + +%install +rm -rf %{buildroot} +env -u CFLAGS -u CPPFLAGS -u CXXFLAGS -u LDFLAGS \ + make dist \ + DESTDIR=%{buildroot} \ + GPROOT=%{gproot} \ + GPDIR=%{gpdir} \ + PARALLEL_MAKE_OPTS=%{?_smp_mflags} \ + -C %{sourcedir}/gpAux + +# Remove executable bit from scripts without shebang +find %{buildroot}%{prefix} -type f \ + \( -name "*.py" -o -name "*.pm" -o -name "*.sh" \) \ + -executable | while read f; do \ + head -1 "$f" | grep -q '^#!' || chmod -x "$f"; \ + done +find %{buildroot}%{prefix} -name "*.md" -executable \ + -exec chmod -x {} + + +# Remove debug info from *.so +find %{buildroot}%{prefix} -type f -name "*.so" -exec strip --strip-debug {} \; + +%files +%{prefix} + +%changelog diff --git a/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py b/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py index a0e1702d3396..f612f706d449 100644 --- a/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py +++ b/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py @@ -67,7 +67,8 @@ def checkTableForeignKey(self, cat): # skip these master-only tables skipped_masteronly = ['gp_relation_node', 'pg_description', 'pg_shdescription', 'pg_stat_last_operation', - 'pg_stat_last_shoperation', 'pg_statistic'] + 'pg_stat_last_shoperation', 'pg_statistic', + 'pg_event_trigger'] if catname in skipped_masteronly: return diff --git a/gpMgmt/bin/gppylib/commands/gp.py b/gpMgmt/bin/gppylib/commands/gp.py index cebc46538495..a973663bc872 100644 --- a/gpMgmt/bin/gppylib/commands/gp.py +++ b/gpMgmt/bin/gppylib/commands/gp.py @@ -1380,6 +1380,8 @@ def start_standbymaster(host, datadir, port, era=None, wrapper=None, wrapper_args=None): logger.info("Starting standby master") + datadir = os.path.normpath(datadir) + logger.info("Checking if standby master is running on host: %s in directory: %s" % (host,datadir)) cmd = Command("recovery_startup", ("python -c " diff --git a/gpMgmt/bin/gppylib/gpcatalog.py b/gpMgmt/bin/gppylib/gpcatalog.py index a237b299f2e0..1429cf723961 100644 --- a/gpMgmt/bin/gppylib/gpcatalog.py +++ b/gpMgmt/bin/gppylib/gpcatalog.py @@ -40,6 +40,7 @@ class GPCatalogException(Exception): 'pg_stat_last_shoperation', 'pg_statistic', 'pg_partition_encoding', + 'pg_event_trigger' ] # Hard coded tables that have different values on every segment diff --git a/gpMgmt/test/behave/mgmt_utils/gpactivatestandby.feature b/gpMgmt/test/behave/mgmt_utils/gpactivatestandby.feature index db4f2df47bce..0041b5ee7711 100644 --- a/gpMgmt/test/behave/mgmt_utils/gpactivatestandby.feature +++ b/gpMgmt/test/behave/mgmt_utils/gpactivatestandby.feature @@ -76,6 +76,20 @@ Feature: gpactivatestandby And the tablespace is valid on the standby master And clean up and revert back to original master + Scenario: master can be made on dir with trailing slash + Given the database is running + And the standby is not initialized + + When the user runs gpinitstandby with options "-S /tmp/standby_data/" + Then gpinitstandby should return a return code of 0 + And verify the standby master entries in catalog + + When the master goes down + And the user runs gpactivatestandby with options "-d /tmp/standby_data/" + Then gpactivatestandby should return a return code of 0 + And verify the standby master is now acting as master + And clean up and revert back to original master + ########################### @concourse_cluster tests ########################### # The @concourse_cluster tag denotes the scenario that requires a remote cluster diff --git a/gpMgmt/test/behave/mgmt_utils/gpexpand.feature b/gpMgmt/test/behave/mgmt_utils/gpexpand.feature index 1bf817c00be4..0e89444bf9fe 100644 --- a/gpMgmt/test/behave/mgmt_utils/gpexpand.feature +++ b/gpMgmt/test/behave/mgmt_utils/gpexpand.feature @@ -820,3 +820,20 @@ Feature: expand the cluster by adding more segments When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--verbose" Then gpexpand should print "[DEBUG]:-Skipping tar file (gpexpand_schema.tar) copy to cdw" escaped to stdout And verify that the cluster has 1 new segments + + @gpexpand_verify_master_only + Scenario: Verify master only tables are not copied to segment + Given the database is not running + And a working directory of the test as '/data/gpdata/gpexpand' + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into + And a cluster is created with no mirrors on "cdw" and "sdw1" + And database "gptest" exists + And the user creates an event trigger test_trigger + And verify that event trigger test_trigger exists + Then verify that the query "SELECT count(*) FROM gp_dist_random('pg_event_trigger');" in database "gptest" returns "0" + When the user runs gpexpand interview to add 1 new segment and 0 new host "ignored.host" + Then the number of segments have been saved + When the user runs gpexpand with the latest gpexpand_inputfile without ret code check + Then gpexpand should return a return code of 0 + And verify that the cluster has 1 new segments + Then verify that the query "SELECT count(*) FROM gp_dist_random('pg_event_trigger');" in database "gptest" returns "0" diff --git a/gpMgmt/test/behave/mgmt_utils/gpinitstandby.feature b/gpMgmt/test/behave/mgmt_utils/gpinitstandby.feature index 8e4e37cdb631..bf9c5557775c 100644 --- a/gpMgmt/test/behave/mgmt_utils/gpinitstandby.feature +++ b/gpMgmt/test/behave/mgmt_utils/gpinitstandby.feature @@ -101,6 +101,16 @@ Feature: Tests for gpinitstandby feature Then gpinitstandby should return a return code of 0 And verify that the file "pg_hba.conf" in the master data directory has "no" line starting with "host.*replication.*(127.0.0.1|::1).*trust" + Scenario: gpinitstandby on dir with trailing slash + Given the database is running + And the standby is not initialized + + When the user runs gpinitstandby with options "-S /tmp/standby_data/" + Then gpinitstandby should return a return code of 0 + And verify the standby master entries in catalog + When execute sql "select datadir from gp_segment_configuration where content = -1 and role = 'm'" in db "postgres" and store result in the context + Then validate that "/tmp/standby_data/" is in the stored rows + @backup_restore_bashrc Scenario: gpinitstandby should not throw error when banner exists on the hsot Given the database is running diff --git a/gpMgmt/test/behave/mgmt_utils/gpstart.feature b/gpMgmt/test/behave/mgmt_utils/gpstart.feature index 4c7c21c74a8d..37ada7c76350 100644 --- a/gpMgmt/test/behave/mgmt_utils/gpstart.feature +++ b/gpMgmt/test/behave/mgmt_utils/gpstart.feature @@ -172,3 +172,20 @@ Feature: gpstart behave tests When the user runs psql with "-c 'drop user foouser;'" against database "postgres" Then psql should return a return code of 0 + + Scenario: gpstart on dir with trailing slash + Given the database is running + And the catalog has a standby master entry + And the standby is not initialized + + When the user runs gpinitstandby with options "-S /tmp/standby_data/" + Then gpinitstandby should return a return code of 0 + And verify the standby master entries in catalog + + When the master goes down + And the user runs "gpstart -a" + Then gpstart should return a return code of 0 + And verify the standby master entries in catalog + + When execute sql "select datadir from gp_segment_configuration where content = -1 and role = 'm'" in db "postgres" and store result in the context + Then validate that "/tmp/standby_data/" is in the stored rows diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index 0f4866327ada..aa2ee38bf507 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -4676,3 +4676,28 @@ def impl(context): def impl(context, second): cmd = Command(name='psql', cmdStr="-c 'SELECT * from generate_series(1, %s) a where pg_sleep(1) is not null;'" % second) cmd.runNoWait() + +@given('the user creates an event trigger {trigger_name}') +def impl(context, trigger_name): + func_name = "%s_fn" % trigger_name + with dbconn.connect(dbconn.DbURL(dbname=context.dbname), unsetSearchPath=False) as conn: + sql = """CREATE OR REPLACE FUNCTION %s() +RETURNS event_trigger +LANGUAGE plpgsql AS $$ +BEGIN + RAISE NOTICE 'Event trigger %s_fn fired after DDL command'; +END; +$$; +""" % (func_name, trigger_name) + + dbconn.execSQL(conn, sql) + sql = "CREATE EVENT TRIGGER %s ON ddl_command_start EXECUTE PROCEDURE %s();" % (trigger_name, func_name) + dbconn.execSQL(conn, sql) + conn.commit() + +@given('verify that event trigger {trigger_name} exists') +def impl(context, trigger_name): + with dbconn.connect(dbconn.DbURL(dbname=context.dbname), unsetSearchPath=False) as conn: + sql = "SELECT evtname FROM pg_event_trigger WHERE evtname = '%s';" % trigger_name + cursor = dbconn.execSQL(conn, sql) + assert cursor.rowcount == 1 diff --git a/gpMgmt/test/coveragerc_behave b/gpMgmt/test/coveragerc_behave index 699be39093ea..f4863d1eb17e 100644 --- a/gpMgmt/test/coveragerc_behave +++ b/gpMgmt/test/coveragerc_behave @@ -11,6 +11,8 @@ omit = */gpload_test/* $GPHOME/lib/python/pygresql/* $GPHOME/lib/python/yaml/* + $GPHOME/lib/python/pg.py + $GPHOME/lib/python/pgdb.py $GPHOME/lib/python/sitecustomize.py $GPHOME/lib/python/subprocess32.py $GPHOME/lib/python/subprocess32.pyc @@ -36,3 +38,9 @@ source3 = /home/gpadmin/gpdb_src/gpMgmt/bin/gppylib /home/gpadmin/gpdb_src/gpMgmt/bin/gppylib $GPHOME/lib/python/gppylib + +source4 = + /home/gpadmin/gpdb_src/gpAux/gpperfmon/src/gpmon + /home/gpadmin/gpdb_src/gpAux/gpperfmon/src/gpmon + $GPHOME/bin + $GPHOME/sbin diff --git a/gpMgmt/test/coveragerc_combine_report b/gpMgmt/test/coveragerc_combine_report index 23729e3b43cc..7e1b8e185365 100644 --- a/gpMgmt/test/coveragerc_combine_report +++ b/gpMgmt/test/coveragerc_combine_report @@ -19,15 +19,23 @@ source3 = $GPHOME/lib/python/gppylib source4 = - /home/gpadmin/gpdb_src/gpcontrib/gp_replica_check/gp_replica_check.py - /home/gpadmin/gpdb_src/gpcontrib/gp_replica_check/gp_replica_check.py - $GPHOME/bin/gp_replica_check.py + /home/gpadmin/gpdb_src/gpcontrib/gp_replica_check + /home/gpadmin/gpdb_src/gpcontrib/gp_replica_check + $GPHOME/bin + +source5 = + /home/gpadmin/gpdb_src/gpAux/gpperfmon/src/gpmon + /home/gpadmin/gpdb_src/gpAux/gpperfmon/src/gpmon + $GPHOME/bin + $GPHOME/sbin [report] omit = */__init__.py* */test/* $GPHOME/lib/python/pygresql/* + $GPHOME/lib/python/pg.py + $GPHOME/lib/python/pgdb.py $GPHOME/lib/python/yaml/* $GPHOME/lib/python/sitecustomize.py $GPHOME/lib/python/subprocess32.py diff --git a/gpMgmt/test/coveragerc_unit b/gpMgmt/test/coveragerc_unit index 6fa5b11654b6..75ca49e5fbdb 100644 --- a/gpMgmt/test/coveragerc_unit +++ b/gpMgmt/test/coveragerc_unit @@ -9,13 +9,15 @@ omit = */sitecustomize.py* */test/* */gpload_test/* - /usr/local/greengage-db-devel/lib/python/pygresql/* - /usr/local/greengage-db-devel/lib/python/yaml/* - /usr/local/greengage-db-devel/lib/python/sitecustomize.py - /usr/local/greengage-db-devel/lib/python/subprocess32.py - /usr/local/greengage-db-devel/lib/python/subprocess32.pyc - /usr/local/greengage-db-devel/bin/lib/pexpect/* - /usr/local/greengage-db-devel/lib/python/psutil/* + $GPHOME/lib/python/pygresql/* + $GPHOME/lib/python/pg.py + $GPHOME/lib/python/pgdb.py + $GPHOME/lib/python/yaml/* + $GPHOME/lib/python/sitecustomize.py + $GPHOME/lib/python/subprocess32.py + $GPHOME/lib/python/subprocess32.pyc + $GPHOME/bin/lib/pexpect/* + $GPHOME/lib/python/psutil/* */dist-packages/* /usr/local/bin/behave /usr/local/bin/coverage @@ -25,19 +27,19 @@ omit = source = /home/gpadmin/gpdb_src/gpMgmt/bin /home/gpadmin/gpdb_src/gpMgmt/bin - /usr/local/greengage-db-devel/bin + $GPHOME/bin source2 = /home/gpadmin/gpdb_src/gpMgmt/sbin /home/gpadmin/gpdb_src/gpMgmt/sbin - /usr/local/greengage-db-devel/sbin + $GPHOME/sbin source3 = /home/gpadmin/gpdb_src/gpMgmt/bin/gppylib /home/gpadmin/gpdb_src/gpMgmt/bin/gppylib - /usr/local/greengage-db-devel/lib/python/gppylib + /$GPHOME/lib/python/gppylib source4 = - /home/gpadmin/gpdb_src/gpcontrib/gp_replica_check/gp_replica_check.py - /home/gpadmin/gpdb_src/gpcontrib/gp_replica_check/gp_replica_check.py - /usr/local/greengage-db-devel/bin/gp_replica_check.py + /home/gpadmin/gpdb_src/gpcontrib/gp_replica_check + /home/gpadmin/gpdb_src/gpcontrib/gp_replica_check + $GPHOME/bin diff --git a/src/backend/cdb/cdbvars.c b/src/backend/cdb/cdbvars.c index 63ea1bcac9bb..f324871761ae 100644 --- a/src/backend/cdb/cdbvars.c +++ b/src/backend/cdb/cdbvars.c @@ -278,6 +278,7 @@ int gp_motion_slice_noop = 0; /* Greengage Database Experimental Feature GUCs */ int gp_distinct_grouping_sets_threshold = 32; +bool gp_enable_explain_rows_out = FALSE; bool gp_enable_explain_allstat = FALSE; bool gp_enable_motion_deadlock_sanity = FALSE; /* planning time sanity * check */ diff --git a/src/backend/commands/explain_gp.c b/src/backend/commands/explain_gp.c index 8765f4c28de7..c69dec48f607 100644 --- a/src/backend/commands/explain_gp.c +++ b/src/backend/commands/explain_gp.c @@ -1857,6 +1857,75 @@ cdbexplain_showExecStats(struct PlanState *planstate, ExplainState *es) } pfree(extraData.data); + /* + * Print "Rows out" + */ + if (gp_enable_explain_rows_out && es->analyze && ns->ninst > 0) + { + double alltuples = 0; + double maxtuples = 0; + int maxseg = -1; + double mintuples = 0; + int minseg = -1; + int workercount = 0; + double avgtuples; + + for (i = 0; i < ns->ninst; i++) + { + CdbExplain_StatInst *nsi = &ns->insts[i]; + double rows; + + if (nsi->pstype == T_Invalid) + continue; + + rows = nsi->nloops > 0 ? nsi->ntuples / nsi->nloops : 0; + + if (workercount == 0 || rows > maxtuples) + { + maxtuples = rows; + maxseg = ns->segindex0 + i; + } + + if (workercount == 0 || rows < mintuples) + { + mintuples = rows; + minseg = ns->segindex0 + i; + } + + alltuples += rows; + workercount++; + } + + if (workercount > 0) + { + avgtuples = alltuples / workercount; + + if (es->format == EXPLAIN_FORMAT_TEXT) + { + appendStringInfoSpaces(es->str, es->indent * 2); + appendStringInfoString(es->str, "Rows out: "); + + appendStringInfo(es->str, + "%.2f rows avg x %d workers, %.0f rows max (seg%d), %.0f rows min (seg%d).\n", + avgtuples, + workercount, + maxtuples, + maxseg, + mintuples, + minseg); + } + else + { + ExplainPropertyInteger("Workers", workercount, es); + ExplainPropertyFloat("Average Rows", avgtuples, 2, es); + ExplainPropertyFloat("Max Rows", maxtuples, 0, es); + ExplainPropertyInteger("Max Rows Segment", maxseg, es); + ExplainPropertyFloat("Min Rows", mintuples, 0, es); + ExplainPropertyInteger("Min Rows Segment", minseg, es); + } + } + } + /* * Dump stats for all workers. */ diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 3f9b85c279b9..c063355b91cb 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -118,6 +118,7 @@ #include "utils/memutils.h" #include "utils/metrics_utils.h" #include "utils/relcache.h" +#include "utils/guc.h" #include "utils/snapmgr.h" #include "utils/syscache.h" #include "utils/tqual.h" @@ -15051,6 +15052,20 @@ ATExecExpandTable(List **wqueue, Relation rel, AlterTableCmd *cmd) (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied: \"%s\" is a system catalog", RelationGetRelationName(rel)))); + /* + * synchronous_commit can be "off" at the session or cluster level for + * reasons unrelated to this command. With it off, a segment's commit never + * waits for its mirror to acknowledge the redistributed data, so a primary + * crash immediately followed by an automatic mirror promotion can silently + * lose/duplicate the rows this command just moved, if not all WAL was sent + * to the mirror. + * Therefore, forbid operation if synchronous_commit is not fully enabled. + */ + if (synchronous_commit != SYNCHRONOUS_COMMIT_ON) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("synchronous_commit should be enabled during EXPAND"))); + oldContext = MemoryContextSwitchTo(GetMemoryChunkContext(rel)); newPolicy = GpPolicyCopy(policy); MemoryContextSwitchTo(oldContext); @@ -15394,6 +15409,17 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied: \"%s\" is a system catalog", RelationGetRelationName(rel)))); + /* + * Like ATExecExpandTable, this command can rewrite and redistribute the + * table's data via an internal CTAS. See the comment there. + * And, forbid operation if synchronous_commit is not fully enabled here + * as well. + */ + if (synchronous_commit != SYNCHRONOUS_COMMIT_ON) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("synchronous_commit should be enabled during SET DISTRIBUTED BY"))); + Assert(PointerIsValid(node)); Assert(IsA(node, List)); diff --git a/src/backend/storage/file/reinit.c b/src/backend/storage/file/reinit.c index debdd37cd6c2..111ddf882ad9 100644 --- a/src/backend/storage/file/reinit.c +++ b/src/backend/storage/file/reinit.c @@ -15,6 +15,7 @@ #include "postgres.h" #include +#include #include "catalog/catalog.h" #include "catalog/pg_tablespace.h" @@ -335,6 +336,25 @@ ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op) dbspacedirname, oidbuf, de->d_name + oidchars + 1 + strlen(forkNames[INIT_FORKNUM])); + /* + * Sometimes, before recovery there is no init fork, but there is + * main fork. So at cleanup it was not removed. Next, at + * recovery init fork will be created and if we will not remove main + * fork here, function copy_file will return error. Let's check main + * fork here and remove it if presented. + * Such situations may happen when pg_basebackup checks main fork + * file at time when init fork is not created yet. pg_basebackup + * does not filter such file and adds it to the backup. Next, init + * fork will be created by reading xlog at recovery. + */ + struct stat statbuf; + if (lstat(dstpath, &statbuf) == 0) + { + if (unlink(dstpath) != 0) + elog(ERROR, "unlink extra main fork %s error: %m", dstpath); + elog(DEBUG2, "unlink extra main fork %s", dstpath); + } + /* OK, we're ready to perform the actual copy. */ elog(DEBUG2, "copying %s to %s", srcpath, dstpath); copy_file(srcpath, dstpath); diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 35315b2a0514..78053eea4dfb 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -5241,7 +5241,6 @@ AtEOXact_GUC(bool isCommit, int nestLevel) * record it and restore QE before next query start */ if (Gp_role == GP_ROLE_DISPATCH - && !IsTransactionBlock() && changed && ((isCommit) || (!isCommit && gp_guc_need_restore)) && (gconf->flags & GUC_GPDB_NEED_SYNC)) @@ -5921,6 +5920,14 @@ set_config_option(const char *name, const char *value, errmsg("unrecognized configuration parameter \"%s\"", name))); return 0; } + /* + * Make SET LOCAL just SET when executing on segments. It is needed for + * correct passage of local GUCs as they might be discarded. Anyway, + * master executes SET as LOCAL, so on next transaction GUC will + * resynchronize and everything will be back in place. + */ + if (Gp_role == GP_ROLE_EXECUTE && action == GUC_ACTION_LOCAL) + action = GUC_ACTION_SET; /* * Check if option can be set by the user. diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index 2d5a687d2208..1ccc7c31698a 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -890,6 +890,17 @@ struct config_bool ConfigureNamesBool_gp[] = NULL, NULL, NULL }, + { + {"gp_enable_explain_rows_out", PGC_USERSET, CLIENT_CONN_OTHER, + gettext_noop("Experimental feature: print avg, min and max rows out in segments in EXPLAIN ANALYZE."), + NULL, + GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE + }, + &gp_enable_explain_rows_out, + false, + NULL, NULL, NULL + }, + { {"gp_enable_sort_limit", PGC_USERSET, QUERY_TUNING_METHOD, gettext_noop("Enable LIMIT operation to be performed while sorting."), diff --git a/src/include/cdb/cdbvars.h b/src/include/cdb/cdbvars.h index 4823798085d4..95ad3cbd0ec9 100644 --- a/src/include/cdb/cdbvars.h +++ b/src/include/cdb/cdbvars.h @@ -744,6 +744,12 @@ extern bool gp_enable_preunique; */ extern bool gp_eager_preunique; +/* + * May Greengage print statistics as average, minimum and maximum rows out + * during EXPLAIN ANALYZE? + */ +extern bool gp_enable_explain_rows_out; + /* May Greengage dump statistics for all segments as a huge ugly string * during EXPLAIN ANALYZE? * diff --git a/src/include/utils/unsync_guc_name.h b/src/include/utils/unsync_guc_name.h index 979ecb2d6365..faef133e92cb 100644 --- a/src/include/utils/unsync_guc_name.h +++ b/src/include/utils/unsync_guc_name.h @@ -176,6 +176,7 @@ "gp_enable_direct_dispatch", "gp_enable_exchange_default_partition", "gp_enable_explain_allstat", + "gp_enable_explain_rows_out", "gp_enable_fast_sri", "gp_enable_global_deadlock_detector", "gp_enable_gpperfmon", diff --git a/src/test/recovery/t/014_unlogged_reinit.pl b/src/test/recovery/t/014_unlogged_reinit.pl index 0af1d5848047..f973d0fc635a 100644 --- a/src/test/recovery/t/014_unlogged_reinit.pl +++ b/src/test/recovery/t/014_unlogged_reinit.pl @@ -7,7 +7,7 @@ use warnings; use PostgresNode; use TestLib; -use Test::More tests => 12; +use Test::More tests => 16; my $node = get_new_node('main'); @@ -79,3 +79,42 @@ 'vm fork in tablespace removed at startup'); ok( !-f "$pgdata/${ts1UnloggedPath}_fsm", 'fsm fork in tablespace removed at startup'); + + +# Create new unlogged tables to check recovery when DB contains main fork only. + +# Make it impossible to get a checkpoint after table creation before a crash. +$node->append_conf('postgresql.conf', <restart; + +$node->safe_psql('postgres', 'CREATE UNLOGGED TABLE base_unlogged2 (id int)'); +$node->safe_psql('postgres', + 'CREATE UNLOGGED TABLE ts1_unlogged2 (id int) TABLESPACE ts1'); + +$baseUnloggedPath = $node->safe_psql('postgres', + q{select pg_relation_filepath('base_unlogged2')}); +$ts1UnloggedPath = $node->safe_psql('postgres', + q{select pg_relation_filepath('ts1_unlogged2')}); + +# Crash the postmaster. +$node->stop('immediate'); + +# Remove init fork to test that it is recreated from init. +unlink("$pgdata/${baseUnloggedPath}_init") + or BAIL_OUT("could not remove \"${baseUnloggedPath}_init\": $!"); +unlink("$pgdata/${ts1UnloggedPath}_init") + or BAIL_OUT("could not remove \"${ts1UnloggedPath}_init\": $!"); + +$node->start; + +# check unlogged table in tablespace +ok( -f "$pgdata/${ts1UnloggedPath}_init", + 'init fork recreated at startup in tablespace'); +ok(-f "$pgdata/$ts1UnloggedPath", + 'main fork in tablespace recreated at startup'); + +# check unlogged table in base +ok(-f "$pgdata/${baseUnloggedPath}_init", 'init fork in base recreated at startup'); +ok(-f "$pgdata/${baseUnloggedPath}", 'main fork in base recreated at startup'); diff --git a/src/test/regress/expected/expand_table.out b/src/test/regress/expected/expand_table.out index 56d181d6222c..91a4eac052bd 100644 --- a/src/test/regress/expected/expand_table.out +++ b/src/test/regress/expected/expand_table.out @@ -1079,3 +1079,122 @@ select numsegments from gp_distribution_policy where localoid='expand_domain_tab reset search_path; drop schema test_reshuffle cascade; -- end_ignore +-- Check that table expand doesn't lead to invalid data distribution +-- if WAL wasn't replicated to a mirror in time, the respective Primary failed, +-- and the mirror with obsolete data is promoted. That could happen if the user +-- disabled 'synchronous_commit'. +create extension if not exists gp_inject_fault; +select gp_debug_set_create_table_default_numsegments(2); + gp_debug_set_create_table_default_numsegments +----------------------------------------------- + 2 +(1 row) + +drop table if exists test; +create table test(a int) distributed by (a); +insert into test select generate_series(1, 100); +set synchronous_commit = off; +select gp_inject_fault('wal_sender_loop', 'suspend', dbid) from gp_segment_configuration where content=0 and role='p'; + gp_inject_fault +----------------- + Success: +(1 row) + +alter table test expand table; +ERROR: synchronous_commit should be enabled during EXPAND +select '\! pg_ctl -D ' || datadir || ' stop -m immediate' as cmd_stop_primary_0 from gp_segment_configuration where content = 0 and role = 'p' +\gset +:cmd_stop_primary_0 +waiting for server to shut down.... done +server stopped +select gp_request_fts_probe_scan(); + gp_request_fts_probe_scan +--------------------------- + t +(1 row) + +select role, preferred_role, status from gp_segment_configuration where content = 0 order by dbid; + role | preferred_role | status +------+----------------+-------- + m | p | d + p | m | u +(2 rows) + +select count(*) from test; + count +------- + 100 +(1 row) + +select count(*), gp_segment_id from test group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 53 | 0 + 47 | 1 +(2 rows) + +reset synchronous_commit; +drop table test; +-- Check the same for SET WITH (REORGANIZE=TRUE), which is used for partitioned table. +select gp_debug_set_create_table_default_numsegments(2); + gp_debug_set_create_table_default_numsegments +----------------------------------------------- + 2 +(1 row) + +create table test (a int) distributed by (a) +partition by range (a) +( + start (0) end (1000) every (1000) +); +NOTICE: CREATE TABLE will create partition "test_1_prt_1" for table "test" +insert into test select generate_series(1, 100); +alter table test expand partition prepare; +set synchronous_commit = off; +select gp_inject_fault('wal_sender_loop', 'suspend', dbid) from gp_segment_configuration where content=0 and role='p'; + gp_inject_fault +----------------- + Success: +(1 row) + +alter table test_1_prt_1 set with (reorganize=true) distributed by (a); +ERROR: synchronous_commit should be enabled during SET DISTRIBUTED BY +select '\! pg_ctl -D ' || datadir || ' stop -m immediate' as cmd_stop_primary_0 from gp_segment_configuration where content = 0 and role = 'p' +\gset +:cmd_stop_primary_0 +waiting for server to shut down.... done +server stopped +select gp_request_fts_probe_scan(); + gp_request_fts_probe_scan +--------------------------- + t +(1 row) + +select role, preferred_role, status from gp_segment_configuration where content = 0 order by dbid; + role | preferred_role | status +------+----------------+-------- + m | p | d + p | m | u +(2 rows) + +select count(*) from test; + count +------- + 100 +(1 row) + +select count(*), gp_segment_id from test group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 53 | 0 + 47 | 1 +(2 rows) + +reset synchronous_commit; +drop table test; +select gp_debug_reset_create_table_default_numsegments(); + gp_debug_reset_create_table_default_numsegments +------------------------------------------------- + +(1 row) + diff --git a/src/test/regress/expected/gp_explain.out b/src/test/regress/expected/gp_explain.out index 77010f95787a..984c2dbe31b6 100644 --- a/src/test/regress/expected/gp_explain.out +++ b/src/test/regress/expected/gp_explain.out @@ -29,13 +29,27 @@ begin end loop; end; $$ language plpgsql; +-- Return EXPLAIN ANALYZE result as xml to manipulate it further. +create or replace function get_explain_analyze_xml_output(explain_query text) +returns xml as +$$ +declare + x xml; +begin + execute 'EXPLAIN (ANALYZE, VERBOSE, FORMAT XML) ' || explain_query + into x; + return x; +end; +$$ language plpgsql; -- -- Test explain_memory_verbosity option -- +set gp_use_legacy_hashops=off; CREATE TABLE explaintest (id int4); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Greengage Database data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. INSERT INTO explaintest SELECT generate_series(1, 10); +reset gp_use_legacy_hashops; EXPLAIN ANALYZE SELECT * FROM explaintest; QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------- @@ -337,21 +351,46 @@ explain analyze SELECT * FROM explaintest; (8 rows) set gp_enable_explain_allstat=DEFAULT; +-- Test explain rows out. +set gp_enable_explain_rows_out=on; +\pset format unaligned +\pset tuples_only on +WITH query_plan (et) AS +( + select get_explain_analyze_output($$ + SELECT * FROM explaintest; + $$) +) +SELECT trim(et) FROM query_plan WHERE et like '%Rows out:%' AND et not like '%(seg-1)%'; +Rows out: 3.33 rows avg x 3 workers, 5 rows max (seg0), 1 rows min (seg1). +-- Rows out on a skewed distribution, so max and min land on different segments. +set gp_use_legacy_hashops=off; +CREATE TABLE explain_rows_skew (id int) DISTRIBUTED BY (id); +INSERT INTO explain_rows_skew SELECT 2 FROM generate_series(1, 100); +INSERT INTO explain_rows_skew SELECT 1 FROM generate_series(1, 10); +INSERT INTO explain_rows_skew VALUES (5); +reset gp_use_legacy_hashops; +ANALYZE explain_rows_skew; +SELECT xpath( + '//*[local-name()="Relation-Name" and text()="explain_rows_skew"]/.. + /*[local-name()="Workers" + or local-name()="Average-Rows" + or local-name()="Max-Rows" + or local-name()="Max-Rows-Segment" + or local-name()="Min-Rows" + or local-name()="Min-Rows-Segment"]/text()', + x) +FROM get_explain_analyze_xml_output($$ + SELECT * FROM explain_rows_skew; + $$) AS query_plan(x); +{3,37.00,100,0,1,2} +\pset tuples_only off +\pset format aligned +reset gp_enable_explain_rows_out; +DROP TABLE explain_rows_skew; -- -- Test output of EXPLAIN ANALYZE for Bitmap index scan's actual rows. -- --- Return EXPLAIN ANALYZE result as xml to manipulate it further. -create or replace function get_explain_analyze_xml_output(explain_query text) -returns xml as -$$ -declare - x xml; -begin - execute 'EXPLAIN (ANALYZE, VERBOSE, FORMAT XML) ' || explain_query - into x; - return x; -end; -$$ language plpgsql; -- force (Dynamic) Bitmap Index Scan set optimizer_enable_dynamictablescan=off; set enable_seqscan=off; diff --git a/src/test/regress/expected/gp_explain_optimizer.out b/src/test/regress/expected/gp_explain_optimizer.out index afc7272c09ad..3bf7a5b76afc 100644 --- a/src/test/regress/expected/gp_explain_optimizer.out +++ b/src/test/regress/expected/gp_explain_optimizer.out @@ -29,13 +29,27 @@ begin end loop; end; $$ language plpgsql; +-- Return EXPLAIN ANALYZE result as xml to manipulate it further. +create or replace function get_explain_analyze_xml_output(explain_query text) +returns xml as +$$ +declare + x xml; +begin + execute 'EXPLAIN (ANALYZE, VERBOSE, FORMAT XML) ' || explain_query + into x; + return x; +end; +$$ language plpgsql; -- -- Test explain_memory_verbosity option -- +set gp_use_legacy_hashops=off; CREATE TABLE explaintest (id int4); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Greengage Database data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. INSERT INTO explaintest SELECT generate_series(1, 10); +reset gp_use_legacy_hashops; EXPLAIN ANALYZE SELECT * FROM explaintest; QUERY PLAN -------------------------------------------------------------------------------------------------------------------------- @@ -368,21 +382,46 @@ explain analyze SELECT * FROM explaintest; (8 rows) set gp_enable_explain_allstat=DEFAULT; +-- Test explain rows out. +set gp_enable_explain_rows_out=on; +\pset format unaligned +\pset tuples_only on +WITH query_plan (et) AS +( + select get_explain_analyze_output($$ + SELECT * FROM explaintest; + $$) +) +SELECT trim(et) FROM query_plan WHERE et like '%Rows out:%' AND et not like '%(seg-1)%'; +Rows out: 3.33 rows avg x 3 workers, 5 rows max (seg0), 1 rows min (seg1). +-- Rows out on a skewed distribution, so max and min land on different segments. +set gp_use_legacy_hashops=off; +CREATE TABLE explain_rows_skew (id int) DISTRIBUTED BY (id); +INSERT INTO explain_rows_skew SELECT 2 FROM generate_series(1, 100); +INSERT INTO explain_rows_skew SELECT 1 FROM generate_series(1, 10); +INSERT INTO explain_rows_skew VALUES (5); +reset gp_use_legacy_hashops; +ANALYZE explain_rows_skew; +SELECT xpath( + '//*[local-name()="Relation-Name" and text()="explain_rows_skew"]/.. + /*[local-name()="Workers" + or local-name()="Average-Rows" + or local-name()="Max-Rows" + or local-name()="Max-Rows-Segment" + or local-name()="Min-Rows" + or local-name()="Min-Rows-Segment"]/text()', + x) +FROM get_explain_analyze_xml_output($$ + SELECT * FROM explain_rows_skew; + $$) AS query_plan(x); +{3,37.00,100,0,1,2} +\pset tuples_only off +\pset format aligned +reset gp_enable_explain_rows_out; +DROP TABLE explain_rows_skew; -- -- Test output of EXPLAIN ANALYZE for Bitmap index scan's actual rows. -- --- Return EXPLAIN ANALYZE result as xml to manipulate it further. -create or replace function get_explain_analyze_xml_output(explain_query text) -returns xml as -$$ -declare - x xml; -begin - execute 'EXPLAIN (ANALYZE, VERBOSE, FORMAT XML) ' || explain_query - into x; - return x; -end; -$$ language plpgsql; -- force (Dynamic) Bitmap Index Scan set optimizer_enable_dynamictablescan=off; set enable_seqscan=off; diff --git a/src/test/regress/expected/plpgsql.out b/src/test/regress/expected/plpgsql.out index e909d02e8a44..aadd00ab92e3 100755 --- a/src/test/regress/expected/plpgsql.out +++ b/src/test/regress/expected/plpgsql.out @@ -5555,3 +5555,24 @@ NOTICE: outer_func() done drop function outer_outer_func(int); drop function outer_func(int); drop function inner_func(int); +-- Test consistency and passage of SET LOCAL GUC to writing commands +create schema s; +do $$ +begin + set local search_path to s; + create table test_table(a int) distributed by (a); + drop table test_table; +end $$; +-- check existence on master and segments +with c as ( + select gp_segment_id, relname, relkind, relnamespace from pg_class + union all + select gp_segment_id, relname, relkind, relnamespace from gp_dist_random('pg_class') +) select gp_segment_id from c +join pg_namespace n on n.oid = c.relnamespace +where c.relname = 'test_table' and c.relkind = 'r' and n.nspname = 's'; + gp_segment_id +--------------- +(0 rows) + +drop schema s cascade; diff --git a/src/test/regress/expected/plpgsql_optimizer.out b/src/test/regress/expected/plpgsql_optimizer.out index c9c9347b8f15..6af96c227cbe 100755 --- a/src/test/regress/expected/plpgsql_optimizer.out +++ b/src/test/regress/expected/plpgsql_optimizer.out @@ -5533,3 +5533,24 @@ NOTICE: outer_func() done drop function outer_outer_func(int); drop function outer_func(int); drop function inner_func(int); +-- Test consistency and passage of SET LOCAL GUC to writing commands +create schema s; +do $$ +begin + set local search_path to s; + create table test_table(a int) distributed by (a); + drop table test_table; +end $$; +-- check existence on master and segments +with c as ( + select gp_segment_id, relname, relkind, relnamespace from pg_class + union all + select gp_segment_id, relname, relkind, relnamespace from gp_dist_random('pg_class') +) select gp_segment_id from c +join pg_namespace n on n.oid = c.relnamespace +where c.relname = 'test_table' and c.relkind = 'r' and n.nspname = 's'; + gp_segment_id +--------------- +(0 rows) + +drop schema s cascade; diff --git a/src/test/regress/greengage_schedule b/src/test/regress/greengage_schedule index ad3f9cb5d81b..7bb10b9961aa 100755 --- a/src/test/regress/greengage_schedule +++ b/src/test/regress/greengage_schedule @@ -159,7 +159,9 @@ test: resource_group_gucs test: wrkloadadmin # expand_table tests may affect the result of 'gp_explain', keep them below that -test: gp_toolkit_ao_funcs trig auth_constraint role portals_updatable plpgsql_cache timeseries pg_stat_last_operation pg_stat_last_shoperation gp_numeric_agg partindex_test partition_pruning runtime_stats expand_table expand_table_ao expand_table_aoco expand_table_regression +test: gp_toolkit_ao_funcs trig auth_constraint role portals_updatable plpgsql_cache timeseries pg_stat_last_operation pg_stat_last_shoperation gp_numeric_agg partindex_test partition_pruning runtime_stats expand_table_ao expand_table_aoco expand_table_regression +# expand_table contains segment stop and recover, so keep it separate from other tests +test: expand_table test: rle rle_delta dsp not_out_of_shmem_exit_slots # direct dispatch tests diff --git a/src/test/regress/sql/expand_table.sql b/src/test/regress/sql/expand_table.sql index 393b2d2b4778..ddfe18c045c3 100644 --- a/src/test/regress/sql/expand_table.sql +++ b/src/test/regress/sql/expand_table.sql @@ -455,3 +455,81 @@ select numsegments from gp_distribution_policy where localoid='expand_domain_tab reset search_path; drop schema test_reshuffle cascade; -- end_ignore + +-- Check that table expand doesn't lead to invalid data distribution +-- if WAL wasn't replicated to a mirror in time, the respective Primary failed, +-- and the mirror with obsolete data is promoted. That could happen if the user +-- disabled 'synchronous_commit'. + +create extension if not exists gp_inject_fault; + +select gp_debug_set_create_table_default_numsegments(2); +drop table if exists test; +create table test(a int) distributed by (a); +insert into test select generate_series(1, 100); + +set synchronous_commit = off; + +select gp_inject_fault('wal_sender_loop', 'suspend', dbid) from gp_segment_configuration where content=0 and role='p'; + +alter table test expand table; + +select '\! pg_ctl -D ' || datadir || ' stop -m immediate' as cmd_stop_primary_0 from gp_segment_configuration where content = 0 and role = 'p' +\gset +:cmd_stop_primary_0 + +select gp_request_fts_probe_scan(); +select role, preferred_role, status from gp_segment_configuration where content = 0 order by dbid; + +select count(*) from test; + +select count(*), gp_segment_id from test group by gp_segment_id order by gp_segment_id; + +reset synchronous_commit; + +-- start_ignore +\! gprecoverseg -aF --no-progress; +\! gprecoverseg -ar; +-- end_ignore + +drop table test; + +-- Check the same for SET WITH (REORGANIZE=TRUE), which is used for partitioned table. +select gp_debug_set_create_table_default_numsegments(2); +create table test (a int) distributed by (a) +partition by range (a) +( + start (0) end (1000) every (1000) +); +insert into test select generate_series(1, 100); + +alter table test expand partition prepare; + +set synchronous_commit = off; + +select gp_inject_fault('wal_sender_loop', 'suspend', dbid) from gp_segment_configuration where content=0 and role='p'; + +alter table test_1_prt_1 set with (reorganize=true) distributed by (a); + +select '\! pg_ctl -D ' || datadir || ' stop -m immediate' as cmd_stop_primary_0 from gp_segment_configuration where content = 0 and role = 'p' +\gset +:cmd_stop_primary_0 + +select gp_request_fts_probe_scan(); +select role, preferred_role, status from gp_segment_configuration where content = 0 order by dbid; + + +select count(*) from test; + +select count(*), gp_segment_id from test group by gp_segment_id order by gp_segment_id; + +reset synchronous_commit; + +-- start_ignore +\! gprecoverseg -aF --no-progress; +\! gprecoverseg -ar; +-- end_ignore + +drop table test; + +select gp_debug_reset_create_table_default_numsegments(); diff --git a/src/test/regress/sql/gp_explain.sql b/src/test/regress/sql/gp_explain.sql index 3deba9eb8beb..281460a45d57 100644 --- a/src/test/regress/sql/gp_explain.sql +++ b/src/test/regress/sql/gp_explain.sql @@ -34,12 +34,27 @@ begin end; $$ language plpgsql; +-- Return EXPLAIN ANALYZE result as xml to manipulate it further. +create or replace function get_explain_analyze_xml_output(explain_query text) +returns xml as +$$ +declare + x xml; +begin + execute 'EXPLAIN (ANALYZE, VERBOSE, FORMAT XML) ' || explain_query + into x; + return x; +end; +$$ language plpgsql; + -- -- Test explain_memory_verbosity option -- +set gp_use_legacy_hashops=off; CREATE TABLE explaintest (id int4); INSERT INTO explaintest SELECT generate_series(1, 10); +reset gp_use_legacy_hashops; EXPLAIN ANALYZE SELECT * FROM explaintest; @@ -162,23 +177,50 @@ set gp_enable_explain_allstat=on; explain analyze SELECT * FROM explaintest; set gp_enable_explain_allstat=DEFAULT; +-- Test explain rows out. +set gp_enable_explain_rows_out=on; + +\pset format unaligned +\pset tuples_only on +WITH query_plan (et) AS +( + select get_explain_analyze_output($$ + SELECT * FROM explaintest; + $$) +) +SELECT trim(et) FROM query_plan WHERE et like '%Rows out:%' AND et not like '%(seg-1)%'; + +-- Rows out on a skewed distribution, so max and min land on different segments. +set gp_use_legacy_hashops=off; +CREATE TABLE explain_rows_skew (id int) DISTRIBUTED BY (id); +INSERT INTO explain_rows_skew SELECT 2 FROM generate_series(1, 100); +INSERT INTO explain_rows_skew SELECT 1 FROM generate_series(1, 10); +INSERT INTO explain_rows_skew VALUES (5); +reset gp_use_legacy_hashops; +ANALYZE explain_rows_skew; + +SELECT xpath( + '//*[local-name()="Relation-Name" and text()="explain_rows_skew"]/.. + /*[local-name()="Workers" + or local-name()="Average-Rows" + or local-name()="Max-Rows" + or local-name()="Max-Rows-Segment" + or local-name()="Min-Rows" + or local-name()="Min-Rows-Segment"]/text()', + x) +FROM get_explain_analyze_xml_output($$ + SELECT * FROM explain_rows_skew; + $$) AS query_plan(x); + +\pset tuples_only off +\pset format aligned +reset gp_enable_explain_rows_out; +DROP TABLE explain_rows_skew; + -- -- Test output of EXPLAIN ANALYZE for Bitmap index scan's actual rows. -- --- Return EXPLAIN ANALYZE result as xml to manipulate it further. -create or replace function get_explain_analyze_xml_output(explain_query text) -returns xml as -$$ -declare - x xml; -begin - execute 'EXPLAIN (ANALYZE, VERBOSE, FORMAT XML) ' || explain_query - into x; - return x; -end; -$$ language plpgsql; - -- force (Dynamic) Bitmap Index Scan set optimizer_enable_dynamictablescan=off; set enable_seqscan=off; diff --git a/src/test/regress/sql/plpgsql.sql b/src/test/regress/sql/plpgsql.sql index 62dc38e92e90..f3a10de5d3ba 100644 --- a/src/test/regress/sql/plpgsql.sql +++ b/src/test/regress/sql/plpgsql.sql @@ -4232,3 +4232,28 @@ select outer_outer_func(20); drop function outer_outer_func(int); drop function outer_func(int); drop function inner_func(int); + +-- Test consistency and passage of SET LOCAL GUC to writing commands +--start_ignore +drop schema if exists s cascade; +--end_ignore + +create schema s; + +do $$ +begin + set local search_path to s; + create table test_table(a int) distributed by (a); + drop table test_table; +end $$; + +-- check existence on master and segments +with c as ( + select gp_segment_id, relname, relkind, relnamespace from pg_class + union all + select gp_segment_id, relname, relkind, relnamespace from gp_dist_random('pg_class') +) select gp_segment_id from c +join pg_namespace n on n.oid = c.relnamespace +where c.relname = 'test_table' and c.relkind = 'r' and n.nspname = 's'; + +drop schema s cascade;