diff --git a/CMakeLists.txt b/CMakeLists.txt index 5d44f3e4e..ddd9505c6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,8 +91,21 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Distro.cmake) if(DEFINED DISKQUOTA_LAST_RELEASE_PATH) message(STATUS "Copy pervious installer from ${DISKQUOTA_LAST_RELEASE_PATH}") file(ARCHIVE_EXTRACT INPUT ${DISKQUOTA_LAST_RELEASE_PATH} PATTERNS "*.so") - file(GLOB DISKQUOTA_PREVIOUS_LIBRARY + file(GLOB so_files_list "${CMAKE_BINARY_DIR}/lib/postgresql/*.so") + + foreach(so_path IN LISTS so_files_list) + get_filename_component(so_name ${so_path} NAME_WLE) + # Replace 'diskquota-x.y' with 'x.y'. 'diskquota' won't be replaced, which belongs to 1.x release. + string(REPLACE "diskquota-" "" so_ver ${so_name}) + # Install the previous so files. Those so files have versions less than current version. + # diskqutoa.so doesn't have version string in the file name. It belongs to 1.x release. + if((${so_ver} STREQUAL "diskquota") OR + ${so_ver} VERSION_LESS ${DISKQUOTA_MAJOR_VERSION}.${DISKQUOTA_MINOR_VERSION}) + list(APPEND DISKQUOTA_PREVIOUS_LIBRARY ${so_path}) + endif() + endforeach() + install(PROGRAMS ${DISKQUOTA_PREVIOUS_LIBRARY} DESTINATION "lib/postgresql/") get_filename_component( diff --git a/VERSION b/VERSION index c043eea77..b1b25a5ff 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.2.1 +2.2.2 diff --git a/concourse/PROJ_NAME b/concourse/PROJ_NAME new file mode 100644 index 000000000..67f1c0899 --- /dev/null +++ b/concourse/PROJ_NAME @@ -0,0 +1 @@ +diskquota diff --git a/concourse/README.md b/concourse/README.md index 259b436bf..1007f4540 100644 --- a/concourse/README.md +++ b/concourse/README.md @@ -81,6 +81,14 @@ To test if the webhook works, use `curl` to send a `POST` request to the hook UR curl --data-raw "foo" ``` +## Update gp-extensions-ci + +We place some of the resources of concourse in a separate repository https://github.com/pivotal/gp-extensions-ci/. And we use that repo as a subtree with prefix ./concourse/lib. This is how to pull from the repo gp-extensions-ci: + +```sh + git subtree pull --prefix concourse/lib git@github.com:pivotal/gp-extensions-ci.git main --squash +``` + # FAQ ## PR pipeline is not triggered. diff --git a/concourse/fly.sh b/concourse/fly.sh deleted file mode 100755 index 75404b6be..000000000 --- a/concourse/fly.sh +++ /dev/null @@ -1,167 +0,0 @@ -#!/bin/bash - -set -e - -fly=${FLY:-"fly"} -echo "'fly' command: ${fly}" -echo "" -proj_name="diskquota" -concourse_team="main" - -usage() { - if [ -n "$1" ]; then - echo "$1" 1>&2 - echo "" 1>&2 - fi - - echo "Usage: $0 -t -c [-p ] [-b branch] [-T]" - echo "Options:" - echo " '-T' adds '_test' suffix to the pipeline type. Useful for pipeline debugging." - exit 1 -} - -# Hacky way to find out which concourse team is being used. -# The team name is needed to generate webhook URL -detect_concourse_team() { - local target="$1" - local fly_rc_file="$HOME/.flyrc" - local found_target=false - while read -r line; - do - line="$(echo -e "${line}" | tr -d '[:space:]')" - if [ ${found_target} != true ] && [ "${line}" = "${target}:" ]; then - found_target=true - fi - if [ ${found_target} = true ] && [[ "${line}" == team:* ]]; then - concourse_team=$(echo "${line}" | cut --delimiter=":" --fields=2) - echo "Use concourse target: ${target}, team: ${concourse_team}" - return - fi - done < "${fly_rc_file}" -} - -# Parse command line options -while getopts ":c:t:p:b:T" o; do - case "${o}" in - c) - # pipeline type/config. pr/merge/dev/rel - pipeline_config=${OPTARG} - ;; - t) - # concourse target - target=${OPTARG} - ;; - p) - # pipeline name - postfix=${OPTARG} - ;; - b) - # branch name - branch=${OPTARG} - ;; - T) - test_suffix="_test" - ;; - *) - usage "" - ;; - esac -done -shift $((OPTIND-1)) - -if [ -z "${target}" ] || [ -z "${pipeline_config}" ]; then - usage "" -fi - -detect_concourse_team "${target}" - -pipeline_type="" -# Decide ytt options to generate pipeline -case ${pipeline_config} in - pr) - pipeline_type="pr" - config_file="pr.yml" - hook_res="${proj_name}_pr" - ;; - merge|commit) - # Default branch is 'gpdb' as it is our main branch - if [ -z "${branch}" ]; then - branch="gpdb" - fi - pipeline_type="merge" - config_file="commit.yml" - hook_res="${proj_name}_commit" - ;; - dev) - if [ -z "${postfix}" ]; then - usage "'-p' needs to be supplied to specify the pipeline name postfix for flying a 'dev' pipeline." - fi - if [ -z "${branch}" ]; then - usage "'-b' needs to be supplied to specify the branch for flying a 'dev' pipeline." - fi - pipeline_type="dev" - config_file="dev.yml" - ;; - release|rel) - # Default branch is 'gpdb' as it is our main branch - if [ -z "${branch}" ]; then - branch="gpdb" - fi - pipeline_type="rel" - config_file="release.yml" - hook_res="${proj_name}_commit" - ;; - *) - usage "" - ;; -esac - -yml_path="/tmp/${proj_name}.yml" -my_path=$(realpath "${BASH_SOURCE[0]}") -ytt_base=$(dirname "${my_path}")/pipeline -# pipeline cannot contain '/' -pipeline_name=${pipeline_name/\//"_"} - -# Generate pipeline name -if [ -n "${test_suffix}" ]; then - pipeline_type="${pipeline_type}_test" -fi -pipeline_name="${pipeline_type}.${proj_name}" -if [ -n "${branch}" ]; then - pipeline_name="${pipeline_name}.${branch}" -fi -if [ -n "${postfix}" ]; then - pipeline_name="${pipeline_name}.${postfix}" -fi -# pipeline cannot contain '/' -pipeline_name=${pipeline_name/\//"_"} - -ytt --data-values-file "${ytt_base}/res_def.yml" \ - -f "${ytt_base}/base.lib.yml" \ - -f "${ytt_base}/job_def.lib.yml" \ - -f "${ytt_base}/trigger_def.lib.yml" \ - -f "${ytt_base}/${config_file}" > "${yml_path}" -echo "Generated pipeline yaml '${yml_path}'." - -echo "" -echo "Fly the pipeline..." -set -v -"${fly}" \ - -t "${target}" \ - sp \ - -p "${pipeline_name}" \ - -c "${yml_path}" \ - -v "${proj_name}-branch=${branch}" -set +v - -if [ "${pipeline_config}" == "dev" ]; then - exit 0 -fi - -concourse_url=$(fly targets | awk "{if (\$1 == \"${target}\") {print \$2}}") -echo "" -echo "================================================================================" -echo "Remeber to set the the webhook URL on GitHub:" -echo "${concourse_url}/api/v1/teams/${concourse_team}/pipelines/${pipeline_name}/resources/${hook_res}/check/webhook?webhook_token=" -echo "You may need to change the base URL if a differnt concourse server is used." -echo "================================================================================" diff --git a/concourse/fly.sh b/concourse/fly.sh new file mode 120000 index 000000000..09b854d19 --- /dev/null +++ b/concourse/fly.sh @@ -0,0 +1 @@ +lib/fly.sh \ No newline at end of file diff --git a/concourse/lib/README.md b/concourse/lib/README.md new file mode 100644 index 000000000..0dd8ca554 --- /dev/null +++ b/concourse/lib/README.md @@ -0,0 +1 @@ +# Common ytt libs for gp-extensions concourse diff --git a/concourse/pipeline/base.lib.yml b/concourse/lib/base.lib.yml similarity index 58% rename from concourse/pipeline/base.lib.yml rename to concourse/lib/base.lib.yml index 1a62132a9..78072de58 100644 --- a/concourse/pipeline/base.lib.yml +++ b/concourse/lib/base.lib.yml @@ -1,4 +1,23 @@ #@ load("@ytt:data", "data") +#@ load("@ytt:struct", "struct") + +#! resources and resource_types could be declared in different data-value files. +#! We check the key prefix, if they are 'resources' and 'resource_types', just +#! merge them into local dictionaries. +#@ resources = [] +#@ resource_types = [] +#@ data_values_dict = struct.decode(data.values) +#@ for key in data_values_dict.keys(): +#@ if key.startswith('resources'): +#@ resources.extend(data_values_dict[key]) +#@ end +#@ end +#@ for key in data_values_dict.keys(): +#@ if key.startswith('resource_type'): +#@ resource_types.extend(data_values_dict[key]) +#@ end +#@ end + #! add_res_by_xxx is to solve the unused resources error for concourse #@ def add_res_by_conf(res_map, job_conf): #@ for key in job_conf: @@ -21,7 +40,7 @@ #@ end #@ #@ def declare_res(res_type_map, res_map): -#@ for val in data.values.resources: +#@ for val in resources: #@ res_name = val["name"] #@ res_type = val["type"] #@ if res_map.get(val["name"]): @@ -32,7 +51,7 @@ #@ end #@ #@ def declare_res_type(res_type_map): -#@ for val in data.values.resource_types: +#@ for val in resource_types: #@ type_name = val["name"] #@ if res_type_map.get(type_name): - #@ val diff --git a/concourse/lib/fly.sh b/concourse/lib/fly.sh new file mode 100755 index 000000000..d68c5c5c9 --- /dev/null +++ b/concourse/lib/fly.sh @@ -0,0 +1,179 @@ +#!/bin/bash + +set -e + +fly=${FLY:-"fly"} +echo "'fly' command: ${fly}" +echo "" + +my_path=$(realpath -s "${BASH_SOURCE[0]}") +my_dir=$(dirname "${my_path}") +proj_name_file="${my_dir}/PROJ_NAME" +if [ ! -f "${proj_name_file}" ]; then + echo "A 'PROJ_NAME' file is needed in '${my_dir}'" + exit 1 +fi +proj_name=$(cat "${proj_name_file}") +concourse_team="main" + +usage() { + if [ -n "$1" ]; then + echo "$1" 1>&2 + echo "" 1>&2 + fi + + echo "Usage: $0 -t -c [-p ] [-b branch] [-T]" + echo "Options:" + echo " '-T' adds '_test' suffix to the pipeline type. Useful for pipeline debugging." + exit 1 +} + +# Hacky way to find out which concourse team is being used. +# The team name is needed to generate webhook URL +detect_concourse_team() { + local target="$1" + local fly_rc_file="$HOME/.flyrc" + local found_target=false + while read -r line; + do + line="$(echo -e "${line}" | tr -d '[:space:]')" + if [ ${found_target} != true ] && [ "${line}" = "${target}:" ]; then + found_target=true + fi + if [ ${found_target} = true ] && [[ "${line}" == team:* ]]; then + concourse_team=$(echo "${line}" | cut --delimiter=":" --fields=2) + echo "Use concourse target: ${target}, team: ${concourse_team}" + return + fi + done < "${fly_rc_file}" +} + +# Parse command line options +while getopts ":c:t:p:b:T" o; do + case "${o}" in + c) + # pipeline type/config. pr/merge/dev/rel + pipeline_config=${OPTARG} + ;; + t) + # concourse target + target=${OPTARG} + ;; + p) + # pipeline name + postfix=${OPTARG} + ;; + b) + # branch name + branch=${OPTARG} + ;; + T) + test_suffix="_test" + ;; + *) + usage "" + ;; + esac +done +shift $((OPTIND-1)) + +if [ -z "${target}" ] || [ -z "${pipeline_config}" ]; then + usage "" +fi + +detect_concourse_team "${target}" + +pipeline_type="" +# Decide ytt options to generate pipeline +case ${pipeline_config} in + pr) + pipeline_type="pr" + config_file="pr.yml" + hook_res="${proj_name}_pr" + ;; + merge|commit) + # Default branch is 'gpdb' as it is our main branch + if [ -z "${branch}" ]; then + branch="gpdb" + fi + pipeline_type="merge" + config_file="commit.yml" + hook_res="${proj_name}_commit" + ;; + dev) + if [ -z "${postfix}" ]; then + usage "'-p' needs to be supplied to specify the pipeline name postfix for flying a 'dev' pipeline." + fi + if [ -z "${branch}" ]; then + usage "'-b' needs to be supplied to specify the branch for flying a 'dev' pipeline." + fi + pipeline_type="dev" + config_file="dev.yml" + ;; + release|rel) + # Default branch is 'gpdb' as it is our main branch + if [ -z "${branch}" ]; then + branch="gpdb" + fi + pipeline_type="rel" + config_file="release.yml" + hook_res="${proj_name}_commit" + ;; + *) + usage "" + ;; +esac + +yml_path="/tmp/${proj_name}.yml" +pipeline_dir="${my_dir}/pipeline" +lib_dir="${my_dir}/lib" +# pipeline cannot contain '/' +pipeline_name=${pipeline_name/\//"_"} + +# Generate pipeline name +if [ -n "${test_suffix}" ]; then + pipeline_type="${pipeline_type}_test" +fi +pipeline_name="${pipeline_type}.${proj_name}" +if [ -n "${branch}" ]; then + pipeline_name="${pipeline_name}.${branch}" +fi +if [ -n "${postfix}" ]; then + pipeline_name="${pipeline_name}.${postfix}" +fi +# pipeline cannot contain '/' +pipeline_name=${pipeline_name/\//"_"} + +ytt \ + --data-values-file "${pipeline_dir}/res_def.yml" \ + --data-values-file "${lib_dir}/res_def_gpdb.yml" \ + --data-values-file "${lib_dir}/res_def_misc.yml" \ + --data-values-file "${lib_dir}/res_types_def.yml" \ + -f "${lib_dir}/base.lib.yml" \ + -f "${pipeline_dir}/job_def.lib.yml" \ + -f "${pipeline_dir}/trigger_def.lib.yml" \ + -f "${pipeline_dir}/${config_file}" > "${yml_path}" +echo "Generated pipeline yaml '${yml_path}'." + +echo "" +echo "Fly the pipeline..." +set -v +"${fly}" \ + -t "${target}" \ + sp \ + -p "${pipeline_name}" \ + -c "${yml_path}" \ + -v "${proj_name}-branch=${branch}" +set +v + +if [ "${pipeline_config}" == "dev" ]; then + exit 0 +fi + +concourse_url=$(fly targets | awk "{if (\$1 == \"${target}\") {print \$2}}") +echo "" +echo "================================================================================" +echo "Remeber to set the the webhook URL on GitHub:" +echo "${concourse_url}/api/v1/teams/${concourse_team}/pipelines/${pipeline_name}/resources/${hook_res}/check/webhook?webhook_token=" +echo "You may need to change the base URL if a different concourse server is used." +echo "================================================================================" diff --git a/concourse/lib/res_def_gpdb.yml b/concourse/lib/res_def_gpdb.yml new file mode 100644 index 000000000..b4384480b --- /dev/null +++ b/concourse/lib/res_def_gpdb.yml @@ -0,0 +1,148 @@ +resources_gpdb: +# Image Resources +# centos6 +- name: centos6-gpdb6-image-build + type: registry-image + source: + repository: gcr.io/data-gpdb-public-images/gpdb6-centos6-build + tag: latest +- name: centos6-gpdb6-image-test + type: registry-image + source: + repository: gcr.io/data-gpdb-public-images/gpdb6-centos6-test + tag: latest +# centos7 +- name: centos7-gpdb6-image-build + type: registry-image + source: + repository: gcr.io/data-gpdb-public-images/gpdb6-centos7-build + tag: latest +- name: centos7-gpdb6-image-test + type: registry-image + source: + repository: gcr.io/data-gpdb-public-images/gpdb6-centos7-test + tag: latest +# rhel8 +- name: rhel8-gpdb6-image-build + type: registry-image + source: + repository: gcr.io/data-gpdb-private-images/gpdb6-rhel8-build + tag: latest + username: _json_key + password: ((data-gpdb-private-images-container-registry-readonly-service-account-key)) +- name: rhel8-gpdb6-image-test + type: registry-image + source: + repository: gcr.io/data-gpdb-private-images/gpdb6-rhel8-test + tag: latest + username: _json_key + password: ((data-gpdb-private-images-container-registry-readonly-service-account-key)) +# Ubuntu18 +- name: ubuntu18-gpdb6-image-build + type: registry-image + source: + repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-build + tag: latest +- name: ubuntu18-gpdb6-image-test + type: registry-image + source: + repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-test + tag: latest +# GPDB7 +# build +- name: rocky8-gpdb7-image-build + type: registry-image + source: + repository: gcr.io/data-gpdb-public-images/gpdb7-rocky8-build + tag: latest +# test +- name: rocky8-gpdb7-image-test + type: registry-image + source: + repository: gcr.io/data-gpdb-public-images/gpdb7-rocky8-test + tag: latest +- name: rhel8-gpdb7-image-test + type: registry-image + source: + repository: gcr.io/data-gpdb-private-images/gpdb7-rhel8-test + tag: latest + username: _json_key + password: ((data-gpdb-private-images-container-registry-readonly-service-account-key)) + +# gpdb binary on gcs is located as different folder for different version +# Latest build with assertion enabled: +# --enable-cassert --enable-tap-tests --enable-debug-extensions +- name: bin_gpdb6_centos6_debug + type: gcs + source: + bucket: pivotal-gpdb-concourse-resources-prod + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: server/published/gpdb6/server-rc-(.*\+dev\.\d+.*)-rhel6_x86_64.debug.tar.gz +- name: bin_gpdb6_centos7_debug + type: gcs + source: + bucket: pivotal-gpdb-concourse-resources-prod + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: server/published/gpdb6/server-rc-(.*\+dev\.\d+.*)-rhel7_x86_64.debug.tar.gz +- name: bin_gpdb6_rhel8_debug + type: gcs + source: + bucket: pivotal-gpdb-concourse-resources-prod + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: server/published/gpdb6/server-rc-(.*\+dev\.\d+.*)-rhel8_x86_64.debug.tar.gz +- name: bin_gpdb6_ubuntu18_debug + type: gcs + source: + bucket: pivotal-gpdb-concourse-resources-prod + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: server/published/gpdb6/server-rc-(.*\+dev\.\d+.*)-ubuntu18.04_x86_64.debug.tar.gz +- name: bin_gpdb7_el8_debug + type: gcs + source: + bucket: pivotal-gpdb-concourse-resources-prod + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: server/published/main/server-rc-(.*\+dev\.\d+.*)-el8_x86_64.debug.tar.gz +# Latest release candidates, no fault-injector, no assertion: +# --disable-debug-extensions --disable-tap-tests --enable-ic-proxy +- name: bin_gpdb6_centos6 + type: gcs + source: + bucket: pivotal-gpdb-concourse-resources-prod + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: server/release-candidates/gpdb6/greenplum-db-server-(6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*\+dev\.\d+)-.*-centos6.tar.gz +- name: bin_gpdb6_centos7 + type: gcs + source: + bucket: pivotal-gpdb-concourse-resources-prod + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: server/release-candidates/gpdb6/greenplum-db-server-(6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*\+dev\.\d+)-.*-centos7.tar.gz +- name: bin_gpdb6_rhel8 + type: gcs + source: + bucket: pivotal-gpdb-concourse-resources-prod + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: server/release-candidates/gpdb6/greenplum-db-server-(6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*\+dev\.\d+)-.*-rhel8.tar.gz +- name: bin_gpdb6_ubuntu18 + type: gcs + source: + bucket: pivotal-gpdb-concourse-resources-prod + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: server/release-candidates/gpdb6/greenplum-db-server-(6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*\+dev\.\d+)-.*-ubuntu18.04.tar.gz +- name: bin_gpdb7_el8 + type: gcs + source: + bucket: pivotal-gpdb-concourse-resources-prod + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: server/release-candidates/gpdb7/greenplum-db-server-(7\.([0-9]|([1-8][0-9])|(9[0-8]))\..*\+dev\.\d+)-.*-el8.tar.gz + +# Greenplum sources +- name: gpdb6_src + type: git + source: + branch: 6X_STABLE + uri: https://github.com/greenplum-db/gpdb.git +- name: gpdb7_src + type: git + source: + branch: main + uri: https://github.com/greenplum-db/gpdb.git diff --git a/concourse/lib/res_def_misc.yml b/concourse/lib/res_def_misc.yml new file mode 100644 index 000000000..3105dd38d --- /dev/null +++ b/concourse/lib/res_def_misc.yml @@ -0,0 +1,7 @@ +resources_misc: + +# Other dependencies +- name: slack_notify_extensions + type: slack-alert + source: + url: ((extension/extensions-slack-webhook)) diff --git a/concourse/lib/res_types_def.yml b/concourse/lib/res_types_def.yml new file mode 100644 index 000000000..07eebf6ea --- /dev/null +++ b/concourse/lib/res_types_def.yml @@ -0,0 +1,21 @@ +resource_types_common: +- name: gcs + type: registry-image + check_every: 1h + source: + repository: frodenas/gcs-resource + tag: latest + +- name: pull-request + type: registry-image + check_every: 1h + source: + repository: teliaoss/github-pr-resource + tag: latest + +- name: slack-alert + type: registry-image + source: + repository: arbourd/concourse-slack-alert-resource + tag: latest + diff --git a/concourse/pipeline/commit.yml b/concourse/pipeline/commit.yml index 823dfcc28..1891caeba 100644 --- a/concourse/pipeline/commit.yml +++ b/concourse/pipeline/commit.yml @@ -19,7 +19,7 @@ #@ centos7_gpdb6_conf(), #@ rhel8_gpdb6_conf(), #@ ubuntu18_gpdb6_conf(), -#@ rhel8_gpdb7_conf(), +#! #@ rhel8_gpdb7_conf(), #@ ] jobs: #@ param = { diff --git a/concourse/pipeline/pr.yml b/concourse/pipeline/pr.yml index 4a715120c..3578131aa 100644 --- a/concourse/pipeline/pr.yml +++ b/concourse/pipeline/pr.yml @@ -22,7 +22,7 @@ #@ centos7_gpdb6_conf(), #@ rhel8_gpdb6_conf(), #@ ubuntu18_gpdb6_conf(), -#@ rhel8_gpdb7_conf(), +#! #@ rhel8_gpdb7_conf(), #@ ] jobs: #@ param = { diff --git a/concourse/pipeline/release.yml b/concourse/pipeline/release.yml index 023e86bd8..6b12b935f 100644 --- a/concourse/pipeline/release.yml +++ b/concourse/pipeline/release.yml @@ -20,7 +20,7 @@ #@ centos7_gpdb6_conf(release_build=True), #@ rhel8_gpdb6_conf(release_build=True), #@ ubuntu18_gpdb6_conf(release_build=True), -#@ rhel8_gpdb7_conf(release_build=True) +#! #@ rhel8_gpdb7_conf(release_build=True) #@ ] jobs: #@ param = { diff --git a/concourse/pipeline/res_def.yml b/concourse/pipeline/res_def.yml index 4e6578d85..1206969af 100644 --- a/concourse/pipeline/res_def.yml +++ b/concourse/pipeline/res_def.yml @@ -1,21 +1,3 @@ -resource_types: -- name: gcs - type: docker-image - check_every: 1h - source: - repository: frodenas/gcs-resource - -- name: pull-request - type: docker-image - check_every: 1h - source: - repository: teliaoss/github-pr-resource - -- name: slack-alert - type: docker-image - source: - repository: arbourd/concourse-slack-alert-resource - resources: # Pull Request - name: diskquota_pr @@ -49,155 +31,6 @@ resources: password: x-oauth-basic -# Greenplum sources -- name: gpdb6_src - type: git - source: - branch: 6X_STABLE - uri: https://github.com/greenplum-db/gpdb.git -- name: gpdb7_src - type: git - source: - branch: main - uri: https://github.com/greenplum-db/gpdb.git - -# Image Resources -# centos6 -- name: centos6-gpdb6-image-build - type: registry-image - source: - repository: gcr.io/data-gpdb-public-images/gpdb6-centos6-build - tag: latest -- name: centos6-gpdb6-image-test - type: registry-image - source: - repository: gcr.io/data-gpdb-public-images/gpdb6-centos6-test - tag: latest -# centos7 -- name: centos7-gpdb6-image-build - type: registry-image - source: - repository: gcr.io/data-gpdb-public-images/gpdb6-centos7-build - tag: latest -- name: centos7-gpdb6-image-test - type: registry-image - source: - repository: gcr.io/data-gpdb-public-images/gpdb6-centos7-test - tag: latest -# rhel8 -- name: rhel8-gpdb6-image-build - type: registry-image - source: - repository: gcr.io/data-gpdb-private-images/gpdb6-rhel8-build - tag: latest - username: _json_key - password: ((data-gpdb-private-images-container-registry-readonly-service-account-key)) -- name: rhel8-gpdb6-image-test - type: registry-image - source: - repository: gcr.io/data-gpdb-private-images/gpdb6-rhel8-test - tag: latest - username: _json_key - password: ((data-gpdb-private-images-container-registry-readonly-service-account-key)) -# Ubuntu18 -- name: ubuntu18-gpdb6-image-build - type: registry-image - source: - repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-build - tag: latest -- name: ubuntu18-gpdb6-image-test - type: registry-image - source: - repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-test - tag: latest -# GPDB7 -# build -- name: rocky8-gpdb7-image-build - type: registry-image - source: - repository: gcr.io/data-gpdb-public-images/gpdb7-rocky8-build - tag: latest -# test -- name: rocky8-gpdb7-image-test - type: registry-image - source: - repository: gcr.io/data-gpdb-public-images/gpdb7-rocky8-test - tag: latest -- name: rhel8-gpdb7-image-test - type: registry-image - source: - repository: gcr.io/data-gpdb-private-images/gpdb7-rhel8-test - tag: latest - username: _json_key - password: ((data-gpdb-private-images-container-registry-readonly-service-account-key)) - -# gpdb binary on gcs is located as different folder for different version -# Latest build with assertion enabled: -# --enable-cassert --enable-tap-tests --enable-debug-extensions -- name: bin_gpdb6_centos6_debug - type: gcs - source: - bucket: pivotal-gpdb-concourse-resources-prod - json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/published/gpdb6/server-rc-(.*)-rhel6_x86_64.debug.tar.gz -- name: bin_gpdb6_centos7_debug - type: gcs - source: - bucket: pivotal-gpdb-concourse-resources-prod - json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/published/gpdb6/server-rc-(.*)-rhel7_x86_64.debug.tar.gz -- name: bin_gpdb6_rhel8_debug - type: gcs - source: - bucket: pivotal-gpdb-concourse-resources-prod - json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/published/gpdb6/server-rc-(.*)-rhel8_x86_64.debug.tar.gz -- name: bin_gpdb6_ubuntu18_debug - type: gcs - source: - bucket: pivotal-gpdb-concourse-resources-prod - json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/published/gpdb6/server-rc-(.*)-ubuntu18.04_x86_64.debug.tar.gz -- name: bin_gpdb7_el8_debug - type: gcs - source: - bucket: pivotal-gpdb-concourse-resources-prod - json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/published/main/server-rc-(.*)-el8_x86_64.debug.tar.gz - -# Latest release candidates, no fault-injector, no assertion: -# --disable-debug-extensions --disable-tap-tests --enable-ic-proxy -- name: bin_gpdb6_centos6 - type: gcs - source: - bucket: pivotal-gpdb-concourse-resources-prod - json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/release-candidates/gpdb6/greenplum-db-server-6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev.*-centos6.tar.gz -- name: bin_gpdb6_centos7 - type: gcs - source: - bucket: pivotal-gpdb-concourse-resources-prod - json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/release-candidates/gpdb6/greenplum-db-server-6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev.*-centos7.tar.gz -- name: bin_gpdb6_rhel8 - type: gcs - source: - bucket: pivotal-gpdb-concourse-resources-prod - json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/release-candidates/gpdb6/greenplum-db-server-6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev.*-rhel8.tar.gz -- name: bin_gpdb6_ubuntu18 - type: gcs - source: - bucket: pivotal-gpdb-concourse-resources-prod - json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/release-candidates/gpdb6/greenplum-db-server-6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev.*-ubuntu18.04.tar.gz -- name: bin_gpdb7_el8 - type: gcs - source: - bucket: pivotal-gpdb-concourse-resources-prod - json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/release-candidates/gpdb7/greenplum-db-server-7\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev.*-el8.tar.gz - # Diskquota releases - name: bin_diskquota_gpdb6_rhel6 type: gcs @@ -350,8 +183,3 @@ resources: bucket: gp-extensions-ci json_key: ((extension/extensions-gcs-service-account-key-dev2)) regexp: dependencies/cmake-(.*)-linux-x86_64.sh - -- name: slack_notify - type: slack-alert - source: - url: ((extension/extensions-slack-webhook)) diff --git a/concourse/pipeline/trigger_def.lib.yml b/concourse/pipeline/trigger_def.lib.yml index c2854d027..607cb7a3f 100644 --- a/concourse/pipeline/trigger_def.lib.yml +++ b/concourse/pipeline/trigger_def.lib.yml @@ -26,16 +26,16 @@ on_error: path: diskquota_src status: failure on_success: - put: diskquota_pr - params: - path: diskquota_src - status: success + put: diskquota_pr + params: + path: diskquota_src + status: success #@ end #! Commit trigger. For master pipelines #@ def commit_trigger(res_map): #@ add_res_by_name(res_map, "diskquota_commit") -#@ add_res_by_name(res_map, "slack_notify") +#@ add_res_by_name(res_map, "slack_notify_extensions") auto_trigger: true to_get: - get: diskquota_src @@ -46,11 +46,11 @@ to_put_post: #@ [] #! Unfortunately it doesn't work with Concourse 5. on_success: on_failure: - put: slack_notify + put: slack_notify_extensions params: alert_type: failed on_error: - put: slack_notify + put: slack_notify_extensions params: alert_type: errored #@ end @@ -74,7 +74,7 @@ on_error: #! Commit trigger. For release pipelines #@ def release_trigger(res_map): #@ add_res_by_name(res_map, "diskquota_commit") -#@ add_res_by_name(res_map, "slack_notify") +#@ add_res_by_name(res_map, "slack_notify_extensions") auto_trigger: true to_get: - get: diskquota_src @@ -86,16 +86,16 @@ to_put_post: repository: diskquota_src tag: diskquota_src/VERSION on_success: - put: slack_notify + put: slack_notify_extensions params: alert_type: success text: A new diskquota release has been pushed! on_failure: - put: slack_notify + put: slack_notify_extensions params: alert_type: failed on_error: - put: slack_notify + put: slack_notify_extensions params: alert_type: errored #@ end diff --git a/concourse/scripts/test_diskquota.sh b/concourse/scripts/test_diskquota.sh index 245196ffc..cda281780 100755 --- a/concourse/scripts/test_diskquota.sh +++ b/concourse/scripts/test_diskquota.sh @@ -39,6 +39,11 @@ function _main() { time cmake --build . --target installcheck fi popd + + # TODO: existing upgrade and alter tests are useless for ADB + # - previous releases incompatible with current on PG-module version check + # - our upgrade process exclude existing of previous versions during upgrade + # time /home/gpadmin/diskquota_src/upgrade_test/alter_test.sh } _main diff --git a/src/diskquota.c b/src/diskquota.c index 4f0fe43c4..834867225 100644 --- a/src/diskquota.c +++ b/src/diskquota.c @@ -42,6 +42,8 @@ #include "utils/snapmgr.h" #include "utils/syscache.h" #include "utils/timestamp.h" +#include "utils/formatting.h" +#include "tcop/pquery.h" PG_MODULE_MAGIC; @@ -147,6 +149,7 @@ static void resetBackgroundWorkerCorruption(void); /* WaitForBackgroundWorkerShutdown is copied from gpdb7 */ static BgwHandleStatus WaitForBackgroundWorkerShutdown(BackgroundWorkerHandle *handle); #endif /* GP_VERSION_NUM */ +static bool is_altering_extension(void); /* * diskquota_launcher_shmem_size @@ -163,6 +166,40 @@ diskquota_launcher_shmem_size(void) size = add_size(size, mul_size(MAX_NUM_MONITORED_DB, sizeof(struct DiskquotaDBEntry))); // hidden memory for dbArray return size; } + +static bool +is_altering_extension(void) +{ + if (ActivePortal == NULL) return false; + /* QD: When the sourceTag is T_AlterExtensionStmt, then return true */ + if (ActivePortal->sourceTag == T_AlterExtensionStmt) return true; + + /* + * QE: The sourceTag won't be T_AlterExtensionStmt, we should check the sourceText. + * If the sourceText contains 'alter extension diskquota update', we consider it is + * a alter extension query. + */ + char *query = asc_tolower(ActivePortal->sourceText, strlen(ActivePortal->sourceText)); + char *pos; + bool match = true; + + pos = strstr(query, "alter"); + if (pos) + pos = strstr(pos, "extension"); + else + match = false; + if (pos) + pos = strstr(pos, "diskquota"); + else + match = false; + if (pos) + pos = strstr(pos, "update"); + else + match = false; + pfree(query); + return match; +} + /* * Entrypoint of diskquota module. * @@ -176,6 +213,15 @@ _PG_init(void) /* diskquota.so must be in shared_preload_libraries to init SHM. */ if (!process_shared_preload_libraries_in_progress) { + /* + * To support the continuous upgrade/downgrade, we should skip the library + * check in _PG_init() during upgrade/downgrade. + */ + if (IsNormalProcessingMode() && is_altering_extension()) + { + ereport(LOG, (errmsg("[diskquota] altering diskquota version to " DISKQUOTA_VERSION "."))); + return; + } ereport(ERROR, (errmsg("[diskquota] booting " DISKQUOTA_VERSION ", but " DISKQUOTA_BINARY_NAME " not in shared_preload_libraries. abort."))); } @@ -339,7 +385,13 @@ disk_quota_worker_main(Datum main_arg) pqsignal(SIGUSR1, disk_quota_sigusr1); if (!MyWorkerInfo->dbEntry->inited) + { + MyWorkerInfo->dbEntry->last_log_time = GetCurrentTimestamp(); ereport(LOG, (errmsg("[diskquota] start disk quota worker process to monitor database:%s", dbname))); + } + /* To avoid last_log_time from being uninitialized. */ + if (MyWorkerInfo->dbEntry->last_log_time > GetCurrentTimestamp()) + MyWorkerInfo->dbEntry->last_log_time = GetCurrentTimestamp(); /* * The shmem exit hook is registered after registering disk_quota_sigterm. * So if the SIGTERM arrives before this statement, the shmem exit hook @@ -483,15 +535,13 @@ disk_quota_worker_main(Datum main_arg) if (!MyWorkerInfo->dbEntry->inited) update_monitordb_status(MyWorkerInfo->dbEntry->dbid, DB_RUNNING); - bool is_gang_destroyed = false; - TimestampTz log_start_timestamp = GetCurrentTimestamp(); - TimestampTz log_end_timestamp; + bool is_gang_destroyed = false; TimestampTz loop_start_timestamp = 0; TimestampTz loop_end_timestamp; + TimestampTz log_time; long sleep_time = diskquota_naptime * 1000; long secs; int usecs; - ereport(LOG, (errmsg("[diskquota] disk quota worker process is monitoring database:%s", dbname))); while (!got_sigterm) { @@ -503,11 +553,11 @@ disk_quota_worker_main(Datum main_arg) * every BGWORKER_LOG_TIME to ensure that we can find the database name * by the bgworker's pid in the log file. */ - log_end_timestamp = GetCurrentTimestamp(); - if (TimestampDifferenceExceeds(log_start_timestamp, log_end_timestamp, BGWORKER_LOG_TIME)) + log_time = GetCurrentTimestamp(); + if (TimestampDifferenceExceeds(MyWorkerInfo->dbEntry->last_log_time, log_time, BGWORKER_LOG_TIME)) { ereport(LOG, (errmsg("[diskquota] disk quota worker process is monitoring database:%s", dbname))); - log_start_timestamp = log_end_timestamp; + MyWorkerInfo->dbEntry->last_log_time = log_time; } /* @@ -1517,10 +1567,8 @@ diskquota_status_binary_version() static const char * diskquota_status_schema_version() { - static char version[64] = {0}; - memset(version, 0, sizeof(version)); - - int ret = SPI_connect(); + static char ret_version[64]; + int ret = SPI_connect(); Assert(ret = SPI_OK_CONNECT); ret = SPI_execute("select extversion from pg_extension where extname = 'diskquota'", true, 0); @@ -1529,30 +1577,33 @@ diskquota_status_schema_version() { ereport(WARNING, (errmsg("[diskquota] when reading installed version lines %ld code = %d", SPI_processed, ret))); - goto out; + goto fail; } if (SPI_processed == 0) { - goto out; + goto fail; } - bool is_null = false; - Datum v = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &is_null); + bool is_null = false; + Datum version_datum = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &is_null); Assert(is_null == false); - char *vv = TextDatumGetCString(v); - if (vv == NULL) + char *version = TextDatumGetCString(version_datum); + if (version == NULL || *version == '\0') { ereport(WARNING, (errmsg("[diskquota] 'extversion' is empty in pg_class.pg_extension. may catalog corrupted"))); - goto out; + goto fail; } - StrNCpy(version, vv, sizeof(version)); + StrNCpy(ret_version, version, sizeof(ret_version) - 1); -out: SPI_finish(); - return version; + return ret_version; + +fail: + SPI_finish(); + return ""; } PG_FUNCTION_INFO_V1(diskquota_status); diff --git a/src/diskquota.h b/src/diskquota.h index 58a00f464..c218148e8 100644 --- a/src/diskquota.h +++ b/src/diskquota.h @@ -215,9 +215,11 @@ struct DiskquotaDBEntry TimestampTz last_run_time; int16 cost; // ms - bool inited; // this entry is inited, will set to true after the worker finish the frist run. - bool in_use; // this slot is in using. AKA dbid != 0 + bool inited; // this entry is inited, will set to true after the worker finish the frist run. + bool in_use; // this slot is in using. AKA dbid != 0 bool corrupted; // consider this entry as invalid to start the worker on + + TimestampTz last_log_time; // the last time log current database info. }; typedef enum MonitorDBStatus @@ -290,4 +292,5 @@ extern HTAB *diskquota_hash_create(const char *tabname, long nelem, HASHC DiskquotaHashFunction hashFunction); extern HTAB *DiskquotaShmemInitHash(const char *name, long init_size, long max_size, HASHCTL *infoP, int hash_flags, DiskquotaHashFunction hash_function); +extern void refresh_monitored_dbid_cache(void); #endif diff --git a/src/gp_activetable.c b/src/gp_activetable.c index cbf6e7b66..23c144fe3 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -750,6 +750,8 @@ get_active_tables_oid(void) Oid relOid; + refresh_monitored_dbid_cache(); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(DiskQuotaActiveTableFileEntry); ctl.entrysize = sizeof(DiskQuotaActiveTableFileEntry); diff --git a/src/monitored_db.c b/src/monitored_db.c index f23cb9a94..c2b29a1ba 100644 --- a/src/monitored_db.c +++ b/src/monitored_db.c @@ -336,3 +336,32 @@ dump_monitored_dbid_cache(long *nitems) Assert(count == 0); return entries; } + +/* + * After primary failure and mirror switching, the monitored_dbid_cache + * is lost on segments. We should refresh the monitored_dbid_cache during + * every diskquota refresh procedure. + */ +void +refresh_monitored_dbid_cache(void) +{ + bool found; + Oid dbid = MyDatabaseId; + LWLockAcquire(diskquota_locks.monitored_dbid_cache_lock, LW_EXCLUSIVE); + MonitorDBEntry entry = hash_search(monitored_dbid_cache, &dbid, HASH_ENTER_NULL, &found); + if (entry == NULL) + { + ereport(WARNING, (errmsg("can't alloc memory on dbid cache, there are too many databases to monitor"))); + } + else if (!found) + { + entry->paused = false; + pg_atomic_init_u32(&(entry->epoch), 0); + pg_atomic_init_u32(&(entry->status), DB_RUNNING); + ereport(LOG, (errmsg("the entry in monitored_dbid_cache is lost due to mirror switching and is added back now, " + "dbid: %d", + dbid))); + } + + LWLockRelease(diskquota_locks.monitored_dbid_cache_lock); +} diff --git a/tests/regress/expected/test_primary_failure.out b/tests/regress/expected/test_primary_failure.out index 5f5c18a23..48160e246 100644 --- a/tests/regress/expected/test_primary_failure.out +++ b/tests/regress/expected/test_primary_failure.out @@ -36,19 +36,27 @@ returns text as $$ else: return subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True, encoding='utf8').replace('.', '') $$ language plpython2u; -CREATE TABLE a(i int) DISTRIBUTED BY (i); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'i' as the Greenplum Database data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. -INSERT INTO a SELECT generate_series(1,100); -INSERT INTO a SELECT generate_series(1,100000); +CREATE TABLE a(i int, j int) DISTRIBUTED BY (i); +-- the entries will be inserted into seg0 +INSERT INTO a SELECT 2, generate_series(1,100); +INSERT INTO a SELECT 2, generate_series(1,100000); SELECT diskquota.wait_for_worker_new_epoch(); wait_for_worker_new_epoch --------------------------- t (1 row) +SELECT tableid::regclass, size, segid FROM diskquota.table_size WHERE tableid = 'a'::regclass ORDER BY segid; + tableid | size | segid +---------+---------+------- + a | 3735552 | -1 + a | 3735552 | 0 + a | 0 | 1 + a | 0 | 2 +(4 rows) + -- expect insert fail -INSERT INTO a SELECT generate_series(1,100); +INSERT INTO a SELECT 2, generate_series(1,100); ERROR: schema's disk space quota exceeded with name: ftsr -- now one of primary is down select pg_ctl((select datadir from gp_segment_configuration c where c.role='p' and c.content=0), 'stop'); @@ -74,7 +82,7 @@ select content, preferred_role, role, status, mode from gp_segment_configuration (2 rows) -- expect insert fail -INSERT INTO a SELECT generate_series(1,100); +INSERT INTO a SELECT 2, generate_series(1,100); ERROR: schema's disk space quota exceeded with name: ftsr -- increase quota SELECT diskquota.set_schema_quota('ftsr', '200 MB'); @@ -83,6 +91,31 @@ SELECT diskquota.set_schema_quota('ftsr', '200 MB'); (1 row) +SELECT diskquota.wait_for_worker_new_epoch(); + wait_for_worker_new_epoch +--------------------------- + t +(1 row) + +-- expect insert success +INSERT INTO a SELECT 2, generate_series(1,10000); +SELECT diskquota.wait_for_worker_new_epoch(); + wait_for_worker_new_epoch +--------------------------- + t +(1 row) + +-- check whether monitored_dbid_cache is refreshed in mirror +-- diskquota.table_size should be updated +SELECT tableid::regclass, size, segid FROM diskquota.table_size WHERE tableid = 'a'::regclass ORDER BY segid; + tableid | size | segid +---------+---------+------- + a | 4096000 | -1 + a | 4096000 | 0 + a | 0 | 1 + a | 0 | 2 +(4 rows) + -- pull up failed primary -- start_ignore select pg_recoverseg((select datadir from gp_segment_configuration c where c.role='p' and c.content=-1), 'a'); @@ -256,10 +289,10 @@ SELECT diskquota.wait_for_worker_new_epoch(); SELECT quota_in_mb, nspsize_in_bytes from diskquota.show_fast_schema_quota_view where schema_name='ftsr'; quota_in_mb | nspsize_in_bytes -------------+------------------ - 200 | 3932160 + 200 | 4096000 (1 row) -INSERT INTO a SELECT generate_series(1,100); +INSERT INTO a SELECT 2, generate_series(1,100); DROP TABLE a; DROP SCHEMA ftsr CASCADE; NOTICE: drop cascades to 2 other objects diff --git a/tests/regress/sql/test_primary_failure.in.sql b/tests/regress/sql/test_primary_failure.in.sql index cbac6e4cd..2dd2689b6 100644 --- a/tests/regress/sql/test_primary_failure.in.sql +++ b/tests/regress/sql/test_primary_failure.in.sql @@ -33,12 +33,16 @@ returns text as $$ return subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True, encoding='utf8').replace('.', '') $$ language @PLPYTHON_LANG_STR@; -CREATE TABLE a(i int) DISTRIBUTED BY (i); -INSERT INTO a SELECT generate_series(1,100); -INSERT INTO a SELECT generate_series(1,100000); +CREATE TABLE a(i int, j int) DISTRIBUTED BY (i); +-- the entries will be inserted into seg0 +INSERT INTO a SELECT 2, generate_series(1,100); +INSERT INTO a SELECT 2, generate_series(1,100000); SELECT diskquota.wait_for_worker_new_epoch(); + +SELECT tableid::regclass, size, segid FROM diskquota.table_size WHERE tableid = 'a'::regclass ORDER BY segid; + -- expect insert fail -INSERT INTO a SELECT generate_series(1,100); +INSERT INTO a SELECT 2, generate_series(1,100); -- now one of primary is down select pg_ctl((select datadir from gp_segment_configuration c where c.role='p' and c.content=0), 'stop'); @@ -50,11 +54,22 @@ select gp_request_fts_probe_scan(); select content, preferred_role, role, status, mode from gp_segment_configuration where content = 0; -- expect insert fail -INSERT INTO a SELECT generate_series(1,100); +INSERT INTO a SELECT 2, generate_series(1,100); -- increase quota SELECT diskquota.set_schema_quota('ftsr', '200 MB'); +SELECT diskquota.wait_for_worker_new_epoch(); + +-- expect insert success +INSERT INTO a SELECT 2, generate_series(1,10000); + +SELECT diskquota.wait_for_worker_new_epoch(); + +-- check whether monitored_dbid_cache is refreshed in mirror +-- diskquota.table_size should be updated +SELECT tableid::regclass, size, segid FROM diskquota.table_size WHERE tableid = 'a'::regclass ORDER BY segid; + -- pull up failed primary -- start_ignore select pg_recoverseg((select datadir from gp_segment_configuration c where c.role='p' and c.content=-1), 'a'); @@ -67,7 +82,7 @@ select content, preferred_role, role, status, mode from gp_segment_configuration SELECT diskquota.wait_for_worker_new_epoch(); SELECT quota_in_mb, nspsize_in_bytes from diskquota.show_fast_schema_quota_view where schema_name='ftsr'; -INSERT INTO a SELECT generate_series(1,100); +INSERT INTO a SELECT 2, generate_series(1,100); DROP TABLE a; DROP SCHEMA ftsr CASCADE; diff --git a/upgrade_test/CMakeLists.txt b/upgrade_test/CMakeLists.txt index 1e3a25628..f151bd66c 100644 --- a/upgrade_test/CMakeLists.txt +++ b/upgrade_test/CMakeLists.txt @@ -30,21 +30,24 @@ regresstarget_add( REGRESS_OPTS --dbname=contrib_regression) -exec_program( - git ${CMAKE_SOURCE_DIR} ARGS - tag | sort --version-sort -r | head -n 1 +execute_process( + COMMAND git describe --tags --abbrev=0 + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE latest_tag + OUTPUT_STRIP_TRAILING_WHITESPACE ) # check whether DDL file (*.sql) is modified -file(GLOB ddl_files ${CMAKE_SOURCE_DIR}/*.sql) +file(GLOB ddl_files ${DISKQUOTA_DDL_DIR}/*.sql) foreach(ddl IN LISTS ddl_files) cmake_path(GET ddl FILENAME ddl) - exec_program( - git ${CMAKE_SOURCE_DIR} ARGS - diff ${latest_tag} --exit-code ${ddl} - OUTPUT_VARIABLE NULL - RETURN_VALUE "${ddl}_modified") + execute_process( + COMMAND + git diff ${latest_tag} --exit-code ${ddl} + OUTPUT_QUIET + WORKING_DIRECTORY ${DISKQUOTA_DDL_DIR} + RESULT_VARIABLE "${ddl}_modified" + ) if("${${ddl}_modified}") message( diff --git a/upgrade_test/alter_test.sh b/upgrade_test/alter_test.sh new file mode 100755 index 000000000..96e29af06 --- /dev/null +++ b/upgrade_test/alter_test.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Test if all the previous diskquota minor versions can be directly upgraded +# to the current version. + +set -ex + +SCRIPT_PATH="${BASH_SOURCE[0]}" +SRC_DIR="$(cd "$(dirname "${SCRIPT_PATH}")"/.. >/dev/null 2>&1 && pwd)" + +# Versions like major.minor +CUR_VERSION=$(cut --delimiter="." --fields=1-2 "${SRC_DIR}/VERSION") +ALL_VERSIONS=$(cd "${SRC_DIR}" && git tag | cut --delimiter="." --fields=1-2 | sort -V -u) +VERSIONS_TO_TEST=() + +test_alter_from() { + local from_ver=$1 + local to_ver=$CUR_VERSION + + gpconfig -c shared_preload_libraries -v "" + gpstop -rai + dropdb diskquota --if-exists + dropdb diskquota_alter_test --if-exists + createdb diskquota + + local from_so_name="diskquota" + if [ "${from_ver}" != "1.0" ];then + from_so_name="diskquota-${from_ver}" + fi + local to_so_name="diskquota-${to_ver}" + + # Preload the old diskquota so + gpconfig -c shared_preload_libraries -v "${from_so_name}" + gpstop -rai + + createdb diskquota_alter_test + + # Test if the extension and be upgraded directly + psql -d diskquota_alter_test -c "CREATE EXTENSION diskquota version '${from_ver}'" + + # Preload the new diskquota so + gpconfig -c shared_preload_libraries -v "${to_so_name}" + gpstop -rai + + psql -d diskquota_alter_test -c "ALTER EXTENSION diskquota update to '${to_ver}'" + # Sleep wait for bgworker starting, otherwise, we will get a warning + # 'cannot remove the database from db list, dbid not found'. + sleep 5 + psql -d diskquota_alter_test -c "DROP EXTENSION diskquota" +} + +# Find all minor versions before current one +while IFS= read -r ver; do + if [ "${ver}" = "${CUR_VERSION}" ]; then + break + fi + if [ "${ver}" = "0.8" ]; then + continue + fi + VERSIONS_TO_TEST+=("${ver}") +done <<< "$ALL_VERSIONS" + +for from_ver in "${VERSIONS_TO_TEST[@]}"; do + test_alter_from "${from_ver}" +done