diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 6db56e583ee4a..b54a5c32de63e 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -2322,11 +2322,11 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_tsan, sequential, 2/2)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_1_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_1_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAxLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAxLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2341,7 +2341,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/6)" - name: Prepare env script run: | @@ -2371,13 +2371,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 1/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 1/6)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_2_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_2_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAyLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAyLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2392,7 +2392,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/6)" - name: Prepare env script run: | @@ -2422,13 +2422,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 2/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 2/6)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_3_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_3_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAzLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAzLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2443,7 +2443,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/6)" - name: Prepare env script run: | @@ -2473,13 +2473,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 3/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 3/6)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_4_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_4_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA0Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA0LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2494,7 +2494,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/6)" - name: Prepare env script run: | @@ -2524,13 +2524,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 4/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 4/6)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_5_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_5_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA1Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA1LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2545,7 +2545,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/6)" - name: Prepare env script run: | @@ -2575,13 +2575,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 5/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 5/6)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_6_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_6_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA2Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA2LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2596,7 +2596,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/6)" - name: Prepare env script run: | @@ -2626,13 +2626,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 6/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 6/6)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_7_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_sequential_1_4: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA3Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 7/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDEvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2647,7 +2647,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 7/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/4)" - name: Prepare env script run: | @@ -2677,13 +2677,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 7/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 1/4)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_8_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_sequential_2_4: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA4Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 8/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDIvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2698,7 +2698,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 8/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/4)" - name: Prepare env script run: | @@ -2728,13 +2728,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 8/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 2/4)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_sequential_1_2: + stateless_tests_amd_msan_wasmedge_sequential_3_4: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDEvMik=') }} - name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/2)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDMvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 3/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2749,7 +2749,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/2)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 3/4)" - name: Prepare env script run: | @@ -2779,13 +2779,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 1/2)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 3/4)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_sequential_2_2: + stateless_tests_amd_msan_wasmedge_sequential_4_4: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDIvMik=') }} - name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/2)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDQvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 4/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2800,7 +2800,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/2)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 4/4)" - name: Prepare env script run: | @@ -2830,7 +2830,7 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 2/2)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 4/4)' --workflow "MasterCI" --ci --timestamp stateless_tests_amd_debug_distributed_plan_s3_storage_parallel: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] @@ -6753,7 +6753,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_llvm_coverage_per_test, build_amd_msan, build_amd_release, build_amd_release_pr_cache_warmup, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_release_pr_cache_warmup, build_arm_tsan, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, sign_release_amd_release, sign_release_arm_release, source_upload, sqllogic_test, sqlstorm_test, sqltest, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_3_3, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_1_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_2_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_3_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_4_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_5_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_6_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_7_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_8_8, stateless_tests_amd_msan_wasmedge_parallel_1_8, stateless_tests_amd_msan_wasmedge_parallel_2_8, stateless_tests_amd_msan_wasmedge_parallel_3_8, stateless_tests_amd_msan_wasmedge_parallel_4_8, stateless_tests_amd_msan_wasmedge_parallel_5_8, stateless_tests_amd_msan_wasmedge_parallel_6_8, stateless_tests_amd_msan_wasmedge_parallel_7_8, stateless_tests_amd_msan_wasmedge_parallel_8_8, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_1_4, stateless_tests_amd_tsan_parallel_2_4, stateless_tests_amd_tsan_parallel_3_4, stateless_tests_amd_tsan_parallel_4_4, stateless_tests_amd_tsan_s3_storage_parallel_1_3, stateless_tests_amd_tsan_s3_storage_parallel_2_3, stateless_tests_amd_tsan_s3_storage_parallel_3_3, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel_1_8, stateless_tests_arm_asan_ubsan_azure_parallel_2_8, stateless_tests_arm_asan_ubsan_azure_parallel_3_8, stateless_tests_arm_asan_ubsan_azure_parallel_4_8, stateless_tests_arm_asan_ubsan_azure_parallel_5_8, stateless_tests_arm_asan_ubsan_azure_parallel_6_8, stateless_tests_arm_asan_ubsan_azure_parallel_7_8, stateless_tests_arm_asan_ubsan_azure_parallel_8_8, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] + needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_llvm_coverage_per_test, build_amd_msan, build_amd_release, build_amd_release_pr_cache_warmup, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_release_pr_cache_warmup, build_arm_tsan, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, sign_release_amd_release, sign_release_arm_release, source_upload, sqllogic_test, sqlstorm_test, sqltest, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_3_3, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_1_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_2_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_3_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_4_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_5_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_6_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_7_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_8_8, stateless_tests_amd_msan_wasmedge_parallel_1_6, stateless_tests_amd_msan_wasmedge_parallel_2_6, stateless_tests_amd_msan_wasmedge_parallel_3_6, stateless_tests_amd_msan_wasmedge_parallel_4_6, stateless_tests_amd_msan_wasmedge_parallel_5_6, stateless_tests_amd_msan_wasmedge_parallel_6_6, stateless_tests_amd_msan_wasmedge_sequential_1_4, stateless_tests_amd_msan_wasmedge_sequential_2_4, stateless_tests_amd_msan_wasmedge_sequential_3_4, stateless_tests_amd_msan_wasmedge_sequential_4_4, stateless_tests_amd_tsan_parallel_1_4, stateless_tests_amd_tsan_parallel_2_4, stateless_tests_amd_tsan_parallel_3_4, stateless_tests_amd_tsan_parallel_4_4, stateless_tests_amd_tsan_s3_storage_parallel_1_3, stateless_tests_amd_tsan_s3_storage_parallel_2_3, stateless_tests_amd_tsan_s3_storage_parallel_3_3, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel_1_8, stateless_tests_arm_asan_ubsan_azure_parallel_2_8, stateless_tests_arm_asan_ubsan_azure_parallel_3_8, stateless_tests_arm_asan_ubsan_azure_parallel_4_8, stateless_tests_arm_asan_ubsan_azure_parallel_5_8, stateless_tests_arm_asan_ubsan_azure_parallel_6_8, stateless_tests_arm_asan_ubsan_azure_parallel_7_8, stateless_tests_arm_asan_ubsan_azure_parallel_8_8, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -6896,16 +6896,16 @@ jobs: - stateless_tests_amd_tsan_parallel_4_4 - stateless_tests_amd_tsan_sequential_1_2 - stateless_tests_amd_tsan_sequential_2_2 - - stateless_tests_amd_msan_wasmedge_parallel_1_8 - - stateless_tests_amd_msan_wasmedge_parallel_2_8 - - stateless_tests_amd_msan_wasmedge_parallel_3_8 - - stateless_tests_amd_msan_wasmedge_parallel_4_8 - - stateless_tests_amd_msan_wasmedge_parallel_5_8 - - stateless_tests_amd_msan_wasmedge_parallel_6_8 - - stateless_tests_amd_msan_wasmedge_parallel_7_8 - - stateless_tests_amd_msan_wasmedge_parallel_8_8 - - stateless_tests_amd_msan_wasmedge_sequential_1_2 - - stateless_tests_amd_msan_wasmedge_sequential_2_2 + - stateless_tests_amd_msan_wasmedge_parallel_1_6 + - stateless_tests_amd_msan_wasmedge_parallel_2_6 + - stateless_tests_amd_msan_wasmedge_parallel_3_6 + - stateless_tests_amd_msan_wasmedge_parallel_4_6 + - stateless_tests_amd_msan_wasmedge_parallel_5_6 + - stateless_tests_amd_msan_wasmedge_parallel_6_6 + - stateless_tests_amd_msan_wasmedge_sequential_1_4 + - stateless_tests_amd_msan_wasmedge_sequential_2_4 + - stateless_tests_amd_msan_wasmedge_sequential_3_4 + - stateless_tests_amd_msan_wasmedge_sequential_4_4 - stateless_tests_amd_debug_distributed_plan_s3_storage_parallel - stateless_tests_amd_debug_distributed_plan_s3_storage_sequential - stateless_tests_amd_tsan_s3_storage_parallel_1_3 diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 734e8779091b6..208ddf97e3e83 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -1357,11 +1357,11 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_debug, sequential)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_1_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_1_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAxLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAxLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1376,7 +1376,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/6)" - name: Prepare env script run: | @@ -1404,13 +1404,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 1/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 1/6)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_2_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_2_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAyLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAyLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1425,7 +1425,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/6)" - name: Prepare env script run: | @@ -1453,13 +1453,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 2/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 2/6)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_3_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_3_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAzLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAzLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1474,7 +1474,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/6)" - name: Prepare env script run: | @@ -1502,13 +1502,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 3/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 3/6)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_4_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_4_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA0Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA0LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1523,7 +1523,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/6)" - name: Prepare env script run: | @@ -1551,13 +1551,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 4/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 4/6)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_5_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_5_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA1Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA1LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1572,7 +1572,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/6)" - name: Prepare env script run: | @@ -1600,13 +1600,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 5/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 5/6)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_6_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_6_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA2Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA2LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1621,7 +1621,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/6)" - name: Prepare env script run: | @@ -1649,13 +1649,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 6/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 6/6)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_7_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_sequential_1_4: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA3Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 7/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDEvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1670,7 +1670,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 7/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/4)" - name: Prepare env script run: | @@ -1698,13 +1698,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 7/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 1/4)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_8_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_sequential_2_4: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA4Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 8/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDIvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1719,7 +1719,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 8/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/4)" - name: Prepare env script run: | @@ -1747,13 +1747,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 8/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 2/4)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_sequential_1_2: + stateless_tests_amd_msan_wasmedge_sequential_3_4: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDEvMik=') }} - name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/2)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDMvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 3/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1768,7 +1768,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/2)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 3/4)" - name: Prepare env script run: | @@ -1796,13 +1796,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 1/2)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 3/4)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_sequential_2_2: + stateless_tests_amd_msan_wasmedge_sequential_4_4: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDIvMik=') }} - name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/2)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDQvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 4/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1817,7 +1817,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/2)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 4/4)" - name: Prepare env script run: | @@ -1845,7 +1845,7 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 2/2)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 4/4)' --workflow "PR" --ci --timestamp stateless_tests_amd_debug_distributed_plan_s3_storage_parallel: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] @@ -5571,48 +5571,6 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'BuzzHouse (amd_msan)' --workflow "PR" --ci --timestamp - parser_memory_check: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64] - needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'UGFyc2VyIG1lbW9yeSBjaGVjaw==') }} - name: "Parser memory check" - outputs: - data: ${{ steps.run.outputs.DATA }} - pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - ref: ${{ env.CHECKOUT_REF }} - - - name: Setup - uses: ./.github/actions/runner_setup - - name: Docker setup - uses: ./.github/actions/docker_setup - with: - test_name: "Parser memory check" - - - name: Prepare env script - run: | - rm -rf ./ci/tmp - mkdir -p ./ci/tmp - cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:.: - - cat > ./ci/tmp/workflow_job.json << 'EOF' - ${{ toJson(job) }} - EOF - cat > ./ci/tmp/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - - name: Run - id: run - run: | - . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Parser memory check' --workflow "PR" --ci --timestamp - promql_compliance: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] @@ -5918,7 +5876,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, bugfix_validation_unit_tests, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, build_wasm_parser, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, ci_tests, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_asan_ubsan_targeted, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, keeper_stress_tests_pr, parser_memory_check, promql_compliance, quick_functional_tests, source_upload, sqllogic_test, sqlstorm_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_wasmedge_parallel_1_8, stateless_tests_amd_msan_wasmedge_parallel_2_8, stateless_tests_amd_msan_wasmedge_parallel_3_8, stateless_tests_amd_msan_wasmedge_parallel_4_8, stateless_tests_amd_msan_wasmedge_parallel_5_8, stateless_tests_amd_msan_wasmedge_parallel_6_8, stateless_tests_amd_msan_wasmedge_parallel_7_8, stateless_tests_amd_msan_wasmedge_parallel_8_8, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_sequential_selected_tests, stateless_tests_amd_tsan_sequential_selected_tests, stateless_tests_arm_asan_ubsan_azure_parallel_1_8, stateless_tests_arm_asan_ubsan_azure_parallel_2_8, stateless_tests_arm_asan_ubsan_azure_parallel_3_8, stateless_tests_arm_asan_ubsan_azure_parallel_4_8, stateless_tests_arm_asan_ubsan_azure_parallel_5_8, stateless_tests_arm_asan_ubsan_azure_parallel_6_8, stateless_tests_arm_asan_ubsan_azure_parallel_7_8, stateless_tests_arm_asan_ubsan_azure_parallel_8_8, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_asan_ubsan_targeted, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] + needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, bugfix_validation_unit_tests, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, build_wasm_parser, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, ci_tests, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_asan_ubsan_targeted, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, keeper_stress_tests_pr, promql_compliance, quick_functional_tests, source_upload, sqllogic_test, sqlstorm_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_wasmedge_parallel_1_6, stateless_tests_amd_msan_wasmedge_parallel_2_6, stateless_tests_amd_msan_wasmedge_parallel_3_6, stateless_tests_amd_msan_wasmedge_parallel_4_6, stateless_tests_amd_msan_wasmedge_parallel_5_6, stateless_tests_amd_msan_wasmedge_parallel_6_6, stateless_tests_amd_msan_wasmedge_sequential_1_4, stateless_tests_amd_msan_wasmedge_sequential_2_4, stateless_tests_amd_msan_wasmedge_sequential_3_4, stateless_tests_amd_msan_wasmedge_sequential_4_4, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_sequential_selected_tests, stateless_tests_amd_tsan_sequential_selected_tests, stateless_tests_arm_asan_ubsan_azure_parallel_1_8, stateless_tests_arm_asan_ubsan_azure_parallel_2_8, stateless_tests_arm_asan_ubsan_azure_parallel_3_8, stateless_tests_arm_asan_ubsan_azure_parallel_4_8, stateless_tests_arm_asan_ubsan_azure_parallel_5_8, stateless_tests_arm_asan_ubsan_azure_parallel_6_8, stateless_tests_arm_asan_ubsan_azure_parallel_7_8, stateless_tests_arm_asan_ubsan_azure_parallel_8_8, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_asan_ubsan_targeted, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -6039,16 +5997,16 @@ jobs: - bugfix_validation_unit_tests - stateless_tests_amd_debug_parallel - stateless_tests_amd_debug_sequential - - stateless_tests_amd_msan_wasmedge_parallel_1_8 - - stateless_tests_amd_msan_wasmedge_parallel_2_8 - - stateless_tests_amd_msan_wasmedge_parallel_3_8 - - stateless_tests_amd_msan_wasmedge_parallel_4_8 - - stateless_tests_amd_msan_wasmedge_parallel_5_8 - - stateless_tests_amd_msan_wasmedge_parallel_6_8 - - stateless_tests_amd_msan_wasmedge_parallel_7_8 - - stateless_tests_amd_msan_wasmedge_parallel_8_8 - - stateless_tests_amd_msan_wasmedge_sequential_1_2 - - stateless_tests_amd_msan_wasmedge_sequential_2_2 + - stateless_tests_amd_msan_wasmedge_parallel_1_6 + - stateless_tests_amd_msan_wasmedge_parallel_2_6 + - stateless_tests_amd_msan_wasmedge_parallel_3_6 + - stateless_tests_amd_msan_wasmedge_parallel_4_6 + - stateless_tests_amd_msan_wasmedge_parallel_5_6 + - stateless_tests_amd_msan_wasmedge_parallel_6_6 + - stateless_tests_amd_msan_wasmedge_sequential_1_4 + - stateless_tests_amd_msan_wasmedge_sequential_2_4 + - stateless_tests_amd_msan_wasmedge_sequential_3_4 + - stateless_tests_amd_msan_wasmedge_sequential_4_4 - stateless_tests_amd_debug_distributed_plan_s3_storage_parallel - stateless_tests_amd_debug_distributed_plan_s3_storage_sequential - stateless_tests_arm_binary_parallel @@ -6129,7 +6087,6 @@ jobs: - buzzhouse_arm_asan_ubsan - buzzhouse_amd_tsan - buzzhouse_amd_msan - - parser_memory_check - promql_compliance - sqllogic_test - sqlstorm_test diff --git a/.github/workflows/pull_request_community.yml b/.github/workflows/pull_request_community.yml index 9968fd9cc635a..47411981ce299 100644 --- a/.github/workflows/pull_request_community.yml +++ b/.github/workflows/pull_request_community.yml @@ -1192,11 +1192,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_1_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_1_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAxLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAxLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1211,7 +1211,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/6)" - name: Prepare env script run: | @@ -1238,14 +1238,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 1/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 1/6)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_1_8 + name: failure-stateless_tests_amd_msan_wasmedge_parallel_1_6 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1256,11 +1256,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_2_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_2_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAyLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAyLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1275,7 +1275,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/6)" - name: Prepare env script run: | @@ -1302,14 +1302,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 2/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 2/6)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_2_8 + name: failure-stateless_tests_amd_msan_wasmedge_parallel_2_6 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1320,11 +1320,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_3_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_3_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAzLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAzLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1339,7 +1339,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/6)" - name: Prepare env script run: | @@ -1366,14 +1366,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 3/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 3/6)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_3_8 + name: failure-stateless_tests_amd_msan_wasmedge_parallel_3_6 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1384,11 +1384,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_4_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_4_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA0Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA0LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1403,7 +1403,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/6)" - name: Prepare env script run: | @@ -1430,14 +1430,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 4/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 4/6)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_4_8 + name: failure-stateless_tests_amd_msan_wasmedge_parallel_4_6 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1448,11 +1448,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_5_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_5_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA1Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA1LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1467,7 +1467,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/6)" - name: Prepare env script run: | @@ -1494,14 +1494,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 5/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 5/6)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_5_8 + name: failure-stateless_tests_amd_msan_wasmedge_parallel_5_6 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1512,11 +1512,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_6_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_6_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA2Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA2LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1531,7 +1531,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/6)" - name: Prepare env script run: | @@ -1558,14 +1558,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 6/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 6/6)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_6_8 + name: failure-stateless_tests_amd_msan_wasmedge_parallel_6_6 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1576,11 +1576,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_7_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_sequential_1_4: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA3Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 7/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDEvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1595,7 +1595,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 7/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/4)" - name: Prepare env script run: | @@ -1622,14 +1622,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 7/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 1/4)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_7_8 + name: failure-stateless_tests_amd_msan_wasmedge_sequential_1_4 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1640,11 +1640,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_8_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_sequential_2_4: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA4Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 8/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDIvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1659,7 +1659,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 8/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/4)" - name: Prepare env script run: | @@ -1686,14 +1686,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 8/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 2/4)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_8_8 + name: failure-stateless_tests_amd_msan_wasmedge_sequential_2_4 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1704,11 +1704,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_sequential_1_2: + stateless_tests_amd_msan_wasmedge_sequential_3_4: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDEvMik=') }} - name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/2)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDMvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 3/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1723,7 +1723,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/2)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 3/4)" - name: Prepare env script run: | @@ -1750,14 +1750,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 1/2)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 3/4)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_sequential_1_2 + name: failure-stateless_tests_amd_msan_wasmedge_sequential_3_4 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1768,11 +1768,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_sequential_2_2: + stateless_tests_amd_msan_wasmedge_sequential_4_4: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDIvMik=') }} - name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/2)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDQvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 4/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1787,7 +1787,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/2)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 4/4)" - name: Prepare env script run: | @@ -1814,14 +1814,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 2/2)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 4/4)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_sequential_2_2 + name: failure-stateless_tests_amd_msan_wasmedge_sequential_4_4 path: | ci/tmp/result_*.json ci/tmp/test_result.txt diff --git a/.github/workflows/regression-reusable-suite.yml b/.github/workflows/regression-reusable-suite.yml index 233ec08d26d0f..51d064ea0f011 100644 --- a/.github/workflows/regression-reusable-suite.yml +++ b/.github/workflows/regression-reusable-suite.yml @@ -93,6 +93,7 @@ jobs: build_sha: ${{ inputs.build_sha }} pr_number: ${{ github.event.number }} artifacts: builds + regression_args: ${{ inputs.regression_args }} # Args args: --test-to-end --no-colors diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 7d0935d60befe..ffcd7283f0808 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -126,7 +126,7 @@ jobs: strategy: fail-fast: false matrix: - SUITE: [aes_encryption, atomic_insert, attach, base_58, clickhouse_keeper_failover,data_types, datetime64_extended_range, disk_level_encryption, dns, engines, example, extended_precision_data_types, functions, jwt_authentication, kafka, kerberos, key_value, lightweight_delete, memory, part_moves_between_shards, selects, session_timezone, settings, version, window_functions] + SUITE: [aes_encryption, atomic_insert, attach, base_58, clickhouse_keeper_failover,data_types, datetime64_extended_range, disk_level_encryption, dns, engines, example, extended_precision_data_types, functions, jwt_authentication, kafka, kerberos, key_value, memory, part_moves_between_shards, selects, session_timezone, settings, version, window_functions] uses: ./.github/workflows/regression-reusable-suite.yml with: ref: ${{ inputs.commit }} @@ -143,6 +143,40 @@ jobs: job_name: ${{ matrix.SUITE }} secrets: inherit + LightweightDelete: + if: | + fromJson(inputs.workflow_config).workflow_config.custom_data.ci_regression_jobs[0] == null || + contains(fromJson(inputs.workflow_config).workflow_config.custom_data.ci_regression_jobs, 'common') + strategy: + fail-fast: false + matrix: + include: + - PART: 1 + FILTER: --only "/lightweight delete/disk space/*" + - PART: 2 + FILTER: --only "/lightweight delete/hard restart/*" + - PART: 3 + FILTER: --skip "/lightweight delete/disk space/*" "/lightweight delete/hard restart/*" "/lightweight delete/alter after delete/*" "/lightweight delete/concurrent alter and delete/*" "/lightweight delete/concurrent delete/*" "/lightweight delete/basic checks/*" "/lightweight delete/performance/*" "/lightweight delete/ontime tests/*" + - PART: 4 + FILTER: --only "/lightweight delete/alter after delete/*" "/lightweight delete/concurrent alter and delete/*" "/lightweight delete/concurrent delete/*" "/lightweight delete/basic checks/*" "/lightweight delete/performance/*" "/lightweight delete/ontime tests/*" + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: lightweight_delete + suite_executable: regression.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: 300 + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + part: ${{ matrix.PART }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: lightweight_delete + extra_args: ${{ matrix.FILTER }} + secrets: inherit + AggregateFunctions: if: | fromJson(inputs.workflow_config).workflow_config.custom_data.ci_regression_jobs[0] == null || @@ -181,6 +215,8 @@ jobs: PART: 1 - ONLY: attach PART: 2 + - ONLY: attach + PART: 3 uses: ./.github/workflows/regression-reusable-suite.yml with: ref: ${{ inputs.commit }} @@ -189,7 +225,7 @@ jobs: suite_executable: regression.py output_format: new-fails flags: --with-analyzer - timeout_minutes: ${{ inputs.timeout_minutes }} + timeout_minutes: 300 runner_arch: ${{ inputs.arch }} runner_type: ${{ inputs.runner_type }} storage_path: /${{ matrix.ONLY }}_partition @@ -534,10 +570,56 @@ jobs: secrets: inherit # CAS (content-addressed storage) is only available in Antalya builds >= 26.6. - # These jobs run the suites that support the `--cas` or `--cas-s3-cache` flag with - # a content-addressed disk as the default MergeTree storage. Gated on the Antalya - # version so they are never scheduled on upstream release builds where the `cas` - # metadata type does not exist. + # The dedicated `cas` suite, plus other suites run with `--cas` or `--cas-s3-cache`. + # Gated on the Antalya version so they are never scheduled on upstream release + # builds where the `cas` metadata type does not exist. + CASSuite: + if: | + contains(fromJson(inputs.workflow_config).workflow_config.custom_data.version.string, 'antalya') && + ( + fromJson(inputs.workflow_config).workflow_config.custom_data.ci_regression_jobs[0] == null || + contains(fromJson(inputs.workflow_config).workflow_config.custom_data.ci_regression_jobs, 'cas') + ) + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: cas + suite_executable: regression.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: 180 + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: cas + secrets: inherit + + CASS3CacheSuite: + if: | + contains(fromJson(inputs.workflow_config).workflow_config.custom_data.version.string, 'antalya') && + ( + fromJson(inputs.workflow_config).workflow_config.custom_data.ci_regression_jobs[0] == null || + contains(fromJson(inputs.workflow_config).workflow_config.custom_data.ci_regression_jobs, 'cas') + ) + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: cas + suite_executable: regression.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: 180 + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: cas_s3_cache + regression_args: --cas-s3-cache + secrets: inherit + CAS: if: | contains(fromJson(inputs.workflow_config).workflow_config.custom_data.version.string, 'antalya') && @@ -668,7 +750,11 @@ jobs: - PART: 1 FILTER: --only "/lightweight delete/disk space/*" - PART: 2 - FILTER: --skip "/lightweight delete/disk space/*" + FILTER: --only "/lightweight delete/hard restart/*" + - PART: 3 + FILTER: --skip "/lightweight delete/disk space/*" "/lightweight delete/hard restart/*" "/lightweight delete/alter after delete/*" "/lightweight delete/concurrent alter and delete/*" "/lightweight delete/concurrent delete/*" "/lightweight delete/basic checks/*" "/lightweight delete/performance/*" "/lightweight delete/ontime tests/*" + - PART: 4 + FILTER: --only "/lightweight delete/alter after delete/*" "/lightweight delete/concurrent alter and delete/*" "/lightweight delete/concurrent delete/*" "/lightweight delete/basic checks/*" "/lightweight delete/performance/*" "/lightweight delete/ontime tests/*" uses: ./.github/workflows/regression-reusable-suite.yml with: ref: ${{ inputs.commit }} @@ -702,7 +788,11 @@ jobs: - PART: 1 FILTER: --only "/lightweight delete/disk space/*" - PART: 2 - FILTER: --skip "/lightweight delete/disk space/*" + FILTER: --only "/lightweight delete/hard restart/*" + - PART: 3 + FILTER: --skip "/lightweight delete/disk space/*" "/lightweight delete/hard restart/*" "/lightweight delete/alter after delete/*" "/lightweight delete/concurrent alter and delete/*" "/lightweight delete/concurrent delete/*" "/lightweight delete/basic checks/*" "/lightweight delete/performance/*" "/lightweight delete/ontime tests/*" + - PART: 4 + FILTER: --only "/lightweight delete/alter after delete/*" "/lightweight delete/concurrent alter and delete/*" "/lightweight delete/concurrent delete/*" "/lightweight delete/basic checks/*" "/lightweight delete/performance/*" "/lightweight delete/ontime tests/*" uses: ./.github/workflows/regression-reusable-suite.yml with: ref: ${{ inputs.commit }} @@ -738,6 +828,8 @@ jobs: PART: 1 - ONLY: attach PART: 2 + - ONLY: attach + PART: 3 uses: ./.github/workflows/regression-reusable-suite.yml with: ref: ${{ inputs.commit }} @@ -750,6 +842,7 @@ jobs: runner_arch: ${{ inputs.arch }} runner_type: ${{ inputs.runner_type }} part: ${{ matrix.PART }} + storage_path: /${{ matrix.ONLY }}_partition build_sha: ${{ inputs.build_sha }} set_commit_status: true job_name: cas_alter_${{ matrix.ONLY }} @@ -773,6 +866,8 @@ jobs: PART: 1 - ONLY: attach PART: 2 + - ONLY: attach + PART: 3 uses: ./.github/workflows/regression-reusable-suite.yml with: ref: ${{ inputs.commit }} @@ -785,6 +880,7 @@ jobs: runner_arch: ${{ inputs.arch }} runner_type: ${{ inputs.runner_type }} part: ${{ matrix.PART }} + storage_path: /${{ matrix.ONLY }}_partition build_sha: ${{ inputs.build_sha }} set_commit_status: true job_name: cas_s3_cache_alter_${{ matrix.ONLY }} diff --git a/.gitmodules b/.gitmodules index 2f3dd2930a0a0..26fec9523a636 100644 --- a/.gitmodules +++ b/.gitmodules @@ -208,7 +208,7 @@ url = https://github.com/ClickHouse/nats.c [submodule "contrib/vectorscan"] path = contrib/vectorscan - url = https://github.com/ClickHouse/vectorscan.git + url = https://github.com/ClickHouse/vectorscan [submodule "contrib/llvm-project"] path = contrib/llvm-project url = https://github.com/ClickHouse/llvm-project @@ -332,7 +332,7 @@ url = https://github.com/ClickHouse/mongo-cxx-driver.git [submodule "contrib/mongo-c-driver"] path = contrib/mongo-c-driver - url = https://github.com/ClickHouse/mongo-c-driver.git + url = https://github.com/ClickHouse/mongo-c-driver [submodule "contrib/sz3"] path = contrib/sz3 url = https://github.com/ClickHouse/SZ3.git @@ -365,7 +365,7 @@ url = https://github.com/ClickHouse/ai-sdk-cpp [submodule "contrib/nlohmann-json"] path = contrib/nlohmann-json - url = https://github.com/nlohmann/json.git + url = https://github.com/ClickHouse/json [submodule "contrib/crc32c"] path = contrib/crc32c url = https://github.com/ClickHouse/crc32c @@ -416,7 +416,7 @@ url = https://github.com/ClickHouse/geometry.hpp [submodule "contrib/zxc"] path = contrib/zxc - url = https://github.com/ClickHouse/zxc + url = https://github.com/hellobertrand/zxc [submodule "contrib/libucontext"] path = contrib/libucontext url = https://github.com/kaniini/libucontext diff --git a/CMakeLists.txt b/CMakeLists.txt index 7860b6cd2ed8e..68b194ad629c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -173,7 +173,8 @@ if (OS_LINUX) if (LINKER_NAME MATCHES "lld" AND NOT SANITIZE AND NOT SANITIZE_COVERAGE AND NOT WITH_COVERAGE) # Fold identical code and data sections to reduce binary size. - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--icf=all") + # `safe` keeps address-taken functions distinct, as C and C++ require. + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--icf=safe") endif () # In Release builds, strip residual debug sections that may come from contrib @@ -345,7 +346,7 @@ if (NOT SANITIZE AND NOT SANITIZE_COVERAGE AND NOT WITH_COVERAGE) set(COMPILER_FLAGS "${COMPILER_FLAGS} -ffunction-sections -fdata-sections") # -faddrsig emits an address-significance table that lets lld's ICF # distinguish address-taken symbols from non-address-taken ones. - # Only useful on Linux where we use lld with --icf=all; Darwin's ld64 + # Only useful on Linux where we use lld with --icf=safe; Darwin's ld64 # does not consume these sections. if (OS_LINUX) set(COMPILER_FLAGS "${COMPILER_FLAGS} -faddrsig") diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h index 475226652f5a7..9e295ce5d299a 100644 --- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h +++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h @@ -149,6 +149,10 @@ namespace Net /// The object is normally guarded by the socket's mutex; a caller that uses it /// directly must ensure the socket is not accessed concurrently. + void markFatalError(); + /// Records that an external operation on the underlying `SSL` object failed fatally. + /// An orderly SSL shutdown must not be attempted afterwards. + X509 * peerCertificate() const; /// Returns the peer's certificate. @@ -233,7 +237,7 @@ namespace Net /// Returns true iff the given host name is the local host /// (either "localhost" or "127.0.0.1"). - bool mustRetry(int rc, Poco::Timespan & remaining_time); + bool mustRetry(int rc, int sslError, int socketError, Poco::Timespan & remaining_time); /// Returns true if the last operation should be retried, /// otherwise false. /// @@ -247,7 +251,7 @@ namespace Net /// not become readable or writable within the sockets /// receive or send timeout. - int handleError(int rc); + int handleError(int rc, int sslError, int socketError, unsigned long errorCode); /// Handles an SSL error by throwing an appropriate exception. void reset(); @@ -281,6 +285,10 @@ namespace Net Poco::AutoPtr _pSocket; Context::Ptr _pContext; bool _needHandshake; + bool _fatalError; + bool _pendingWrite = false; + /// Whether the last `SSL_write` returned `SSL_ERROR_WANT_WRITE`. OpenSSL keeps that record pending + /// until `SSL_write` is retried, even after other operations change what `SSL_get_error` reports. std::string _peerHostName; Session::Ptr _pSession; const BIO_METHOD * _bioMethod = nullptr; diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureStreamSocketImpl.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureStreamSocketImpl.h index a222ee01914c7..c5420492ec663 100644 --- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureStreamSocketImpl.h +++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureStreamSocketImpl.h @@ -183,6 +183,9 @@ namespace Net /// Returns the underlying OpenSSL SSL object, or null if the SSL handshake /// has not been performed yet. + void markFatalError(); + /// Records that an external operation on the underlying `SSL` object failed fatally. + void setLazyHandshake(bool flag = true); /// Enable lazy SSL handshake. If enabled, the SSL handshake /// will be performed the first time date is sent or @@ -301,6 +304,12 @@ namespace Net } + inline void SecureStreamSocketImpl::markFatalError() + { + _impl.markFatalError(); + } + + inline Session::Ptr SecureStreamSocketImpl::currentSession() { return _impl.currentSession(); diff --git a/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp b/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp index 80d85855fe3c9..94b66cdcf5f1e 100644 --- a/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp +++ b/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp @@ -340,9 +340,7 @@ void SSLManager::initDefaultContext(bool server) { _ptrDefaultClientContext->enableSessionCache(cacheSessions); } - /// Only an outbound connection has a requested host name to check: on an accepted socket this - /// would run against the client's address, which a client certificate does not name. - bool extendedVerification = config.getBool(prefix + CFG_EXTENDED_VERIFICATION, !server); + bool extendedVerification = config.getBool(prefix + CFG_EXTENDED_VERIFICATION, false); if (server) _ptrDefaultServerContext->enableExtendedCertificateVerification(extendedVerification); else diff --git a/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp b/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp index 60a55fec88bb8..61e0a720ba610 100644 --- a/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp +++ b/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp @@ -25,6 +25,7 @@ #include "Poco/NumberFormatter.h" #include "Poco/NumberParser.h" #include "Poco/Format.h" +#include #include #include @@ -60,11 +61,52 @@ struct RemainingTimeCounter Poco::Timestamp start; }; +struct SSLOperationResult +{ + int rc = 0; + int sslError = SSL_ERROR_NONE; + int socketError = 0; + unsigned long errorCode = 0; +}; + + +template +SSLOperationResult performSSLOperation(SSL * ssl, Operation && operation, bool zeroIsError = true) +{ + /// The error queue contract for TLS I/O and `SSL_get_error`: + /// https://docs.openssl.org/3.5/man3/SSL_get_error/ + /// The queue is cleared with `ERR_clear_error`: + /// https://docs.openssl.org/3.5/man3/ERR_clear_error/ + ERR_clear_error(); + + SSLOperationResult result; + /// `errno` is only meaningful if it was set by this operation. In particular, a custom `BIO` + /// can report `SSL_ERROR_SYSCALL` without changing it, so do not inherit another syscall result. + errno = 0; + result.rc = operation(); + + if (result.rc < 0 || (zeroIsError && result.rc == 0)) + { + /// Save `errno` before calling another function. `SSL_get_error` must be the first + /// OpenSSL call after the operation and must observe the queue created by that operation. + result.socketError = errno; + result.sslError = SSL_get_error(ssl, result.rc); + result.errorCode = ERR_get_error(); + } + + /// `errorCode` above preserves the diagnostic used by `handleError`. Do not leave any + /// additional entries in the thread-local queue for another connection on this thread. + ERR_clear_error(); + return result; +} + + SecureSocketImpl::SecureSocketImpl(Poco::AutoPtr pSocketImpl, Context::Ptr pContext): _pSSL(nullptr), _pSocket(pSocketImpl), _pContext(pContext), - _needHandshake(false) + _needHandshake(false), + _fatalError(false) { poco_check_ptr (_pSocket); poco_check_ptr (_pContext); @@ -111,6 +153,13 @@ void SecureSocketImpl::setMutex(std::unique_ptr mutex) } +void SecureSocketImpl::markFatalError() +{ + ScopedLock lock(*_mutex); + _fatalError = true; +} + + const BIO_METHOD * SecureSocketImpl::getBioMethod() const { return _bioMethod ? _bioMethod : BIO_s_socket(); @@ -121,6 +170,8 @@ void SecureSocketImpl::acceptSSL() { ScopedLock lock(*_mutex); poco_assert (!_pSSL); + _fatalError = false; + _pendingWrite = false; BIO* pBIO = BIO_new(getBioMethod()); if (!pBIO) throw SSLException("Cannot create BIO object"); @@ -190,6 +241,8 @@ void SecureSocketImpl::connectSSL(bool performHandshake) ScopedLock lock(*_mutex); poco_assert (!_pSSL); poco_assert (_pSocket->initialized()); + _fatalError = false; + _pendingWrite = false; BIO* pBIO = BIO_new(getBioMethod()); if (!pBIO) throw SSLException("Cannot create SSL BIO object"); @@ -218,31 +271,39 @@ void SecureSocketImpl::connectSSL(bool performHandshake) SSL_set_session(_pSSL, _pSession->sslSession()); } + SSL_set_connect_state(_pSSL); + _needHandshake = true; + try { if (performHandshake && _pSocket->getBlocking()) { - int ret; + SSLOperationResult result; Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); do { RemainingTimeCounter counter(remaining_time); - ret = SSL_connect(_pSSL); + result = performSSLOperation(_pSSL, [this] + { + return SSL_connect(_pSSL); + }); + } + while (mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + if (result.rc <= 0) + { + if (handleError(result.rc, result.sslError, result.socketError, result.errorCode) < 0) + throw Poco::TimeoutException("SSL handshake timed out"); + throw SSLConnectionUnexpectedlyClosedException(); } - while (mustRetry(ret, remaining_time)); - handleError(ret); + _needHandshake = false; verifyPeerCertificate(); } - else - { - SSL_set_connect_state(_pSSL); - _needHandshake = true; - } } catch (...) { SSL_free(_pSSL); _pSSL = 0; + _fatalError = false; throw; } } @@ -271,6 +332,16 @@ void SecureSocketImpl::shutdown() ScopedLock lock(*_mutex); if (_pSSL) { + if (_fatalError) + { + /// OpenSSL forbids `SSL_shutdown` after a fatal TLS or syscall error. + /// https://docs.openssl.org/3.5/man3/SSL_shutdown/ + /// Close the underlying transport without attempting an orderly TLS shutdown. + if (_pSocket->getBlocking()) + _pSocket->shutdown(); + return; + } + // Don't shut down the socket more than once. int shutdownState = SSL_get_shutdown(_pSSL); bool shutdownSent = (shutdownState & SSL_SENT_SHUTDOWN) == SSL_SENT_SHUTDOWN; @@ -283,8 +354,24 @@ void SecureSocketImpl::shutdown() // most web browsers, so we just set the shutdown // flag by calling SSL_shutdown() once and be // done with it. - int rc = SSL_shutdown(_pSSL); - if (rc < 0) handleError(rc); + /// A zero result is not an error for `SSL_shutdown` and must not be passed to + /// `SSL_get_error`; it means that `close_notify` was sent but not received yet. + SSLOperationResult result; + Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); + do + { + RemainingTimeCounter counter(remaining_time); + result = performSSLOperation(_pSSL, [this] + { + return SSL_shutdown(_pSSL); + }, false); + } + /// OpenSSL does not dispatch the `close_notify` alert while a record write from an + /// earlier `SSL_write` is still pending, so retrying cannot make progress then. + while (!_pendingWrite && result.rc < 0 + && mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + if (result.rc < 0) + handleError(result.rc, result.sslError, result.socketError, result.errorCode); if (_pSocket->getBlocking()) { _pSocket->shutdown(); @@ -328,24 +415,22 @@ int SecureSocketImpl::sendBytes(const void* buffer, int length, int flags) return rc; } + SSLOperationResult result; Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); do { RemainingTimeCounter counter(remaining_time); - rc = SSL_write(_pSSL, buffer, length); + result = performSSLOperation(_pSSL, [this, buffer, length] + { + return SSL_write(_pSSL, buffer, length); + }); + _pendingWrite = result.sslError == SSL_ERROR_WANT_WRITE; } - while (mustRetry(rc, remaining_time)); + while (mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + rc = result.rc; if (rc <= 0) { - // At this stage we still can have last not yet received SSL message containing SSL error - // so make a read to force SSL to process possible SSL error - if (SSL_get_error(_pSSL, rc) == SSL_ERROR_SYSCALL && SocketImpl::lastError() == POCO_ECONNRESET) - { - char c = 0; - SSL_read(_pSSL, &c, 1); - } - - rc = handleError(rc); + rc = handleError(rc, result.sslError, result.socketError, result.errorCode); if (rc == 0) throw SSLConnectionUnexpectedlyClosedException(); if (rc < 0 && _pSocket->getBlocking()) throw Poco::TimeoutException("SSL_write timed out"); @@ -379,6 +464,7 @@ int SecureSocketImpl::receiveBytes(void* buffer, int length, int flags) return rc; } + SSLOperationResult result; Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); do { @@ -386,12 +472,16 @@ int SecureSocketImpl::receiveBytes(void* buffer, int length, int flags) /// so thread can be blocked on recv/send and epoll_wait several times /// until SSL_read will return rc > 0. Let's use our own time counter. RemainingTimeCounter counter(remaining_time); - rc = SSL_read(_pSSL, buffer, length); + result = performSSLOperation(_pSSL, [this, buffer, length] + { + return SSL_read(_pSSL, buffer, length); + }); } - while (mustRetry(rc, remaining_time)); + while (mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + rc = result.rc; if (rc <= 0) { - rc = handleError(rc); + rc = handleError(rc, result.sslError, result.socketError, result.errorCode); if (rc < 0 && _pSocket->getBlocking()) throw Poco::TimeoutException("SSL_read timed out"); return rc; @@ -419,16 +509,21 @@ int SecureSocketImpl::completeHandshake() poco_check_ptr (_pSSL); int rc; + SSLOperationResult result; Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); do { RemainingTimeCounter counter(remaining_time); - rc = SSL_do_handshake(_pSSL); + result = performSSLOperation(_pSSL, [this] + { + return SSL_do_handshake(_pSSL); + }); } - while (mustRetry(rc, remaining_time)); + while (mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + rc = result.rc; if (rc <= 0) { - rc = handleError(rc); + rc = handleError(rc, result.sslError, result.socketError, result.errorCode); if (rc < 0 && _pSocket->getBlocking()) throw Poco::TimeoutException("SSL handshake timed out"); return rc; @@ -536,15 +631,13 @@ Poco::Timespan SecureSocketImpl::getMaxTimeoutOrLimit() return remaining_time; } -bool SecureSocketImpl::mustRetry(int rc, Poco::Timespan& remaining_time) +bool SecureSocketImpl::mustRetry(int rc, int sslError, int socketError, Poco::Timespan& remaining_time) { if (remaining_time == 0) return false; ScopedLock lock(*_mutex); if (rc <= 0) { - int sslError = SSL_get_error(_pSSL, rc); - int socketError = _pSocket->lastError(); switch (sslError) { case SSL_ERROR_WANT_READ: @@ -571,21 +664,20 @@ bool SecureSocketImpl::mustRetry(int rc, Poco::Timespan& remaining_time) case SSL_ERROR_SYSCALL: return socketError == POCO_EAGAIN || socketError == POCO_EINTR; default: - return socketError == POCO_EINTR; + /// `errno` is only meaningful for `SSL_ERROR_SYSCALL`; other errors must not + /// be retried because of a leftover `EINTR`, especially `SSL_ERROR_SSL`. + return false; } } return false; } -int SecureSocketImpl::handleError(int rc) +int SecureSocketImpl::handleError(int rc, int sslError, int error, unsigned long errorCode) { ScopedLock lock(*_mutex); if (rc > 0) return rc; - int sslError = SSL_get_error(_pSSL, rc); - int error = SocketImpl::lastError(); - switch (sslError) { case SSL_ERROR_ZERO_RETURN: @@ -598,18 +690,21 @@ int SecureSocketImpl::handleError(int rc) case SSL_ERROR_WANT_ACCEPT: case SSL_ERROR_WANT_X509_LOOKUP: // these should not occur + _fatalError = true; poco_bugcheck(); return rc; case SSL_ERROR_SYSCALL: + _fatalError = true; if (error != 0) { SocketImpl::error(error); } - // fallthrough + [[fallthrough]]; + case SSL_ERROR_SSL: default: { - long lastError = ERR_get_error(); - if (lastError == 0) + _fatalError = true; + if (errorCode == 0) { if (rc == 0) { @@ -631,7 +726,7 @@ int SecureSocketImpl::handleError(int rc) else { char buffer[256]; - ERR_error_string_n(lastError, buffer, sizeof(buffer)); + ERR_error_string_n(errorCode, buffer, sizeof(buffer)); std::string msg(buffer); throw SSLException(msg); } @@ -658,6 +753,7 @@ void SecureSocketImpl::reset() SSL_free(_pSSL); _pSSL = nullptr; } + _fatalError = false; } diff --git a/base/poco/XML/CMakeLists.txt b/base/poco/XML/CMakeLists.txt index 0bcbb22dc4925..1d3c8d332f1b1 100644 --- a/base/poco/XML/CMakeLists.txt +++ b/base/poco/XML/CMakeLists.txt @@ -9,6 +9,7 @@ add_library (Poco::XML::Expat ALIAS _poco_xml_expat) target_compile_options (_poco_xml_expat PRIVATE -Wno-cast-qual + -Wno-disabled-macro-expansion -Wno-empty-translation-unit -Wno-extra-semi-stmt -Wno-implicit-fallthrough @@ -49,6 +50,7 @@ target_compile_options (_poco_xml -Wno-unreachable-code -Wno-unreachable-code-break -Wno-unreachable-code-fallthrough + -Wno-unreachable-code-return -Wno-unused-macros -Wno-unused-parameter -Wno-zero-as-null-pointer-constant diff --git a/base/poco/XML/include/Poco/XML/expat.h b/base/poco/XML/include/Poco/XML/expat.h index ede83ff6dd328..02a3fe98b2ac5 100644 --- a/base/poco/XML/include/Poco/XML/expat.h +++ b/base/poco/XML/include/Poco/XML/expat.h @@ -20,6 +20,7 @@ Copyright (c) 2023 Sony Corporation / Snild Dolkow Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp> Copyright (c) 2025 Matthew Fernandez + Copyright (c) 2026 Braian Plaku Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -40,12 +41,14 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef Expat_INCLUDED # define Expat_INCLUDED 1 -# include // for uint8_t +# include // for int64_t, uint8_t, uint64_t # include # include "expat_external.h" @@ -137,6 +140,26 @@ enum XML_Error { XML_ERROR_NOT_STARTED, }; +/* Added in 2.9.0. */ +enum XML_Prop_Error { + XML_PROP_ERROR_NONE = 0, // i.e. success + + XML_PROP_ERROR_INVALID_KEY = 1, // i.e. the property is not known at all + XML_PROP_ERROR_INVALID_TYPE = 2, // i.e. the property is known but expects a + // different type (so a different + // getter/setter would need to be called) + XML_PROP_ERROR_INVALID_VALUE = 3, // i.e. the value is invalid (with a setter + // function) or a NULL-pointer (with a + // getter function) + + XML_PROP_ERROR_PARSER_NULL = 4, // i.e. the parser argument is NULL + XML_PROP_ERROR_PARSER_NOT_ROOT = 5, // i.e. the parser is not a root parser + // but a subparser (and a root parser is + // needed) + + /* potentially more error codes upcoming here */ +}; + enum XML_Content_Type { XML_CTYPE_EMPTY = 1, XML_CTYPE_ANY, @@ -764,29 +787,6 @@ XML_GetSpecifiedAttributeCount(XML_Parser parser); XMLPARSEAPI(int) XML_GetIdAttributeIndex(XML_Parser parser); -# ifdef XML_ATTR_INFO -/* Source file byte offsets for the start and end of attribute names and values. - The value indices are exclusive of surrounding quotes; thus in a UTF-8 source - file an attribute value of "blah" will yield: - info->valueEnd - info->valueStart = 4 bytes. -*/ -typedef struct { - XML_Index nameStart; /* Offset to beginning of the attribute name. */ - XML_Index nameEnd; /* Offset after the attribute name's last byte. */ - XML_Index valueStart; /* Offset to beginning of the attribute value. */ - XML_Index valueEnd; /* Offset after the attribute value's last byte. */ -} XML_AttrInfo; - -/* Returns an array of XML_AttrInfo structures for the attribute/value pairs - passed in last call to the XML_StartElementHandler that were specified - in the start-tag rather than defaulted. Each attribute/value pair counts - as 1; thus the number of entries in the array is - XML_GetSpecifiedAttributeCount(parser) / 2. -*/ -XMLPARSEAPI(const XML_AttrInfo *) -XML_GetAttributeInfo(XML_Parser parser); -# endif - /* Parses some input. Returns XML_STATUS_ERROR if a fatal error is detected. The last call to XML_Parse must have isFinal true; len may be zero for this call (or any other). @@ -929,7 +929,9 @@ XML_SetParamEntityParsing(XML_Parser parser, Returns 1 if successful, 0 when called after parsing has started. Note: If parser == NULL, the function will do nothing and return 0. DEPRECATED since Expat 2.8.0. + Please use XML_SetHashSalt16Bytes instead. */ +XML_ATTR_DEPRECATED("please use XML_SetHashSalt16Bytes instead") XMLPARSEAPI(int) XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt); @@ -969,15 +971,24 @@ XML_GetErrorCode(XML_Parser parser); return 0 to indicate an error. Note: XML_GetCurrentByteIndex returns -1 to indicate an error. */ +XML_ATTR_DEPRECATED("please use XML_GetCurrentLineNumber64 instead") XMLPARSEAPI(XML_Size) XML_GetCurrentLineNumber(XML_Parser parser); +XMLPARSEAPI(uint64_t) XML_GetCurrentLineNumber64(XML_Parser parser); +XML_ATTR_DEPRECATED("please use XML_GetCurrentColumnNumber64 instead") XMLPARSEAPI(XML_Size) XML_GetCurrentColumnNumber(XML_Parser parser); +XMLPARSEAPI(uint64_t) XML_GetCurrentColumnNumber64(XML_Parser parser); +XML_ATTR_DEPRECATED("please use XML_GetCurrentByteIndex64 instead") XMLPARSEAPI(XML_Index) XML_GetCurrentByteIndex(XML_Parser parser); +XMLPARSEAPI(int64_t) XML_GetCurrentByteIndex64(XML_Parser parser); /* Return the number of bytes in the current event. Returns 0 if the event is in an internal entity. */ +XML_ATTR_DEPRECATED("please use XML_GetCurrentByteCount64 instead") XMLPARSEAPI(int) XML_GetCurrentByteCount(XML_Parser parser); +XMLPARSEAPI(uint64_t) +XML_GetCurrentByteCount64(XML_Parser parser); /* If XML_CONTEXT_BYTES is >=1, returns the input buffer, sets the integer pointed to by offset to the offset within this buffer @@ -989,8 +1000,11 @@ XML_GetCurrentByteCount(XML_Parser parser); NOTE: The character pointer returned should not be used outside the handler that makes the call. */ +XML_ATTR_DEPRECATED("please use XML_GetInputContext64 instead") XMLPARSEAPI(const char *) XML_GetInputContext(XML_Parser parser, int *offset, int *size); +XMLPARSEAPI(const char *) +XML_GetInputContext64(XML_Parser parser, int64_t *offset, uint64_t *size); /* For backwards compatibility with previous versions. */ # define XML_GetErrorLineNumber XML_GetCurrentLineNumber @@ -1045,11 +1059,15 @@ enum XML_FeatureEnum { XML_FEATURE_UNICODE_WCHAR_T, XML_FEATURE_DTD, XML_FEATURE_CONTEXT_BYTES, + /* Added in Expat 1.95.5, removed in Expat 2.9.0. */ XML_FEATURE_MIN_SIZE, XML_FEATURE_SIZEOF_XML_CHAR, XML_FEATURE_SIZEOF_XML_LCHAR, + /* Added in Expat 2.0.0. */ XML_FEATURE_NS, + /* Added in Expat 2.0.1. */ XML_FEATURE_LARGE_SIZE, + /* Added in Expat 2.1.0, removed in Expat 2.9.0. */ XML_FEATURE_ATTR_INFO, /* Added in Expat 2.4.0. */ XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, @@ -1071,6 +1089,41 @@ typedef struct { XMLPARSEAPI(const XML_Feature *) XML_GetFeatureList(void); +/* Added in Expat 2.9.0. */ +enum XML_Parser_Property { +# if defined(XML_TESTING) + XML_PROP_INVALID = 0, +# endif +# if XML_GE == 1 + XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD = 1, // of type `uint64_t` + XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION = 2, // of type `double` + XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD = 3, // of type `uint64_t` + XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION = 4, // of type `double` +# endif + XML_PROP_REPARSE_DEFERRAL_ENABLED = 5, // of type `XML_Bool` + /* more properties upcoming here */ +}; + +/* Added in Expat 2.9.0. */ +XMLPARSEAPI(enum XML_Prop_Error) +XML_GetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool *value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_GetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double *value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_GetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t *value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_SetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_SetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_SetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t value); + # if defined(XML_DTD) || (defined(XML_GE) && XML_GE == 1) /* Added in Expat 2.4.0 for XML_DTD defined and * added in Expat 2.6.0 for XML_GE == 1. */ @@ -1103,8 +1156,8 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); See https://semver.org */ # define XML_MAJOR_VERSION 2 -# define XML_MINOR_VERSION 8 -# define XML_MICRO_VERSION 2 +# define XML_MINOR_VERSION 9 +# define XML_MICRO_VERSION 0 # ifdef __cplusplus } diff --git a/base/poco/XML/include/Poco/XML/expat_external.h b/base/poco/XML/include/Poco/XML/expat_external.h index 1c882c3a41ecd..9fb80149acd3a 100644 --- a/base/poco/XML/include/Poco/XML/expat_external.h +++ b/base/poco/XML/include/Poco/XML/expat_external.h @@ -16,6 +16,7 @@ Copyright (c) 2017 Rhodri James Copyright (c) 2018 Yury Gribov Copyright (c) 2026 Matthew Fernandez + Copyright (c) 2026 Braian Plaku Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -36,6 +37,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef Expat_External_INCLUDED @@ -123,6 +126,17 @@ # define XML_ATTR_ALLOC_SIZE(x) # endif +/* Marks a function that Expat still provides but that callers should move off + of. */ +# if defined(__clang__) || defined(__GNUC__) +# define XML_ATTR_DEPRECATED(message) \ + __attribute__((__deprecated__(message))) +# elif defined(_MSC_VER) +# define XML_ATTR_DEPRECATED(message) __declspec(deprecated(message)) +# else +# define XML_ATTR_DEPRECATED(message) // empty i.e. no deprecation +# endif + # define XMLPARSEAPI(type) XMLIMPORT type XMLCALL # ifdef __cplusplus @@ -152,6 +166,9 @@ typedef char XML_LChar; # endif /* XML_UNICODE */ # ifdef XML_LARGE_SIZE /* Use large integers for file/stream positions. */ +# if defined(__clang__) || defined(__GNUC__) +# warning Macro XML_LARGE_SIZE is deprecated, please use the 64bit location API functions instead. +# endif typedef long long XML_Index; typedef unsigned long long XML_Size; # else diff --git a/base/poco/XML/src/ascii.h b/base/poco/XML/src/ascii.h index 3e1f61ec03a80..56c8807aa5746 100644 --- a/base/poco/XML/src/ascii.h +++ b/base/poco/XML/src/ascii.h @@ -31,6 +31,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #define ASCII_A 0x41 diff --git a/base/poco/XML/src/asciitab.h b/base/poco/XML/src/asciitab.h index ea92a318ffde0..4708a7295bf6c 100644 --- a/base/poco/XML/src/asciitab.h +++ b/base/poco/XML/src/asciitab.h @@ -30,6 +30,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ /* 0x00 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML, diff --git a/base/poco/XML/src/fallthrough.h b/base/poco/XML/src/fallthrough.h index cae4c6274da3a..819a42d1d34c1 100644 --- a/base/poco/XML/src/fallthrough.h +++ b/base/poco/XML/src/fallthrough.h @@ -27,6 +27,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef FALLTHROUGH_H diff --git a/base/poco/XML/src/hash_table.h b/base/poco/XML/src/hash_table.h new file mode 100644 index 0000000000000..ee637623c2f04 --- /dev/null +++ b/base/poco/XML/src/hash_table.h @@ -0,0 +1,76 @@ +/* Hash table related internal API + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Sebastian Pipping + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#if ! defined(HASH_TABLE_H) +# define HASH_TABLE_H 1 + +# include "Poco/XML/expat.h" // for XML_Bool, XML_Parser +# include "internal.h" // for XML_NONTESTING_STATIC + +# include // for size_t + +typedef const XML_Char *KEY; + +typedef struct { + KEY name; +} NAMED; + +typedef struct { + NAMED **v; + unsigned char power; + size_t size; + size_t used; + XML_Parser parser; +} HASH_TABLE; + +typedef struct { + NAMED **p; + NAMED **end; +} HASH_TABLE_ITER; + +XML_NONTESTING_STATIC NAMED *lookupWithLength(XML_Parser parser, + HASH_TABLE *table, KEY name, + size_t nameLen, + size_t createSize); +XML_NONTESTING_STATIC NAMED *lookup(XML_Parser parser, HASH_TABLE *table, + KEY name, size_t createSize); + +XML_NONTESTING_STATIC void hashTableInit(HASH_TABLE *table, XML_Parser parser); +XML_NONTESTING_STATIC void hashTableClear(HASH_TABLE *table); +XML_NONTESTING_STATIC void hashTableDestroy(HASH_TABLE *table); +XML_NONTESTING_STATIC void hashTableIterInit(HASH_TABLE_ITER *iter, + const HASH_TABLE *table); +XML_NONTESTING_STATIC NAMED *hashTableIterNext(HASH_TABLE_ITER *iter); + +XML_NONTESTING_STATIC XML_Bool keyeq(KEY s1, size_t s1len, KEY s2); +XML_NONTESTING_STATIC size_t keylen(KEY s); + +#endif // ! defined(HASH_TABLE_H) diff --git a/base/poco/XML/src/iasciitab.h b/base/poco/XML/src/iasciitab.h index fdadd766afb73..652b3c03e02da 100644 --- a/base/poco/XML/src/iasciitab.h +++ b/base/poco/XML/src/iasciitab.h @@ -30,6 +30,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ /* Like asciitab.h, except that 0xD has code BT_S rather than BT_CR */ diff --git a/base/poco/XML/src/internal.h b/base/poco/XML/src/internal.h index f9ffbc72cf810..d0465449a3991 100644 --- a/base/poco/XML/src/internal.h +++ b/base/poco/XML/src/internal.h @@ -6,13 +6,6 @@ The following calling convention macros are defined for frequently called functions: - FASTCALL - Used for those internal functions that have a simple - body and a low number of arguments and local variables. - - PTRCALL - Used for functions called though function pointers. - - PTRFASTCALL - Like PTRCALL, but for low number of arguments. - inline - Used for selected internal functions for which inlining may improve performance on some platforms. @@ -33,6 +26,9 @@ Copyright (c) 2019 David Loffredo Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp> + Copyright (c) 2026 Matthew Wozniczka + Copyright (c) 2026 Braian Plaku + Copyright (c) 2026 Florian Schmaus Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -53,117 +49,83 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - -#if defined(__GNUC__) && defined(__i386__) && ! defined(__MINGW32__) -/* We'll use this version by default only where we know it helps. - - regparm() generates warnings on Solaris boxes. See SF bug #692878. - - Instability reported with egcs on a RedHat Linux 7.3. - Let's comment out: - #define FASTCALL __attribute__((stdcall, regparm(3))) - and let's try this: -*/ -# define FASTCALL __attribute__((regparm(3))) -# define PTRFASTCALL __attribute__((regparm(3))) -#endif -/* Using __fastcall seems to have an unexpected negative effect under - MS VC++, especially for function pointers, so we won't use it for - now on that platform. It may be reconsidered for a future release - if it can be made more effective. - Likely reason: __fastcall on Windows is like stdcall, therefore - the compiler cannot perform stack optimizations for call clusters. + SPDX-License-Identifier: MIT */ -/* Make sure all of these are defined if they aren't already. */ - -#ifndef FASTCALL -# define FASTCALL -#endif - -#ifndef PTRCALL -# define PTRCALL -#endif - -#ifndef PTRFASTCALL -# define PTRFASTCALL -#endif +#if ! defined(INTERNAL_H) +# define INTERNAL_H 1 -#ifndef XML_MIN_SIZE -# if ! defined(__cplusplus) && ! defined(inline) +# if ! defined(inline) # ifdef __GNUC__ # define inline __inline # endif /* __GNUC__ */ # endif -#endif /* XML_MIN_SIZE */ -#ifdef __cplusplus -# define inline inline -#else # ifndef inline # define inline # endif -#endif -#include // ULONG_MAX -#include // size_t - -#if defined(_WIN32) \ - && (! defined(__USE_MINGW_ANSI_STDIO) \ - || (1 - __USE_MINGW_ANSI_STDIO - 1 == 0)) -# define EXPAT_FMT_LLX(midpart) "%" midpart "I64x" -# define EXPAT_FMT_ULL(midpart) "%" midpart "I64u" -# if defined(_WIN64) // Note: modifiers "td" and "zu" do not work for MinGW -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "I64d" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "I64u" -# else -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" +# if ! defined(XML_NONTESTING_STATIC) +# if defined(XML_TESTING) +# define XML_NONTESTING_STATIC // empty i.e. not static +# else +# define XML_NONTESTING_STATIC static +# endif # endif -#else -# define EXPAT_FMT_LLX(midpart) "%" midpart "llx" -# define EXPAT_FMT_ULL(midpart) "%" midpart "llu" -# if ! defined(ULONG_MAX) -# error Compiler did not define ULONG_MAX for us -# elif ULONG_MAX == 18446744073709551615u // 2^64-1 -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "ld" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "lu" -# elif defined(__wasm32__) // 32bit mode Emscripten or WASI SDK -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "ld" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "zu" + +# include // ULONG_MAX +# include // size_t + +# if defined(_WIN32) \ + && (! defined(__USE_MINGW_ANSI_STDIO) \ + || (1 - __USE_MINGW_ANSI_STDIO - 1 == 0)) +# define EXPAT_FMT_LLX(midpart) "%" midpart "I64x" +# define EXPAT_FMT_ULL(midpart) "%" midpart "I64u" +# if defined(_WIN64) // Note: modifiers "td" and "zu" do not work for MinGW +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "I64d" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "I64u" +# else +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" +# endif # else -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" +# include // PRIdPTR, PRIuPTR +# define EXPAT_FMT_LLX(midpart) "%" midpart "llx" +# define EXPAT_FMT_ULL(midpart) "%" midpart "llu" +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart PRIdPTR +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart PRIuPTR # endif -#endif -#ifndef UNUSED_P -# define UNUSED_P(p) (void)p -#endif +# ifndef UNUSED_P +# define UNUSED_P(p) (void)p +# endif /* NOTE BEGIN If you ever patch these defaults to greater values for non-attack XML payload in your environment, please file a bug report with libexpat. Thank you! */ -#define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT \ - 100.0f -#define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT \ - 8388608 // 8 MiB, 2^23 +# define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT \ + 100.0f +# define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT \ + 8388608 // 8 MiB, 2^23 -#define EXPAT_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT 100.0f -#define EXPAT_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT \ - 67108864 // 64 MiB, 2^26 +# define EXPAT_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT 100.0f +# define EXPAT_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT \ + 67108864 // 64 MiB, 2^26 // NOTE: If function expat_alloc was user facing, EXPAT_MALLOC_ALIGNMENT would // have to take sizeof(long double) into account -#define EXPAT_MALLOC_ALIGNMENT sizeof(long long) // largest parser (sub)member -#define EXPAT_MALLOC_PADDING ((EXPAT_MALLOC_ALIGNMENT) - sizeof(size_t)) +union expat_align { + long long l; + void *p; +}; +# define EXPAT_MALLOC_ALIGNMENT sizeof(union expat_align) +# define EXPAT_MALLOC_PADDING ((EXPAT_MALLOC_ALIGNMENT) - sizeof(size_t)) /* NOTE END */ -#include "Poco/XML/expat.h" // so we can use type XML_Parser below +# include "Poco/XML/expat.h" // so we can use type XML_Parser below #ifdef __cplusplus extern "C" { @@ -172,24 +134,29 @@ extern "C" { void _INTERNAL_trim_to_complete_utf8_characters(const char *from, const char **fromLimRef); -#if defined(XML_GE) && XML_GE == 1 +# if defined(XML_GE) && XML_GE == 1 unsigned long long testingAccountingGetCountBytesDirect(XML_Parser parser); unsigned long long testingAccountingGetCountBytesIndirect(XML_Parser parser); const char *unsignedCharToPrintable(unsigned char c); -#endif +# endif extern -#if ! defined(XML_TESTING) +# if ! defined(XML_TESTING) const -#endif +# endif XML_Bool g_reparseDeferralEnabledDefault; // written ONLY in runtests.c -#if defined(XML_TESTING) +# if defined(XML_TESTING) + +int xmlSetHashSalt(XML_Parser parser, unsigned long hash_salt); + void *expat_malloc(XML_Parser parser, size_t size, int sourceLine); void expat_free(XML_Parser parser, void *ptr, int sourceLine); void *expat_realloc(XML_Parser parser, void *ptr, size_t size, int sourceLine); extern unsigned int g_bytesScanned; // used for testing only -#endif +# endif #ifdef __cplusplus } #endif + +#endif // not defined INTERNAL_H diff --git a/base/poco/XML/src/latin1tab.h b/base/poco/XML/src/latin1tab.h index 59cd962cbdec3..48bbf4c7fd6ca 100644 --- a/base/poco/XML/src/latin1tab.h +++ b/base/poco/XML/src/latin1tab.h @@ -30,6 +30,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ /* 0x80 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER, diff --git a/base/poco/XML/src/nametab.h b/base/poco/XML/src/nametab.h index 616bc29662bb3..1047453942a65 100644 --- a/base/poco/XML/src/nametab.h +++ b/base/poco/XML/src/nametab.h @@ -28,6 +28,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ static const unsigned namingBitmap[] = { diff --git a/base/poco/XML/src/siphash.h b/base/poco/XML/src/siphash.h index 906a37613e9e1..cca88c11fd4f1 100644 --- a/base/poco/XML/src/siphash.h +++ b/base/poco/XML/src/siphash.h @@ -8,6 +8,8 @@ * * 1. https://www.131002.net/siphash/siphash24.c * 2. https://www.131002.net/siphash/ + * + * SPDX-License-Identifier: CC0-1.0 * -------------------------------------------------------------------------- * HISTORY: * diff --git a/base/poco/XML/src/utf8tab.h b/base/poco/XML/src/utf8tab.h index 825be4eb1bbc4..60848da01d8b7 100644 --- a/base/poco/XML/src/utf8tab.h +++ b/base/poco/XML/src/utf8tab.h @@ -30,6 +30,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ /* 0x80 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL, diff --git a/base/poco/XML/src/xcs.h b/base/poco/XML/src/xcs.h new file mode 100644 index 0000000000000..1e63a9b1de2f6 --- /dev/null +++ b/base/poco/XML/src/xcs.h @@ -0,0 +1,52 @@ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Sebastian Pipping + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#if ! defined(EXPAT_XCS_H) + +# include // size_t +# include "Poco/XML/expat_external.h" // for XML_Char + +// `xcs.c` is compiled as C while `xmlparse.cpp` is compiled as C++, so the +// declarations need C linkage to match the definitions. +#ifdef __cplusplus +extern "C" { +#endif + +extern size_t xcslen(const XML_Char *s); + +extern int xcscmp(const XML_Char *a, const XML_Char *b); +extern int xcsncmp(const XML_Char *a, const XML_Char *b, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif // ! defined(EXPAT_XCS_H) diff --git a/base/poco/XML/src/xmlparse.cpp b/base/poco/XML/src/xmlparse.cpp index bbbf902f8688f..ac428105490cc 100644 --- a/base/poco/XML/src/xmlparse.cpp +++ b/base/poco/XML/src/xmlparse.cpp @@ -1,4 +1,4 @@ -/* 5de44e6750c6cc78818f06ed552f522a1241df0299395250e1792cb339389daf (2.8.2+) +/* e3ca845466942140c9af531ac57808c4650116df5d6dbeda3b6546d2548a586e (2.9.0+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -50,6 +50,14 @@ Copyright (c) 2026 Nick Begg Copyright (c) 2026 Kartik Kenchi Copyright (c) 2026 Haris Hussain + Copyright (c) 2026 Evgeny Kotkov + Copyright (c) 2026 Darren Carreras + Copyright (c) 2026 Alberto Maschietto + Copyright (c) 2026 Zeyou Liu + Copyright (c) 2026 Stan Ulbrych + Copyright (c) 2026 Braian Plaku + Copyright (c) 2026 Filippo Tedeschi + Copyright (c) 2026 Junki Lee Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -70,6 +78,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #define XML_BUILDING_EXPAT 1 @@ -93,12 +103,13 @@ #include #include /* memset(), memcpy() */ #include -#include /* INT_MAX, LLONG_MAX, LONG_MAX, UINT_MAX */ +#include /* INT_MAX, UINT_MAX */ #include /* fprintf */ #include /* getenv */ -#include /* SIZE_MAX, uintptr_t */ +#include /* SIZE_MAX, UINT64_MAX, uint64_t, uintptr_t */ #include /* isnan */ #include +#include /* wcsncmp */ #define EXPAT_POCO 1 @@ -122,7 +133,7 @@ #include "ascii.h" #include "Poco/XML/expat.h" #include "siphash.h" -#include "xcsinc.c" +#include "xcs.h" #if defined(HAVE_ARC4RANDOM) # include "random_arc4random.h" @@ -217,12 +228,6 @@ typedef char ICHAR; #endif -#ifdef XML_LARGE_SIZE -# define XML_INDEX_MAX LLONG_MAX -#else -# define XML_INDEX_MAX LONG_MAX -#endif - /* Round up n to be a multiple of sz, where sz is a power of 2. */ #define ROUND_UP(n, sz) (((n) + ((sz) - 1)) & ~((sz) - 1)) @@ -232,25 +237,10 @@ typedef char ICHAR; #define EXPAT_MIN(a, b) (((a) < (b)) ? (a) : (b)) #include "internal.h" +#include "hash_table.h" #include "xmltok.h" #include "xmlrole.h" -typedef const XML_Char *KEY; - -typedef struct { - KEY name; -} NAMED; - -typedef struct { - NAMED **v; - unsigned char power; - size_t size; - size_t used; - XML_Parser parser; -} HASH_TABLE; - -static size_t keylen(KEY s); - static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key); /* For probing (after a collision) we need a step size relative prime @@ -266,11 +256,6 @@ static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key); #define PROBE_STEP(hash, mask, power) \ ((unsigned char)((SECOND_HASH(hash, mask, power)) | 1)) -typedef struct { - NAMED **p; - NAMED **end; -} HASH_TABLE_ITER; - #define INIT_TAG_BUF_SIZE 32 /* must be a multiple of sizeof(XML_Char) */ #define INIT_DATA_BUF_SIZE 1024 #define INIT_ATTS_SIZE 16 @@ -339,13 +324,13 @@ typedef struct { const XML_Char *base; const XML_Char *publicId; const XML_Char *notation; - XML_Bool open; - XML_Bool hasMore; /* true if entity has not been completely processed */ - /* An entity can be open while being already completely processed (hasMore == - XML_FALSE). The reason is the delayed closing of entities until their inner + bool open; + bool hasMore; /* true if entity has not been completely processed */ + /* An entity can be open while being already completely processed (!hasMore). + The reason is the delayed closing of entities until their inner entities are processed and closed */ - XML_Bool is_param; - XML_Bool is_internal; /* true if declared in internal subset outside PE */ + bool is_param; + bool is_internal; /* true if declared in internal subset outside PE */ } ENTITY; typedef struct { @@ -380,8 +365,8 @@ typedef struct { typedef struct attribute_id { XML_Char *name; PREFIX *prefix; - XML_Bool maybeTokenized; - XML_Bool xmlns; + bool maybeTokenized; + bool xmlns; } ATTRIBUTE_ID; typedef struct { @@ -390,6 +375,22 @@ typedef struct { const XML_Char *value; } DEFAULT_ATTRIBUTE; +// This structure allows mapping attribute names to instances of +// `DEFAULT_ATTRIBUTE`. +typedef struct { + // Member `name` goes first to make this structure compatible with structure + // `NAMED` (further up), which is needed to support use of structure + // `NAME_AND_DEFAULT_ATTRIBUTE` in a hash table as implemented by function + // `lookup` (further down). + const XML_Char *name; + // We would store a `DEFAULT_ATTRIBUTE *` here but the backing array + // can be reallocated which would invalidate the pointer. Using an index + // into the array instead, avoids that problem. + size_t attIndex; + // This is set to `false` by function `lookup`. + bool initialized; +} NAME_AND_DEFAULT_ATTRIBUTE; + typedef struct { unsigned long version; unsigned long hash; @@ -403,7 +404,7 @@ typedef struct { size_t nDefaultAtts; size_t allocDefaultAtts; DEFAULT_ATTRIBUTE *defaultAtts; - HASH_TABLE defaultAttsNames; + HASH_TABLE defaultAttForName; } ELEMENT_TYPE; typedef struct { @@ -421,12 +422,12 @@ typedef struct { XML_Bool standalone; #ifdef XML_DTD /* indicates if external PE has been read */ - XML_Bool paramEntityRead; + bool paramEntityRead; HASH_TABLE paramEntities; #endif /* XML_DTD */ PREFIX defaultPrefix; /* === scaffolding for building content model === */ - XML_Bool in_eldecl; + bool in_eldecl; CONTENT_SCAFFOLD *scaffold; unsigned contentStringLen; unsigned scaffSize; @@ -448,7 +449,7 @@ typedef struct open_internal_entity { struct open_internal_entity *next; ENTITY *entity; int startTagLevel; - XML_Bool betweenDecl; /* WFC: PE Between Declarations */ + bool betweenDecl; /* WFC: PE Between Declarations */ enum EntityType type; } OPEN_INTERNAL_ENTITY; @@ -470,8 +471,8 @@ typedef struct accounting { } ACCOUNTING; typedef struct MALLOC_TRACKER { - XmlBigCount bytesAllocated; - XmlBigCount peakBytesAllocated; // updated live only for debug level >=2 + size_t bytesAllocated; + size_t peakBytesAllocated; // updated live only for debug level >=2 unsigned long debugLevel; float maximumAmplificationFactor; // >=1.0 XmlBigCount activationThresholdBytes; @@ -485,8 +486,8 @@ typedef struct entity_stats { } ENTITY_STATS; #endif /* XML_GE == 1 */ -typedef enum XML_Error PTRCALL Processor(XML_Parser parser, const char *start, - const char *end, const char **endPtr); +typedef enum XML_Error Processor(XML_Parser parser, const char *start, + const char *end, const char **endPtr); static Processor prologProcessor; static Processor prologInitProcessor; @@ -518,7 +519,7 @@ static enum XML_Error doProlog(XML_Parser parser, const ENCODING *enc, XML_Bool haveMore, XML_Bool allowClosingDoctype, enum XML_Account account); static enum XML_Error processEntity(XML_Parser parser, ENTITY *entity, - XML_Bool betweenDecl, enum EntityType type); + bool betweenDecl, enum EntityType type); static enum XML_Error doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const char *start, const char *end, const char **endPtr, @@ -578,7 +579,7 @@ static void reportDefault(XML_Parser parser, const ENCODING *enc, static const XML_Char *getContext(XML_Parser parser); static XML_Bool setContext(XML_Parser parser, const XML_Char *context); -static void FASTCALL normalizePublicId(XML_Char *s); +static void normalizePublicId(XML_Char *s); static DTD *dtdCreate(XML_Parser parser); /* do not call if m_parentParser != NULL */ @@ -588,34 +589,24 @@ static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, XML_Parser parser); static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, STRING_POOL *newPool, const HASH_TABLE *oldTable); -static NAMED *lookup(XML_Parser parser, HASH_TABLE *table, KEY name, - size_t createSize); -static void FASTCALL hashTableInit(HASH_TABLE *table, XML_Parser parser); -static void FASTCALL hashTableClear(HASH_TABLE *table); -static void FASTCALL hashTableDestroy(HASH_TABLE *table); -static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *iter, - const HASH_TABLE *table); -static NAMED *FASTCALL hashTableIterNext(HASH_TABLE_ITER *iter); - -static void FASTCALL poolInit(STRING_POOL *pool, XML_Parser parser); -static void FASTCALL poolClear(STRING_POOL *pool); -static void FASTCALL poolDestroy(STRING_POOL *pool); + +static void poolInit(STRING_POOL *pool, XML_Parser parser); +static void poolClear(STRING_POOL *pool); +static void poolDestroy(STRING_POOL *pool); static XML_Char *poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end); static XML_Char *poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end); -static XML_Bool FASTCALL poolGrow(STRING_POOL *pool); -static bool FASTCALL poolGrowUntil(STRING_POOL *pool, size_t needed); -static const XML_Char *FASTCALL poolCopyString(STRING_POOL *pool, - const XML_Char *s); -static const XML_Char *FASTCALL poolCopyStringNoFinish(STRING_POOL *pool, - const XML_Char *s); +static XML_Bool poolGrow(STRING_POOL *pool); +static bool poolGrowUntil(STRING_POOL *pool, size_t needed); +static const XML_Char *poolCopyString(STRING_POOL *pool, const XML_Char *s); +static const XML_Char *poolCopyStringNoFinish(STRING_POOL *pool, + const XML_Char *s); static const XML_Char *poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n); -static const XML_Char *FASTCALL poolAppendString(STRING_POOL *pool, - const XML_Char *s); +static const XML_Char *poolAppendString(STRING_POOL *pool, const XML_Char *s); -static int FASTCALL nextScaffoldPart(XML_Parser parser); +static int nextScaffoldPart(XML_Parser parser); static XML_Content *build_model(XML_Parser parser); static ELEMENT_TYPE *getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, const char *end); @@ -664,6 +655,25 @@ static bool poolAppendChar(STRING_POOL *pool, XML_Char c); static bool poolAppendChars(STRING_POOL *pool, const XML_Char *s, size_t len); +#if XML_GE == 1 +static enum XML_Prop_Error setBillionLaughsAttackProtectionMaximumAmplification( + XML_Parser parser, float maximumAmplificationFactor); + +static enum XML_Prop_Error setBillionLaughsAttackProtectionActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes); + +static enum XML_Prop_Error +setAllocTrackerMaximumAmplification(XML_Parser parser, + float maximumAmplificationFactor); + +static enum XML_Prop_Error +setAllocTrackerActivationThreshold(XML_Parser parser, + unsigned long long activationThresholdBytes); +#endif /* XML_GE == 1 */ + +static enum XML_Prop_Error setReparseDeferralEnabled(XML_Parser parser, + XML_Bool enabled); + #define poolStart(pool) ((pool)->start) #define poolLength(pool) ((pool)->ptr - (pool)->start) #define poolChop(pool) ((void)--(pool->ptr)) @@ -725,7 +735,7 @@ struct XML_ParserStruct { char *m_bufferEnd; // past last character to be parsed const char *m_bufferLim; // allocated end of m_buffer - XML_Index m_parseEndByteIndex; + uint64_t m_parseEndByteIndex; const char *m_parseEndPtr; size_t m_partialTokenBytesBefore; /* used in heuristic to avoid O(n^2) */ XML_Bool m_reparseDeferralEnabled; @@ -764,6 +774,8 @@ struct XML_ParserStruct { void *m_unknownEncodingMem; void *m_unknownEncodingData; void *m_unknownEncodingHandlerData; + // Application callback invoked by callUnknownEncodingConvert. + int(XMLCALL *m_unknownEncodingConvert)(void *, const char *); void(XMLCALL *m_unknownEncodingRelease)(void *); PROLOG_STATE m_prologState; Processor *m_processor; @@ -801,9 +813,6 @@ struct XML_ParserStruct { NS_ATT *m_nsAtts; unsigned long m_nsAttsVersion; unsigned char m_nsAttsPower; -#ifdef XML_ATTR_INFO - XML_AttrInfo *m_attInfo; -#endif POSITION m_position; STRING_POOL m_tempPool; STRING_POOL m_temp2Pool; @@ -840,30 +849,30 @@ struct XML_ParserStruct { #if XML_GE == 1 static void -expat_heap_stat(XML_Parser rootParser, char operator, XmlBigCount absDiff, - XmlBigCount newTotal, XmlBigCount peakTotal, int sourceLine) { +expat_heap_stat(XML_Parser rootParser, char operator, size_t absDiff, + size_t newTotal, size_t peakTotal, int sourceLine) { // NOTE: This can be +infinity or -nan const float amplification = (float)newTotal / (float)rootParser->m_accounting.countBytesDirect; fprintf( stderr, - "expat: Allocations(%p): Direct " EXPAT_FMT_ULL("10") ", allocated %c" EXPAT_FMT_ULL( - "10") " to " EXPAT_FMT_ULL("10") " (" EXPAT_FMT_ULL("10") " peak), amplification %8.2f (xmlparse.c:%d)\n", + "expat: Allocations(%p): Direct " EXPAT_FMT_ULL("10") ", allocated %c" EXPAT_FMT_SIZE_T( + "10") " to " EXPAT_FMT_SIZE_T("10") " (" EXPAT_FMT_SIZE_T("10") " peak), amplification %8.2f (xmlparse.c:%d)\n", (void *)rootParser, rootParser->m_accounting.countBytesDirect, operator, absDiff, newTotal, peakTotal, (double)amplification, sourceLine); } static bool -expat_heap_increase_tolerable(XML_Parser rootParser, XmlBigCount increase, +expat_heap_increase_tolerable(XML_Parser rootParser, size_t increase, int sourceLine) { assert(rootParser != NULL); assert(increase > 0); - XmlBigCount newTotal = 0; + size_t newTotal = 0; bool tolerable = true; // Detect integer overflow - if ((XmlBigCount)-1 - rootParser->m_alloc_tracker.bytesAllocated < increase) { + if (SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated < increase) { tolerable = false; } else { newTotal = rootParser->m_alloc_tracker.bytesAllocated + increase; @@ -903,8 +912,7 @@ expat_malloc(XML_Parser parser, size_t size, int sourceLine) { const size_t bytesToAllocate = sizeof(size_t) + EXPAT_MALLOC_PADDING + size; - if ((XmlBigCount)-1 - rootParser->m_alloc_tracker.bytesAllocated - < bytesToAllocate) { + if (SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated < bytesToAllocate) { return NULL; // i.e. signal integer overflow as out-of-memory } @@ -1014,9 +1022,10 @@ expat_realloc(XML_Parser parser, void *ptr, size_t size, int sourceLine) { } } - // NOTE: Integer overflow detection has already been done for us - // by expat_heap_increase_tolerable(..) above - assert(SIZE_MAX - sizeof(size_t) - EXPAT_MALLOC_PADDING >= size); + // Detect and prevent integer overflow + if (size > SIZE_MAX - sizeof(size_t) - EXPAT_MALLOC_PADDING) { + return NULL; + } // Actually allocate mallocedPtr = parser->m_mem.realloc_fcn( @@ -1028,8 +1037,7 @@ expat_realloc(XML_Parser parser, void *ptr, size_t size, int sourceLine) { // Update accounting if (isIncrease) { - assert((XmlBigCount)-1 - rootParser->m_alloc_tracker.bytesAllocated - >= absDiff); + assert(SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated >= absDiff); rootParser->m_alloc_tracker.bytesAllocated += absDiff; } else { // i.e. decrease assert(rootParser->m_alloc_tracker.bytesAllocated >= absDiff); @@ -1203,6 +1211,25 @@ isCalledFromInsideHandler(XML_Parser parser) { return parser->m_handlerCallDepth > 0; } +static void +callUnknownEncodingRelease(XML_Parser parser) { + beforeHandler(parser); + parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); + afterHandler(parser); + parser->m_unknownEncodingRelease = NULL; + parser->m_unknownEncodingData = NULL; +} + +static int XMLCALL +callUnknownEncodingConvert(void *data, const char *p) { + XML_Parser parser = (XML_Parser)data; + beforeHandler(parser); + const int result + = parser->m_unknownEncodingConvert(parser->m_unknownEncodingData, p); + afterHandler(parser); + return result; +} + static enum XML_Error callProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { @@ -1403,20 +1430,9 @@ parserCreate(const XML_Char *encodingName, FREE(parser, parser); return NULL; } -#ifdef XML_ATTR_INFO - parser->m_attInfo = (XML_AttrInfo*)MALLOC(parser, parser->m_attsSize * sizeof(XML_AttrInfo)); - if (parser->m_attInfo == NULL) { - FREE(parser, parser->m_atts); - FREE(parser, parser); - return NULL; - } -#endif parser->m_dataBuf = (XML_Char*)MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char)); if (parser->m_dataBuf == NULL) { FREE(parser, parser->m_atts); -#ifdef XML_ATTR_INFO - FREE(parser, parser->m_attInfo); -#endif FREE(parser, parser); return NULL; } @@ -1429,9 +1445,6 @@ parserCreate(const XML_Char *encodingName, if (parser->m_dtd == NULL) { FREE(parser, parser->m_dataBuf); FREE(parser, parser->m_atts); -#ifdef XML_ATTR_INFO - FREE(parser, parser->m_attInfo); -#endif FREE(parser, parser); return NULL; } @@ -1550,6 +1563,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_inheritedBindings = NULL; parser->m_nSpecifiedAtts = 0; parser->m_unknownEncodingMem = NULL; + parser->m_unknownEncodingConvert = NULL; parser->m_unknownEncodingRelease = NULL; parser->m_unknownEncodingData = NULL; parser->m_parsingStatus.parsing = XML_INITIALIZED; @@ -1579,7 +1593,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { } /* moves list of bindings to m_freeBindingList */ -static void FASTCALL +static void moveToFreeBindingList(XML_Parser parser, BINDING *bindings) { while (bindings) { BINDING *b = bindings; @@ -1627,6 +1641,10 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { if (parser->m_parentParser) return XML_FALSE; + // The application-defined release callback may access the parser, so it + // must run before any parser state is freed. + if (parser->m_unknownEncodingRelease) + callUnknownEncodingRelease(parser); /* move m_tagStack to m_freeTagList */ tStk = parser->m_tagStack; while (tStk) { @@ -1647,14 +1665,16 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { moveEntityList(&parser->m_freeEntities, &parser->m_openValueEntities); moveToFreeBindingList(parser, parser->m_inheritedBindings); FREE(parser, parser->m_unknownEncodingMem); - if (parser->m_unknownEncodingRelease) - parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); poolClear(&parser->m_tempPool); poolClear(&parser->m_temp2Pool); FREE(parser, (void *)parser->m_protocolEncodingName); parser->m_protocolEncodingName = NULL; parserInit(parser, encodingName); dtdReset(parser->m_dtd, parser); + if (encodingName && ! parser->m_protocolEncodingName) { + parser->m_errorCode = XML_ERROR_NO_MEMORY; + return XML_FALSE; + } return XML_TRUE; } @@ -1870,7 +1890,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, return parser; } -static void FASTCALL +static void destroyBindings(BINDING *bindings, XML_Parser parser) { for (;;) { BINDING *b = bindings; @@ -1887,6 +1907,10 @@ XML_ParserFree(XML_Parser parser) { TAG *tagList; if ((parser == NULL) || isCalledFromInsideHandler(parser)) return; + // The application-defined release callback may access the parser, so it + // must run before any parser state is freed. + if (parser->m_unknownEncodingRelease) + callUnknownEncodingRelease(parser); /* free m_tagStack and m_freeTagList */ tagList = parser->m_tagStack; for (;;) { @@ -1947,9 +1971,6 @@ XML_ParserFree(XML_Parser parser) { #endif /* XML_DTD */ dtdDestroy(parser->m_dtd, (XML_Bool)! parser->m_parentParser, parser); FREE(parser, parser->m_atts); -#ifdef XML_ATTR_INFO - FREE(parser, parser->m_attInfo); -#endif FREE(parser, parser->m_groupConnector); // NOTE: We are avoiding FREE(..) here because parser->m_buffer // is not being allocated with MALLOC(..) but with plain @@ -1958,8 +1979,6 @@ XML_ParserFree(XML_Parser parser) { FREE(parser, parser->m_dataBuf); FREE(parser, parser->m_nsAtts); FREE(parser, parser->m_unknownEncodingMem); - if (parser->m_unknownEncodingRelease) - parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); FREE(parser, parser); } @@ -2040,15 +2059,6 @@ XML_GetIdAttributeIndex(XML_Parser parser) { return parser->m_idAttIndex; } -#ifdef XML_ATTR_INFO -const XML_AttrInfo *XMLCALL -XML_GetAttributeInfo(XML_Parser parser) { - if (parser == NULL) - return NULL; - return parser->m_attInfo; -} -#endif - void XMLCALL XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start, XML_EndElementHandler end) { @@ -2269,9 +2279,10 @@ XML_SetParamEntityParsing(XML_Parser parser, #endif } -// DEPRECATED since Expat 2.8.0. -int XMLCALL -XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { +/* The body of XML_SetHashSalt, so that Expat's own tests can reach it + without tripping the deprecation of the public function. */ +XML_NONTESTING_STATIC int +xmlSetHashSalt(XML_Parser parser, unsigned long hash_salt) { if (parser == NULL) return 0; @@ -2297,6 +2308,12 @@ XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { return 1; } +// DEPRECATED since Expat 2.8.0. +int XMLCALL +XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { + return xmlSetHashSalt(parser, hash_salt); +} + XML_Bool XMLCALL XML_SetHashSalt16Bytes(XML_Parser parser, const uint8_t entropy[16]) { if (parser == NULL) @@ -2353,7 +2370,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { int nLeftOver; enum XML_Status result; /* Detect overflow (a+b > MAX <==> b > MAX-a) */ - if (len > XML_INDEX_MAX - parser->m_parseEndByteIndex) { + if ((uint64_t)len > UINT64_MAX - parser->m_parseEndByteIndex) { parser->m_errorCode = XML_ERROR_NO_MEMORY; parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_processor = errorProcessor; @@ -2470,8 +2487,13 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_parsingStatus.parsing = XML_PARSING; } + if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd)) { + parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; + return XML_STATUS_ERROR; + } + // Detect and avoid integer overflow - if (len > XML_INDEX_MAX - parser->m_parseEndByteIndex) { + if ((uint64_t)len > UINT64_MAX - parser->m_parseEndByteIndex) { parser->m_errorCode = XML_ERROR_NO_MEMORY; parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_processor = errorProcessor; @@ -2729,35 +2751,57 @@ XML_GetErrorCode(XML_Parser parser) { return parser->m_errorCode; } -XML_Index XMLCALL -XML_GetCurrentByteIndex(XML_Parser parser) { +int64_t XMLCALL +XML_GetCurrentByteIndex64(XML_Parser parser) { if (parser == NULL) return -1; - if (parser->m_eventPtr) - return (XML_Index)(parser->m_parseEndByteIndex - - (parser->m_parseEndPtr - parser->m_eventPtr)); + if (parser->m_eventPtr) { + return (int64_t)(parser->m_parseEndByteIndex + - (parser->m_parseEndPtr - parser->m_eventPtr)); + } return -1; } -int XMLCALL -XML_GetCurrentByteCount(XML_Parser parser) { +// DEPRECATED since Expat 2.9.0. +XML_Index XMLCALL +XML_GetCurrentByteIndex(XML_Parser parser) { + // NOTE: XML_Index is known to wrap around for >2 GiB content + // on 32bit machines and 64bit Windows, unless (non-default and + // uncommon) XML_LARGE_SIZE is defined. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + return (XML_Index)XML_GetCurrentByteIndex64(parser); +} + +uint64_t XMLCALL +XML_GetCurrentByteCount64(XML_Parser parser) { if (parser == NULL) return 0; - if (parser->m_eventEndPtr && parser->m_eventPtr) - return (int)(parser->m_eventEndPtr - parser->m_eventPtr); + if (parser->m_eventEndPtr && parser->m_eventPtr) { + return parser->m_eventEndPtr - parser->m_eventPtr; + } return 0; } +// DEPRECATED since Expat 2.9.0. +int XMLCALL +XML_GetCurrentByteCount(XML_Parser parser) { + // NOTE: int is known to wrap around for >2 GiB content. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + return (int)XML_GetCurrentByteCount64(parser); +} + const char *XMLCALL -XML_GetInputContext(XML_Parser parser, int *offset, int *size) { +XML_GetInputContext64(XML_Parser parser, int64_t *offset, uint64_t *size) { #if XML_CONTEXT_BYTES > 0 if (parser == NULL) return NULL; if (parser->m_eventPtr && parser->m_buffer) { if (offset != NULL) - *offset = (int)(parser->m_eventPtr - parser->m_buffer); + *offset = parser->m_eventPtr - parser->m_buffer; if (size != NULL) - *size = (int)(parser->m_bufferEnd - parser->m_buffer); + *size = parser->m_bufferEnd - parser->m_buffer; return parser->m_buffer; } #else @@ -2768,8 +2812,41 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) { return NULL; } -XML_Size XMLCALL -XML_GetCurrentLineNumber(XML_Parser parser) { +// DEPRECATED since Expat 2.9.0. +const char *XMLCALL +XML_GetInputContext(XML_Parser parser, int *offset, int *size) { +#if XML_CONTEXT_BYTES > 0 + if (parser == NULL) + return NULL; + + int64_t offset64; + uint64_t size64; + + const char *const buffer = XML_GetInputContext64(parser, &offset64, &size64); + + if (buffer == NULL) + return NULL; + + // NOTE: int is known to wrap around for >2 GiB content. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + if (offset != NULL) + *offset = (int)offset64; + + if (size != NULL) + *size = (int)size64; + + return buffer; +#else + (void)parser; + (void)offset; + (void)size; +#endif /* XML_CONTEXT_BYTES > 0 */ + return NULL; +} + +uint64_t XMLCALL +XML_GetCurrentLineNumber64(XML_Parser parser) { if (parser == NULL) return 0; if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) { @@ -2780,8 +2857,19 @@ XML_GetCurrentLineNumber(XML_Parser parser) { return parser->m_position.lineNumber + 1; } +// DEPRECATED since Expat 2.9.0. XML_Size XMLCALL -XML_GetCurrentColumnNumber(XML_Parser parser) { +XML_GetCurrentLineNumber(XML_Parser parser) { + // NOTE: XML_Size is known to wrap around for >4 GiB content + // on 32bit machines and 64bit Windows, unless (non-default and + // uncommon) XML_LARGE_SIZE is defined. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + return (XML_Size)XML_GetCurrentLineNumber64(parser); +} + +uint64_t XMLCALL +XML_GetCurrentColumnNumber64(XML_Parser parser) { if (parser == NULL) return 0; if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) { @@ -2792,6 +2880,17 @@ XML_GetCurrentColumnNumber(XML_Parser parser) { return parser->m_position.columnNumber; } +// DEPRECATED since Expat 2.9.0. +XML_Size XMLCALL +XML_GetCurrentColumnNumber(XML_Parser parser) { + // NOTE: XML_Size is known to wrap around for >4 GiB content + // on 32bit machines and 64bit Windows, unless (non-default and + // uncommon) XML_LARGE_SIZE is defined. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + return (XML_Size)XML_GetCurrentColumnNumber64(parser); +} + void XMLCALL XML_FreeContentModel(XML_Parser parser, XML_Content *model) { if (parser == NULL) @@ -2835,7 +2934,7 @@ XML_MemFree(XML_Parser parser, void *ptr) { void XMLCALL XML_DefaultCurrent(XML_Parser parser) { - if (parser == NULL) + if (parser == NULL || ! isCalledFromInsideHandler(parser)) return; if (parser->m_defaultHandler) { if (parser->m_openInternalEntities) @@ -3006,18 +3105,12 @@ XML_GetFeatureList(void) { {XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"), XML_CONTEXT_BYTES}, #endif -#ifdef XML_MIN_SIZE - {XML_FEATURE_MIN_SIZE, XML_L("XML_MIN_SIZE"), 0}, -#endif #ifdef XML_NS {XML_FEATURE_NS, XML_L("XML_NS"), 0}, #endif #ifdef XML_LARGE_SIZE {XML_FEATURE_LARGE_SIZE, XML_L("XML_LARGE_SIZE"), 0}, #endif -#ifdef XML_ATTR_INFO - {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0}, -#endif #if XML_GE == 1 /* Added in Expat 2.4.0 for XML_DTD defined and * added in Expat 2.6.0 for XML_GE == 1. */ @@ -3044,59 +3137,294 @@ XML_GetFeatureList(void) { } #if XML_GE == 1 +static enum XML_Prop_Error +setBillionLaughsAttackProtectionMaximumAmplification( + XML_Parser parser, float maximumAmplificationFactor) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + if (isnan(maximumAmplificationFactor) || (maximumAmplificationFactor < 1.0f)) + return XML_PROP_ERROR_INVALID_VALUE; + + parser->m_accounting.maximumAmplificationFactor = maximumAmplificationFactor; + + return XML_PROP_ERROR_NONE; +} + XML_Bool XMLCALL XML_SetBillionLaughsAttackProtectionMaximumAmplification( XML_Parser parser, float maximumAmplificationFactor) { - if ((parser == NULL) || (parser->m_parentParser != NULL) - || isnan(maximumAmplificationFactor) - || (maximumAmplificationFactor < 1.0f)) { - return XML_FALSE; - } - parser->m_accounting.maximumAmplificationFactor = maximumAmplificationFactor; - return XML_TRUE; + return (setBillionLaughsAttackProtectionMaximumAmplification( + parser, maximumAmplificationFactor) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +static enum XML_Prop_Error +setBillionLaughsAttackProtectionActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + parser->m_accounting.activationThresholdBytes = activationThresholdBytes; + + return XML_PROP_ERROR_NONE; } XML_Bool XMLCALL XML_SetBillionLaughsAttackProtectionActivationThreshold( XML_Parser parser, unsigned long long activationThresholdBytes) { - if ((parser == NULL) || (parser->m_parentParser != NULL)) { - return XML_FALSE; - } - parser->m_accounting.activationThresholdBytes = activationThresholdBytes; - return XML_TRUE; + return (setBillionLaughsAttackProtectionActivationThreshold( + parser, activationThresholdBytes) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +static enum XML_Prop_Error +setAllocTrackerMaximumAmplification(XML_Parser parser, + float maximumAmplificationFactor) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + if (isnan(maximumAmplificationFactor) || (maximumAmplificationFactor < 1.0f)) + return XML_PROP_ERROR_INVALID_VALUE; + + parser->m_alloc_tracker.maximumAmplificationFactor + = maximumAmplificationFactor; + + return XML_PROP_ERROR_NONE; } XML_Bool XMLCALL XML_SetAllocTrackerMaximumAmplification(XML_Parser parser, float maximumAmplificationFactor) { - if ((parser == NULL) || (parser->m_parentParser != NULL) - || isnan(maximumAmplificationFactor) - || (maximumAmplificationFactor < 1.0f)) { - return XML_FALSE; - } - parser->m_alloc_tracker.maximumAmplificationFactor - = maximumAmplificationFactor; - return XML_TRUE; + return (setAllocTrackerMaximumAmplification(parser, + maximumAmplificationFactor) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +static enum XML_Prop_Error +setAllocTrackerActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + parser->m_alloc_tracker.activationThresholdBytes = activationThresholdBytes; + + return XML_PROP_ERROR_NONE; } XML_Bool XMLCALL XML_SetAllocTrackerActivationThreshold( XML_Parser parser, unsigned long long activationThresholdBytes) { - if ((parser == NULL) || (parser->m_parentParser != NULL)) { - return XML_FALSE; - } - parser->m_alloc_tracker.activationThresholdBytes = activationThresholdBytes; - return XML_TRUE; + return (setAllocTrackerActivationThreshold(parser, activationThresholdBytes) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; } #endif /* XML_GE == 1 */ +static enum XML_Prop_Error +setReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (enabled != XML_TRUE && enabled != XML_FALSE) + return XML_PROP_ERROR_INVALID_VALUE; + + parser->m_reparseDeferralEnabled = enabled; + + return XML_PROP_ERROR_NONE; +} + XML_Bool XMLCALL XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled) { - if (parser != NULL && (enabled == XML_TRUE || enabled == XML_FALSE)) { - parser->m_reparseDeferralEnabled = enabled; - return XML_TRUE; + return (setReparseDeferralEnabled(parser, enabled) == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +enum XML_Prop_Error XMLCALL +XML_SetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + return XML_PROP_ERROR_INVALID_TYPE; +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return setReparseDeferralEnabled(parser, value); + default: + return XML_PROP_ERROR_INVALID_KEY; } - return XML_FALSE; + + assert(0 && "considered unreachable"); +} + +enum XML_Prop_Error XMLCALL +XML_SetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + +#if XML_GE == 0 + UNUSED_P(value); +#endif + + switch (property) { +#if XML_GE == 1 + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + return setAllocTrackerMaximumAmplification(parser, (float)value); + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + return setBillionLaughsAttackProtectionMaximumAmplification(parser, + (float)value); + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; + } + + assert(0 && "considered unreachable"); +} + +enum XML_Prop_Error XMLCALL +XML_SetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + +#if XML_GE == 0 + UNUSED_P(value); +#endif + + switch (property) { +#if XML_GE == 1 + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + return setAllocTrackerActivationThreshold(parser, + (unsigned long long)value); + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + return setBillionLaughsAttackProtectionActivationThreshold( + parser, (unsigned long long)value); + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; + } + + assert(0 && "considered unreachable"); +} + +enum XML_Prop_Error XMLCALL +XML_GetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool *value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (value == NULL) + return XML_PROP_ERROR_INVALID_VALUE; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + return XML_PROP_ERROR_INVALID_TYPE; +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + *value = parser->m_reparseDeferralEnabled; + break; + default: + return XML_PROP_ERROR_INVALID_KEY; + } + + return XML_PROP_ERROR_NONE; +} + +enum XML_Prop_Error XMLCALL +XML_GetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double *value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (value == NULL) + return XML_PROP_ERROR_INVALID_VALUE; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + *value = (double)parser->m_accounting.maximumAmplificationFactor; + break; + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + *value = (double)parser->m_alloc_tracker.maximumAmplificationFactor; + break; + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; + } + + return XML_PROP_ERROR_NONE; +} + +enum XML_Prop_Error XMLCALL +XML_GetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t *value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (value == NULL) + return XML_PROP_ERROR_INVALID_VALUE; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + *value = parser->m_accounting.activationThresholdBytes; + break; + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + *value = parser->m_alloc_tracker.activationThresholdBytes; + break; + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; + } + + return XML_PROP_ERROR_NONE; } /* Initially tag->rawName always points into the parse buffer; @@ -3153,7 +3481,7 @@ storeRawNames(XML_Parser parser) { return XML_TRUE; } -static enum XML_Error PTRCALL +static enum XML_Error contentProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result = doContent( @@ -3167,7 +3495,7 @@ contentProcessor(XML_Parser parser, const char *start, const char *end, return result; } -static enum XML_Error PTRCALL +static enum XML_Error externalEntityInitProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result = initializeEncoding(parser); @@ -3177,7 +3505,7 @@ externalEntityInitProcessor(XML_Parser parser, const char *start, return externalEntityInitProcessor2(parser, start, end, endPtr); } -static enum XML_Error PTRCALL +static enum XML_Error externalEntityInitProcessor2(XML_Parser parser, const char *start, const char *end, const char **endPtr) { const char *next = start; /* XmlContentTok doesn't always set the last arg */ @@ -3222,7 +3550,7 @@ externalEntityInitProcessor2(XML_Parser parser, const char *start, return externalEntityInitProcessor3(parser, start, end, endPtr); } -static enum XML_Error PTRCALL +static enum XML_Error externalEntityInitProcessor3(XML_Parser parser, const char *start, const char *end, const char **endPtr) { int tok; @@ -3274,7 +3602,7 @@ externalEntityInitProcessor3(XML_Parser parser, const char *start, return externalEntityContentProcessor(parser, start, end, endPtr); } -static enum XML_Error PTRCALL +static enum XML_Error externalEntityContentProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result @@ -3438,9 +3766,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, return result; } else if (parser->m_externalEntityRefHandler) { const XML_Char *context; - entity->open = XML_TRUE; + entity->open = true; context = getContext(parser); - entity->open = XML_FALSE; + entity->open = false; if (! context) return XML_ERROR_NO_MEMORY; beforeHandler(parser); @@ -3865,8 +4193,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sizeof(ELEMENT_TYPE)); if (! elementType) return XML_ERROR_NO_MEMORY; - if (! elementType->defaultAttsNames.parser) - hashTableInit(&(elementType->defaultAttsNames), parser); + if (! elementType->defaultAttForName.parser) + hashTableInit(&(elementType->defaultAttForName), parser); if (parser->m_ns && ! setElementTypePrefix(parser, elementType)) return XML_ERROR_NO_MEMORY; } @@ -3909,21 +4237,6 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, return XML_ERROR_NO_MEMORY; } parser->m_atts = temp; -#ifdef XML_ATTR_INFO - /* Detect and prevent integer overflow. */ - if (parser->m_attsSize > SIZE_MAX / sizeof(XML_AttrInfo)) { - parser->m_attsSize = oldAttsSize; - return XML_ERROR_NO_MEMORY; - } - - XML_AttrInfo *const temp2 = REALLOC( - parser, parser->m_attInfo, parser->m_attsSize * sizeof(XML_AttrInfo)); - if (temp2 == NULL) { - parser->m_attsSize = oldAttsSize; - return XML_ERROR_NO_MEMORY; - } - parser->m_attInfo = temp2; -#endif if (n > oldAttsSize) { /* Detect and prevent integer overflow. */ if (n > (size_t)INT_MAX) @@ -3936,25 +4249,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, const XML_Char **const appAtts = (const XML_Char **)parser->m_atts; for (size_t i = 0; i < n; i++) { ATTRIBUTE *currAtt = &parser->m_atts[i]; -#ifdef XML_ATTR_INFO - XML_AttrInfo *currAttInfo = &parser->m_attInfo[i]; -#endif /* add the name and value to the attribute list */ ATTRIBUTE_ID *attId = getAttributeId(parser, enc, currAtt->name, currAtt->name + XmlNameLength(enc, currAtt->name)); if (! attId) return XML_ERROR_NO_MEMORY; -#ifdef XML_ATTR_INFO - currAttInfo->nameStart - = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->name); - currAttInfo->nameEnd - = currAttInfo->nameStart + XmlNameLength(enc, currAtt->name); - currAttInfo->valueStart = parser->m_parseEndByteIndex - - (parser->m_parseEndPtr - currAtt->valuePtr); - currAttInfo->valueEnd = parser->m_parseEndByteIndex - - (parser->m_parseEndPtr - currAtt->valueEnd); -#endif /* Detect duplicate attributes by their QNames. This does not work when namespace processing is turned on and different prefixes for the same namespace are used. For this case we have a check further down. @@ -3971,11 +4271,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* figure out whether declared as other than CDATA */ if (attId->maybeTokenized) { - for (size_t j = 0; j < nDefaultAtts; j++) { - if (attId == elementType->defaultAtts[j].id) { - isCdata = elementType->defaultAtts[j].isCdata; - break; - } + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(elementType->defaultAttForName), attId->name, 0); + if (nameAndDefaultAttribute != NULL) { + assert(nameAndDefaultAttribute->attIndex < elementType->nDefaultAtts); + const DEFAULT_ATTRIBUTE *const att + = elementType->defaultAtts + nameAndDefaultAttribute->attIndex; + isCdata = att->isCdata; } } @@ -4066,8 +4369,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, unsigned int nsAttsSize = 1u << parser->m_nsAttsPower; unsigned char oldNsAttsPower = parser->m_nsAttsPower; /* size of hash table must be at least 2 * (# of prefixed attributes) */ - if ((nPrefixes << 1) - >> parser->m_nsAttsPower) { /* true for m_nsAttsPower = 0 */ + if (parser->m_nsAttsPower == 0 + || (nPrefixes >> (parser->m_nsAttsPower - 1))) { /* hash table size must also be a power of 2 and >= 8 */ while (nPrefixes >> parser->m_nsAttsPower++) ; @@ -4525,6 +4828,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, /* Detect and prevent integer overflow */ if (len > SIZE_MAX - EXPAND_SPARE || len + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { + FREE(parser, b); return XML_ERROR_NO_MEMORY; } @@ -4562,7 +4866,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, /* The idea here is to avoid using stack for each CDATA section when the whole file is parsed with one call. */ -static enum XML_Error PTRCALL +static enum XML_Error cdataSectionProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result = doCdataSection( @@ -4728,7 +5032,7 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, /* The idea here is to avoid using stack for each IGNORE section when the whole file is parsed with one call. */ -static enum XML_Error PTRCALL +static enum XML_Error ignoreSectionProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result @@ -4966,30 +5270,39 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName) { const int status = parser->m_unknownEncodingHandler( parser->m_unknownEncodingHandlerData, encodingName, &info); afterHandler(parser); + + parser->m_unknownEncodingRelease = info.release; + parser->m_unknownEncodingData = info.data; + if (status) { ENCODING *enc; parser->m_unknownEncodingMem = (void*)MALLOC(parser, XmlSizeOfUnknownEncoding()); if (! parser->m_unknownEncodingMem) { - if (info.release) - info.release(info.data); + if (parser->m_unknownEncodingRelease) + callUnknownEncodingRelease(parser); + else + parser->m_unknownEncodingData = NULL; return XML_ERROR_NO_MEMORY; } + parser->m_unknownEncodingConvert = info.convert; enc = (parser->m_ns ? XmlInitUnknownEncodingNS : XmlInitUnknownEncoding)( - parser->m_unknownEncodingMem, info.map, info.convert, info.data); + parser->m_unknownEncodingMem, info.map, + info.convert ? callUnknownEncodingConvert : NULL, parser); if (enc) { - parser->m_unknownEncodingData = info.data; - parser->m_unknownEncodingRelease = info.release; parser->m_encoding = enc; return XML_ERROR_NONE; } + parser->m_unknownEncodingConvert = NULL; } - if (info.release != NULL) - info.release(info.data); + if (parser->m_unknownEncodingRelease != NULL) + callUnknownEncodingRelease(parser); + else + parser->m_unknownEncodingData = NULL; } return XML_ERROR_UNKNOWN_ENCODING; } -static enum XML_Error PTRCALL +static enum XML_Error prologInitProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { enum XML_Error result = initializeEncoding(parser); @@ -5001,7 +5314,7 @@ prologInitProcessor(XML_Parser parser, const char *s, const char *end, #ifdef XML_DTD -static enum XML_Error PTRCALL +static enum XML_Error externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { enum XML_Error result = initializeEncoding(parser); @@ -5010,7 +5323,7 @@ externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end, /* we know now that XML_Parse(Buffer) has been called, so we consider the external parameter entity read */ - parser->m_dtd->paramEntityRead = XML_TRUE; + parser->m_dtd->paramEntityRead = true; if (parser->m_prologState.inEntityValue) { parser->m_processor = entityValueInitProcessor; @@ -5021,7 +5334,7 @@ externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end, } } -static enum XML_Error PTRCALL +static enum XML_Error entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { int tok; @@ -5105,7 +5418,7 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, } } -static enum XML_Error PTRCALL +static enum XML_Error externalParEntProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { const char *next = s; @@ -5151,7 +5464,7 @@ externalParEntProcessor(XML_Parser parser, const char *s, const char *end, XML_ACCOUNT_DIRECT); } -static enum XML_Error PTRCALL +static enum XML_Error entityValueProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { const char *start = s; @@ -5198,7 +5511,7 @@ entityValueProcessor(XML_Parser parser, const char *s, const char *end, #endif /* XML_DTD */ -static enum XML_Error PTRCALL +static enum XML_Error prologProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { const char *next = s; @@ -5444,7 +5757,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } if (parser->m_useForeignDTD) entity->base = parser->m_curBase; - dtd->paramEntityRead = XML_FALSE; + dtd->paramEntityRead = false; beforeHandler(parser); const int status = parser->m_externalEntityRefHandler( parser->m_externalEntityRefHandlerArg, 0, entity->base, @@ -5494,7 +5807,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (! entity) return XML_ERROR_NO_MEMORY; entity->base = parser->m_curBase; - dtd->paramEntityRead = XML_FALSE; + dtd->paramEntityRead = false; beforeHandler(parser); const int status = parser->m_externalEntityRefHandler( parser->m_externalEntityRefHandlerArg, 0, entity->base, @@ -5837,7 +6150,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } else { poolFinish(&dtd->pool); parser->m_declEntity->publicId = NULL; - parser->m_declEntity->is_param = XML_FALSE; + parser->m_declEntity->is_param = false; /* if we have a parent parser or are reading an internal parameter entity, then the entity declaration is not considered "internal" */ @@ -5867,7 +6180,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } else { poolFinish(&dtd->pool); parser->m_declEntity->publicId = NULL; - parser->m_declEntity->is_param = XML_TRUE; + parser->m_declEntity->is_param = true; /* if we have a parent parser or are reading an internal parameter entity, then the entity declaration is not considered "internal" */ @@ -6102,8 +6415,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_RECURSIVE_ENTITY_REF; if (entity->textPtr) { enum XML_Error result; - XML_Bool betweenDecl - = (role == XML_ROLE_PARAM_ENTITY_REF ? XML_TRUE : XML_FALSE); + bool betweenDecl = (role == XML_ROLE_PARAM_ENTITY_REF); result = processEntity(parser, entity, betweenDecl, ENTITY_INTERNAL); if (result != XML_ERROR_NONE) return result; @@ -6111,8 +6423,8 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, break; } if (parser->m_externalEntityRefHandler) { - dtd->paramEntityRead = XML_FALSE; - entity->open = XML_TRUE; + dtd->paramEntityRead = false; + entity->open = true; entityTrackingOnOpen(parser, entity, __LINE__); beforeHandler(parser); const int status = parser->m_externalEntityRefHandler( @@ -6121,11 +6433,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, afterHandler(parser); if (! status) { entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; return XML_ERROR_EXTERNAL_ENTITY_HANDLING; } entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; handleDefault = XML_FALSE; if (! dtd->paramEntityRead) { dtd->keepProcessing = dtd->standalone; @@ -6155,7 +6467,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_NO_MEMORY; dtd->scaffLevel = 0; dtd->scaffCount = 0; - dtd->in_eldecl = XML_TRUE; + dtd->in_eldecl = true; handleDefault = XML_FALSE; } break; @@ -6184,7 +6496,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, afterHandler(parser); handleDefault = XML_FALSE; } - dtd->in_eldecl = XML_FALSE; + dtd->in_eldecl = false; } break; @@ -6266,7 +6578,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, parser->m_handlerArg, parser->m_declElementType->name, model); afterHandler(parser); } - dtd->in_eldecl = XML_FALSE; + dtd->in_eldecl = false; dtd->contentStringLen = 0; } } @@ -6335,7 +6647,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, /* not reached */ } -static enum XML_Error PTRCALL +static enum XML_Error epilogProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { parser->m_processor = epilogProcessor; @@ -6414,7 +6726,7 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, } static enum XML_Error -processEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl, +processEntity(XML_Parser parser, ENTITY *entity, bool betweenDecl, enum EntityType type) { OPEN_INTERNAL_ENTITY *openEntity, **openEntityList; OPEN_INTERNAL_ENTITY **const freeEntityList = &parser->m_freeEntities; @@ -6449,8 +6761,8 @@ processEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl, if (! openEntity) return XML_ERROR_NO_MEMORY; } - entity->open = XML_TRUE; - entity->hasMore = XML_TRUE; + entity->open = true; + entity->hasMore = true; #if XML_GE == 1 entityTrackingOnOpen(parser, entity, __LINE__); #endif @@ -6472,7 +6784,7 @@ processEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl, return XML_ERROR_NONE; } -static enum XML_Error PTRCALL +static enum XML_Error internalEntityProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { UNUSED_P(s); @@ -6522,7 +6834,7 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, // Entity is complete. We cannot close it here since we need to first // process its possible inner entities (which are added to the // m_openInternalEntities during doProlog or doContent calls above) - entity->hasMore = XML_FALSE; + entity->hasMore = false; if (! entity->is_param && (openEntity->startTagLevel != parser->m_tagLevel)) { return XML_ERROR_ASYNC_ENTITY; @@ -6540,7 +6852,7 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, // to false. This means we can directly remove the head of // m_openInternalEntities assert(parser->m_openInternalEntities == openEntity); - entity->open = XML_FALSE; + entity->open = false; parser->m_openInternalEntities = parser->m_openInternalEntities->next; /* put openEntity back in list of free instances */ @@ -6554,7 +6866,7 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, return XML_ERROR_NONE; } -static enum XML_Error PTRCALL +static enum XML_Error errorProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { UNUSED_P(s); @@ -6595,16 +6907,17 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, // Check if entity is complete, if not, mark down how much of it is // processed. A XML_SUSPENDED check here is not required as // appendAttributeValue will never suspend the parser. - if (textEnd != nextInEntity) { + if (nextInEntity < textEnd) { entity->processed = (int)(nextInEntity - (const char *)entity->textPtr); continue; } + assert(nextInEntity == textEnd); // Entity is complete. We cannot close it here since we need to first // process its possible inner entities (which are added to the // m_openAttributeEntities during appendAttributeValue) - entity->hasMore = XML_FALSE; + entity->hasMore = false; continue; } // End of entity processing, "if" block skips the rest @@ -6617,7 +6930,7 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, // with hasMore set to false. This means we can directly remove the head // of m_openAttributeEntities assert(parser->m_openAttributeEntities == openEntity); - entity->open = XML_FALSE; + entity->open = false; parser->m_openAttributeEntities = parser->m_openAttributeEntities->next; /* put openEntity back in list of free instances */ @@ -6912,8 +7225,8 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, } if (entity->systemId) { if (parser->m_externalEntityRefHandler) { - dtd->paramEntityRead = XML_FALSE; - entity->open = XML_TRUE; + dtd->paramEntityRead = false; + entity->open = true; entityTrackingOnOpen(parser, entity, __LINE__); beforeHandler(parser); const int status = parser->m_externalEntityRefHandler( @@ -6922,12 +7235,12 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, afterHandler(parser); if (! status) { entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; result = XML_ERROR_EXTERNAL_ENTITY_HANDLING; goto endEntityValue; } entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; if (! dtd->paramEntityRead) dtd->keepProcessing = dtd->standalone; } else @@ -7064,7 +7377,7 @@ callStoreEntityValue(XML_Parser parser, const ENCODING *enc, // Entity is complete. We cannot close it here since we need to first // process its possible inner entities (which are added to the // m_openValueEntities during storeEntityValue) - entity->hasMore = XML_FALSE; + entity->hasMore = false; continue; } // End of entity processing, "if" block skips the rest @@ -7077,7 +7390,7 @@ callStoreEntityValue(XML_Parser parser, const ENCODING *enc, // with hasMore set to false. This means we can directly remove the head // of m_openValueEntities assert(parser->m_openValueEntities == openEntity); - entity->open = XML_FALSE; + entity->open = false; parser->m_openValueEntities = parser->m_openValueEntities->next; /* put openEntity back in list of free instances */ @@ -7126,7 +7439,7 @@ storeSelfEntityValue(XML_Parser parser, ENTITY *entity) { #endif /* XML_GE == 0 */ -static void FASTCALL +static void normalizeLines(XML_Char *s) { XML_Char *p; for (;; s++) { @@ -7258,7 +7571,7 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, /* The handling of default attributes gets messed up if we have a default which duplicates a non-default. */ NAMED *const nameFound - = lookup(parser, &(type->defaultAttsNames), attId->name, 0); + = lookup(parser, &(type->defaultAttForName), attId->name, 0); if (nameFound) return 1; if (isId && ! type->idAtt && ! attId->xmlns) @@ -7292,13 +7605,26 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, att->value = value; att->isCdata = isCdata; if (! isCdata) - attId->maybeTokenized = XML_TRUE; + attId->maybeTokenized = true; - NAMED *const nameAddedOrFound - = lookup(parser, &(type->defaultAttsNames), attId->name, sizeof(NAMED)); - if (! nameAddedOrFound) + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(type->defaultAttForName), attId->name, + sizeof(NAME_AND_DEFAULT_ATTRIBUTE)); + if (! nameAndDefaultAttribute) return 0; + assert(nameAndDefaultAttribute->name == attId->name); + + // NOTE: The XML 1.0r4 spec says: + // "When more than one definition is provided for the same attribute of a + // given element type, the first declaration is binding and later + // declarations are ignored." + if (! nameAndDefaultAttribute->initialized) { + nameAndDefaultAttribute->attIndex = type->nDefaultAtts; + nameAndDefaultAttribute->initialized = true; + } + type->nDefaultAtts += 1; return 1; } @@ -7364,7 +7690,7 @@ getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start, else id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, name + 6, sizeof(PREFIX)); - id->xmlns = XML_TRUE; + id->xmlns = true; } else { int i; for (i = 0; name[i]; i++) { @@ -7499,7 +7825,7 @@ setContext(XML_Parser parser, const XML_Char *context) { e = (ENTITY *)lookup(parser, &dtd->generalEntities, poolStart(&parser->m_tempPool), 0); if (e) - e->open = XML_TRUE; + e->open = true; if (*s != XML_T('\0')) s++; context = s; @@ -7554,7 +7880,7 @@ setContext(XML_Parser parser, const XML_Char *context) { return XML_TRUE; } -static void FASTCALL +static void normalizePublicId(XML_Char *publicId) { XML_Char *p = publicId; XML_Char *s; @@ -7587,13 +7913,13 @@ dtdCreate(XML_Parser parser) { hashTableInit(&(p->attributeIds), parser); hashTableInit(&(p->prefixes), parser); #ifdef XML_DTD - p->paramEntityRead = XML_FALSE; + p->paramEntityRead = false; hashTableInit(&(p->paramEntities), parser); #endif /* XML_DTD */ p->defaultPrefix.name = NULL; p->defaultPrefix.binding = NULL; - p->in_eldecl = XML_FALSE; + p->in_eldecl = false; p->scaffIndex = NULL; p->scaffIndexSize = 0; p->scaffold = NULL; @@ -7616,12 +7942,12 @@ dtdReset(DTD *p, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; - hashTableDestroy(&(e->defaultAttsNames)); + hashTableDestroy(&(e->defaultAttForName)); FREE(parser, e->defaultAtts); } hashTableClear(&(p->generalEntities)); #ifdef XML_DTD - p->paramEntityRead = XML_FALSE; + p->paramEntityRead = false; hashTableClear(&(p->paramEntities)); #endif /* XML_DTD */ hashTableClear(&(p->elementTypes)); @@ -7632,7 +7958,7 @@ dtdReset(DTD *p, XML_Parser parser) { p->defaultPrefix.name = NULL; p->defaultPrefix.binding = NULL; - p->in_eldecl = XML_FALSE; + p->in_eldecl = false; FREE(parser, p->scaffIndex); p->scaffIndex = NULL; @@ -7658,7 +7984,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; - hashTableDestroy(&(e->defaultAttsNames)); + hashTableDestroy(&(e->defaultAttForName)); FREE(parser, e->defaultAtts); } hashTableDestroy(&(p->generalEntities)); @@ -7751,8 +8077,8 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, if (! newE) return 0; - if (! newE->defaultAttsNames.parser) - hashTableInit(&(newE->defaultAttsNames), parser); + if (! newE->defaultAttForName.parser) + hashTableInit(&(newE->defaultAttForName), parser); if (oldE->nDefaultAtts) { /* Detect and prevent integer overflow. */ @@ -7785,11 +8111,22 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, } else newE->defaultAtts[i].value = NULL; - NAMED *const nameAddedOrFound = lookup(parser, &(newE->defaultAttsNames), - attributeName, sizeof(NAMED)); - if (! nameAddedOrFound) { + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(newE->defaultAttForName), attributeName, + sizeof(NAME_AND_DEFAULT_ATTRIBUTE)); + if (! nameAndDefaultAttribute) { return 0; } + + // NOTE: The XML 1.0r4 spec says: + // "When more than one definition is provided for the same attribute of a + // given element type, the first declaration is binding and later + // declarations are ignored." + if (! nameAndDefaultAttribute->initialized) { + nameAndDefaultAttribute->attIndex = i; + nameAndDefaultAttribute->initialized = true; + } } } @@ -7886,23 +8223,27 @@ copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, #define INIT_POWER 6 -static XML_Bool FASTCALL -keyeq(KEY s1, KEY s2) { +// Compares two strings `s1` and `s2` whereas: +// - `s2` is zero-terminated but +// - `s1` is made up of exactly (not just up to) `s1len` non-zero characters. +XML_NONTESTING_STATIC XML_Bool +keyeq(KEY s1, size_t s1len, KEY s2) { #ifdef XML_UNICODE # ifdef XML_UNICODE_WCHAR_T - return (wcscmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; + return (wcsncmp(s1, s2, s1len) == 0 && s2[s1len] == L'\0') ? XML_TRUE + : XML_FALSE; # else - for (; *s1 == *s2; s1++, s2++) - if (*s1 == 0) - return XML_TRUE; - return XML_FALSE; + for (; s1len > 0 && *s1 == *s2; s1len--, s1++, s2++) + ; /* no loop body! */ + return ((s1len == 0) && (*s2 == 0)) ? XML_TRUE : XML_FALSE; # endif #else - return (strcmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; + return (strncmp(s1, s2, s1len) == 0 && s2[s1len] == '\0') ? XML_TRUE + : XML_FALSE; #endif } -static size_t +XML_NONTESTING_STATIC size_t keylen(KEY s) { return xcslen(s); } @@ -7915,19 +8256,39 @@ copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key) { *key = rootParser->m_hash_secret_salt_128; } -static unsigned long FASTCALL -hash(XML_Parser parser, KEY s) { +static unsigned long +hash(XML_Parser parser, KEY s, size_t keyLen) { struct siphash state; struct sipkey key; (void)sip24_valid; copy_salt_to_sipkey(parser, &key); sip24_init(&state, &key); - sip24_update(&state, s, keylen(s) * sizeof(XML_Char)); + sip24_update(&state, s, keyLen * sizeof(XML_Char)); return (unsigned long)sip24_final(&state); } -static NAMED * -lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { +// Function `lookupWithLength` can be used to either… +// +// a) check whether an element with key `name` exists in the given hash table +// (read-only mode where `createSize == 0`) or +// +// b) check whether an element with key `name` exists in the given hash table +// *and* insert it if missing (i.e. read-write mode where `createSize != 0`. +// +// When inserting, a block of `createSize` number of bytes will be allocated +// and set to zero, and the resulting block of memory will be considered +// to start with a `NAMED` structure, and `->name = name;` is performed. +// The fact that all other bytes in the structure are initially zero can +// be used to tell cases "existed and found" and "newly inserted" apart +// with the structure returned. +// +// NOTE: Read-only lookup does not need zero-terminated keys but +// read-write mode does, because keys can be re-hashed later and the +// hash table does not store key length information. +// +XML_NONTESTING_STATIC NAMED * +lookupWithLength(XML_Parser parser, HASH_TABLE *table, KEY name, size_t nameLen, + size_t createSize) { size_t i; if (table->size == 0) { size_t tsize; @@ -7943,14 +8304,14 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { return NULL; } memset(table->v, 0, tsize); - i = hash(parser, name) & ((unsigned long)table->size - 1); + i = hash(parser, name, nameLen) & ((unsigned long)table->size - 1); } else { - unsigned long h = hash(parser, name); + unsigned long h = hash(parser, name, nameLen); unsigned long mask = (unsigned long)table->size - 1; unsigned char step = 0; i = h & mask; while (table->v[i]) { - if (keyeq(name, table->v[i]->name)) + if (keyeq(name, nameLen, table->v[i]->name)) return table->v[i]; if (! step) step = PROBE_STEP(h, mask, table->power); @@ -7983,7 +8344,8 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { memset(newV, 0, tsize); for (i = 0; i < table->size; i++) if (table->v[i]) { - unsigned long newHash = hash(parser, table->v[i]->name); + KEY const key = table->v[i]->name; + unsigned long newHash = hash(parser, key, keylen(key)); size_t j = newHash & newMask; step = 0; while (newV[j]) { @@ -8006,16 +8368,37 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { } } } + assert(createSize >= sizeof(NAMED)); table->v[i] = (NAMED*)MALLOC(table->parser, createSize); if (! table->v[i]) return NULL; memset(table->v[i], 0, createSize); - table->v[i]->name = name; + table->v[i]->name = name; // NOTE: This requires and assumes zero termination! (table->used)++; return table->v[i]; } -static void FASTCALL +// Function `lookup` can be used to either… +// +// a) check whether an element with key `name` exists in the given hash table +// (read-only mode where `createSize == 0`) or +// +// b) check whether an element with key `name` exists in the given hash table +// *and* insert it if missing (i.e. read-write mode where `createSize != 0`. +// +// When inserting, a block of `createSize` number of bytes will be allocated +// and set to zero, and the resulting block of memory will be considered +// to start with a `NAMED` structure, and `->name = name;` is performed. +// The fact that all other bytes in the structure are initially zero can +// be used to tell cases "existed and found" and "newly inserted" apart +// with the structure returned. +// +XML_NONTESTING_STATIC NAMED * +lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { + return lookupWithLength(parser, table, name, keylen(name), createSize); +} + +XML_NONTESTING_STATIC void hashTableClear(HASH_TABLE *table) { size_t i; for (i = 0; i < table->size; i++) { @@ -8025,7 +8408,7 @@ hashTableClear(HASH_TABLE *table) { table->used = 0; } -static void FASTCALL +XML_NONTESTING_STATIC void hashTableDestroy(HASH_TABLE *table) { size_t i; for (i = 0; i < table->size; i++) @@ -8033,7 +8416,7 @@ hashTableDestroy(HASH_TABLE *table) { FREE(table->parser, table->v); } -static void FASTCALL +XML_NONTESTING_STATIC void hashTableInit(HASH_TABLE *p, XML_Parser parser) { p->power = 0; p->size = 0; @@ -8042,13 +8425,13 @@ hashTableInit(HASH_TABLE *p, XML_Parser parser) { p->parser = parser; } -static void FASTCALL +XML_NONTESTING_STATIC void hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table) { iter->p = table->v; iter->end = iter->p ? iter->p + table->size : NULL; } -static NAMED *FASTCALL +XML_NONTESTING_STATIC NAMED * hashTableIterNext(HASH_TABLE_ITER *iter) { while (iter->p != iter->end) { NAMED *tem = *(iter->p)++; @@ -8058,7 +8441,7 @@ hashTableIterNext(HASH_TABLE_ITER *iter) { return NULL; } -static void FASTCALL +static void poolInit(STRING_POOL *pool, XML_Parser parser) { pool->blocks = NULL; pool->freeBlocks = NULL; @@ -8068,7 +8451,7 @@ poolInit(STRING_POOL *pool, XML_Parser parser) { pool->parser = parser; } -static void FASTCALL +static void poolClear(STRING_POOL *pool) { if (! pool->freeBlocks) pool->freeBlocks = pool->blocks; @@ -8087,7 +8470,7 @@ poolClear(STRING_POOL *pool) { pool->end = NULL; } -static void FASTCALL +static void poolDestroy(STRING_POOL *pool) { BLOCK *p = pool->blocks; while (p) { @@ -8120,7 +8503,7 @@ poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr, return pool->start; } -static const XML_Char *FASTCALL +static const XML_Char * poolCopyString(STRING_POOL *pool, const XML_Char *s) { if (! poolAppendChars(pool, s, xcslen(s) + /*null terminator*/ 1)) return NULL; @@ -8131,7 +8514,7 @@ poolCopyString(STRING_POOL *pool, const XML_Char *s) { // A version of `poolCopyString` that does not call `poolFinish` // and reverts any partial advancement upon failure. -static const XML_Char *FASTCALL +static const XML_Char * poolCopyStringNoFinish(STRING_POOL *pool, const XML_Char *s) { const XML_Char *const original = s; do { @@ -8169,7 +8552,7 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { return s; } -static const XML_Char *FASTCALL +static const XML_Char * poolAppendString(STRING_POOL *pool, const XML_Char *s) { if (! poolAppendChars(pool, s, xcslen(s))) return NULL; @@ -8214,7 +8597,7 @@ poolBytesToAllocateFor(int blockSize) { } } -static XML_Bool FASTCALL +static XML_Bool poolGrow(STRING_POOL *pool) { if (pool->freeBlocks) { if (pool->start == NULL) { @@ -8233,7 +8616,7 @@ poolGrow(STRING_POOL *pool) { pool->freeBlocks = tem; memcpy(pool->blocks->s, pool->start, (pool->end - pool->start) * sizeof(XML_Char)); - pool->ptr = pool->blocks->s + (pool->ptr - pool->start); + pool->ptr = pool->blocks->s + EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start); pool->start = pool->blocks->s; pool->end = pool->start + pool->blocks->size; return XML_TRUE; @@ -8246,7 +8629,8 @@ poolGrow(STRING_POOL *pool) { /* NOTE: Needs to be calculated prior to calling `realloc` to avoid dangling pointers: */ - const ptrdiff_t offsetInsideBlock = pool->ptr - pool->start; + const ptrdiff_t offsetInsideBlock + = EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start); if (blockSize < 0) { /* This condition traps a situation where either more than @@ -8309,15 +8693,16 @@ poolGrow(STRING_POOL *pool) { tem->next = pool->blocks; pool->blocks = tem; if (pool->ptr != pool->start) - memcpy(tem->s, pool->start, (pool->ptr - pool->start) * sizeof(XML_Char)); - pool->ptr = tem->s + (pool->ptr - pool->start); + memcpy(tem->s, pool->start, + EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start) * sizeof(XML_Char)); + pool->ptr = tem->s + EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start); pool->start = tem->s; pool->end = tem->s + blockSize; } return XML_TRUE; } -static bool FASTCALL +static bool poolGrowUntil(STRING_POOL *pool, size_t needed) { for (;;) { const size_t available = pool->end - pool->ptr; @@ -8330,7 +8715,7 @@ poolGrowUntil(STRING_POOL *pool, size_t needed) { } } -static int FASTCALL +static int nextScaffoldPart(XML_Parser parser) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ CONTENT_SCAFFOLD *me; @@ -8552,8 +8937,8 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, sizeof(ELEMENT_TYPE)); if (! ret) return NULL; - if (! ret->defaultAttsNames.parser) - hashTableInit(&(ret->defaultAttsNames), getRootParserOf(parser, NULL)); + if (! ret->defaultAttForName.parser) + hashTableInit(&(ret->defaultAttForName), getRootParserOf(parser, NULL)); if (ret->name != name) poolDiscard(&dtd->pool); else { diff --git a/base/poco/XML/src/xmlrole.c b/base/poco/XML/src/xmlrole.c index 2c6133f236a9f..15aff759b45d3 100644 --- a/base/poco/XML/src/xmlrole.c +++ b/base/poco/XML/src/xmlrole.c @@ -12,7 +12,7 @@ Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2002-2003 Fred L. Drake, Jr. Copyright (c) 2005-2009 Steven Solie - Copyright (c) 2016-2023 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2017 Rhodri James Copyright (c) 2019 David Loffredo Copyright (c) 2021 Donghee Na @@ -36,6 +36,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #include "expat_config.h" @@ -118,9 +120,8 @@ static const char KW_SYSTEM[] # define setTopLevel(state) ((state)->handler = internalSubset) #endif /* not XML_DTD */ -typedef int PTRCALL PROLOG_HANDLER(PROLOG_STATE *state, int tok, - const char *ptr, const char *end, - const ENCODING *enc); +typedef int PROLOG_HANDLER(PROLOG_STATE *state, int tok, const char *ptr, + const char *end, const ENCODING *enc); static PROLOG_HANDLER prolog0, prolog1, prolog2, doctype0, doctype1, doctype2, doctype3, doctype4, doctype5, internalSubset, entity0, entity1, entity2, @@ -134,9 +135,9 @@ static PROLOG_HANDLER prolog0, prolog1, prolog2, doctype0, doctype1, doctype2, #endif /* XML_DTD */ declClose, error; -static int FASTCALL common(PROLOG_STATE *state, int tok); +static int common(PROLOG_STATE *state, int tok); -static int PTRCALL +static int prolog0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -167,7 +168,7 @@ prolog0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int prolog1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -199,7 +200,7 @@ prolog1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int prolog2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -219,7 +220,7 @@ prolog2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -236,7 +237,7 @@ doctype0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -262,7 +263,7 @@ doctype1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -278,7 +279,7 @@ doctype2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -294,7 +295,7 @@ doctype3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -313,7 +314,7 @@ doctype4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -329,7 +330,7 @@ doctype5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int internalSubset(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -374,7 +375,7 @@ internalSubset(PROLOG_STATE *state, int tok, const char *ptr, const char *end, #ifdef XML_DTD -static int PTRCALL +static int externalSubset0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { state->handler = externalSubset1; @@ -383,7 +384,7 @@ externalSubset0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return externalSubset1(state, tok, ptr, end, enc); } -static int PTRCALL +static int externalSubset1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -411,7 +412,7 @@ externalSubset1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, #endif /* XML_DTD */ -static int PTRCALL +static int entity0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -430,7 +431,7 @@ entity0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -446,7 +447,7 @@ entity1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -470,7 +471,7 @@ entity2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -486,7 +487,7 @@ entity3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -502,7 +503,7 @@ entity4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -521,7 +522,7 @@ entity5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -538,7 +539,7 @@ entity6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -562,7 +563,7 @@ entity7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity8(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -578,7 +579,7 @@ entity8(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity9(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -594,7 +595,7 @@ entity9(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity10(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -610,7 +611,7 @@ entity10(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -626,7 +627,7 @@ notation0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -646,7 +647,7 @@ notation1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -662,7 +663,7 @@ notation2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -679,7 +680,7 @@ notation3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -699,7 +700,7 @@ notation4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -716,7 +717,7 @@ attlist0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -736,7 +737,7 @@ attlist1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -766,7 +767,7 @@ attlist2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -784,7 +785,7 @@ attlist3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -803,7 +804,7 @@ attlist4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -819,7 +820,7 @@ attlist5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -835,7 +836,7 @@ attlist6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -855,7 +856,7 @@ attlist7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, } /* default value */ -static int PTRCALL +static int attlist8(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -885,7 +886,7 @@ attlist8(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist9(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -901,7 +902,7 @@ attlist9(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -918,7 +919,7 @@ element0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -944,7 +945,7 @@ element1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -978,7 +979,7 @@ element2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1002,7 +1003,7 @@ element3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1019,7 +1020,7 @@ element4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1039,7 +1040,7 @@ element5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1068,7 +1069,7 @@ element6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1117,7 +1118,7 @@ element7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, #ifdef XML_DTD -static int PTRCALL +static int condSect0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -1137,7 +1138,7 @@ condSect0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int condSect1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1154,7 +1155,7 @@ condSect1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int condSect2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1172,7 +1173,7 @@ condSect2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, #endif /* XML_DTD */ -static int PTRCALL +static int declClose(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1208,7 +1209,7 @@ declClose(PROLOG_STATE *state, int tok, const char *ptr, const char *end, * * LCOV_EXCL_START */ -static int PTRCALL +static int error(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(state); @@ -1220,7 +1221,7 @@ error(PROLOG_STATE *state, int tok, const char *ptr, const char *end, } /* LCOV_EXCL_STOP */ -static int FASTCALL +static int common(PROLOG_STATE *state, int tok) { #ifdef XML_DTD if (! state->documentEntity && tok == XML_TOK_PARAM_ENTITY_REF) diff --git a/base/poco/XML/src/xmlrole.h b/base/poco/XML/src/xmlrole.h index 4cb28ddf9ca4f..9bb732df7773f 100644 --- a/base/poco/XML/src/xmlrole.h +++ b/base/poco/XML/src/xmlrole.h @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Karl Waclawek Copyright (c) 2002 Fred L. Drake, Jr. - Copyright (c) 2017-2025 Sebastian Pipping + Copyright (c) 2017-2026 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -31,6 +31,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef XmlRole_INCLUDED @@ -110,8 +112,8 @@ enum { }; typedef struct prolog_state { - int(PTRCALL *handler)(struct prolog_state *state, int tok, const char *ptr, - const char *end, const ENCODING *enc); + int (*handler)(struct prolog_state *state, int tok, const char *ptr, + const char *end, const ENCODING *enc); unsigned level; int role_none; # ifdef XML_DTD diff --git a/base/poco/XML/src/xmltok.c b/base/poco/XML/src/xmltok.c index fe8a780b625d3..c36dde216214e 100644 --- a/base/poco/XML/src/xmltok.c +++ b/base/poco/XML/src/xmltok.c @@ -12,7 +12,7 @@ Copyright (c) 2002 Greg Stein Copyright (c) 2002-2016 Karl Waclawek Copyright (c) 2005-2009 Steven Solie - Copyright (c) 2016-2024 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2016 Pascal Cuoq Copyright (c) 2016 Don Lewis Copyright (c) 2017 Rhodri James @@ -26,6 +26,8 @@ Copyright (c) 2023 Hanno Böck Copyright (c) 2025 Alfonso Gregory Copyright (c) 2026 Nick Begg + Copyright (c) 2026 Kartik Kenchi + Copyright (c) 2026 Afonso Januário Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -46,6 +48,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #include "expat_config.h" @@ -134,20 +138,20 @@ : ((p)[1] & 0x80) == 0 \ || ((*p) == 0xF4 ? (p)[1] > 0x8F : ((p)[1] & 0xC0) == 0xC0))) -static int PTRFASTCALL +static int isNever(const ENCODING *enc, const char *p) { UNUSED_P(enc); UNUSED_P(p); return 0; } -static int PTRFASTCALL +static int utf8_isName2(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_GET_NAMING2(namePages, (const unsigned char *)p); } -static int PTRFASTCALL +static int utf8_isName3(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_GET_NAMING3(namePages, (const unsigned char *)p); @@ -155,13 +159,13 @@ utf8_isName3(const ENCODING *enc, const char *p) { #define utf8_isName4 isNever -static int PTRFASTCALL +static int utf8_isNmstrt2(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_GET_NAMING2(nmstrtPages, (const unsigned char *)p); } -static int PTRFASTCALL +static int utf8_isNmstrt3(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_GET_NAMING3(nmstrtPages, (const unsigned char *)p); @@ -169,19 +173,19 @@ utf8_isNmstrt3(const ENCODING *enc, const char *p) { #define utf8_isNmstrt4 isNever -static int PTRFASTCALL +static int utf8_isInvalid2(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_INVALID2((const unsigned char *)p); } -static int PTRFASTCALL +static int utf8_isInvalid3(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_INVALID3((const unsigned char *)p); } -static int PTRFASTCALL +static int utf8_isInvalid4(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_INVALID4((const unsigned char *)p); @@ -190,36 +194,20 @@ utf8_isInvalid4(const ENCODING *enc, const char *p) { struct normal_encoding { ENCODING enc; unsigned char type[256]; -#ifdef XML_MIN_SIZE - int(PTRFASTCALL *byteType)(const ENCODING *, const char *); - int(PTRFASTCALL *isNameMin)(const ENCODING *, const char *); - int(PTRFASTCALL *isNmstrtMin)(const ENCODING *, const char *); - int(PTRFASTCALL *byteToAscii)(const ENCODING *, const char *); - int(PTRCALL *charMatches)(const ENCODING *, const char *, int); -#endif /* XML_MIN_SIZE */ - int(PTRFASTCALL *isName2)(const ENCODING *, const char *); - int(PTRFASTCALL *isName3)(const ENCODING *, const char *); - int(PTRFASTCALL *isName4)(const ENCODING *, const char *); - int(PTRFASTCALL *isNmstrt2)(const ENCODING *, const char *); - int(PTRFASTCALL *isNmstrt3)(const ENCODING *, const char *); - int(PTRFASTCALL *isNmstrt4)(const ENCODING *, const char *); - int(PTRFASTCALL *isInvalid2)(const ENCODING *, const char *); - int(PTRFASTCALL *isInvalid3)(const ENCODING *, const char *); - int(PTRFASTCALL *isInvalid4)(const ENCODING *, const char *); + int (*isName2)(const ENCODING *, const char *); + int (*isName3)(const ENCODING *, const char *); + int (*isName4)(const ENCODING *, const char *); + int (*isNmstrt2)(const ENCODING *, const char *); + int (*isNmstrt3)(const ENCODING *, const char *); + int (*isNmstrt4)(const ENCODING *, const char *); + int (*isInvalid2)(const ENCODING *, const char *); + int (*isInvalid3)(const ENCODING *, const char *); + int (*isInvalid4)(const ENCODING *, const char *); }; #define AS_NORMAL_ENCODING(enc) ((const struct normal_encoding *)(enc)) -#ifdef XML_MIN_SIZE - -# define STANDARD_VTABLE(E) \ - E##byteType, E##isNameMin, E##isNmstrtMin, E##byteToAscii, E##charMatches, - -#else - -# define STANDARD_VTABLE(E) /* as nothing */ - -#endif +#define STANDARD_VTABLE(E) /* as nothing */ #define NORMAL_VTABLE(E) \ E##isName2, E##isName3, E##isName4, E##isNmstrt2, E##isNmstrt3, \ @@ -230,80 +218,33 @@ struct normal_encoding { /* isNmstrt2 */ NULL, /* isNmstrt3 */ NULL, /* isNmstrt4 */ NULL, \ /* isInvalid2 */ NULL, /* isInvalid3 */ NULL, /* isInvalid4 */ NULL -static int FASTCALL checkCharRefNumber(int result); +#define UTF16_NULL_VTABLE(E) NULL_VTABLE + +static int checkCharRefNumber(int result); #include "xmltok_impl.h" #include "ascii.h" -#ifdef XML_MIN_SIZE -# define sb_isNameMin isNever -# define sb_isNmstrtMin isNever -#endif - -#ifdef XML_MIN_SIZE -# define MINBPC(enc) ((enc)->minBytesPerChar) -#else /* minimum bytes per character */ -# define MINBPC(enc) 1 -#endif +#define MINBPC(enc) 1 #define SB_BYTE_TYPE(enc, p) \ (((const struct normal_encoding *)(enc))->type[(unsigned char)*(p)]) -#ifdef XML_MIN_SIZE -static int PTRFASTCALL -sb_byteType(const ENCODING *enc, const char *p) { - return SB_BYTE_TYPE(enc, p); -} -# define BYTE_TYPE(enc, p) (AS_NORMAL_ENCODING(enc)->byteType(enc, p)) -#else -# define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p) -#endif +#define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p) -#ifdef XML_MIN_SIZE -# define BYTE_TO_ASCII(enc, p) (AS_NORMAL_ENCODING(enc)->byteToAscii(enc, p)) -static int PTRFASTCALL -sb_byteToAscii(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return *p; -} -#else -# define BYTE_TO_ASCII(enc, p) (*(p)) -#endif +#define BYTE_TO_ASCII(enc, p) (*(p)) #define IS_NAME_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isName##n(enc, p)) #define IS_NMSTRT_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isNmstrt##n(enc, p)) -#ifdef XML_MIN_SIZE -# define IS_INVALID_CHAR(enc, p, n) \ - (AS_NORMAL_ENCODING(enc)->isInvalid##n \ - && AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) -#else -# define IS_INVALID_CHAR(enc, p, n) \ - (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) -#endif +#define IS_INVALID_CHAR(enc, p, n) \ + (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) -#ifdef XML_MIN_SIZE -# define IS_NAME_CHAR_MINBPC(enc, p) \ - (AS_NORMAL_ENCODING(enc)->isNameMin(enc, p)) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) \ - (AS_NORMAL_ENCODING(enc)->isNmstrtMin(enc, p)) -#else -# define IS_NAME_CHAR_MINBPC(enc, p) (0) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) (0) -#endif +#define IS_NAME_CHAR_MINBPC(enc, p) (0) +#define IS_NMSTRT_CHAR_MINBPC(enc, p) (0) -#ifdef XML_MIN_SIZE -# define CHAR_MATCHES(enc, p, c) \ - (AS_NORMAL_ENCODING(enc)->charMatches(enc, p, c)) -static int PTRCALL -sb_charMatches(const ENCODING *enc, const char *p, int c) { - UNUSED_P(enc); - return *p == c; -} -#else /* c is an ASCII character */ -# define CHAR_MATCHES(enc, p, c) (*(p) == (c)) -#endif +#define CHAR_MATCHES(enc, p, c) (*(p) == (c)) #define PREFIX(ident) normal_##ident #define XML_TOK_IMPL_C @@ -366,7 +307,7 @@ _INTERNAL_trim_to_complete_utf8_characters(const char *from, *fromLimRef = fromLim; } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result utf8_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim) { bool input_incomplete = false; @@ -405,7 +346,7 @@ utf8_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, return XML_CONVERT_COMPLETED; } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result utf8_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { enum XML_Convert_Result res = XML_CONVERT_COMPLETED; @@ -503,7 +444,7 @@ static const struct normal_encoding internal_utf8_encoding }, STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)}; -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result latin1_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim) { UNUSED_P(enc); @@ -526,7 +467,7 @@ latin1_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, } } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result latin1_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { UNUSED_P(enc); @@ -561,7 +502,7 @@ static const struct normal_encoding latin1_encoding }, STANDARD_VTABLE(sb_) NULL_VTABLE}; -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result ascii_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim) { UNUSED_P(enc); @@ -596,7 +537,7 @@ static const struct normal_encoding ascii_encoding }, STANDARD_VTABLE(sb_) NULL_VTABLE}; -static int PTRFASTCALL +static int unicode_byte_type(char hi, char lo) { switch ((unsigned char)hi) { /* 0xD800-0xDBFF first 16-bit code unit or high surrogate (W1) */ @@ -623,7 +564,7 @@ unicode_byte_type(char hi, char lo) { } #define DEFINE_UTF16_TO_UTF8(E) \ - static enum XML_Convert_Result PTRCALL E##toUtf8( \ + static enum XML_Convert_Result E##toUtf8( \ const ENCODING *enc, const char **fromP, const char *fromLim, \ char **toP, const char *toLim) { \ const char *from = *fromP; \ @@ -700,15 +641,16 @@ unicode_byte_type(char hi, char lo) { } #define DEFINE_UTF16_TO_UTF16(E) \ - static enum XML_Convert_Result PTRCALL E##toUtf16( \ + static enum XML_Convert_Result E##toUtf16( \ const ENCODING *enc, const char **fromP, const char *fromLim, \ unsigned short **toP, const unsigned short *toLim) { \ enum XML_Convert_Result res = XML_CONVERT_COMPLETED; \ UNUSED_P(enc); \ fromLim = *fromP + (((fromLim - *fromP) >> 1) << 1); /* shrink to even */ \ - /* Avoid copying first half only of surrogate */ \ + /* Avoid copying the first half (2 bytes) of surrogate pairs (4 bytes) */ \ if (fromLim - *fromP > ((toLim - *toP) << 1) \ - && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) { \ + && /* are the last two bytes a high surrogate (0xD800-0xDBFF)? */ \ + (GET_HI(fromLim - 2) & 0xFC) == 0xD8) { \ fromLim -= 2; \ res = XML_CONVERT_INPUT_INCOMPLETE; \ } \ @@ -746,70 +688,38 @@ DEFINE_UTF16_TO_UTF16(big2_) UCS2_GET_NAMING(namePages, (unsigned char)p[1], (unsigned char)p[0]) #define LITTLE2_IS_NMSTRT_CHAR_MINBPC(p) \ UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[1], (unsigned char)p[0]) +/* A 4-byte UTF-16 character is a surrogate pair; byteType only reports BT_LEAD4 + for a high surrogate, so the pair is invalid unless the second unit is a low + surrogate (U+DC00..U+DFFF, i.e. high byte 0xDC..0xDF). */ +#define LITTLE2_IS_INVALID_CHAR(p, n) \ + ((n) == 4 && ((unsigned char)(p)[3] & 0xFC) != 0xDC) -#ifdef XML_MIN_SIZE - -static int PTRFASTCALL -little2_byteType(const ENCODING *enc, const char *p) { - return LITTLE2_BYTE_TYPE(enc, p); -} - -static int PTRFASTCALL -little2_byteToAscii(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return LITTLE2_BYTE_TO_ASCII(p); -} - -static int PTRCALL -little2_charMatches(const ENCODING *enc, const char *p, int c) { - UNUSED_P(enc); - return LITTLE2_CHAR_MATCHES(p, c); -} - -static int PTRFASTCALL -little2_isNameMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return LITTLE2_IS_NAME_CHAR_MINBPC(p); -} - -static int PTRFASTCALL -little2_isNmstrtMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return LITTLE2_IS_NMSTRT_CHAR_MINBPC(p); -} - -# undef VTABLE -# define VTABLE VTABLE1, little2_toUtf8, little2_toUtf16 +#undef PREFIX +#define PREFIX(ident) little2_##ident +#define MINBPC(enc) 2 +/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ +#define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p) +#define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(p) +#define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(p, c) +#define IS_NAME_CHAR(enc, p, n) 0 +#define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(p) +#define IS_NMSTRT_CHAR(enc, p, n) (0) +#define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(p) +#define IS_INVALID_CHAR(enc, p, n) LITTLE2_IS_INVALID_CHAR(p, n) -#else /* not XML_MIN_SIZE */ +#define XML_TOK_IMPL_C +#include "xmltok_impl.c" +#undef XML_TOK_IMPL_C -# undef PREFIX -# define PREFIX(ident) little2_##ident -# define MINBPC(enc) 2 -/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ -# define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p) -# define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(p) -# define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(p, c) -# define IS_NAME_CHAR(enc, p, n) 0 -# define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(p) -# define IS_NMSTRT_CHAR(enc, p, n) (0) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(p) - -# define XML_TOK_IMPL_C -# include "xmltok_impl.c" -# undef XML_TOK_IMPL_C - -# undef MINBPC -# undef BYTE_TYPE -# undef BYTE_TO_ASCII -# undef CHAR_MATCHES -# undef IS_NAME_CHAR -# undef IS_NAME_CHAR_MINBPC -# undef IS_NMSTRT_CHAR -# undef IS_NMSTRT_CHAR_MINBPC -# undef IS_INVALID_CHAR - -#endif /* not XML_MIN_SIZE */ +#undef MINBPC +#undef BYTE_TYPE +#undef BYTE_TO_ASCII +#undef CHAR_MATCHES +#undef IS_NAME_CHAR +#undef IS_NAME_CHAR_MINBPC +#undef IS_NMSTRT_CHAR +#undef IS_NMSTRT_CHAR_MINBPC +#undef IS_INVALID_CHAR #ifdef XML_NS @@ -825,7 +735,7 @@ static const struct normal_encoding little2_encoding_ns # include "asciitab.h" # include "latin1tab.h" }, - STANDARD_VTABLE(little2_) NULL_VTABLE}; + STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)}; #endif @@ -843,7 +753,7 @@ static const struct normal_encoding little2_encoding #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(little2_) NULL_VTABLE}; + STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)}; #if BYTEORDER != 4321 @@ -855,7 +765,7 @@ static const struct normal_encoding internal_little2_encoding_ns # include "iasciitab.h" # include "latin1tab.h" }, - STANDARD_VTABLE(little2_) NULL_VTABLE}; + STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)}; # endif @@ -867,7 +777,7 @@ static const struct normal_encoding internal_little2_encoding # undef BT_COLON # include "latin1tab.h" }, - STANDARD_VTABLE(little2_) NULL_VTABLE}; + STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)}; #endif @@ -879,70 +789,38 @@ static const struct normal_encoding internal_little2_encoding UCS2_GET_NAMING(namePages, (unsigned char)p[0], (unsigned char)p[1]) #define BIG2_IS_NMSTRT_CHAR_MINBPC(p) \ UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[0], (unsigned char)p[1]) +/* A 4-byte UTF-16 character is a surrogate pair; byteType only reports BT_LEAD4 + for a high surrogate, so the pair is invalid unless the second unit is a low + surrogate (U+DC00..U+DFFF, i.e. high byte 0xDC..0xDF). */ +#define BIG2_IS_INVALID_CHAR(p, n) \ + ((n) == 4 && ((unsigned char)(p)[2] & 0xFC) != 0xDC) -#ifdef XML_MIN_SIZE - -static int PTRFASTCALL -big2_byteType(const ENCODING *enc, const char *p) { - return BIG2_BYTE_TYPE(enc, p); -} - -static int PTRFASTCALL -big2_byteToAscii(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return BIG2_BYTE_TO_ASCII(p); -} - -static int PTRCALL -big2_charMatches(const ENCODING *enc, const char *p, int c) { - UNUSED_P(enc); - return BIG2_CHAR_MATCHES(p, c); -} - -static int PTRFASTCALL -big2_isNameMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return BIG2_IS_NAME_CHAR_MINBPC(p); -} - -static int PTRFASTCALL -big2_isNmstrtMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return BIG2_IS_NMSTRT_CHAR_MINBPC(p); -} - -# undef VTABLE -# define VTABLE VTABLE1, big2_toUtf8, big2_toUtf16 +#undef PREFIX +#define PREFIX(ident) big2_##ident +#define MINBPC(enc) 2 +/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ +#define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p) +#define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(p) +#define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(p, c) +#define IS_NAME_CHAR(enc, p, n) 0 +#define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(p) +#define IS_NMSTRT_CHAR(enc, p, n) (0) +#define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(p) +#define IS_INVALID_CHAR(enc, p, n) BIG2_IS_INVALID_CHAR(p, n) -#else /* not XML_MIN_SIZE */ +#define XML_TOK_IMPL_C +#include "xmltok_impl.c" +#undef XML_TOK_IMPL_C -# undef PREFIX -# define PREFIX(ident) big2_##ident -# define MINBPC(enc) 2 -/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ -# define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p) -# define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(p) -# define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(p, c) -# define IS_NAME_CHAR(enc, p, n) 0 -# define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(p) -# define IS_NMSTRT_CHAR(enc, p, n) (0) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(p) - -# define XML_TOK_IMPL_C -# include "xmltok_impl.c" -# undef XML_TOK_IMPL_C - -# undef MINBPC -# undef BYTE_TYPE -# undef BYTE_TO_ASCII -# undef CHAR_MATCHES -# undef IS_NAME_CHAR -# undef IS_NAME_CHAR_MINBPC -# undef IS_NMSTRT_CHAR -# undef IS_NMSTRT_CHAR_MINBPC -# undef IS_INVALID_CHAR - -#endif /* not XML_MIN_SIZE */ +#undef MINBPC +#undef BYTE_TYPE +#undef BYTE_TO_ASCII +#undef CHAR_MATCHES +#undef IS_NAME_CHAR +#undef IS_NAME_CHAR_MINBPC +#undef IS_NMSTRT_CHAR +#undef IS_NMSTRT_CHAR_MINBPC +#undef IS_INVALID_CHAR #ifdef XML_NS @@ -958,7 +836,7 @@ static const struct normal_encoding big2_encoding_ns # include "asciitab.h" # include "latin1tab.h" }, - STANDARD_VTABLE(big2_) NULL_VTABLE}; + STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)}; #endif @@ -976,7 +854,7 @@ static const struct normal_encoding big2_encoding #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(big2_) NULL_VTABLE}; + STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)}; #if BYTEORDER != 1234 @@ -988,7 +866,7 @@ static const struct normal_encoding internal_big2_encoding_ns # include "iasciitab.h" # include "latin1tab.h" }, - STANDARD_VTABLE(big2_) NULL_VTABLE}; + STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)}; # endif @@ -1000,13 +878,13 @@ static const struct normal_encoding internal_big2_encoding # undef BT_COLON # include "latin1tab.h" }, - STANDARD_VTABLE(big2_) NULL_VTABLE}; + STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)}; #endif #undef PREFIX -static int FASTCALL +static int streqci(const char *s1, const char *s2) { for (;;) { char c1 = *s1++; @@ -1027,7 +905,7 @@ streqci(const char *s1, const char *s2) { return 1; } -static void PTRCALL +static void initUpdatePosition(const ENCODING *enc, const char *ptr, const char *end, POSITION *pos) { UNUSED_P(enc); @@ -1045,7 +923,7 @@ toAscii(const ENCODING *enc, const char *ptr, const char *end) { return buf[0]; } -static int FASTCALL +static int isSpace(int c) { switch (c) { case 0x20: @@ -1151,6 +1029,34 @@ static const char KW_yes[] = {ASCII_y, ASCII_e, ASCII_s, '\0'}; static const char KW_no[] = {ASCII_n, ASCII_o, '\0'}; +static const char KW_1_dot[] = {ASCII_1, ASCII_PERIOD, '\0'}; + +/* Checks a version pseudo-attribute value against the VersionNum production. + XML 1.0 Fourth Edition only allows the literal "1.0", but the Fifth + Edition relaxed this to "1." followed by one or more digits, since Expat + only implements 1.0 itself but plans to track the Fifth Edition's laxer + grammar here so that "1.1" and similar aren't rejected only to have that + rejection reverted later. Returns true for a value matching "1.[0-9]+". + val/valEnd bound the value itself; valEnd is the upper bound used when + decoding the individual characters between them. */ +static bool +checkXmlDeclVersionNum(const ENCODING *enc, const char *val, + const char *valEnd) { + if (valEnd - val < 2 * enc->minBytesPerChar + || ! XmlNameMatchesAscii(enc, val, val + 2 * enc->minBytesPerChar, + KW_1_dot)) + return false; + val += 2 * enc->minBytesPerChar; + if (val == valEnd) + return false; + for (; val != valEnd; val += enc->minBytesPerChar) { + int c = toAscii(enc, val, valEnd); + if (c < ASCII_0 || c > ASCII_9) + return false; + } + return true; +} + static int doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *, const char *), @@ -1178,6 +1084,22 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *, *versionPtr = val; if (versionEndPtr) *versionEndPtr = ptr; + /* The version number must not be empty; VersionNum requires at least + one character. The encoding and standalone pseudo-attributes below + already reject an empty value, so keep version consistent. */ + if (val == ptr - enc->minBytesPerChar) { + *badPtr = val; + return 0; + } + /* Expat implements XML 1.0 only, so any version outside the "1.0"/"1.x" + family is rejected. Following the Fifth Edition's VersionNum + production (rather than the Fourth Edition's exact "1.0") avoids + rejecting "1.1" now only to have to revert that once Expat tracks + the newer edition. */ + if (! checkXmlDeclVersionNum(enc, val, ptr - enc->minBytesPerChar)) { + *badPtr = val; + return 0; + } if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) { *badPtr = ptr; return 0; @@ -1232,7 +1154,7 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *, return 1; } -static int FASTCALL +static int checkCharRefNumber(int result) { switch (result >> 8) { case 0xD8: @@ -1256,7 +1178,7 @@ checkCharRefNumber(int result) { return result; } -int FASTCALL +int XmlUtf8Encode(int c, char *buf) { enum { /* minN is minimum legal resulting value for N byte sequence */ @@ -1292,7 +1214,7 @@ XmlUtf8Encode(int c, char *buf) { return 0; /* LCOV_EXCL_LINE: this case too is eliminated before calling */ } -int FASTCALL +int XmlUtf16Encode(int charNum, unsigned short *buf) { if (charNum < 0) return 0; @@ -1324,7 +1246,7 @@ XmlSizeOfUnknownEncoding(void) { return sizeof(struct unknown_encoding); } -static int PTRFASTCALL +static int unknown_isName(const ENCODING *enc, const char *p) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); int c = uenc->convert(uenc->userData, p); @@ -1333,7 +1255,7 @@ unknown_isName(const ENCODING *enc, const char *p) { return UCS2_GET_NAMING(namePages, c >> 8, c & 0xFF); } -static int PTRFASTCALL +static int unknown_isNmstrt(const ENCODING *enc, const char *p) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); int c = uenc->convert(uenc->userData, p); @@ -1342,14 +1264,14 @@ unknown_isNmstrt(const ENCODING *enc, const char *p) { return UCS2_GET_NAMING(nmstrtPages, c >> 8, c & 0xFF); } -static int PTRFASTCALL +static int unknown_isInvalid(const ENCODING *enc, const char *p) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); int c = uenc->convert(uenc->userData, p); return (c & ~0xFFFF) || checkCharRefNumber(c) < 0; } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result unknown_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); @@ -1379,7 +1301,7 @@ unknown_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, } } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result unknown_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); @@ -1503,7 +1425,7 @@ static const char KW_UTF_16LE[] = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, ASCII_L, ASCII_E, '\0'}; -static int FASTCALL +static int getEncodingIndex(const char *name) { static const char *const encodingNames[] = { KW_ISO_8859_1, KW_US_ASCII, KW_UTF_8, KW_UTF_16, KW_UTF_16BE, KW_UTF_16LE, diff --git a/base/poco/XML/src/xmltok.h b/base/poco/XML/src/xmltok.h index f949ebf9c1714..4f6fb3850cafc 100644 --- a/base/poco/XML/src/xmltok.h +++ b/base/poco/XML/src/xmltok.h @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Fred L. Drake, Jr. Copyright (c) 2002-2005 Karl Waclawek - Copyright (c) 2016-2024 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2017 Rhodri James Licensed under the MIT license: @@ -32,11 +32,15 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef XmlTok_INCLUDED # define XmlTok_INCLUDED 1 +# include // uint64_t + # ifdef __cplusplus extern "C" { # endif @@ -145,8 +149,8 @@ extern "C" { typedef struct position { /* first line and first column are 0 not 1 */ - XML_Size lineNumber; - XML_Size columnNumber; + uint64_t lineNumber; + uint64_t columnNumber; } POSITION; typedef struct { @@ -159,41 +163,40 @@ typedef struct { struct encoding; typedef struct encoding ENCODING; -typedef int(PTRCALL *SCANNER)(const ENCODING *, const char *, const char *, - const char **); +typedef int (*SCANNER)(const ENCODING *, const char *, const char *, + const char **); enum XML_Convert_Result { XML_CONVERT_COMPLETED = 0, XML_CONVERT_INPUT_INCOMPLETE = 1, - XML_CONVERT_OUTPUT_EXHAUSTED - = 2 /* and therefore potentially input remaining as well */ + XML_CONVERT_OUTPUT_EXHAUSTED = 2 /* and therefore potentially input remaining + as well */ }; struct encoding { SCANNER scanners[XML_N_STATES]; SCANNER literalScanners[XML_N_LITERAL_TYPES]; - int(PTRCALL *nameMatchesAscii)(const ENCODING *, const char *, const char *, - const char *); - int(PTRFASTCALL *nameLength)(const ENCODING *, const char *); - const char *(PTRFASTCALL *skipS)(const ENCODING *, const char *); - int(PTRCALL *getAtts)(const ENCODING *enc, const char *ptr, int attsMax, - ATTRIBUTE *atts); - int(PTRFASTCALL *charRefNumber)(const ENCODING *enc, const char *ptr); - int(PTRCALL *predefinedEntityName)(const ENCODING *, const char *, - const char *); - void(PTRCALL *updatePosition)(const ENCODING *, const char *ptr, - const char *end, POSITION *); - int(PTRCALL *isPublicId)(const ENCODING *enc, const char *ptr, - const char *end, const char **badPtr); - enum XML_Convert_Result(PTRCALL *utf8Convert)(const ENCODING *enc, - const char **fromP, - const char *fromLim, char **toP, - const char *toLim); - enum XML_Convert_Result(PTRCALL *utf16Convert)(const ENCODING *enc, - const char **fromP, - const char *fromLim, - unsigned short **toP, - const unsigned short *toLim); + int (*nameMatchesAscii)(const ENCODING *, const char *, const char *, + const char *); + int (*nameLength)(const ENCODING *, const char *); + const char *(*skipS)(const ENCODING *, const char *); + int (*getAtts)(const ENCODING *enc, const char *ptr, int attsMax, + ATTRIBUTE *atts); + int (*charRefNumber)(const ENCODING *enc, const char *ptr); + int (*predefinedEntityName)(const ENCODING *, const char *, const char *); + void (*updatePosition)(const ENCODING *, const char *ptr, const char *end, + POSITION *); + int (*isPublicId)(const ENCODING *enc, const char *ptr, const char *end, + const char **badPtr); + enum XML_Convert_Result (*utf8Convert)(const ENCODING *enc, + const char **fromP, + const char *fromLim, char **toP, + const char *toLim); + enum XML_Convert_Result (*utf16Convert)(const ENCODING *enc, + const char **fromP, + const char *fromLim, + unsigned short **toP, + const unsigned short *toLim); int minBytesPerChar; char isUtf8; char isUtf16; @@ -293,8 +296,8 @@ int XmlInitEncoding(INIT_ENCODING *p, const ENCODING **encPtr, const char *name); const ENCODING *XmlGetUtf8InternalEncoding(void); const ENCODING *XmlGetUtf16InternalEncoding(void); -int FASTCALL XmlUtf8Encode(int charNumber, char *buf); -int FASTCALL XmlUtf16Encode(int charNumber, unsigned short *buf); +int XmlUtf8Encode(int charNumber, char *buf); +int XmlUtf16Encode(int charNumber, unsigned short *buf); int XmlSizeOfUnknownEncoding(void); typedef int(XMLCALL *CONVERTER)(void *userData, const char *p); @@ -314,6 +317,7 @@ const ENCODING *XmlGetUtf8InternalEncodingNS(void); const ENCODING *XmlGetUtf16InternalEncodingNS(void); ENCODING *XmlInitUnknownEncodingNS(void *mem, const int *table, CONVERTER convert, void *userData); + # ifdef __cplusplus } # endif diff --git a/base/poco/XML/src/xmltok_impl.c b/base/poco/XML/src/xmltok_impl.c index d41b8e9e1a0c2..571af3698af7a 100644 --- a/base/poco/XML/src/xmltok_impl.c +++ b/base/poco/XML/src/xmltok_impl.c @@ -1,4 +1,4 @@ -/* This file is included (from xmltok.c, 1-3 times depending on XML_MIN_SIZE)! +/* This file is included (from xmltok.c, 3 times)! __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Fred L. Drake, Jr. Copyright (c) 2002-2016 Karl Waclawek - Copyright (c) 2016-2022 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2017 Rhodri James Copyright (c) 2018 Benjamin Peterson Copyright (c) 2018 Anton Maklakov @@ -18,6 +18,7 @@ Copyright (c) 2020 Boris Kolpackov Copyright (c) 2022 Martin Ettl Copyright (c) 2026 Nick Begg + Copyright (c) 2026 Kartik Kenchi Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -38,14 +39,12 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifdef XML_TOK_IMPL_C -# ifndef IS_INVALID_CHAR // i.e. for UTF-16 and XML_MIN_SIZE not defined -# define IS_INVALID_CHAR(enc, ptr, n) (0) -# endif - # define INVALID_LEAD_CASE(n, ptr, nextTokPtr) \ case BT_LEAD##n: \ if (end - ptr < n) \ @@ -142,7 +141,7 @@ /* ptr points to character following "= end) @@ -428,7 +427,7 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, const char *end, /* ptr points to character following "= end) @@ -920,7 +919,7 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, /* ptr points to character following "%" */ -static int PTRCALL +static int PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { REQUIRE_CHAR(enc, ptr, end); @@ -950,7 +949,7 @@ PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_PARTIAL; } -static int PTRCALL +static int PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { REQUIRE_CHAR(enc, ptr, end); @@ -980,7 +979,7 @@ PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end, return -XML_TOK_POUND_NAME; } -static int PTRCALL +static int PREFIX(scanLit)(int open, const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { while (HAS_CHAR(enc, ptr, end)) { @@ -1014,7 +1013,7 @@ PREFIX(scanLit)(int open, const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_PARTIAL; } -static int PTRCALL +static int PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { int tok; @@ -1258,7 +1257,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, return -tok; } -static int PTRCALL +static int PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { const char *start; @@ -1327,7 +1326,7 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_DATA_CHARS; } -static int PTRCALL +static int PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { const char *start; @@ -1394,7 +1393,7 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *end, # ifdef XML_DTD -static int PTRCALL +static int PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { int level = 0; @@ -1446,7 +1445,7 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, const char *end, # endif /* XML_DTD */ -static int PTRCALL +static int PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end, const char **badPtr) { ptr += MINBPC(enc); @@ -1506,7 +1505,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end, first attsMax attributes are stored in atts. */ -static int PTRCALL +static int PREFIX(getAtts)(const ENCODING *enc, const char *ptr, int attsMax, ATTRIBUTE *atts) { enum { other, inName, inValue } state = inName; @@ -1599,7 +1598,7 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr, int attsMax, /* not reached */ } -static int PTRFASTCALL +static int PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) { int result = 0; /* skip &# */ @@ -1657,7 +1656,7 @@ PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) { return checkCharRefNumber(result); } -static int PTRCALL +static int PREFIX(predefinedEntityName)(const ENCODING *enc, const char *ptr, const char *end) { UNUSED_P(enc); @@ -1711,7 +1710,7 @@ PREFIX(predefinedEntityName)(const ENCODING *enc, const char *ptr, return 0; } -static int PTRCALL +static int PREFIX(nameMatchesAscii)(const ENCODING *enc, const char *ptr1, const char *end1, const char *ptr2) { UNUSED_P(enc); @@ -1730,7 +1729,7 @@ PREFIX(nameMatchesAscii)(const ENCODING *enc, const char *ptr1, return ptr1 == end1; } -static int PTRFASTCALL +static int PREFIX(nameLength)(const ENCODING *enc, const char *ptr) { const char *start = ptr; for (;;) { @@ -1760,7 +1759,7 @@ PREFIX(nameLength)(const ENCODING *enc, const char *ptr) { } } -static const char *PTRFASTCALL +static const char * PREFIX(skipS)(const ENCODING *enc, const char *ptr) { for (;;) { switch (BYTE_TYPE(enc, ptr)) { @@ -1775,7 +1774,7 @@ PREFIX(skipS)(const ENCODING *enc, const char *ptr) { } } -static void PTRCALL +static void PREFIX(updatePosition)(const ENCODING *enc, const char *ptr, const char *end, POSITION *pos) { while (HAS_CHAR(enc, ptr, end)) { diff --git a/base/poco/XML/src/xmltok_impl.h b/base/poco/XML/src/xmltok_impl.h index eb18b31625d34..00fd893c060ab 100644 --- a/base/poco/XML/src/xmltok_impl.h +++ b/base/poco/XML/src/xmltok_impl.h @@ -29,6 +29,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ enum { diff --git a/base/poco/XML/src/xmltok_ns.c b/base/poco/XML/src/xmltok_ns.c index 0a23d7e8c4980..9601f53b68d8a 100644 --- a/base/poco/XML/src/xmltok_ns.c +++ b/base/poco/XML/src/xmltok_ns.c @@ -11,7 +11,8 @@ Copyright (c) 2002 Greg Stein Copyright (c) 2002 Fred L. Drake, Jr. Copyright (c) 2002-2006 Karl Waclawek - Copyright (c) 2017-2021 Sebastian Pipping + Copyright (c) 2017-2026 Sebastian Pipping + Copyright (c) 2025 Alfonso Gregory Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -32,6 +33,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifdef XML_TOK_NS_C @@ -61,14 +64,14 @@ static const ENCODING *const NS(encodings)[] = { &ns(utf8_encoding).enc /* NO_ENC */ }; -static int PTRCALL +static int NS(initScanProlog)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_PROLOG_STATE, ptr, end, nextTokPtr); } -static int PTRCALL +static int NS(initScanContent)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_CONTENT_STATE, diff --git a/base/poco/XML/src/xcsinc.c b/base/poco/XML/src/xmlxcs.c similarity index 65% rename from base/poco/XML/src/xcsinc.c rename to base/poco/XML/src/xmlxcs.c index 3597c2480bc96..ae5e042d778ff 100644 --- a/base/poco/XML/src/xcsinc.c +++ b/base/poco/XML/src/xmlxcs.c @@ -1,4 +1,4 @@ -/* This file is included from other .c files! +/* __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -6,7 +6,7 @@ \___/_/\_\ .__/ \__,_|\__| |_| XML parser - Copyright (c) 2022 Sebastian Pipping + Copyright (c) 2022-2026 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -27,9 +27,21 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ -static size_t +#include "xcs.h" + +#if defined(XML_UNICODE) +# if defined(XML_UNICODE_WCHAR_T) +# include // for wcscmp, wcslen, wcsncmp +# endif +#else +# include // for strcmp, strlen, strncmp +#endif + +size_t xcslen(const XML_Char *s) { #ifdef XML_UNICODE # ifdef XML_UNICODE_WCHAR_T @@ -46,3 +58,33 @@ xcslen(const XML_Char *s) { return strlen(s); #endif } + +int +xcscmp(const XML_Char *a, const XML_Char *b) { +#if defined(XML_UNICODE) +# if defined(XML_UNICODE_WCHAR_T) + return wcscmp(a, b); +# else + for (; a[0] && b[0] && a[0] == b[0]; a++, b++) + ; + return a[0] - b[0]; +# endif +#else + return strcmp(a, b); +#endif +} + +int +xcsncmp(const XML_Char *a, const XML_Char *b, size_t len) { +#if defined(XML_UNICODE) +# if defined(XML_UNICODE_WCHAR_T) + return wcsncmp(a, b, len); +# else + for (; len > 0 && a[0] && b[0] && a[0] == b[0]; len--, a++, b++) { + } + return (len == 0) ? 0 : (a[0] - b[0]); +# endif +#else + return strncmp(a, b, len); +#endif +} diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index 0d96a727b19ff..eaec79a77d782 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -975,25 +975,25 @@ class JobConfigs: for total_batches in (2,) for batch in range(1, total_batches + 1) ], - # One shard can lose worker concurrency mid-suite under MSan and - # overrun the shared 3.5h functional-test budget. - *[ + # NOTE (strtgbb): 64g keeps MSan at 6 workers, Lower mem can cause workers to be killed + *[ Job.ParamSet( parameter=f"amd_msan, WasmEdge, parallel, {batch}/{total_batches}", - runs_on=RunnerLabels.FUNC_TESTER_AMD, + runs_on=RunnerLabels.AMD_LARGE, requires=[ArtifactNames.CH_AMD_MSAN_GH], timeout=3600 * 4, ) - for total_batches in (8,) + for total_batches in (6,) for batch in range(1, total_batches + 1) ], + # NOTE (strtgbb): 2/2 was ~3.3h of sequential work and overran 3.5h in teardown. *[ Job.ParamSet( parameter=f"amd_msan, WasmEdge, sequential, {batch}/{total_batches}", runs_on=RunnerLabels.AMD_SMALL_MEM, requires=[ArtifactNames.CH_AMD_MSAN_GH], ) - for total_batches in (2,) + for total_batches in (4,) for batch in range(1, total_batches + 1) ], Job.ParamSet( diff --git a/ci/jobs/check_style.py b/ci/jobs/check_style.py index 26767e7eb84d1..2015f1d6ecd26 100644 --- a/ci/jobs/check_style.py +++ b/ci/jobs/check_style.py @@ -3,6 +3,7 @@ import math import multiprocessing import os +import pathlib import re import shlex from concurrent.futures import ProcessPoolExecutor @@ -878,6 +879,125 @@ def check_catch_all(files) -> str: return "\n".join(violations) +# Storage classes whose tables can be deferred behind `StorageTableProxy`, which means a pointer +# taken from `DatabaseCatalog` may be the proxy rather than the engine. +DEFERRABLE_STORAGE_CLASSES = ( + "MergeTreeData", + "StorageMergeTree", + "StorageReplicatedMergeTree", + "StorageSharedMergeTree", + "StorageSetOrJoinBase", + "StorageSet", + "StorageJoin", + "StorageSharedSet", + "StorageSharedJoin", + "StorageEmbeddedRocksDB", + "IKeyValueEntity", + "IStorageURLBase", + "IBackgroundOperation", + "StorageWithCommonVirtualColumns", + "StorageLog", + "StorageStripeLog", + "StorageURL", + "StorageObjectStorage", + "StorageKeeperMap", + "StorageMySQL", + "StoragePostgreSQL", + "StorageMongoDB", + "StorageRedis", + "StorageSQLite", + "StorageXDBC", + "StorageHive", + "StorageArrowFlight", + "StorageYTsaurus", + "StorageBigQuery", + "StorageKafka", + "StorageKafka2", + "StorageFileLog", + "StorageRabbitMQ", + "StorageNATS", + "StorageObjectStorageQueue", + "IStreamingStorage", +) + +# Casts on an operand that cannot be a catalog pointer, so no proxy can be in the way. +_NOT_A_CATALOG_POINTER = re.compile( + r"^(?:\*?this\b" + r"|shared_from_this\(\)" + r"|&?\w*(?:snapshot|storage_snapshot)->storage\b" + r"|&?\w*reading->getMergeTreeData\(\)" + r")" +) + + +def _without_comments(text): + """Blanks out comments, keeping the line layout so offsets and line numbers still match.""" + return re.sub( + r"//[^\n]*|/\*.*?\*/", lambda m: re.sub(r"[^\n]", " ", m.group(0)), text, flags=re.S + ) + + +def _cast_operand(text, open_paren): + """The argument of a cast whose '(' is at `open_paren`, or None when unbalanced.""" + depth = 0 + for i in range(open_paren, min(open_paren + 2000, len(text))): + depth += (text[i] == "(") - (text[i] == ")") + if not depth: + return " ".join(text[open_paren + 1 : i].split()) + return None + + +def check_storage_casts(files) -> str: + """Require `castStorage` for casts to an engine that supports deferred loading. + + Such a table lives behind `StorageTableProxy` until its first access and the catalog keeps + handing out that proxy afterwards, so a direct cast fails for the whole life of the table. + """ + types = "|".join(DEFERRABLE_STORAGE_CLASSES) + cast_head = re.compile( + r"\b(?Pdynamic_cast|typeid_cast|dynamic_pointer_cast|static_pointer_cast)\s*<\s*" + r"(?:const\s+)?(?:" + types + r")\s*[*&]?\s*>\s*\(" + ) + # `IStorage::as()` is a `typeid_cast` on the receiver, so the operand is what precedes it. + as_cast = re.compile( + r"(?P[\w.\[\]()]+)(?:->|\.)as\s*<\s*(?:const\s+)?(?:" + types + r")\s*>\s*\(\s*\)" + ) + resolvers = ("castStorage", "resolveStorageProxy", "resolveStorageProxyLoading") + + violations = [] + for path in files: + # The helpers and the proxy itself have to reach the nested storage directly. + if not path.endswith((".cpp", ".h")) or path.endswith( + ("StorageProxy.h", "StorageTableProxy.h", "StorageTableFunction.h") + ): + continue + try: + lines = pathlib.Path(path).read_text(errors="replace").splitlines() + except OSError: + continue + + text = _without_comments("\n".join(lines)) + casts = [(m, m.group("cast"), _cast_operand(text, m.end() - 1)) for m in cast_head.finditer(text)] + casts += [(m, "as", m.group("operand")) for m in as_cast.finditer(text)] + + for match, cast, operand in casts: + if not operand or _NOT_A_CATALOG_POINTER.match(operand) or any(r in operand for r in resolvers): + continue + line = text.count("\n", 0, match.start()) + 1 + # The marker goes on the cast or, when the line is long, the one above it. + if any("NOLINT(storage-cast)" in lines[i] for i in (line - 1, line - 2) if i >= 0): + continue + violations.append( + f"{path}:{line}: {cast} to a deferrable storage engine on `{operand[:60]}`. Such a table " + "is reached through StorageTableProxy, so this cast fails for the whole life of the table " + "and whatever it guards is silently skipped. Use castStorage(ptr, " + "DeferredTable::Load) when the query names this table, or DeferredTable::Skip " + "when this walks every table and must not load one. If the pointer cannot come from " + "DatabaseCatalog, say why in a `/// NOLINT(storage-cast)` comment." + ) + return "\n".join(violations) + + def check_file_names(files): files_set = set() for file in files: @@ -1371,6 +1491,15 @@ def parse_args(): files=cpp_files, ) ) + testname = "storage_casts" + if testpattern.lower() in testname.lower(): + results.append( + run_check_concurrent( + check_name=testname, + check_function=check_storage_casts, + files=cpp_files, + ) + ) testname = "compose_images_from_dockerhub" if testpattern.lower() in testname.lower(): results.append( diff --git a/ci/jobs/scripts/check_style/experimental_settings_ignore.txt b/ci/jobs/scripts/check_style/experimental_settings_ignore.txt index 4c4a0ec48a103..0b63ce3eb10b4 100644 --- a/ci/jobs/scripts/check_style/experimental_settings_ignore.txt +++ b/ci/jobs/scripts/check_style/experimental_settings_ignore.txt @@ -14,6 +14,7 @@ allow_deprecated_database_ordinary allow_deprecated_snowflake_conversion_functions allow_distributed_ddl allow_drop_detached +allow_executable_tables allow_execute_multiif_columnar allow_experimental_ai_functions allow_experimental_alter_materialized_view_structure diff --git a/ci/workflows/pull_request.py b/ci/workflows/pull_request.py index ab1c92dc9763e..fd402def0eaf9 100644 --- a/ci/workflows/pull_request.py +++ b/ci/workflows/pull_request.py @@ -215,7 +215,8 @@ # job.set_run_after(CORE_BLOCKING_JOB_NAMES) # for job in JobConfigs.performance_comparison_with_master_head_jobs # ], # NOTE (strtgbb): failed previously due to GH secrets not being handled properly, try again later - JobConfigs.parser_memory_check_job, + # NOTE (strtgbb): ooks up master clickhouse-examples on upstream S3 + # JobConfigs.parser_memory_check_job, # ClickBench runs on PRs only when files in its digest change # (see `clickbench_jobs.digest_config`), so the cost is bounded. # *[ diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 95803bd73fbb5..83560c8679930 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,13 +2,13 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54523) +SET(VERSION_REVISION 54540) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 11) -SET(VERSION_GITHASH bedf2ab54b8a0c34afbf2d907eaf933324f43cd6) -SET(VERSION_DESCRIBE v26.8.11.10001.altinitytest) -SET(VERSION_STRING 26.8.11.10001.altinitytest) +SET(VERSION_PATCH 28) +SET(VERSION_GITHASH 551899e257e06d9e3c977481944b69695adb3c32) +SET(VERSION_DESCRIBE v26.8.28.10001.altinitytest) +SET(VERSION_STRING 26.8.28.10001.altinitytest) # end of autochange SET(VERSION_TWEAK 10001) diff --git a/cmake/strip_rust_symbols.sh b/cmake/strip_rust_symbols.sh index cb91889088bfa..4987b189590fa 100755 --- a/cmake/strip_rust_symbols.sh +++ b/cmake/strip_rust_symbols.sh @@ -50,7 +50,8 @@ for sym in "$@"; do done # Localize all symbols except the public ones, then strip unneeded locals -"$OBJCOPY" $KEEP_FLAGS --strip-unneeded "$WORK_DIR/combined.o" "$WORK_DIR/stripped.o" +# Drop .llvm_addrsig: after ld -r and objcopy its sh_link is 0, so lld --icf=safe ignores it with a warning +"$OBJCOPY" $KEEP_FLAGS --strip-unneeded --remove-section=.llvm_addrsig "$WORK_DIR/combined.o" "$WORK_DIR/stripped.o" # Repackage as .a (replace original) rm -f "$LIB_PATH" diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 4253bf5fad424..537a9f057984d 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -266,8 +266,12 @@ option(ENABLE_GOOGLE_CLOUD_CPP "Enable Google Cloud Cpp" ${ENABLE_GOOGLE_CLOUD_C # `crc32c` is a small portable library used by both `google-cloud-cpp` and the snappy framing format # in `SnappyWriteBuffer`/`SnappyFramedReadBuffer`, so it is always built when libraries are enabled. add_contrib (crc32c-cmake crc32c) +# `nlohmann::json` is header-only and is used by both `google-cloud-cpp` and `ai-sdk-cpp`, which are +# enabled independently of each other. It is added for everyone so that there is one copy of it in +# the build: two copies of the same version define the same symbols with different layouts, which +# the linker resolves silently - see `contrib/ai-sdk-cpp-cmake/CMakeLists.txt`. +add_contrib (nlohmann-json-cmake nlohmann-json) if(ENABLE_GOOGLE_CLOUD_CPP) - add_contrib (nlohmann-json-cmake nlohmann-json) add_contrib (google-cloud-cpp-cmake google-cloud-cpp) # requires grpc, protobuf, absl, nlohmann's json, crc32c else() message(STATUS "Not using Google Cloud Cpp") diff --git a/contrib/NuRaft b/contrib/NuRaft index a0eda0069d2da..884294e9dbee1 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit a0eda0069d2dac0852c453cbe7b0285eda218479 +Subproject commit 884294e9dbee183b53601bbe2c8c5753cec1a3db diff --git a/contrib/ai-sdk-cpp-cmake/CMakeLists.txt b/contrib/ai-sdk-cpp-cmake/CMakeLists.txt index 6bcf697023abf..72230124d7e1f 100644 --- a/contrib/ai-sdk-cpp-cmake/CMakeLists.txt +++ b/contrib/ai-sdk-cpp-cmake/CMakeLists.txt @@ -38,13 +38,16 @@ set(AI_SDK_ANTHROPIC_SOURCES "${AI_SDK_SOURCE_DIR}/src/providers/anthropic/anthropic_factory.cpp" ) -# Add nlohmann_json from submodule -set(NLOHMANN_JSON_SOURCE_DIR "${AI_SDK_THIRD_PARTY_DIR}/nlohmann_json_patched") -add_library(_ai_sdk_nlohmann_json INTERFACE) -target_include_directories(_ai_sdk_nlohmann_json SYSTEM INTERFACE - "${NLOHMANN_JSON_SOURCE_DIR}/include" -) -add_library(ai_sdk_nlohmann_json::ai_sdk_nlohmann_json ALIAS _ai_sdk_nlohmann_json) +# `nlohmann::json` comes from `contrib/nlohmann-json`, not from the `third_party/nlohmann_json_patched` +# copy this library vendors. Both are version 3.12.0, so both define the same +# `nlohmann::json_abi_v3_12_0` symbols - and the two definitions differ: the `contrib` copy is the +# ClickHouse fork that reads numbers without consulting the locale, whose `lexer` holds a +# `number_buffer` member that the vendored copy has no field for. Linking both is an ODR violation +# that the linker resolves by keeping one definition of each symbol, so a `parser` laid out by one +# header ends up in a `parse` compiled from the other, which reads and destroys members past the +# end of it: `AddressSanitizer` reports a stack-buffer-overflow and the client aborts on the first +# response of the model. The vendored copy exists to keep the number parsing away from the locale, +# which the fork does properly, so there is nothing left to keep it for. # Add httplib from submodule (header-only) set(HTTPLIB_SOURCE_DIR "${AI_SDK_THIRD_PARTY_DIR}/httplib-header-only") @@ -79,7 +82,7 @@ target_include_directories(_ai-sdk-cpp-core SYSTEM target_link_libraries(_ai-sdk-cpp-core PUBLIC - ai_sdk_nlohmann_json::ai_sdk_nlohmann_json + ch_contrib::nlohmann_json PRIVATE httplib::httplib concurrentqueue @@ -105,7 +108,7 @@ target_include_directories(_ai-sdk-cpp-openai SYSTEM target_link_libraries(_ai-sdk-cpp-openai PUBLIC _ai-sdk-cpp-core - ai_sdk_nlohmann_json::ai_sdk_nlohmann_json + ch_contrib::nlohmann_json PRIVATE httplib::httplib concurrentqueue @@ -133,7 +136,7 @@ target_include_directories(_ai-sdk-cpp-anthropic SYSTEM target_link_libraries(_ai-sdk-cpp-anthropic PUBLIC _ai-sdk-cpp-core - ai_sdk_nlohmann_json::ai_sdk_nlohmann_json + ch_contrib::nlohmann_json PRIVATE httplib::httplib concurrentqueue diff --git a/contrib/cctz b/contrib/cctz index 8e694da054a9a..fa5fe114a87a3 160000 --- a/contrib/cctz +++ b/contrib/cctz @@ -1 +1 @@ -Subproject commit 8e694da054a9a31d98392bf03ee188b04f810d0a +Subproject commit fa5fe114a87a34c055853adf54dde305979acea0 diff --git a/contrib/croaring b/contrib/croaring index 025ae3f7add16..f2b1a55bcfa49 160000 --- a/contrib/croaring +++ b/contrib/croaring @@ -1 +1 @@ -Subproject commit 025ae3f7add169bc820dcfd46fa9304f382ec40a +Subproject commit f2b1a55bcfa49ec986539b05b603e26a3adcc95c diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 00d8349091850..34eeb17b00322 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -198,6 +198,26 @@ else () message (FATAL_ERROR "internal jemalloc: This arch is not supported") endif () +if (OS_LINUX AND ARCH_AARCH64) + # aarch64 kernels are built with 4KiB, 16KiB or 64KiB pages. jemalloc's page size is fixed at + # compile time and must not be smaller than the kernel page size, otherwise jemalloc fails to + # boot. 64KiB runs everywhere, at the cost of a higher memory footprint on smaller-page kernels + # (page-sized slabs, page-granular purging); pick a smaller size when the target kernel is known. + set (JEMALLOC_AARCH64_PAGE_SIZE_KIB 64 CACHE STRING "jemalloc page size on aarch64, in KiB (4, 16 or 64). Must not be less than the kernel page size.") + if (JEMALLOC_AARCH64_PAGE_SIZE_KIB EQUAL 4) + set (JEMALLOC_LG_PAGE 12) + elseif (JEMALLOC_AARCH64_PAGE_SIZE_KIB EQUAL 16) + set (JEMALLOC_LG_PAGE 14) + elseif (JEMALLOC_AARCH64_PAGE_SIZE_KIB EQUAL 64) + set (JEMALLOC_LG_PAGE 16) + else () + message (FATAL_ERROR "Invalid JEMALLOC_AARCH64_PAGE_SIZE_KIB=${JEMALLOC_AARCH64_PAGE_SIZE_KIB}, must be 4, 16 or 64") + endif () + # A PMD-level THP maps (page size / 8) entries of one page each: 2MiB/32MiB/512MiB for 4/16/64 KiB pages. + math (EXPR JEMALLOC_LG_HUGEPAGE "2 * ${JEMALLOC_LG_PAGE} - 3") + message (STATUS "jemalloc aarch64 page size: ${JEMALLOC_AARCH64_PAGE_SIZE_KIB}KiB (LG_PAGE=${JEMALLOC_LG_PAGE}, LG_HUGEPAGE=${JEMALLOC_LG_HUGEPAGE})") +endif () + configure_file(${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal/jemalloc_internal_defs.h.in ${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal/jemalloc_internal_defs.h) target_include_directories(_jemalloc SYSTEM PRIVATE diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in index 9e495d40d94aa..9a98fbd3810a0 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -204,7 +204,7 @@ /* #undef LG_QUANTUM */ /* One page is 2^LG_PAGE bytes. */ -#define LG_PAGE 16 +#define LG_PAGE @JEMALLOC_LG_PAGE@ /* Maximum number of regions in a slab. */ /* #undef CONFIG_LG_SLAB_MAXREGS */ @@ -214,7 +214,7 @@ * system does not explicitly support huge pages; system calls that require * explicit huge page support are separately configured. */ -#define LG_HUGEPAGE 29 +#define LG_HUGEPAGE @JEMALLOC_LG_HUGEPAGE@ /* * If defined, adjacent virtual memory mappings with identical attributes diff --git a/contrib/jemalloc-cmake/include_linux_aarch64_musl/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_aarch64_musl/jemalloc/internal/jemalloc_internal_defs.h.in index 3a8ef54f40eab..0ac9fb9c4526d 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64_musl/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_aarch64_musl/jemalloc/internal/jemalloc_internal_defs.h.in @@ -200,7 +200,7 @@ /* #undef LG_QUANTUM */ /* One page is 2^LG_PAGE bytes. */ -#define LG_PAGE 16 +#define LG_PAGE @JEMALLOC_LG_PAGE@ /* Maximum number of regions in a slab. */ /* #undef CONFIG_LG_SLAB_MAXREGS */ @@ -210,7 +210,7 @@ * system does not explicitly support huge pages; system calls that require * explicit huge page support are separately configured. */ -#define LG_HUGEPAGE 29 +#define LG_HUGEPAGE @JEMALLOC_LG_HUGEPAGE@ /* * If defined, adjacent virtual memory mappings with identical attributes diff --git a/contrib/librdkafka-cmake/CMakeLists.txt b/contrib/librdkafka-cmake/CMakeLists.txt index 57ccb00086b02..b2945e968d8d6 100644 --- a/contrib/librdkafka-cmake/CMakeLists.txt +++ b/contrib/librdkafka-cmake/CMakeLists.txt @@ -32,7 +32,7 @@ set(SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_cert.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_cgrp.c" - "${RDKAFKA_SOURCE_DIR}/rdkafka_conf.c" +# "${RDKAFKA_SOURCE_DIR}/rdkafka_conf.c" # compiled via the rdkafka_conf_sensitive.c wrapper below "${RDKAFKA_SOURCE_DIR}/rdkafka_coord.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_error.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_event.c" @@ -84,6 +84,7 @@ set(SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_transport.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_txnmgr.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_zstd.c" # WITH_ZSTD + "${CMAKE_CURRENT_SOURCE_DIR}/rdkafka_conf_sensitive.c" # ClickHouse wrapper compiling rdkafka_conf.c, see the comment in the file "${RDKAFKA_SOURCE_DIR}/rdlist.c" "${RDKAFKA_SOURCE_DIR}/rdlog.c" "${RDKAFKA_SOURCE_DIR}/rdmap.c" diff --git a/contrib/librdkafka-cmake/include/chrdkafka_conf_sensitive.h b/contrib/librdkafka-cmake/include/chrdkafka_conf_sensitive.h new file mode 100644 index 0000000000000..83b99dce955ca --- /dev/null +++ b/contrib/librdkafka-cmake/include/chrdkafka_conf_sensitive.h @@ -0,0 +1,14 @@ +/// See contrib/librdkafka-cmake/rdkafka_conf_sensitive.c +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +/// The names of the configuration properties that librdkafka marks with the _RK_SENSITIVE flag, +/// i.e. whose values must not appear in logs, as a NULL-terminated array of static strings. +const char * const * chrd_kafka_conf_sensitive_properties(void); + +#ifdef __cplusplus +} +#endif diff --git a/contrib/librdkafka-cmake/rdkafka_conf_sensitive.c b/contrib/librdkafka-cmake/rdkafka_conf_sensitive.c new file mode 100644 index 0000000000000..740f444ad0a17 --- /dev/null +++ b/contrib/librdkafka-cmake/rdkafka_conf_sensitive.c @@ -0,0 +1,31 @@ +/// Wrapper around librdkafka's rdkafka_conf.c: compiles the file (this is the only place it is +/// compiled, see the SRCS list in contrib/librdkafka-cmake/CMakeLists.txt) and adds an accessor +/// for the static rd_kafka_properties table, which is the source of truth for the _RK_SENSITIVE +/// flag. ClickHouse uses the flag to hide the values of sensitive properties (e.g. sasl.password) +/// in its logs. No public librdkafka API exposes the flag: the redacting dump +/// (rd_kafka_anyconf_dump with redact_sensitive) is static, and the public rd_kafka_conf_dump +/// does not redact. + +#include "rdkafka_conf.c" + +#include + +/* The table ends with a terminator entry, so the count is an upper bound and the + * array is always NULL-terminated. The names point into the static table. */ +static const char * chrd_sensitive_names[sizeof(rd_kafka_properties) / sizeof(*rd_kafka_properties)]; +static pthread_once_t chrd_sensitive_names_once = PTHREAD_ONCE_INIT; + +static void chrd_fill_sensitive_names(void) +{ + const struct rd_kafka_property * prop = NULL; + size_t n = 0; + for (prop = rd_kafka_properties; prop->name; prop++) + if (prop->scope & _RK_SENSITIVE) + chrd_sensitive_names[n++] = prop->name; +} + +const char * const * chrd_kafka_conf_sensitive_properties(void) +{ + pthread_once(&chrd_sensitive_names_once, chrd_fill_sensitive_names); + return chrd_sensitive_names; +} diff --git a/contrib/mariadb-connector-c b/contrib/mariadb-connector-c index 111ec1a5958cf..b3c09ab333c2e 160000 --- a/contrib/mariadb-connector-c +++ b/contrib/mariadb-connector-c @@ -1 +1 @@ -Subproject commit 111ec1a5958cf984f50a28b0fb82a4087918d677 +Subproject commit b3c09ab333c2ef9277291e721871a2bafca6e8bf diff --git a/contrib/mariadb-connector-c-cmake/CMakeLists.txt b/contrib/mariadb-connector-c-cmake/CMakeLists.txt index 576d02de4b5e3..1f0e9a8695b20 100644 --- a/contrib/mariadb-connector-c-cmake/CMakeLists.txt +++ b/contrib/mariadb-connector-c-cmake/CMakeLists.txt @@ -45,7 +45,7 @@ MATH(EXPR MARIADB_VERSION_ID "${MARIADB_CLIENT_VERSION_MAJOR} * 10000 + set(CPACK_PACKAGE_VERSION_MAJOR 3) set(CPACK_PACKAGE_VERSION_MINOR 4) -set(CPACK_PACKAGE_VERSION_PATCH 9) +set(CPACK_PACKAGE_VERSION_PATCH 10) set(CPACK_PACKAGE_VERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}") MATH(EXPR MARIADB_PACKAGE_VERSION_ID "${CPACK_PACKAGE_VERSION_MAJOR} * 10000 + ${CPACK_PACKAGE_VERSION_MINOR} * 100 + diff --git a/contrib/mongo-c-driver b/contrib/mongo-c-driver index a00f0b91bd24f..70de9e880072b 160000 --- a/contrib/mongo-c-driver +++ b/contrib/mongo-c-driver @@ -1 +1 @@ -Subproject commit a00f0b91bd24f8ddbe62ec3864cb3704385670f5 +Subproject commit 70de9e880072b120ccbb48c77de334775747d7a9 diff --git a/contrib/mongo-c-driver-cmake/CMakeLists.txt b/contrib/mongo-c-driver-cmake/CMakeLists.txt index a79ef97c8fe2c..43ac3f5cd0283 100644 --- a/contrib/mongo-c-driver-cmake/CMakeLists.txt +++ b/contrib/mongo-c-driver-cmake/CMakeLists.txt @@ -6,12 +6,12 @@ endif() set(libbson_VERSION_MAJOR 2) set(libbson_VERSION_MINOR 5) -set(libbson_VERSION_PATCH 4) -set(libbson_VERSION 2.5.4) +set(libbson_VERSION_PATCH 5) +set(libbson_VERSION 2.5.5) set(libmongoc_VERSION_MAJOR 2) set(libmongoc_VERSION_MINOR 5) -set(libmongoc_VERSION_PATCH 4) -set(libmongoc_VERSION 2.5.4) +set(libmongoc_VERSION_PATCH 5) +set(libmongoc_VERSION 2.5.5) set(LIBBSON_SOURCES_ROOT "${ClickHouse_SOURCE_DIR}/contrib/mongo-c-driver/src") set(LIBBSON_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/libbson/src") diff --git a/contrib/nats-io b/contrib/nats-io index 3e3a3f10572ed..b88704dfeb55e 160000 --- a/contrib/nats-io +++ b/contrib/nats-io @@ -1 +1 @@ -Subproject commit 3e3a3f10572ed9ff1b7ad17873881877cb62f301 +Subproject commit b88704dfeb55e0f1dee165b9c1daccb90d5816d4 diff --git a/contrib/nlohmann-json b/contrib/nlohmann-json index 55f93686c0152..4438a2a0cda77 160000 --- a/contrib/nlohmann-json +++ b/contrib/nlohmann-json @@ -1 +1 @@ -Subproject commit 55f93686c01528224f448c19128836e7df245f72 +Subproject commit 4438a2a0cda77bf4cd74652423e03363e003a0cc diff --git a/contrib/nlohmann-json-cmake/CMakeLists.txt b/contrib/nlohmann-json-cmake/CMakeLists.txt index 752b845bc819e..863c35363cd12 100644 --- a/contrib/nlohmann-json-cmake/CMakeLists.txt +++ b/contrib/nlohmann-json-cmake/CMakeLists.txt @@ -1,8 +1,4 @@ set(JSON_DIR ${ClickHouse_SOURCE_DIR}/contrib/nlohmann-json) add_library(_nlohmann_json INTERFACE) -set_property( - TARGET _nlohmann_json - APPEND - PROPERTY INTERFACE_INCLUDE_DIRECTORIES - ${JSON_DIR}/single_include) +target_include_directories(_nlohmann_json SYSTEM INTERFACE ${JSON_DIR}/single_include) add_library(ch_contrib::nlohmann_json ALIAS _nlohmann_json) diff --git a/contrib/replxx b/contrib/replxx index c2de583a3cd41..00cd994b363c3 160000 --- a/contrib/replxx +++ b/contrib/replxx @@ -1 +1 @@ -Subproject commit c2de583a3cd41f7b476cb625e954c191b6fcc448 +Subproject commit 00cd994b363c3fc32f80d9d266e4d2acb4159abf diff --git a/contrib/thrift b/contrib/thrift index 13d30d6a6cdfb..26bf3f0c3cade 160000 --- a/contrib/thrift +++ b/contrib/thrift @@ -1 +1 @@ -Subproject commit 13d30d6a6cdfbd44c07977b8b762e88d9d5eb6df +Subproject commit 26bf3f0c3cade6f2c9fac82d4962751466282c23 diff --git a/contrib/vectorscan b/contrib/vectorscan index e6993b7003a19..4f863adbc2aa1 160000 --- a/contrib/vectorscan +++ b/contrib/vectorscan @@ -1 +1 @@ -Subproject commit e6993b7003a19806902fa69ac57dcc2df826b946 +Subproject commit 4f863adbc2aa1129d473483d1a70134da07b5074 diff --git a/contrib/vectorscan-cmake/CMakeLists.txt b/contrib/vectorscan-cmake/CMakeLists.txt index 2d8a05af15024..ab06d8a8ac601 100644 --- a/contrib/vectorscan-cmake/CMakeLists.txt +++ b/contrib/vectorscan-cmake/CMakeLists.txt @@ -37,8 +37,8 @@ set (SRCS "${LIBRARY_DIR}/src/fdr/fdr_confirm_compile.cpp" "${LIBRARY_DIR}/src/fdr/fdr_engine_description.cpp" "${LIBRARY_DIR}/src/fdr/flood_compile.cpp" - "${LIBRARY_DIR}/src/fdr/teddy.c" - "${LIBRARY_DIR}/src/fdr/teddy_avx2.c" + "${LIBRARY_DIR}/src/fdr/teddy.cpp" + "${LIBRARY_DIR}/src/fdr/teddy_fat.cpp" "${LIBRARY_DIR}/src/fdr/teddy_compile.cpp" "${LIBRARY_DIR}/src/fdr/teddy_engine_description.cpp" @@ -238,11 +238,20 @@ set (SRCS # weird reasons different constants in the output. # # Also, please use upstream versions of colm and ragel, the packages in Ubuntu 22.04 seem to produce wrong output on ARM. +# +# The generated scanners read the input through `const char *` but compare it against `unsigned char` tables (the .rl files declare +# `alphtype unsigned char`), so they must be compiled with `-funsigned-char`, as the original build system does, otherwise bytes >= 0x80 +# never match on platforms where `char` is signed. if (ARCH_AMD64) list (APPEND SRCS "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/amd64/Parser.cpp" "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/amd64/control_verbs.cpp" ) + set_source_files_properties( + "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/amd64/Parser.cpp" + "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/amd64/control_verbs.cpp" + COMPILE_FLAGS -funsigned-char + ) elseif (ARCH_AARCH64) list (APPEND SRCS "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/aarch64/Parser.cpp" @@ -251,7 +260,7 @@ elseif (ARCH_AARCH64) set_source_files_properties( "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/aarch64/Parser.cpp" "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/aarch64/control_verbs.cpp" - COMPILE_FLAGS -Wno-c++11-narrowing + COMPILE_FLAGS "-funsigned-char -Wno-c++11-narrowing" ) endif() diff --git a/contrib/vectorscan-cmake/common/hs_version.h b/contrib/vectorscan-cmake/common/hs_version.h index 3d266484095e4..6b69038d3f09c 100644 --- a/contrib/vectorscan-cmake/common/hs_version.h +++ b/contrib/vectorscan-cmake/common/hs_version.h @@ -32,12 +32,12 @@ /** * A version string to identify this release of Hyperscan. */ -#define HS_VERSION_STRING "5.4.11 2024-07-04" +#define HS_VERSION_STRING "5.4.13 2026-08-23" #define HS_VERSION_32BIT ((5 << 24) | (1 << 16) | (7 << 8) | 0) #define HS_MAJOR 5 #define HS_MINOR 4 -#define HS_PATCH 11 +#define HS_PATCH 13 #endif /* HS_VERSION_H_C6428FAF8E3713 */ diff --git a/contrib/vectorscan-cmake/rageled_files/aarch64/Parser.cpp b/contrib/vectorscan-cmake/rageled_files/aarch64/Parser.cpp index fd58ead412837..05494b959c23a 100644 --- a/contrib/vectorscan-cmake/rageled_files/aarch64/Parser.cpp +++ b/contrib/vectorscan-cmake/rageled_files/aarch64/Parser.cpp @@ -164,8 +164,8 @@ namespace ue2 { assert(child); ComponentSequence *seq = child.get(); - parent->addComponent(move(child)); - return seq; + parent->addComponent(std::move(child)); + return seq; // cppcheck-suppress returnDanglingLifetime } static @@ -176,7 +176,7 @@ namespace ue2 { assert(cc); cc->add(c); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } else { currentSeq->addComponent(getLiteralComponentClass(c, mode.caseless)); } @@ -191,7 +191,7 @@ namespace ue2 { assert(cc); cc->add(accum); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } else { if (accum > 255) { throw LocatedParseError(err_msg); @@ -225,7 +225,7 @@ namespace ue2 { static unichar readUtf8CodePoint2c(const char *s) { - auto *ts = (const u8 *)s; + auto *ts = reinterpret_cast(s); assert(ts[0] >= 0xc0 && ts[0] < 0xe0); assert(ts[1] >= 0x80 && ts[1] < 0xc0); unichar val = ts[0] & 0x1f; @@ -238,7 +238,7 @@ namespace ue2 { static unichar readUtf8CodePoint3c(const char *s) { - auto *ts = (const u8 *)s; + auto *ts = reinterpret_cast(s); assert(ts[0] >= 0xe0 && ts[0] < 0xf0); assert(ts[1] >= 0x80 && ts[1] < 0xc0); assert(ts[2] >= 0x80 && ts[2] < 0xc0); @@ -254,7 +254,7 @@ namespace ue2 { static unichar readUtf8CodePoint4c(const char *s) { - auto *ts = (const u8 *)s; + auto *ts = reinterpret_cast(s); assert(ts[0] >= 0xf0 && ts[0] < 0xf8); assert(ts[1] >= 0x80 && ts[1] < 0xc0); assert(ts[2] >= 0x80 && ts[2] < 0xc0); @@ -272,7 +272,7 @@ namespace ue2 { } -#line 1909 "Parser.rl" +#line 1916 "Parser.rl" @@ -352,18 +352,18 @@ static const short _regex_actions[] = { 289, 1, 290, 1, 291, 1, 292, 1, 293, 1, 294, 1, 295, 1, 296, 1, 297, 1, 298, 1, 299, 1, 300, 1, - 301, 1, 302, 1, 303, 1, 307, 1, - 308, 1, 309, 1, 310, 1, 311, 1, + 301, 1, 302, 1, 303, 1, 304, 1, + 305, 1, 309, 1, 310, 1, 311, 1, 312, 1, 313, 1, 314, 1, 315, 1, 316, 1, 317, 1, 318, 1, 319, 1, 320, 1, 321, 1, 322, 1, 323, 1, 324, 1, 325, 1, 326, 1, 327, 1, 328, 1, 329, 1, 330, 1, 331, 1, 332, 1, 333, 1, 334, 1, 335, 1, - 336, 1, 337, 1, 338, 1, 342, 1, - 343, 1, 344, 1, 345, 1, 346, 1, + 336, 1, 337, 1, 338, 1, 339, 1, + 340, 1, 344, 1, 345, 1, 346, 1, 347, 1, 348, 1, 349, 1, 350, 1, - 352, 1, 353, 1, 354, 1, 355, 1, + 351, 1, 352, 1, 354, 1, 355, 1, 356, 1, 357, 1, 358, 1, 359, 1, 360, 1, 361, 1, 362, 1, 363, 1, 364, 1, 365, 1, 366, 1, 367, 1, @@ -384,17 +384,18 @@ static const short _regex_actions[] = { 424, 1, 425, 1, 426, 1, 427, 1, 428, 1, 429, 1, 430, 1, 431, 1, 432, 1, 433, 1, 434, 1, 435, 1, - 436, 2, 3, 0, 2, 4, 5, 2, - 5, 1, 2, 9, 10, 2, 9, 238, - 2, 9, 239, 2, 9, 339, 2, 10, - 1, 2, 10, 340, 2, 10, 341, 2, - 11, 241, 2, 11, 351, 2, 12, 241, - 2, 12, 351, 2, 13, 241, 2, 13, - 351, 2, 14, 375, 2, 14, 376, 2, - 25, 0, 2, 25, 3, 2, 25, 6, - 2, 25, 14, 3, 25, 5, 306, 3, - 25, 10, 305, 3, 25, 14, 15, 4, - 25, 9, 304, 10, 0 + 436, 1, 437, 1, 438, 2, 3, 0, + 2, 4, 5, 2, 5, 1, 2, 9, + 10, 2, 9, 238, 2, 9, 239, 2, + 9, 341, 2, 10, 1, 2, 10, 342, + 2, 10, 343, 2, 11, 241, 2, 11, + 353, 2, 12, 241, 2, 12, 353, 2, + 13, 241, 2, 13, 353, 2, 14, 377, + 2, 14, 378, 2, 25, 0, 2, 25, + 3, 2, 25, 6, 2, 25, 14, 3, + 25, 5, 308, 3, 25, 10, 307, 3, + 25, 14, 15, 4, 25, 9, 306, 10, + 0 }; static const short _regex_key_offsets[] = { @@ -500,14 +501,14 @@ static const short _regex_actions[] = { 1790, 1795, 1804, 1806, 1807, 1809, 1814, 1817, 1819, 1821, 1822, 1824, 1834, 1840, 1841, 1846, 1850, 1858, 1860, 1869, 1873, 1874, 1875, 1879, - 1880, 1883, 1883, 1890, 1904, 1907, 1946, 1948, - 1950, 1952, 1954, 1955, 1955, 1956, 1957, 1964, - 1970, 1976, 1978, 1980, 1982, 1985, 1987, 1998, - 1999, 2001, 2003, 2005, 2016, 2017, 2019, 2021, - 2023, 2024, 0 + 1880, 1883, 1883, 1890, 1905, 1906, 1909, 1948, + 1950, 1952, 1954, 1956, 1957, 1957, 1958, 1959, + 1966, 1972, 1978, 1980, 1982, 1984, 1987, 1989, + 2000, 2001, 2003, 2005, 2007, 2018, 2019, 2021, + 2023, 2025, 2026, 0 }; - static const char _regex_trans_keys[] = { + static const unsigned char _regex_trans_keys[] = { 41u, 33u, 35u, 38u, 39u, 40u, 41u, 43u, 45u, 58u, 60u, 61u, 62u, 63u, 67u, 80u, 105u, 109u, 115u, 120u, 123u, 48u, 57u, 41u, @@ -744,24 +745,24 @@ static const short _regex_actions[] = { 107u, 109u, 111u, 117u, 121u, 97u, 101u, 104u, 105u, 103u, 97u, 97u, 112u, 115u, 119u, 105u, 108u, 112u, 115u, 67u, 76u, 77u, 78u, 80u, - 83u, 90u, 45u, 91u, 92u, 93u, 128u, 191u, - 192u, 223u, 224u, 239u, 240u, 247u, 248u, 255u, - 46u, 58u, 61u, 48u, 68u, 69u, 72u, 76u, - 78u, 80u, 81u, 83u, 85u, 86u, 87u, 97u, - 98u, 99u, 100u, 101u, 102u, 103u, 104u, 108u, - 110u, 111u, 112u, 114u, 115u, 116u, 117u, 118u, - 119u, 120u, 49u, 55u, 56u, 57u, 65u, 90u, - 105u, 122u, 48u, 55u, 48u, 55u, 48u, 55u, - 48u, 55u, 123u, 123u, 123u, 123u, 48u, 57u, + 83u, 90u, 38u, 45u, 91u, 92u, 93u, 128u, + 191u, 192u, 223u, 224u, 239u, 240u, 247u, 248u, + 255u, 38u, 46u, 58u, 61u, 48u, 68u, 69u, + 72u, 76u, 78u, 80u, 81u, 83u, 85u, 86u, + 87u, 97u, 98u, 99u, 100u, 101u, 102u, 103u, + 104u, 108u, 110u, 111u, 112u, 114u, 115u, 116u, + 117u, 118u, 119u, 120u, 49u, 55u, 56u, 57u, + 65u, 90u, 105u, 122u, 48u, 55u, 48u, 55u, + 48u, 55u, 48u, 55u, 123u, 123u, 123u, 123u, + 48u, 57u, 65u, 70u, 97u, 102u, 48u, 57u, 65u, 70u, 97u, 102u, 48u, 57u, 65u, 70u, - 97u, 102u, 48u, 57u, 65u, 70u, 97u, 102u, - 128u, 191u, 128u, 191u, 128u, 191u, 92u, 93u, - 94u, 69u, 81u, 92u, 128u, 191u, 192u, 223u, - 224u, 239u, 240u, 247u, 248u, 255u, 69u, 128u, - 191u, 128u, 191u, 128u, 191u, 92u, 128u, 191u, + 97u, 102u, 128u, 191u, 128u, 191u, 128u, 191u, + 92u, 93u, 94u, 69u, 81u, 92u, 128u, 191u, 192u, 223u, 224u, 239u, 240u, 247u, 248u, 255u, - 69u, 128u, 191u, 128u, 191u, 128u, 191u, 41u, - 10u, 0u + 69u, 128u, 191u, 128u, 191u, 128u, 191u, 92u, + 128u, 191u, 192u, 223u, 224u, 239u, 240u, 247u, + 248u, 255u, 69u, 128u, 191u, 128u, 191u, 128u, + 191u, 41u, 10u, 0u }; static const signed char _regex_single_lengths[] = { @@ -867,11 +868,11 @@ static const short _regex_actions[] = { 5, 9, 2, 1, 2, 5, 3, 2, 2, 1, 2, 10, 6, 1, 5, 4, 8, 2, 9, 4, 1, 1, 4, 1, - 3, 0, 7, 4, 3, 31, 0, 0, - 0, 0, 1, 0, 1, 1, 1, 0, - 0, 0, 0, 0, 3, 2, 1, 1, - 0, 0, 0, 1, 1, 0, 0, 0, - 1, 1, 0 + 3, 0, 7, 5, 1, 3, 31, 0, + 0, 0, 0, 1, 0, 1, 1, 1, + 0, 0, 0, 0, 0, 3, 2, 1, + 1, 0, 0, 0, 1, 1, 0, 0, + 0, 1, 1, 0 }; static const signed char _regex_range_lengths[] = { @@ -977,11 +978,11 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 5, 0, 4, 1, 1, - 1, 1, 0, 0, 0, 0, 3, 3, - 3, 1, 1, 1, 0, 0, 5, 0, - 1, 1, 1, 5, 0, 1, 1, 1, - 0, 0, 0 + 0, 0, 0, 5, 0, 0, 4, 1, + 1, 1, 1, 0, 0, 0, 0, 3, + 3, 3, 1, 1, 1, 0, 0, 5, + 0, 1, 1, 1, 5, 0, 1, 1, + 1, 0, 0, 0 }; static const short _regex_index_offsets[] = { @@ -1087,11 +1088,11 @@ static const short _regex_actions[] = { 2412, 2418, 2428, 2431, 2433, 2436, 2442, 2446, 2449, 2452, 2454, 2457, 2468, 2475, 2477, 2483, 2488, 2497, 2500, 2510, 2515, 2517, 2519, 2524, - 2526, 2530, 2531, 2539, 2549, 2553, 2589, 2591, - 2593, 2595, 2597, 2599, 2600, 2602, 2604, 2609, - 2613, 2617, 2619, 2621, 2623, 2627, 2630, 2637, - 2639, 2641, 2643, 2645, 2652, 2654, 2656, 2658, - 2660, 2662, 0 + 2526, 2530, 2531, 2539, 2550, 2552, 2556, 2592, + 2594, 2596, 2598, 2600, 2602, 2603, 2605, 2607, + 2612, 2616, 2620, 2622, 2624, 2626, 2630, 2633, + 2640, 2642, 2644, 2646, 2648, 2655, 2657, 2659, + 2661, 2663, 2665, 0 }; static const signed char _regex_trans_cond_spaces[] = { @@ -1412,8 +1413,8 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 0, - 0, 0, 0, 0, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1422,12 +1423,12 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 0, -1, 0, -1, + 0, -1, -1, 2, 2, -1, -1, -1, + -1, -1, 0, 0, 0, 0, 0, -1, + -1, -1, 0, -1, 0, -1, 0, -1, + -1, 0, 0, 0, 0, 0, -1, -1, -1, 0, -1, 0, -1, 0, -1, -1, - 2, 2, -1, -1, -1, -1, -1, 0, - 0, 0, 0, 0, -1, -1, -1, 0, - -1, 0, -1, 0, -1, -1, 0, 0, - 0, 0, 0, -1, -1, -1, 0, -1, - 0, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1534,7 +1535,7 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 0 + -1, -1, -1, -1, -1, -1, 0 }; static const short _regex_trans_offsets[] = { @@ -1856,7 +1857,7 @@ static const short _regex_actions[] = { 2541, 2542, 2543, 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, 2553, 2554, 2555, 2556, 2557, 2558, 2559, 2560, 2561, 2562, 2563, 2564, - 2566, 2568, 2570, 2572, 2574, 2575, 2576, 2577, + 2565, 2567, 2569, 2571, 2573, 2575, 2576, 2577, 2578, 2579, 2580, 2581, 2582, 2583, 2584, 2585, 2586, 2587, 2588, 2589, 2590, 2591, 2592, 2593, 2594, 2595, 2596, 2597, 2598, 2599, 2600, 2601, @@ -1865,12 +1866,12 @@ static const short _regex_actions[] = { 2618, 2619, 2620, 2621, 2622, 2623, 2624, 2625, 2626, 2627, 2628, 2629, 2630, 2631, 2632, 2633, 2634, 2635, 2636, 2637, 2638, 2639, 2640, 2641, - 2642, 2643, 2645, 2646, 2648, 2649, 2651, 2652, - 2653, 2655, 2657, 2658, 2659, 2660, 2661, 2662, - 2664, 2666, 2668, 2670, 2672, 2673, 2674, 2675, - 2677, 2678, 2680, 2681, 2683, 2684, 2685, 2687, - 2689, 2691, 2693, 2695, 2696, 2697, 2698, 2700, - 2701, 2703, 2704, 2706, 2707, 2708, 2709, 2710, + 2642, 2643, 2644, 2645, 2646, 2648, 2649, 2651, + 2652, 2654, 2655, 2656, 2658, 2660, 2661, 2662, + 2663, 2664, 2665, 2667, 2669, 2671, 2673, 2675, + 2676, 2677, 2678, 2680, 2681, 2683, 2684, 2686, + 2687, 2688, 2690, 2692, 2694, 2696, 2698, 2699, + 2700, 2701, 2703, 2704, 2706, 2707, 2709, 2710, 2711, 2712, 2713, 2714, 2715, 2716, 2717, 2718, 2719, 2720, 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, 2729, 2730, 2731, 2732, 2733, 2734, @@ -1977,7 +1978,7 @@ static const short _regex_actions[] = { 3535, 3536, 3537, 3538, 3539, 3540, 3541, 3542, 3543, 3544, 3545, 3546, 3547, 3548, 3549, 3550, 3551, 3552, 3553, 3554, 3555, 3556, 3557, 3558, - 3559, 3560, 0 + 3559, 3560, 3561, 3562, 3563, 3564, 0 }; static const signed char _regex_trans_lengths[] = { @@ -2298,8 +2299,8 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 2, - 2, 2, 2, 2, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2308,12 +2309,12 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 2, 1, 2, 1, + 2, 1, 1, 2, 2, 1, 1, 1, + 1, 1, 2, 2, 2, 2, 2, 1, + 1, 1, 2, 1, 2, 1, 2, 1, + 1, 2, 2, 2, 2, 2, 1, 1, 1, 2, 1, 2, 1, 2, 1, 1, - 2, 2, 1, 1, 1, 1, 1, 2, - 2, 2, 2, 2, 1, 1, 1, 2, - 1, 2, 1, 2, 1, 1, 2, 2, - 2, 2, 2, 1, 1, 1, 2, 1, - 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2420,7 +2421,7 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 0 + 1, 1, 1, 1, 1, 1, 0 }; static const signed char _regex_cond_keys[] = { @@ -2744,8 +2745,8 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, - 0, 1, 0, 1, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2754,15 +2755,15 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 1, - 0, 0, 1, 0, 0, 0, 1, 0, - 1, 0, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 1, + 0, 0, 1, 0, 0, 1, 0, 0, + 0, 1, 0, 1, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 0, 0, 0, 1, + 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, - 0, 0, 0, 0, 1, 0, 0, 1, - 0, 0, 1, 0, 0, 0, 1, 0, - 1, 0, 1, 0, 1, 0, 1, 0, - 0, 0, 0, 1, 0, 0, 1, 0, - 0, 1, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 0, 0, 0, 1, 0, + 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2869,7 +2870,7 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0 + 0, 0, 0, 0, 0, 0 }; static const short _regex_cond_targs[] = { @@ -3144,9 +3145,9 @@ static const short _regex_actions[] = { 732, 733, 819, 731, 732, 733, 731, 735, 819, 819, 735, 819, 819, 736, 736, 736, 819, 819, 819, 819, 819, 739, 819, 819, - 819, 819, 838, 838, 838, 838, 742, 838, - 838, 838, 838, 843, 843, 843, 843, 745, - 843, 843, 843, 843, 746, 746, 746, 746, + 819, 819, 839, 839, 839, 839, 742, 839, + 839, 839, 839, 844, 844, 844, 844, 745, + 844, 844, 844, 844, 746, 746, 746, 746, 746, 747, 746, 759, 760, 746, 761, 762, 763, 746, 782, 746, 746, 746, 746, 746, 784, 746, 785, 746, 786, 746, 746, 746, @@ -3193,26 +3194,26 @@ static const short _regex_actions[] = { 548, 790, 554, 790, 555, 556, 557, 558, 790, 790, 790, 790, 790, 790, 790, 0, 818, 818, 818, 818, 818, 818, 818, 818, - 819, 820, 821, 819, 819, 819, 819, 833, - 819, 834, 819, 835, 819, 819, 819, 563, - 566, 731, 819, 822, 819, 819, 819, 819, - 819, 826, 819, 819, 819, 819, 819, 819, - 819, 827, 819, 819, 819, 819, 819, 819, - 819, 828, 829, 819, 819, 819, 819, 819, - 819, 830, 824, 819, 819, 819, 819, 823, - 819, 819, 819, 825, 819, 819, 819, 819, - 819, 819, 734, 819, 819, 819, 832, 831, - 831, 831, 819, 819, 819, 819, 819, 736, - 736, 736, 819, 819, 819, 819, 819, 737, - 819, 819, 738, 819, 837, 836, 836, 836, - 836, 836, 836, 836, 836, 839, 838, 838, - 838, 840, 838, 841, 838, 842, 838, 838, - 838, 838, 838, 838, 838, 838, 838, 740, - 838, 838, 741, 838, 844, 843, 843, 843, - 845, 843, 846, 843, 847, 843, 843, 843, - 843, 843, 843, 843, 843, 843, 743, 843, - 843, 744, 843, 848, 848, 849, 849, 0, - 746, 746, 746, 746, 746, 746, 746, 746, + 820, 819, 821, 822, 819, 819, 819, 819, + 834, 819, 835, 819, 836, 819, 819, 819, + 819, 819, 563, 566, 731, 819, 823, 819, + 819, 819, 819, 819, 827, 819, 819, 819, + 819, 819, 819, 819, 828, 819, 819, 819, + 819, 819, 819, 819, 829, 830, 819, 819, + 819, 819, 819, 819, 831, 825, 819, 819, + 819, 819, 824, 819, 819, 819, 826, 819, + 819, 819, 819, 819, 819, 734, 819, 819, + 819, 833, 832, 832, 832, 819, 819, 819, + 819, 819, 736, 736, 736, 819, 819, 819, + 819, 819, 737, 819, 819, 738, 819, 838, + 837, 837, 837, 837, 837, 837, 837, 837, + 840, 839, 839, 839, 841, 839, 842, 839, + 843, 839, 839, 839, 839, 839, 839, 839, + 839, 839, 740, 839, 839, 741, 839, 845, + 844, 844, 844, 846, 844, 847, 844, 848, + 844, 844, 844, 844, 844, 844, 844, 844, + 844, 743, 844, 844, 744, 844, 849, 849, + 850, 850, 0, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, @@ -3220,7 +3221,8 @@ static const short _regex_actions[] = { 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, - 787, 787, 787, 787, 787, 787, 790, 790, + 746, 746, 746, 787, 787, 787, 787, 787, + 787, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, @@ -3281,8 +3283,7 @@ static const short _regex_actions[] = { 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, - 790, 790, 790, 790, 790, 790, 559, 560, - 561, 562, 819, 819, 819, 819, 819, 819, + 790, 559, 560, 561, 562, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, @@ -3304,62 +3305,62 @@ static const short _regex_actions[] = { 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, - 819, 819, 819, 838, 838, 838, 843, 843, - 843, 746, 746, 746, 746, 746, 746, 746, + 819, 819, 819, 819, 819, 819, 839, 839, + 839, 844, 844, 844, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, - 746, 746, 787, 787, 787, 790, 790, 790, + 746, 746, 746, 746, 746, 787, 787, 787, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, - 817, 818, 819, 819, 819, 819, 819, 819, + 790, 790, 790, 817, 818, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, - 819, 819, 819, 836, 836, 838, 838, 838, - 838, 838, 843, 843, 843, 843, 843, 848, - 849, 0 + 819, 819, 819, 819, 819, 819, 819, 837, + 837, 839, 839, 839, 839, 839, 844, 844, + 844, 844, 844, 849, 850, 0 }; static const short _regex_cond_actions[] = { - 827, 631, 731, 723, 45, 903, 903, 897, - 45, 912, 900, 903, 729, 741, 0, 45, - 45, 923, 923, 923, 923, 737, 45, 765, - 747, 0, 0, 0, 0, 841, 743, 3, + 831, 635, 735, 727, 45, 907, 907, 901, + 45, 916, 904, 907, 733, 745, 0, 45, + 45, 927, 927, 927, 927, 741, 45, 769, + 751, 0, 0, 0, 0, 845, 747, 3, + 3, 3, 3, 845, 3, 3, 3, 3, + 843, 0, 3, 3, 3, 3, 843, 765, + 843, 765, 3, 3, 3, 3, 843, 763, + 3, 874, 3, 3, 843, 765, 3, 874, + 3, 3, 843, 3, 3, 3, 3, 843, + 0, 3, 3, 3, 3, 843, 755, 0, + 753, 843, 759, 757, 843, 0, 761, 3, + 3, 3, 3, 843, 0, 0, 0, 0, + 843, 761, 0, 0, 0, 0, 843, 761, + 3, 3, 3, 3, 843, 749, 0, 845, + 729, 731, 27, 27, 27, 27, 845, 747, + 3, 3, 3, 3, 845, 767, 0, 845, + 3, 3, 3, 3, 845, 3, 3, 3, + 3, 845, 677, 3, 3, 3, 3, 845, + 0, 0, 0, 0, 845, 27, 27, 27, + 27, 845, 729, 0, 731, 25, 25, 25, + 25, 845, 0, 0, 833, 0, 0, 0, + 607, 0, 0, 0, 0, 0, 0, 833, + 0, 0, 0, 605, 0, 0, 0, 0, + 0, 0, 833, 0, 0, 0, 607, 0, + 0, 0, 0, 679, 0, 17, 835, 679, + 0, 7, 3, 874, 3, 3, 835, 17, + 835, 667, 17, 835, 667, 17, 835, 667, + 835, 3, 665, 874, 3, 3, 835, 3, + 665, 874, 3, 3, 835, 3, 665, 3, + 3, 3, 835, 3, 669, 3, 3, 3, + 835, 3, 3, 3, 3, 841, 675, 3, 3, 3, 3, 841, 3, 3, 3, 3, - 839, 0, 3, 3, 3, 3, 839, 761, - 839, 761, 3, 3, 3, 3, 839, 759, - 3, 870, 3, 3, 839, 761, 3, 870, - 3, 3, 839, 3, 3, 3, 3, 839, - 0, 3, 3, 3, 3, 839, 751, 0, - 749, 839, 755, 753, 839, 0, 757, 3, - 3, 3, 3, 839, 0, 0, 0, 0, - 839, 757, 0, 0, 0, 0, 839, 757, - 3, 3, 3, 3, 839, 745, 0, 841, - 725, 727, 27, 27, 27, 27, 841, 743, - 3, 3, 3, 3, 841, 763, 0, 841, - 3, 3, 3, 3, 841, 3, 3, 3, - 3, 841, 673, 3, 3, 3, 3, 841, - 0, 0, 0, 0, 841, 27, 27, 27, - 27, 841, 725, 0, 727, 25, 25, 25, - 25, 841, 0, 0, 829, 0, 0, 0, - 603, 0, 0, 0, 0, 0, 0, 829, - 0, 0, 0, 601, 0, 0, 0, 0, - 0, 0, 829, 0, 0, 0, 603, 0, - 0, 0, 0, 675, 0, 17, 831, 675, - 0, 7, 3, 870, 3, 3, 831, 17, - 831, 663, 17, 831, 663, 17, 831, 663, - 831, 3, 661, 870, 3, 3, 831, 3, - 661, 870, 3, 3, 831, 3, 661, 3, - 3, 3, 831, 3, 665, 3, 3, 3, - 831, 3, 3, 3, 3, 837, 671, 3, - 3, 3, 3, 837, 3, 3, 3, 3, - 837, 669, 3, 3, 3, 3, 837, 3, - 3, 3, 3, 837, 3, 667, 3, 3, - 3, 837, 0, 833, 677, 0, 833, 679, - 0, 0, 0, 835, 0, 29, 11, 845, - 31, 13, 845, 0, 13, 845, 843, 769, - 843, 843, 0, 843, 843, 771, 843, 53, + 841, 673, 3, 3, 3, 3, 841, 3, + 3, 3, 3, 841, 3, 671, 3, 3, + 3, 841, 0, 837, 681, 0, 837, 683, + 0, 0, 0, 839, 0, 29, 11, 849, + 31, 13, 849, 0, 13, 849, 847, 773, + 847, 847, 0, 847, 847, 775, 847, 53, 0, 53, 0, 0, 51, 0, 53, 0, 0, 49, 0, 0, 47, 0, 315, 359, 0, 0, 359, 0, 359, 0, 359, 127, @@ -3489,139 +3490,139 @@ static const short _regex_actions[] = { 359, 0, 359, 0, 359, 309, 359, 311, 359, 119, 359, 121, 359, 123, 359, 125, 359, 0, 39, 33, 35, 37, 39, 39, - 35, 0, 0, 539, 0, 0, 0, 377, - 0, 0, 0, 0, 0, 0, 539, 0, + 35, 0, 0, 543, 0, 0, 0, 377, + 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 539, 0, + 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 435, 0, 0, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 381, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 385, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 389, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 393, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 397, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 401, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 405, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 409, - 0, 0, 0, 539, 0, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 413, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 417, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 421, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 425, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 429, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 433, 0, 0, 0, 539, - 0, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 379, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 383, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 387, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 391, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 395, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 399, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 403, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 407, 0, - 0, 0, 539, 0, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 411, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 415, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 419, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 423, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 427, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 431, 0, 0, 0, 539, 0, + 0, 543, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 381, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 385, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 389, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 393, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 397, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 401, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 405, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 409, + 0, 0, 0, 543, 0, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 413, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 417, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 421, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 425, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 429, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 433, 0, 0, 0, 543, + 0, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 379, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 383, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 387, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 391, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 395, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 399, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 403, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 407, 0, + 0, 0, 543, 0, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 411, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 415, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 419, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 423, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 427, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 431, 0, 0, 0, 543, 0, 0, 0, 377, 0, 0, 0, 0, 0, - 533, 471, 0, 533, 475, 0, 0, 0, - 535, 537, 503, 537, 537, 0, 537, 537, - 505, 537, 569, 557, 569, 569, 0, 569, - 569, 559, 569, 587, 575, 587, 587, 0, - 587, 587, 577, 587, 633, 775, 777, 629, - 637, 45, 597, 0, 0, 609, 0, 45, - 0, 635, 909, 599, 775, 777, 773, 777, - 0, 777, 45, 777, 45, 777, 773, 777, - 0, 0, 779, 0, 0, 0, 0, 821, - 3, 3, 3, 3, 821, 1, 3, 1, - 0, 3, 3, 855, 855, 3, 3, 819, - 0, 821, 27, 27, 27, 27, 0, 821, - 745, 0, 821, 735, 733, 3, 3, 3, - 3, 821, 739, 821, 763, 0, 821, 1, - 1, 0, 821, 725, 0, 727, 25, 25, - 25, 25, 821, 615, 613, 783, 619, 617, - 785, 623, 621, 787, 0, 0, 0, 781, - 5, 639, 647, 611, 695, 607, 717, 699, - 715, 683, 683, 0, 605, 713, 691, 683, - 703, 687, 719, 641, 657, 645, 0, 693, - 659, 655, 906, 697, 45, 683, 651, 45, - 0, 653, 689, 649, 683, 701, 685, 7, - 643, 852, 915, 721, 15, 791, 867, 791, - 858, 919, 795, 927, 919, 793, 919, 847, - 709, 711, 681, 0, 7, 0, 849, 17, - 801, 17, 799, 876, 799, 17, 797, 873, - 797, 1, 1, 1, 815, 0, 803, 705, - 707, 45, 19, 23, 21, 805, 882, 894, - 888, 805, 0, 0, 0, 807, 11, 825, - 627, 625, 789, 823, 767, 823, 823, 0, - 823, 823, 0, 823, 55, 45, 45, 53, + 537, 471, 0, 537, 475, 0, 0, 0, + 539, 541, 503, 541, 541, 0, 541, 541, + 505, 541, 573, 561, 573, 573, 0, 573, + 573, 563, 573, 591, 579, 591, 591, 0, + 591, 591, 581, 591, 637, 779, 781, 633, + 641, 45, 601, 0, 0, 613, 0, 45, + 0, 639, 913, 603, 779, 781, 777, 781, + 0, 781, 45, 781, 45, 781, 777, 781, + 0, 0, 783, 0, 0, 0, 0, 825, + 3, 3, 3, 3, 825, 1, 3, 1, + 0, 3, 3, 859, 859, 3, 3, 823, + 0, 825, 27, 27, 27, 27, 0, 825, + 749, 0, 825, 739, 737, 3, 3, 3, + 3, 825, 743, 825, 767, 0, 825, 1, + 1, 0, 825, 729, 0, 731, 25, 25, + 25, 25, 825, 619, 617, 787, 623, 621, + 789, 627, 625, 791, 0, 0, 0, 785, + 5, 643, 651, 615, 699, 611, 721, 703, + 719, 687, 687, 0, 609, 717, 695, 687, + 707, 691, 723, 645, 661, 649, 0, 697, + 663, 659, 910, 701, 45, 687, 655, 45, + 0, 657, 693, 653, 687, 705, 689, 7, + 647, 856, 919, 725, 15, 795, 871, 795, + 862, 923, 799, 931, 923, 797, 923, 851, + 713, 715, 685, 0, 7, 0, 853, 17, + 805, 17, 803, 880, 803, 17, 801, 877, + 801, 1, 1, 1, 819, 0, 807, 709, + 711, 45, 19, 23, 21, 809, 886, 898, + 892, 809, 0, 0, 0, 811, 11, 829, + 631, 629, 793, 827, 771, 827, 827, 0, + 827, 827, 0, 827, 55, 45, 45, 53, 0, 53, 0, 0, 0, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, @@ -3642,47 +3643,48 @@ static const short _regex_actions[] = { 0, 339, 0, 339, 0, 0, 0, 0, 339, 313, 339, 113, 115, 117, 335, 39, 361, 363, 365, 367, 369, 371, 373, 375, - 491, 45, 0, 511, 509, 507, 509, 0, - 509, 45, 509, 45, 509, 507, 509, 0, - 0, 0, 531, 5, 489, 439, 457, 493, - 493, 0, 437, 485, 493, 461, 481, 451, - 441, 0, 487, 453, 449, 495, 455, 493, - 445, 45, 0, 447, 483, 443, 493, 459, - 479, 7, 9, 473, 497, 497, 499, 15, - 517, 861, 517, 15, 519, 864, 519, 467, - 469, 477, 0, 521, 463, 465, 45, 19, - 23, 21, 523, 879, 891, 885, 523, 0, - 0, 0, 525, 529, 501, 529, 529, 0, - 529, 529, 0, 529, 0, 549, 543, 549, - 541, 549, 547, 545, 551, 0, 563, 561, - 563, 0, 563, 45, 563, 45, 563, 561, - 563, 553, 567, 565, 555, 565, 565, 0, - 565, 565, 0, 565, 0, 581, 579, 581, - 0, 581, 45, 581, 45, 581, 579, 581, - 571, 585, 583, 573, 583, 583, 0, 583, - 583, 0, 583, 589, 591, 593, 595, 0, - 827, 827, 841, 841, 839, 839, 839, 839, - 839, 839, 839, 839, 839, 839, 839, 839, - 839, 839, 841, 841, 841, 841, 841, 841, - 841, 841, 841, 841, 829, 829, 829, 829, - 829, 829, 829, 829, 829, 831, 831, 831, - 831, 831, 831, 831, 831, 831, 831, 831, - 831, 837, 837, 837, 837, 837, 837, 833, - 833, 835, 845, 845, 845, 843, 843, 843, - 59, 59, 59, 59, 59, 59, 359, 359, + 0, 491, 45, 0, 513, 511, 507, 511, + 0, 511, 45, 511, 45, 511, 507, 511, + 509, 535, 0, 0, 0, 533, 5, 489, + 439, 457, 493, 493, 0, 437, 485, 493, + 461, 481, 451, 441, 0, 487, 453, 449, + 495, 455, 493, 445, 45, 0, 447, 483, + 443, 493, 459, 479, 7, 9, 473, 497, + 497, 499, 15, 519, 865, 519, 15, 521, + 868, 521, 467, 469, 477, 0, 523, 463, + 465, 45, 19, 23, 21, 525, 883, 895, + 889, 525, 0, 0, 0, 527, 531, 501, + 531, 531, 0, 531, 531, 0, 531, 0, + 553, 547, 553, 545, 553, 551, 549, 555, + 0, 567, 565, 567, 0, 567, 45, 567, + 45, 567, 565, 567, 557, 571, 569, 559, + 569, 569, 0, 569, 569, 0, 569, 0, + 585, 583, 585, 0, 585, 45, 585, 45, + 585, 583, 585, 575, 589, 587, 577, 587, + 587, 0, 587, 587, 0, 587, 593, 595, + 597, 599, 0, 831, 831, 845, 845, 843, + 843, 843, 843, 843, 843, 843, 843, 843, + 843, 843, 843, 843, 843, 845, 845, 845, + 845, 845, 845, 845, 845, 845, 845, 833, + 833, 833, 833, 833, 833, 833, 833, 833, + 835, 835, 835, 835, 835, 835, 835, 835, + 835, 835, 835, 835, 841, 841, 841, 841, + 841, 841, 837, 837, 839, 849, 849, 849, + 847, 847, 847, 59, 59, 59, 59, 59, + 59, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 359, 359, 359, - 359, 341, 341, 341, 341, 341, 341, 341, + 359, 359, 359, 359, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, - 341, 341, 341, 341, 343, 343, 343, 343, - 343, 343, 341, 341, 341, 341, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 343, + 343, 343, 343, 343, 343, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, - 341, 341, 341, 359, 359, 359, 359, 359, + 341, 341, 341, 341, 341, 341, 359, 359, + 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, @@ -3690,8 +3692,8 @@ static const short _regex_actions[] = { 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, + 359, 359, 359, 357, 357, 357, 357, 357, 359, 359, 359, 359, 359, 359, 359, 359, - 357, 357, 357, 357, 357, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, @@ -3702,72 +3704,71 @@ static const short _regex_actions[] = { 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, - 345, 345, 345, 345, 347, 347, 347, 347, - 347, 347, 347, 347, 347, 347, 347, 349, - 349, 349, 349, 349, 349, 349, 349, 349, + 345, 345, 345, 345, 345, 345, 345, 347, 347, 347, 347, 347, 347, 347, 347, 347, - 347, 347, 347, 347, 351, 351, 351, 351, - 351, 351, 351, 351, 351, 351, 359, 359, + 347, 347, 349, 349, 349, 349, 349, 349, + 349, 349, 349, 347, 347, 347, 347, 347, + 347, 347, 347, 347, 347, 347, 347, 351, + 351, 351, 351, 351, 351, 351, 351, 351, + 351, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 353, 353, 353, 353, 353, 353, 353, 353, 353, 353, 353, - 353, 353, 359, 359, 359, 359, 359, 359, - 359, 355, 355, 355, 355, 355, 355, 355, + 353, 353, 353, 353, 353, 359, 359, 359, + 359, 359, 359, 359, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, - 355, 355, 355, 355, 355, 359, 359, 359, + 355, 355, 355, 355, 355, 355, 355, 355, + 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 359, 39, 39, - 39, 39, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 533, 533, 535, - 537, 537, 537, 569, 569, 569, 587, 587, - 587, 0, 779, 821, 821, 819, 821, 821, - 821, 821, 821, 821, 821, 821, 783, 785, - 787, 781, 817, 791, 791, 795, 793, 847, - 811, 809, 801, 799, 799, 797, 797, 815, - 803, 813, 805, 805, 807, 825, 789, 823, - 823, 823, 0, 57, 57, 0, 339, 339, - 319, 321, 339, 339, 339, 339, 337, 339, - 339, 339, 323, 325, 327, 329, 339, 331, - 339, 333, 339, 339, 339, 339, 339, 335, - 0, 0, 0, 531, 531, 517, 517, 519, - 519, 513, 527, 521, 515, 523, 523, 525, - 529, 529, 529, 0, 551, 0, 567, 565, - 565, 565, 0, 585, 583, 583, 583, 0, - 0, 0 + 359, 39, 39, 39, 39, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 537, 537, 539, 541, 541, 541, 573, 573, + 573, 591, 591, 591, 0, 783, 825, 825, + 823, 825, 825, 825, 825, 825, 825, 825, + 825, 787, 789, 791, 785, 821, 795, 795, + 799, 797, 851, 815, 813, 805, 803, 803, + 801, 801, 819, 807, 817, 809, 809, 811, + 829, 793, 827, 827, 827, 0, 57, 57, + 0, 339, 339, 319, 321, 339, 339, 339, + 339, 337, 339, 339, 339, 323, 325, 327, + 329, 339, 331, 339, 333, 339, 339, 339, + 339, 339, 335, 0, 0, 0, 535, 533, + 535, 519, 519, 521, 521, 515, 529, 523, + 517, 525, 525, 527, 531, 531, 531, 0, + 555, 0, 571, 569, 569, 569, 0, 589, + 587, 587, 587, 0, 0, 0 }; static const signed char _regex_to_state_actions[] = { @@ -3875,9 +3876,9 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 41, 0, 41, 0, - 0, 0, 0, 41, 0, 0, 0, 0, - 41, 41, 0 + 0, 0, 0, 0, 0, 41, 0, 41, + 0, 0, 0, 0, 41, 0, 0, 0, + 0, 41, 41, 0 }; static const signed char _regex_from_state_actions[] = { @@ -3985,119 +3986,119 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 43, 0, 43, 0, - 0, 0, 0, 43, 0, 0, 0, 0, - 43, 43, 0 + 0, 0, 0, 0, 0, 43, 0, 43, + 0, 0, 0, 0, 43, 0, 0, 0, + 0, 43, 43, 0 }; static const short _regex_eof_trans[] = { - 2665, 2666, 2667, 2668, 2669, 2670, 2671, 2672, - 2673, 2674, 2675, 2676, 2677, 2678, 2679, 2680, - 2681, 2682, 2683, 2684, 2685, 2686, 2687, 2688, - 2689, 2690, 2691, 2692, 2693, 2694, 2695, 2696, - 2697, 2698, 2699, 2700, 2701, 2702, 2703, 2704, - 2705, 2706, 2707, 2708, 2709, 2710, 2711, 2712, - 2713, 2714, 2715, 2716, 2717, 2718, 2719, 2720, - 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, - 2729, 2730, 2731, 2732, 2733, 2734, 2735, 2736, - 2737, 2738, 2739, 2740, 2741, 2742, 2743, 2744, - 2745, 2746, 2747, 2748, 2749, 2750, 2751, 2752, - 2753, 2754, 2755, 2756, 2757, 2758, 2759, 2760, - 2761, 2762, 2763, 2764, 2765, 2766, 2767, 2768, - 2769, 2770, 2771, 2772, 2773, 2774, 2775, 2776, - 2777, 2778, 2779, 2780, 2781, 2782, 2783, 2784, - 2785, 2786, 2787, 2788, 2789, 2790, 2791, 2792, - 2793, 2794, 2795, 2796, 2797, 2798, 2799, 2800, - 2801, 2802, 2803, 2804, 2805, 2806, 2807, 2808, - 2809, 2810, 2811, 2812, 2813, 2814, 2815, 2816, - 2817, 2818, 2819, 2820, 2821, 2822, 2823, 2824, - 2825, 2826, 2827, 2828, 2829, 2830, 2831, 2832, - 2833, 2834, 2835, 2836, 2837, 2838, 2839, 2840, - 2841, 2842, 2843, 2844, 2845, 2846, 2847, 2848, - 2849, 2850, 2851, 2852, 2853, 2854, 2855, 2856, - 2857, 2858, 2859, 2860, 2861, 2862, 2863, 2864, - 2865, 2866, 2867, 2868, 2869, 2870, 2871, 2872, - 2873, 2874, 2875, 2876, 2877, 2878, 2879, 2880, - 2881, 2882, 2883, 2884, 2885, 2886, 2887, 2888, - 2889, 2890, 2891, 2892, 2893, 2894, 2895, 2896, - 2897, 2898, 2899, 2900, 2901, 2902, 2903, 2904, - 2905, 2906, 2907, 2908, 2909, 2910, 2911, 2912, - 2913, 2914, 2915, 2916, 2917, 2918, 2919, 2920, - 2921, 2922, 2923, 2924, 2925, 2926, 2927, 2928, - 2929, 2930, 2931, 2932, 2933, 2934, 2935, 2936, - 2937, 2938, 2939, 2940, 2941, 2942, 2943, 2944, - 2945, 2946, 2947, 2948, 2949, 2950, 2951, 2952, - 2953, 2954, 2955, 2956, 2957, 2958, 2959, 2960, - 2961, 2962, 2963, 2964, 2965, 2966, 2967, 2968, - 2969, 2970, 2971, 2972, 2973, 2974, 2975, 2976, - 2977, 2978, 2979, 2980, 2981, 2982, 2983, 2984, - 2985, 2986, 2987, 2988, 2989, 2990, 2991, 2992, - 2993, 2994, 2995, 2996, 2997, 2998, 2999, 3000, - 3001, 3002, 3003, 3004, 3005, 3006, 3007, 3008, - 3009, 3010, 3011, 3012, 3013, 3014, 3015, 3016, - 3017, 3018, 3019, 3020, 3021, 3022, 3023, 3024, - 3025, 3026, 3027, 3028, 3029, 3030, 3031, 3032, - 3033, 3034, 3035, 3036, 3037, 3038, 3039, 3040, - 3041, 3042, 3043, 3044, 3045, 3046, 3047, 3048, - 3049, 3050, 3051, 3052, 3053, 3054, 3055, 3056, - 3057, 3058, 3059, 3060, 3061, 3062, 3063, 3064, - 3065, 3066, 3067, 3068, 3069, 3070, 3071, 3072, - 3073, 3074, 3075, 3076, 3077, 3078, 3079, 3080, - 3081, 3082, 3083, 3084, 3085, 3086, 3087, 3088, - 3089, 3090, 3091, 3092, 3093, 3094, 3095, 3096, - 3097, 3098, 3099, 3100, 3101, 3102, 3103, 3104, - 3105, 3106, 3107, 3108, 3109, 3110, 3111, 3112, - 3113, 3114, 3115, 3116, 3117, 3118, 3119, 3120, - 3121, 3122, 3123, 3124, 3125, 3126, 3127, 3128, - 3129, 3130, 3131, 3132, 3133, 3134, 3135, 3136, - 3137, 3138, 3139, 3140, 3141, 3142, 3143, 3144, - 3145, 3146, 3147, 3148, 3149, 3150, 3151, 3152, - 3153, 3154, 3155, 3156, 3157, 3158, 3159, 3160, - 3161, 3162, 3163, 3164, 3165, 3166, 3167, 3168, - 3169, 3170, 3171, 3172, 3173, 3174, 3175, 3176, - 3177, 3178, 3179, 3180, 3181, 3182, 3183, 3184, - 3185, 3186, 3187, 3188, 3189, 3190, 3191, 3192, - 3193, 3194, 3195, 3196, 3197, 3198, 3199, 3200, - 3201, 3202, 3203, 3204, 3205, 3206, 3207, 3208, - 3209, 3210, 3211, 3212, 3213, 3214, 3215, 3216, - 3217, 3218, 3219, 3220, 3221, 3222, 3223, 3224, - 3225, 3226, 3227, 3228, 3229, 3230, 3231, 3232, - 3233, 3234, 3235, 3236, 3237, 3238, 3239, 3240, - 3241, 3242, 3243, 3244, 3245, 3246, 3247, 3248, - 3249, 3250, 3251, 3252, 3253, 3254, 3255, 3256, - 3257, 3258, 3259, 3260, 3261, 3262, 3263, 3264, - 3265, 3266, 3267, 3268, 3269, 3270, 3271, 3272, - 3273, 3274, 3275, 3276, 3277, 3278, 3279, 3280, - 3281, 3282, 3283, 3284, 3285, 3286, 3287, 3288, - 3289, 3290, 3291, 3292, 3293, 3294, 3295, 3296, - 3297, 3298, 3299, 3300, 3301, 3302, 3303, 3304, - 3305, 3306, 3307, 3308, 3309, 3310, 3311, 3312, - 3313, 3314, 3315, 3316, 3317, 3318, 3319, 3320, - 3321, 3322, 3323, 3324, 3325, 3326, 3327, 3328, - 3329, 3330, 3331, 3332, 3333, 3334, 3335, 3336, - 3337, 3338, 3339, 3340, 3341, 3342, 3343, 3344, - 3345, 3346, 3347, 3348, 3349, 3350, 3351, 3352, - 3353, 3354, 3355, 3356, 3357, 3358, 3359, 3360, - 3361, 3362, 3363, 3364, 3365, 3366, 3367, 3368, - 3369, 3370, 3371, 3372, 3373, 3374, 3375, 3376, - 3377, 3378, 3379, 3380, 3381, 3382, 3383, 3384, - 3385, 3386, 3387, 3388, 3389, 3390, 3391, 3392, - 3393, 3394, 3395, 3396, 3397, 3398, 3399, 3400, - 3401, 3402, 3403, 3404, 3405, 3406, 3407, 3408, - 3409, 3410, 3411, 3412, 3413, 3414, 3415, 3416, - 3417, 3418, 3419, 3420, 3421, 3422, 3423, 3424, - 3425, 3426, 3427, 3428, 3429, 3430, 3431, 3432, - 3433, 3434, 3435, 3436, 3437, 3438, 3439, 3440, - 3441, 3442, 3443, 3444, 3445, 3446, 3447, 3448, - 3449, 3450, 3451, 3452, 3453, 3454, 3455, 3456, - 3457, 3458, 3459, 3460, 3461, 3462, 3463, 3464, - 3465, 3466, 3467, 3468, 3469, 3470, 3471, 3472, - 3473, 3474, 3475, 3476, 3477, 3478, 3479, 3480, - 3481, 3482, 3483, 3484, 3485, 3486, 3487, 3488, - 3489, 3490, 3491, 3492, 3493, 3494, 3495, 3496, - 3497, 3498, 3499, 3500, 3501, 3502, 3503, 3504, - 3505, 3506, 3507, 3508, 3509, 3510, 3511, 3512, - 3513, 3514, 0 + 2668, 2669, 2670, 2671, 2672, 2673, 2674, 2675, + 2676, 2677, 2678, 2679, 2680, 2681, 2682, 2683, + 2684, 2685, 2686, 2687, 2688, 2689, 2690, 2691, + 2692, 2693, 2694, 2695, 2696, 2697, 2698, 2699, + 2700, 2701, 2702, 2703, 2704, 2705, 2706, 2707, + 2708, 2709, 2710, 2711, 2712, 2713, 2714, 2715, + 2716, 2717, 2718, 2719, 2720, 2721, 2722, 2723, + 2724, 2725, 2726, 2727, 2728, 2729, 2730, 2731, + 2732, 2733, 2734, 2735, 2736, 2737, 2738, 2739, + 2740, 2741, 2742, 2743, 2744, 2745, 2746, 2747, + 2748, 2749, 2750, 2751, 2752, 2753, 2754, 2755, + 2756, 2757, 2758, 2759, 2760, 2761, 2762, 2763, + 2764, 2765, 2766, 2767, 2768, 2769, 2770, 2771, + 2772, 2773, 2774, 2775, 2776, 2777, 2778, 2779, + 2780, 2781, 2782, 2783, 2784, 2785, 2786, 2787, + 2788, 2789, 2790, 2791, 2792, 2793, 2794, 2795, + 2796, 2797, 2798, 2799, 2800, 2801, 2802, 2803, + 2804, 2805, 2806, 2807, 2808, 2809, 2810, 2811, + 2812, 2813, 2814, 2815, 2816, 2817, 2818, 2819, + 2820, 2821, 2822, 2823, 2824, 2825, 2826, 2827, + 2828, 2829, 2830, 2831, 2832, 2833, 2834, 2835, + 2836, 2837, 2838, 2839, 2840, 2841, 2842, 2843, + 2844, 2845, 2846, 2847, 2848, 2849, 2850, 2851, + 2852, 2853, 2854, 2855, 2856, 2857, 2858, 2859, + 2860, 2861, 2862, 2863, 2864, 2865, 2866, 2867, + 2868, 2869, 2870, 2871, 2872, 2873, 2874, 2875, + 2876, 2877, 2878, 2879, 2880, 2881, 2882, 2883, + 2884, 2885, 2886, 2887, 2888, 2889, 2890, 2891, + 2892, 2893, 2894, 2895, 2896, 2897, 2898, 2899, + 2900, 2901, 2902, 2903, 2904, 2905, 2906, 2907, + 2908, 2909, 2910, 2911, 2912, 2913, 2914, 2915, + 2916, 2917, 2918, 2919, 2920, 2921, 2922, 2923, + 2924, 2925, 2926, 2927, 2928, 2929, 2930, 2931, + 2932, 2933, 2934, 2935, 2936, 2937, 2938, 2939, + 2940, 2941, 2942, 2943, 2944, 2945, 2946, 2947, + 2948, 2949, 2950, 2951, 2952, 2953, 2954, 2955, + 2956, 2957, 2958, 2959, 2960, 2961, 2962, 2963, + 2964, 2965, 2966, 2967, 2968, 2969, 2970, 2971, + 2972, 2973, 2974, 2975, 2976, 2977, 2978, 2979, + 2980, 2981, 2982, 2983, 2984, 2985, 2986, 2987, + 2988, 2989, 2990, 2991, 2992, 2993, 2994, 2995, + 2996, 2997, 2998, 2999, 3000, 3001, 3002, 3003, + 3004, 3005, 3006, 3007, 3008, 3009, 3010, 3011, + 3012, 3013, 3014, 3015, 3016, 3017, 3018, 3019, + 3020, 3021, 3022, 3023, 3024, 3025, 3026, 3027, + 3028, 3029, 3030, 3031, 3032, 3033, 3034, 3035, + 3036, 3037, 3038, 3039, 3040, 3041, 3042, 3043, + 3044, 3045, 3046, 3047, 3048, 3049, 3050, 3051, + 3052, 3053, 3054, 3055, 3056, 3057, 3058, 3059, + 3060, 3061, 3062, 3063, 3064, 3065, 3066, 3067, + 3068, 3069, 3070, 3071, 3072, 3073, 3074, 3075, + 3076, 3077, 3078, 3079, 3080, 3081, 3082, 3083, + 3084, 3085, 3086, 3087, 3088, 3089, 3090, 3091, + 3092, 3093, 3094, 3095, 3096, 3097, 3098, 3099, + 3100, 3101, 3102, 3103, 3104, 3105, 3106, 3107, + 3108, 3109, 3110, 3111, 3112, 3113, 3114, 3115, + 3116, 3117, 3118, 3119, 3120, 3121, 3122, 3123, + 3124, 3125, 3126, 3127, 3128, 3129, 3130, 3131, + 3132, 3133, 3134, 3135, 3136, 3137, 3138, 3139, + 3140, 3141, 3142, 3143, 3144, 3145, 3146, 3147, + 3148, 3149, 3150, 3151, 3152, 3153, 3154, 3155, + 3156, 3157, 3158, 3159, 3160, 3161, 3162, 3163, + 3164, 3165, 3166, 3167, 3168, 3169, 3170, 3171, + 3172, 3173, 3174, 3175, 3176, 3177, 3178, 3179, + 3180, 3181, 3182, 3183, 3184, 3185, 3186, 3187, + 3188, 3189, 3190, 3191, 3192, 3193, 3194, 3195, + 3196, 3197, 3198, 3199, 3200, 3201, 3202, 3203, + 3204, 3205, 3206, 3207, 3208, 3209, 3210, 3211, + 3212, 3213, 3214, 3215, 3216, 3217, 3218, 3219, + 3220, 3221, 3222, 3223, 3224, 3225, 3226, 3227, + 3228, 3229, 3230, 3231, 3232, 3233, 3234, 3235, + 3236, 3237, 3238, 3239, 3240, 3241, 3242, 3243, + 3244, 3245, 3246, 3247, 3248, 3249, 3250, 3251, + 3252, 3253, 3254, 3255, 3256, 3257, 3258, 3259, + 3260, 3261, 3262, 3263, 3264, 3265, 3266, 3267, + 3268, 3269, 3270, 3271, 3272, 3273, 3274, 3275, + 3276, 3277, 3278, 3279, 3280, 3281, 3282, 3283, + 3284, 3285, 3286, 3287, 3288, 3289, 3290, 3291, + 3292, 3293, 3294, 3295, 3296, 3297, 3298, 3299, + 3300, 3301, 3302, 3303, 3304, 3305, 3306, 3307, + 3308, 3309, 3310, 3311, 3312, 3313, 3314, 3315, + 3316, 3317, 3318, 3319, 3320, 3321, 3322, 3323, + 3324, 3325, 3326, 3327, 3328, 3329, 3330, 3331, + 3332, 3333, 3334, 3335, 3336, 3337, 3338, 3339, + 3340, 3341, 3342, 3343, 3344, 3345, 3346, 3347, + 3348, 3349, 3350, 3351, 3352, 3353, 3354, 3355, + 3356, 3357, 3358, 3359, 3360, 3361, 3362, 3363, + 3364, 3365, 3366, 3367, 3368, 3369, 3370, 3371, + 3372, 3373, 3374, 3375, 3376, 3377, 3378, 3379, + 3380, 3381, 3382, 3383, 3384, 3385, 3386, 3387, + 3388, 3389, 3390, 3391, 3392, 3393, 3394, 3395, + 3396, 3397, 3398, 3399, 3400, 3401, 3402, 3403, + 3404, 3405, 3406, 3407, 3408, 3409, 3410, 3411, + 3412, 3413, 3414, 3415, 3416, 3417, 3418, 3419, + 3420, 3421, 3422, 3423, 3424, 3425, 3426, 3427, + 3428, 3429, 3430, 3431, 3432, 3433, 3434, 3435, + 3436, 3437, 3438, 3439, 3440, 3441, 3442, 3443, + 3444, 3445, 3446, 3447, 3448, 3449, 3450, 3451, + 3452, 3453, 3454, 3455, 3456, 3457, 3458, 3459, + 3460, 3461, 3462, 3463, 3464, 3465, 3466, 3467, + 3468, 3469, 3470, 3471, 3472, 3473, 3474, 3475, + 3476, 3477, 3478, 3479, 3480, 3481, 3482, 3483, + 3484, 3485, 3486, 3487, 3488, 3489, 3490, 3491, + 3492, 3493, 3494, 3495, 3496, 3497, 3498, 3499, + 3500, 3501, 3502, 3503, 3504, 3505, 3506, 3507, + 3508, 3509, 3510, 3511, 3512, 3513, 3514, 3515, + 3516, 3517, 3518, 0 }; static const int regex_start = 746; @@ -4108,15 +4109,15 @@ static const short _regex_actions[] = { static const int regex_en_readBracedUCP = 559; static const int regex_en_readUCPSingle = 818; static const int regex_en_charClassGuts = 819; - static const int regex_en_readClass = 836; - static const int regex_en_readQuotedLiteral = 838; - static const int regex_en_readQuotedClass = 843; - static const int regex_en_readComment = 848; - static const int regex_en_readNewlineTerminatedComment = 849; + static const int regex_en_readClass = 837; + static const int regex_en_readQuotedLiteral = 839; + static const int regex_en_readQuotedClass = 844; + static const int regex_en_readComment = 849; + static const int regex_en_readNewlineTerminatedComment = 850; static const int regex_en_main = 746; -#line 1911 "Parser.rl" +#line 1918 "Parser.rl" /** \brief Main parser call, returns root Component or nullptr. */ @@ -4189,7 +4190,7 @@ static const short _regex_actions[] = { try { // Embed the Ragel machine here -#line 4187 "Parser.cpp" +#line 4188 "Parser.cpp" { cs = (int)regex_start; top = 0; @@ -4198,17 +4199,17 @@ static const short _regex_actions[] = { act = 0; } -#line 1982 "Parser.rl" +#line 1989 "Parser.rl" -#line 4196 "Parser.cpp" +#line 4197 "Parser.cpp" { int _cpc; int _klen; const signed char * _ckeys; unsigned int _trans = 0; unsigned int _cond = 0; - const char * _keys; + const unsigned char * _keys; const short * _acts; unsigned int _nacts; _resume: {} @@ -4224,7 +4225,7 @@ static const short _regex_actions[] = { #line 1 "NONE" {ts = p;}} -#line 4218 "Parser.cpp" +#line 4219 "Parser.cpp" break; } @@ -4244,9 +4245,9 @@ static const short _regex_actions[] = { _klen = (int)_regex_single_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + _klen - 1; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + _klen - 1; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _keys += _klen; @@ -4268,9 +4269,9 @@ static const short _regex_actions[] = { _klen = (int)_regex_range_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + (_klen<<1) - 2; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + (_klen<<1) - 2; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _trans += (unsigned int)_klen; @@ -4300,19 +4301,19 @@ static const short _regex_actions[] = { case 0: { if ( ( mode.utf8 ) -#line 4293 "Parser.cpp" +#line 4294 "Parser.cpp" ) _cpc += 1; break; } case 1: { if ( ( mode.ignore_space ) -#line 4298 "Parser.cpp" +#line 4299 "Parser.cpp" ) _cpc += 1; break; } case 2: { if ( ( inCharClassEarly ) -#line 4303 "Parser.cpp" +#line 4304 "Parser.cpp" ) _cpc += 1; break; } @@ -4323,7 +4324,7 @@ static const short _regex_actions[] = { const signed char *_mid; while ( 1 ) { if ( _upper < _lower ) { - _cond = 3561; + _cond = 3565; break; } @@ -4352,159 +4353,159 @@ static const short _regex_actions[] = { { case 0: { { -#line 285 "Parser.rl" +#line 286 "Parser.rl" label.clear();} -#line 4345 "Parser.cpp" +#line 4346 "Parser.cpp" break; } case 1: { { -#line 286 "Parser.rl" +#line 287 "Parser.rl" label.push_back((( (*( p)))));} -#line 4353 "Parser.cpp" +#line 4354 "Parser.cpp" break; } case 2: { { -#line 287 "Parser.rl" +#line 288 "Parser.rl" octAccumulator = 0;} -#line 4361 "Parser.cpp" +#line 4362 "Parser.cpp" break; } case 3: { { -#line 288 "Parser.rl" +#line 289 "Parser.rl" accumulator = 0;} -#line 4369 "Parser.cpp" +#line 4370 "Parser.cpp" break; } case 4: { { -#line 289 "Parser.rl" +#line 290 "Parser.rl" octAccumulator = 0; pushOct(&octAccumulator, (( (*( p))))); } -#line 4380 "Parser.cpp" +#line 4381 "Parser.cpp" break; } case 5: { { -#line 293 "Parser.rl" +#line 294 "Parser.rl" accumulator = 0; pushDec(&accumulator, (( (*( p))))); } -#line 4391 "Parser.cpp" +#line 4392 "Parser.cpp" break; } case 6: { { -#line 297 "Parser.rl" +#line 298 "Parser.rl" repeatN = 0; repeatM = 0; } -#line 4399 "Parser.cpp" +#line 4400 "Parser.cpp" break; } case 7: { { -#line 298 "Parser.rl" +#line 299 "Parser.rl" pushDec(&repeatN, (( (*( p))))); } -#line 4407 "Parser.cpp" +#line 4408 "Parser.cpp" break; } case 8: { { -#line 299 "Parser.rl" +#line 300 "Parser.rl" pushDec(&repeatM, (( (*( p))))); } -#line 4415 "Parser.cpp" +#line 4416 "Parser.cpp" break; } case 9: { { -#line 300 "Parser.rl" +#line 301 "Parser.rl" pushOct(&octAccumulator, (( (*( p))))); } -#line 4423 "Parser.cpp" +#line 4424 "Parser.cpp" break; } case 10: { { -#line 301 "Parser.rl" +#line 302 "Parser.rl" pushDec(&accumulator, (( (*( p))))); } -#line 4431 "Parser.cpp" +#line 4432 "Parser.cpp" break; } case 11: { { -#line 302 "Parser.rl" +#line 303 "Parser.rl" accumulator *= 16; accumulator += (( (*( p)))) - '0'; } -#line 4442 "Parser.cpp" +#line 4443 "Parser.cpp" break; } case 12: { { -#line 306 "Parser.rl" +#line 307 "Parser.rl" accumulator *= 16; accumulator += 10 + (( (*( p)))) - 'a'; } -#line 4453 "Parser.cpp" +#line 4454 "Parser.cpp" break; } case 13: { { -#line 310 "Parser.rl" +#line 311 "Parser.rl" accumulator *= 16; accumulator += 10 + (( (*( p)))) - 'A'; } -#line 4464 "Parser.cpp" +#line 4465 "Parser.cpp" break; } case 14: { { -#line 430 "Parser.rl" +#line 431 "Parser.rl" newMode = mode; } -#line 4474 "Parser.cpp" +#line 4475 "Parser.cpp" break; } case 15: { { -#line 437 "Parser.rl" +#line 438 "Parser.rl" switch ((( (*( p))))) { case 'i': @@ -4525,13 +4526,13 @@ static const short _regex_actions[] = { } } -#line 4500 "Parser.cpp" +#line 4501 "Parser.cpp" break; } case 16: { { -#line 456 "Parser.rl" +#line 457 "Parser.rl" switch ((( (*( p))))) { case 'i': @@ -4552,42 +4553,42 @@ static const short _regex_actions[] = { } } -#line 4526 "Parser.cpp" +#line 4527 "Parser.cpp" break; } case 17: { { -#line 510 "Parser.rl" +#line 511 "Parser.rl" repeatM = repeatN;} -#line 4534 "Parser.cpp" +#line 4535 "Parser.cpp" break; } case 18: { { -#line 510 "Parser.rl" +#line 511 "Parser.rl" repeatM = ComponentRepeat::NoLimit;} -#line 4542 "Parser.cpp" +#line 4543 "Parser.cpp" break; } case 19: { { -#line 722 "Parser.rl" +#line 723 "Parser.rl" negated = !negated; } -#line 4550 "Parser.cpp" +#line 4551 "Parser.cpp" break; } case 20: { { -#line 723 "Parser.rl" +#line 724 "Parser.rl" {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -4596,30 +4597,30 @@ static const short _regex_actions[] = { } stack[top] = cs; top += 1;cs = 790;goto _again;}} -#line 4566 "Parser.cpp" +#line 4567 "Parser.cpp" break; } case 21: { { -#line 724 "Parser.rl" +#line 725 "Parser.rl" if (!inCharClass) { // not inside [..] currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } -#line 4579 "Parser.cpp" +#line 4580 "Parser.cpp" break; } case 22: { { -#line 730 "Parser.rl" +#line 731 "Parser.rl" throw LocatedParseError("Malformed property"); } -#line 4587 "Parser.cpp" +#line 4588 "Parser.cpp" break; } @@ -4628,60 +4629,60 @@ static const short _regex_actions[] = { #line 1 "NONE" {te = p+1;}} -#line 4595 "Parser.cpp" +#line 4596 "Parser.cpp" break; } case 26: { { -#line 550 "Parser.rl" +#line 551 "Parser.rl" {te = p+1;{ -#line 550 "Parser.rl" +#line 551 "Parser.rl" throw LocatedParseError("(*UTF8) must be at start of " "expression, encountered"); } }} -#line 4609 "Parser.cpp" +#line 4610 "Parser.cpp" break; } case 27: { { -#line 554 "Parser.rl" +#line 555 "Parser.rl" {te = p+1;{ -#line 554 "Parser.rl" +#line 555 "Parser.rl" throw LocatedParseError("(*UTF) must be at start of " "expression, encountered"); } }} -#line 4623 "Parser.cpp" +#line 4624 "Parser.cpp" break; } case 28: { { -#line 558 "Parser.rl" +#line 559 "Parser.rl" {te = p+1;{ -#line 558 "Parser.rl" +#line 559 "Parser.rl" throw LocatedParseError("(*UCP) must be at start of " "expression, encountered"); } }} -#line 4637 "Parser.cpp" +#line 4638 "Parser.cpp" break; } case 29: { { -#line 564 "Parser.rl" +#line 565 "Parser.rl" {te = p+1;{ -#line 564 "Parser.rl" +#line 565 "Parser.rl" ParseMode temp_mode; assert(ts - 2 >= ptr); // parser needs the '(*' at the start too. @@ -4691,2626 +4692,2626 @@ static const short _regex_actions[] = { } }} -#line 4654 "Parser.cpp" +#line 4655 "Parser.cpp" break; } case 30: { { -#line 571 "Parser.rl" +#line 572 "Parser.rl" {te = p+1;{ -#line 571 "Parser.rl" +#line 572 "Parser.rl" throw LocatedParseError("Unknown control verb"); } }} -#line 4667 "Parser.cpp" +#line 4668 "Parser.cpp" break; } case 31: { { -#line 571 "Parser.rl" +#line 572 "Parser.rl" {te = p;p = p - 1;{ -#line 571 "Parser.rl" +#line 572 "Parser.rl" throw LocatedParseError("Unknown control verb"); } }} -#line 4680 "Parser.cpp" +#line 4681 "Parser.cpp" break; } case 32: { { -#line 571 "Parser.rl" +#line 572 "Parser.rl" {p = ((te))-1; { -#line 571 "Parser.rl" +#line 572 "Parser.rl" throw LocatedParseError("Unknown control verb"); } }} -#line 4694 "Parser.cpp" +#line 4695 "Parser.cpp" break; } case 33: { { -#line 581 "Parser.rl" +#line 582 "Parser.rl" {te = p+1;{ -#line 581 "Parser.rl" +#line 582 "Parser.rl" currentCls->add(CLASS_UCP_CC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4705 "Parser.cpp" +#line 4706 "Parser.cpp" break; } case 34: { { -#line 582 "Parser.rl" +#line 583 "Parser.rl" {te = p+1;{ -#line 582 "Parser.rl" +#line 583 "Parser.rl" currentCls->add(CLASS_UCP_CF, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4716 "Parser.cpp" +#line 4717 "Parser.cpp" break; } case 35: { { -#line 583 "Parser.rl" +#line 584 "Parser.rl" {te = p+1;{ -#line 583 "Parser.rl" +#line 584 "Parser.rl" currentCls->add(CLASS_UCP_CN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4727 "Parser.cpp" +#line 4728 "Parser.cpp" break; } case 36: { { -#line 585 "Parser.rl" +#line 586 "Parser.rl" {te = p+1;{ -#line 585 "Parser.rl" +#line 586 "Parser.rl" currentCls->add(CLASS_UCP_CS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4738 "Parser.cpp" +#line 4739 "Parser.cpp" break; } case 37: { { -#line 587 "Parser.rl" +#line 588 "Parser.rl" {te = p+1;{ -#line 587 "Parser.rl" +#line 588 "Parser.rl" currentCls->add(CLASS_UCP_LL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4749 "Parser.cpp" +#line 4750 "Parser.cpp" break; } case 38: { { -#line 588 "Parser.rl" +#line 589 "Parser.rl" {te = p+1;{ -#line 588 "Parser.rl" +#line 589 "Parser.rl" currentCls->add(CLASS_UCP_LM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4760 "Parser.cpp" +#line 4761 "Parser.cpp" break; } case 39: { { -#line 589 "Parser.rl" +#line 590 "Parser.rl" {te = p+1;{ -#line 589 "Parser.rl" +#line 590 "Parser.rl" currentCls->add(CLASS_UCP_LO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4771 "Parser.cpp" +#line 4772 "Parser.cpp" break; } case 40: { { -#line 590 "Parser.rl" +#line 591 "Parser.rl" {te = p+1;{ -#line 590 "Parser.rl" +#line 591 "Parser.rl" currentCls->add(CLASS_UCP_LT, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4782 "Parser.cpp" +#line 4783 "Parser.cpp" break; } case 41: { { -#line 591 "Parser.rl" +#line 592 "Parser.rl" {te = p+1;{ -#line 591 "Parser.rl" +#line 592 "Parser.rl" currentCls->add(CLASS_UCP_LU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4793 "Parser.cpp" +#line 4794 "Parser.cpp" break; } case 42: { { -#line 592 "Parser.rl" +#line 593 "Parser.rl" {te = p+1;{ -#line 592 "Parser.rl" +#line 593 "Parser.rl" currentCls->add(CLASS_UCP_L_AND, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4804 "Parser.cpp" +#line 4805 "Parser.cpp" break; } case 43: { { -#line 594 "Parser.rl" +#line 595 "Parser.rl" {te = p+1;{ -#line 594 "Parser.rl" +#line 595 "Parser.rl" currentCls->add(CLASS_UCP_MC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4815 "Parser.cpp" +#line 4816 "Parser.cpp" break; } case 44: { { -#line 596 "Parser.rl" +#line 597 "Parser.rl" {te = p+1;{ -#line 596 "Parser.rl" +#line 597 "Parser.rl" currentCls->add(CLASS_UCP_MN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4826 "Parser.cpp" +#line 4827 "Parser.cpp" break; } case 45: { { -#line 598 "Parser.rl" +#line 599 "Parser.rl" {te = p+1;{ -#line 598 "Parser.rl" +#line 599 "Parser.rl" currentCls->add(CLASS_UCP_ND, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4837 "Parser.cpp" +#line 4838 "Parser.cpp" break; } case 46: { { -#line 599 "Parser.rl" +#line 600 "Parser.rl" {te = p+1;{ -#line 599 "Parser.rl" +#line 600 "Parser.rl" currentCls->add(CLASS_UCP_NL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4848 "Parser.cpp" +#line 4849 "Parser.cpp" break; } case 47: { { -#line 600 "Parser.rl" +#line 601 "Parser.rl" {te = p+1;{ -#line 600 "Parser.rl" +#line 601 "Parser.rl" currentCls->add(CLASS_UCP_NO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4859 "Parser.cpp" +#line 4860 "Parser.cpp" break; } case 48: { { -#line 602 "Parser.rl" +#line 603 "Parser.rl" {te = p+1;{ -#line 602 "Parser.rl" +#line 603 "Parser.rl" currentCls->add(CLASS_UCP_PC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4870 "Parser.cpp" +#line 4871 "Parser.cpp" break; } case 49: { { -#line 603 "Parser.rl" +#line 604 "Parser.rl" {te = p+1;{ -#line 603 "Parser.rl" +#line 604 "Parser.rl" currentCls->add(CLASS_UCP_PD, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4881 "Parser.cpp" +#line 4882 "Parser.cpp" break; } case 50: { { -#line 604 "Parser.rl" +#line 605 "Parser.rl" {te = p+1;{ -#line 604 "Parser.rl" +#line 605 "Parser.rl" currentCls->add(CLASS_UCP_PE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4892 "Parser.cpp" +#line 4893 "Parser.cpp" break; } case 51: { { -#line 605 "Parser.rl" +#line 606 "Parser.rl" {te = p+1;{ -#line 605 "Parser.rl" +#line 606 "Parser.rl" currentCls->add(CLASS_UCP_PF, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4903 "Parser.cpp" +#line 4904 "Parser.cpp" break; } case 52: { { -#line 606 "Parser.rl" +#line 607 "Parser.rl" {te = p+1;{ -#line 606 "Parser.rl" +#line 607 "Parser.rl" currentCls->add(CLASS_UCP_PI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4914 "Parser.cpp" +#line 4915 "Parser.cpp" break; } case 53: { { -#line 607 "Parser.rl" +#line 608 "Parser.rl" {te = p+1;{ -#line 607 "Parser.rl" +#line 608 "Parser.rl" currentCls->add(CLASS_UCP_PO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4925 "Parser.cpp" +#line 4926 "Parser.cpp" break; } case 54: { { -#line 608 "Parser.rl" +#line 609 "Parser.rl" {te = p+1;{ -#line 608 "Parser.rl" +#line 609 "Parser.rl" currentCls->add(CLASS_UCP_PS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4936 "Parser.cpp" +#line 4937 "Parser.cpp" break; } case 55: { { -#line 610 "Parser.rl" +#line 611 "Parser.rl" {te = p+1;{ -#line 610 "Parser.rl" +#line 611 "Parser.rl" currentCls->add(CLASS_UCP_SC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4947 "Parser.cpp" +#line 4948 "Parser.cpp" break; } case 56: { { -#line 611 "Parser.rl" +#line 612 "Parser.rl" {te = p+1;{ -#line 611 "Parser.rl" +#line 612 "Parser.rl" currentCls->add(CLASS_UCP_SK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4958 "Parser.cpp" +#line 4959 "Parser.cpp" break; } case 57: { { -#line 612 "Parser.rl" +#line 613 "Parser.rl" {te = p+1;{ -#line 612 "Parser.rl" +#line 613 "Parser.rl" currentCls->add(CLASS_UCP_SM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4969 "Parser.cpp" +#line 4970 "Parser.cpp" break; } case 58: { { -#line 613 "Parser.rl" +#line 614 "Parser.rl" {te = p+1;{ -#line 613 "Parser.rl" +#line 614 "Parser.rl" currentCls->add(CLASS_UCP_SO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4980 "Parser.cpp" +#line 4981 "Parser.cpp" break; } case 59: { { -#line 615 "Parser.rl" +#line 616 "Parser.rl" {te = p+1;{ -#line 615 "Parser.rl" +#line 616 "Parser.rl" currentCls->add(CLASS_UCP_ZL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4991 "Parser.cpp" +#line 4992 "Parser.cpp" break; } case 60: { { -#line 616 "Parser.rl" +#line 617 "Parser.rl" {te = p+1;{ -#line 616 "Parser.rl" +#line 617 "Parser.rl" currentCls->add(CLASS_UCP_ZP, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5002 "Parser.cpp" +#line 5003 "Parser.cpp" break; } case 61: { { -#line 617 "Parser.rl" +#line 618 "Parser.rl" {te = p+1;{ -#line 617 "Parser.rl" +#line 618 "Parser.rl" currentCls->add(CLASS_UCP_ZS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5013 "Parser.cpp" +#line 5014 "Parser.cpp" break; } case 62: { { -#line 618 "Parser.rl" +#line 619 "Parser.rl" {te = p+1;{ -#line 618 "Parser.rl" +#line 619 "Parser.rl" currentCls->add(CLASS_UCP_XAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5024 "Parser.cpp" +#line 5025 "Parser.cpp" break; } case 63: { { -#line 619 "Parser.rl" +#line 620 "Parser.rl" {te = p+1;{ -#line 619 "Parser.rl" +#line 620 "Parser.rl" currentCls->add(CLASS_UCP_XPS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5035 "Parser.cpp" +#line 5036 "Parser.cpp" break; } case 64: { { -#line 620 "Parser.rl" +#line 621 "Parser.rl" {te = p+1;{ -#line 620 "Parser.rl" +#line 621 "Parser.rl" currentCls->add(CLASS_UCP_XSP, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5046 "Parser.cpp" +#line 5047 "Parser.cpp" break; } case 65: { { -#line 621 "Parser.rl" +#line 622 "Parser.rl" {te = p+1;{ -#line 621 "Parser.rl" +#line 622 "Parser.rl" currentCls->add(CLASS_UCP_XWD, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5057 "Parser.cpp" +#line 5058 "Parser.cpp" break; } case 66: { { -#line 622 "Parser.rl" +#line 623 "Parser.rl" {te = p+1;{ -#line 622 "Parser.rl" +#line 623 "Parser.rl" currentCls->add(CLASS_SCRIPT_ARABIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5068 "Parser.cpp" +#line 5069 "Parser.cpp" break; } case 67: { { -#line 623 "Parser.rl" +#line 624 "Parser.rl" {te = p+1;{ -#line 623 "Parser.rl" +#line 624 "Parser.rl" currentCls->add(CLASS_SCRIPT_ARMENIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5079 "Parser.cpp" +#line 5080 "Parser.cpp" break; } case 68: { { -#line 624 "Parser.rl" +#line 625 "Parser.rl" {te = p+1;{ -#line 624 "Parser.rl" +#line 625 "Parser.rl" currentCls->add(CLASS_SCRIPT_AVESTAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5090 "Parser.cpp" +#line 5091 "Parser.cpp" break; } case 69: { { -#line 625 "Parser.rl" +#line 626 "Parser.rl" {te = p+1;{ -#line 625 "Parser.rl" +#line 626 "Parser.rl" currentCls->add(CLASS_SCRIPT_BALINESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5101 "Parser.cpp" +#line 5102 "Parser.cpp" break; } case 70: { { -#line 626 "Parser.rl" +#line 627 "Parser.rl" {te = p+1;{ -#line 626 "Parser.rl" +#line 627 "Parser.rl" currentCls->add(CLASS_SCRIPT_BAMUM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5112 "Parser.cpp" +#line 5113 "Parser.cpp" break; } case 71: { { -#line 627 "Parser.rl" +#line 628 "Parser.rl" {te = p+1;{ -#line 627 "Parser.rl" +#line 628 "Parser.rl" currentCls->add(CLASS_SCRIPT_BATAK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5123 "Parser.cpp" +#line 5124 "Parser.cpp" break; } case 72: { { -#line 628 "Parser.rl" +#line 629 "Parser.rl" {te = p+1;{ -#line 628 "Parser.rl" +#line 629 "Parser.rl" currentCls->add(CLASS_SCRIPT_BENGALI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5134 "Parser.cpp" +#line 5135 "Parser.cpp" break; } case 73: { { -#line 629 "Parser.rl" +#line 630 "Parser.rl" {te = p+1;{ -#line 629 "Parser.rl" +#line 630 "Parser.rl" currentCls->add(CLASS_SCRIPT_BOPOMOFO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5145 "Parser.cpp" +#line 5146 "Parser.cpp" break; } case 74: { { -#line 630 "Parser.rl" +#line 631 "Parser.rl" {te = p+1;{ -#line 630 "Parser.rl" +#line 631 "Parser.rl" currentCls->add(CLASS_SCRIPT_BRAHMI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5156 "Parser.cpp" +#line 5157 "Parser.cpp" break; } case 75: { { -#line 631 "Parser.rl" +#line 632 "Parser.rl" {te = p+1;{ -#line 631 "Parser.rl" +#line 632 "Parser.rl" currentCls->add(CLASS_SCRIPT_BRAILLE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5167 "Parser.cpp" +#line 5168 "Parser.cpp" break; } case 76: { { -#line 632 "Parser.rl" +#line 633 "Parser.rl" {te = p+1;{ -#line 632 "Parser.rl" +#line 633 "Parser.rl" currentCls->add(CLASS_SCRIPT_BUGINESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5178 "Parser.cpp" +#line 5179 "Parser.cpp" break; } case 77: { { -#line 633 "Parser.rl" +#line 634 "Parser.rl" {te = p+1;{ -#line 633 "Parser.rl" +#line 634 "Parser.rl" currentCls->add(CLASS_SCRIPT_BUHID, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5189 "Parser.cpp" +#line 5190 "Parser.cpp" break; } case 78: { { -#line 634 "Parser.rl" +#line 635 "Parser.rl" {te = p+1;{ -#line 634 "Parser.rl" +#line 635 "Parser.rl" currentCls->add(CLASS_SCRIPT_CANADIAN_ABORIGINAL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5200 "Parser.cpp" +#line 5201 "Parser.cpp" break; } case 79: { { -#line 635 "Parser.rl" +#line 636 "Parser.rl" {te = p+1;{ -#line 635 "Parser.rl" +#line 636 "Parser.rl" currentCls->add(CLASS_SCRIPT_CARIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5211 "Parser.cpp" +#line 5212 "Parser.cpp" break; } case 80: { { -#line 636 "Parser.rl" +#line 637 "Parser.rl" {te = p+1;{ -#line 636 "Parser.rl" +#line 637 "Parser.rl" currentCls->add(CLASS_SCRIPT_CHAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5222 "Parser.cpp" +#line 5223 "Parser.cpp" break; } case 81: { { -#line 637 "Parser.rl" +#line 638 "Parser.rl" {te = p+1;{ -#line 637 "Parser.rl" +#line 638 "Parser.rl" currentCls->add(CLASS_SCRIPT_CHEROKEE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5233 "Parser.cpp" +#line 5234 "Parser.cpp" break; } case 82: { { -#line 638 "Parser.rl" +#line 639 "Parser.rl" {te = p+1;{ -#line 638 "Parser.rl" +#line 639 "Parser.rl" currentCls->add(CLASS_SCRIPT_COMMON, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5244 "Parser.cpp" +#line 5245 "Parser.cpp" break; } case 83: { { -#line 639 "Parser.rl" +#line 640 "Parser.rl" {te = p+1;{ -#line 639 "Parser.rl" +#line 640 "Parser.rl" currentCls->add(CLASS_SCRIPT_COPTIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5255 "Parser.cpp" +#line 5256 "Parser.cpp" break; } case 84: { { -#line 640 "Parser.rl" +#line 641 "Parser.rl" {te = p+1;{ -#line 640 "Parser.rl" +#line 641 "Parser.rl" currentCls->add(CLASS_SCRIPT_CUNEIFORM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5266 "Parser.cpp" +#line 5267 "Parser.cpp" break; } case 85: { { -#line 641 "Parser.rl" +#line 642 "Parser.rl" {te = p+1;{ -#line 641 "Parser.rl" +#line 642 "Parser.rl" currentCls->add(CLASS_SCRIPT_CYPRIOT, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5277 "Parser.cpp" +#line 5278 "Parser.cpp" break; } case 86: { { -#line 642 "Parser.rl" +#line 643 "Parser.rl" {te = p+1;{ -#line 642 "Parser.rl" +#line 643 "Parser.rl" currentCls->add(CLASS_SCRIPT_CYRILLIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5288 "Parser.cpp" +#line 5289 "Parser.cpp" break; } case 87: { { -#line 643 "Parser.rl" +#line 644 "Parser.rl" {te = p+1;{ -#line 643 "Parser.rl" +#line 644 "Parser.rl" currentCls->add(CLASS_SCRIPT_DESERET, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5299 "Parser.cpp" +#line 5300 "Parser.cpp" break; } case 88: { { -#line 644 "Parser.rl" +#line 645 "Parser.rl" {te = p+1;{ -#line 644 "Parser.rl" +#line 645 "Parser.rl" currentCls->add(CLASS_SCRIPT_DEVANAGARI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5310 "Parser.cpp" +#line 5311 "Parser.cpp" break; } case 89: { { -#line 645 "Parser.rl" +#line 646 "Parser.rl" {te = p+1;{ -#line 645 "Parser.rl" +#line 646 "Parser.rl" currentCls->add(CLASS_SCRIPT_EGYPTIAN_HIEROGLYPHS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5321 "Parser.cpp" +#line 5322 "Parser.cpp" break; } case 90: { { -#line 646 "Parser.rl" +#line 647 "Parser.rl" {te = p+1;{ -#line 646 "Parser.rl" +#line 647 "Parser.rl" currentCls->add(CLASS_SCRIPT_ETHIOPIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5332 "Parser.cpp" +#line 5333 "Parser.cpp" break; } case 91: { { -#line 647 "Parser.rl" +#line 648 "Parser.rl" {te = p+1;{ -#line 647 "Parser.rl" +#line 648 "Parser.rl" currentCls->add(CLASS_SCRIPT_GEORGIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5343 "Parser.cpp" +#line 5344 "Parser.cpp" break; } case 92: { { -#line 648 "Parser.rl" +#line 649 "Parser.rl" {te = p+1;{ -#line 648 "Parser.rl" +#line 649 "Parser.rl" currentCls->add(CLASS_SCRIPT_GLAGOLITIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5354 "Parser.cpp" +#line 5355 "Parser.cpp" break; } case 93: { { -#line 649 "Parser.rl" +#line 650 "Parser.rl" {te = p+1;{ -#line 649 "Parser.rl" +#line 650 "Parser.rl" currentCls->add(CLASS_SCRIPT_GOTHIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5365 "Parser.cpp" +#line 5366 "Parser.cpp" break; } case 94: { { -#line 650 "Parser.rl" +#line 651 "Parser.rl" {te = p+1;{ -#line 650 "Parser.rl" +#line 651 "Parser.rl" currentCls->add(CLASS_SCRIPT_GREEK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5376 "Parser.cpp" +#line 5377 "Parser.cpp" break; } case 95: { { -#line 651 "Parser.rl" +#line 652 "Parser.rl" {te = p+1;{ -#line 651 "Parser.rl" +#line 652 "Parser.rl" currentCls->add(CLASS_SCRIPT_GUJARATI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5387 "Parser.cpp" +#line 5388 "Parser.cpp" break; } case 96: { { -#line 652 "Parser.rl" +#line 653 "Parser.rl" {te = p+1;{ -#line 652 "Parser.rl" +#line 653 "Parser.rl" currentCls->add(CLASS_SCRIPT_GURMUKHI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5398 "Parser.cpp" +#line 5399 "Parser.cpp" break; } case 97: { { -#line 654 "Parser.rl" +#line 655 "Parser.rl" {te = p+1;{ -#line 654 "Parser.rl" +#line 655 "Parser.rl" currentCls->add(CLASS_SCRIPT_HANGUL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5409 "Parser.cpp" +#line 5410 "Parser.cpp" break; } case 98: { { -#line 655 "Parser.rl" +#line 656 "Parser.rl" {te = p+1;{ -#line 655 "Parser.rl" +#line 656 "Parser.rl" currentCls->add(CLASS_SCRIPT_HANUNOO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5420 "Parser.cpp" +#line 5421 "Parser.cpp" break; } case 99: { { -#line 656 "Parser.rl" +#line 657 "Parser.rl" {te = p+1;{ -#line 656 "Parser.rl" +#line 657 "Parser.rl" currentCls->add(CLASS_SCRIPT_HEBREW, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5431 "Parser.cpp" +#line 5432 "Parser.cpp" break; } case 100: { { -#line 657 "Parser.rl" +#line 658 "Parser.rl" {te = p+1;{ -#line 657 "Parser.rl" +#line 658 "Parser.rl" currentCls->add(CLASS_SCRIPT_HIRAGANA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5442 "Parser.cpp" +#line 5443 "Parser.cpp" break; } case 101: { { -#line 658 "Parser.rl" +#line 659 "Parser.rl" {te = p+1;{ -#line 658 "Parser.rl" +#line 659 "Parser.rl" currentCls->add(CLASS_SCRIPT_IMPERIAL_ARAMAIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5453 "Parser.cpp" +#line 5454 "Parser.cpp" break; } case 102: { { -#line 659 "Parser.rl" +#line 660 "Parser.rl" {te = p+1;{ -#line 659 "Parser.rl" +#line 660 "Parser.rl" currentCls->add(CLASS_SCRIPT_INHERITED, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5464 "Parser.cpp" +#line 5465 "Parser.cpp" break; } case 103: { { -#line 660 "Parser.rl" +#line 661 "Parser.rl" {te = p+1;{ -#line 660 "Parser.rl" +#line 661 "Parser.rl" currentCls->add(CLASS_SCRIPT_INSCRIPTIONAL_PAHLAVI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5475 "Parser.cpp" +#line 5476 "Parser.cpp" break; } case 104: { { -#line 661 "Parser.rl" +#line 662 "Parser.rl" {te = p+1;{ -#line 661 "Parser.rl" +#line 662 "Parser.rl" currentCls->add(CLASS_SCRIPT_INSCRIPTIONAL_PARTHIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5486 "Parser.cpp" +#line 5487 "Parser.cpp" break; } case 105: { { -#line 662 "Parser.rl" +#line 663 "Parser.rl" {te = p+1;{ -#line 662 "Parser.rl" +#line 663 "Parser.rl" currentCls->add(CLASS_SCRIPT_JAVANESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5497 "Parser.cpp" +#line 5498 "Parser.cpp" break; } case 106: { { -#line 663 "Parser.rl" +#line 664 "Parser.rl" {te = p+1;{ -#line 663 "Parser.rl" +#line 664 "Parser.rl" currentCls->add(CLASS_SCRIPT_KAITHI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5508 "Parser.cpp" +#line 5509 "Parser.cpp" break; } case 107: { { -#line 664 "Parser.rl" +#line 665 "Parser.rl" {te = p+1;{ -#line 664 "Parser.rl" +#line 665 "Parser.rl" currentCls->add(CLASS_SCRIPT_KANNADA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5519 "Parser.cpp" +#line 5520 "Parser.cpp" break; } case 108: { { -#line 665 "Parser.rl" +#line 666 "Parser.rl" {te = p+1;{ -#line 665 "Parser.rl" +#line 666 "Parser.rl" currentCls->add(CLASS_SCRIPT_KATAKANA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5530 "Parser.cpp" +#line 5531 "Parser.cpp" break; } case 109: { { -#line 666 "Parser.rl" +#line 667 "Parser.rl" {te = p+1;{ -#line 666 "Parser.rl" +#line 667 "Parser.rl" currentCls->add(CLASS_SCRIPT_KAYAH_LI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5541 "Parser.cpp" +#line 5542 "Parser.cpp" break; } case 110: { { -#line 667 "Parser.rl" +#line 668 "Parser.rl" {te = p+1;{ -#line 667 "Parser.rl" +#line 668 "Parser.rl" currentCls->add(CLASS_SCRIPT_KHAROSHTHI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5552 "Parser.cpp" +#line 5553 "Parser.cpp" break; } case 111: { { -#line 668 "Parser.rl" +#line 669 "Parser.rl" {te = p+1;{ -#line 668 "Parser.rl" +#line 669 "Parser.rl" currentCls->add(CLASS_SCRIPT_KHMER, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5563 "Parser.cpp" +#line 5564 "Parser.cpp" break; } case 112: { { -#line 669 "Parser.rl" +#line 670 "Parser.rl" {te = p+1;{ -#line 669 "Parser.rl" +#line 670 "Parser.rl" currentCls->add(CLASS_SCRIPT_LAO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5574 "Parser.cpp" +#line 5575 "Parser.cpp" break; } case 113: { { -#line 670 "Parser.rl" +#line 671 "Parser.rl" {te = p+1;{ -#line 670 "Parser.rl" +#line 671 "Parser.rl" currentCls->add(CLASS_SCRIPT_LATIN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5585 "Parser.cpp" +#line 5586 "Parser.cpp" break; } case 114: { { -#line 671 "Parser.rl" +#line 672 "Parser.rl" {te = p+1;{ -#line 671 "Parser.rl" +#line 672 "Parser.rl" currentCls->add(CLASS_SCRIPT_LEPCHA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5596 "Parser.cpp" +#line 5597 "Parser.cpp" break; } case 115: { { -#line 672 "Parser.rl" +#line 673 "Parser.rl" {te = p+1;{ -#line 672 "Parser.rl" +#line 673 "Parser.rl" currentCls->add(CLASS_SCRIPT_LIMBU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5607 "Parser.cpp" +#line 5608 "Parser.cpp" break; } case 116: { { -#line 673 "Parser.rl" +#line 674 "Parser.rl" {te = p+1;{ -#line 673 "Parser.rl" +#line 674 "Parser.rl" currentCls->add(CLASS_SCRIPT_LINEAR_B, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5618 "Parser.cpp" +#line 5619 "Parser.cpp" break; } case 117: { { -#line 674 "Parser.rl" +#line 675 "Parser.rl" {te = p+1;{ -#line 674 "Parser.rl" +#line 675 "Parser.rl" currentCls->add(CLASS_SCRIPT_LISU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5629 "Parser.cpp" +#line 5630 "Parser.cpp" break; } case 118: { { -#line 675 "Parser.rl" +#line 676 "Parser.rl" {te = p+1;{ -#line 675 "Parser.rl" +#line 676 "Parser.rl" currentCls->add(CLASS_SCRIPT_LYCIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5640 "Parser.cpp" +#line 5641 "Parser.cpp" break; } case 119: { { -#line 676 "Parser.rl" +#line 677 "Parser.rl" {te = p+1;{ -#line 676 "Parser.rl" +#line 677 "Parser.rl" currentCls->add(CLASS_SCRIPT_LYDIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5651 "Parser.cpp" +#line 5652 "Parser.cpp" break; } case 120: { { -#line 677 "Parser.rl" +#line 678 "Parser.rl" {te = p+1;{ -#line 677 "Parser.rl" +#line 678 "Parser.rl" currentCls->add(CLASS_SCRIPT_MALAYALAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5662 "Parser.cpp" +#line 5663 "Parser.cpp" break; } case 121: { { -#line 678 "Parser.rl" +#line 679 "Parser.rl" {te = p+1;{ -#line 678 "Parser.rl" +#line 679 "Parser.rl" currentCls->add(CLASS_SCRIPT_MANDAIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5673 "Parser.cpp" +#line 5674 "Parser.cpp" break; } case 122: { { -#line 679 "Parser.rl" +#line 680 "Parser.rl" {te = p+1;{ -#line 679 "Parser.rl" +#line 680 "Parser.rl" currentCls->add(CLASS_SCRIPT_MEETEI_MAYEK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5684 "Parser.cpp" +#line 5685 "Parser.cpp" break; } case 123: { { -#line 680 "Parser.rl" +#line 681 "Parser.rl" {te = p+1;{ -#line 680 "Parser.rl" +#line 681 "Parser.rl" currentCls->add(CLASS_SCRIPT_MONGOLIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5695 "Parser.cpp" +#line 5696 "Parser.cpp" break; } case 124: { { -#line 681 "Parser.rl" +#line 682 "Parser.rl" {te = p+1;{ -#line 681 "Parser.rl" +#line 682 "Parser.rl" currentCls->add(CLASS_SCRIPT_MYANMAR, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5706 "Parser.cpp" +#line 5707 "Parser.cpp" break; } case 125: { { -#line 682 "Parser.rl" +#line 683 "Parser.rl" {te = p+1;{ -#line 682 "Parser.rl" +#line 683 "Parser.rl" currentCls->add(CLASS_SCRIPT_NEW_TAI_LUE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5717 "Parser.cpp" +#line 5718 "Parser.cpp" break; } case 126: { { -#line 683 "Parser.rl" +#line 684 "Parser.rl" {te = p+1;{ -#line 683 "Parser.rl" +#line 684 "Parser.rl" currentCls->add(CLASS_SCRIPT_NKO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5728 "Parser.cpp" +#line 5729 "Parser.cpp" break; } case 127: { { -#line 684 "Parser.rl" +#line 685 "Parser.rl" {te = p+1;{ -#line 684 "Parser.rl" +#line 685 "Parser.rl" currentCls->add(CLASS_SCRIPT_OGHAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5739 "Parser.cpp" +#line 5740 "Parser.cpp" break; } case 128: { { -#line 685 "Parser.rl" +#line 686 "Parser.rl" {te = p+1;{ -#line 685 "Parser.rl" +#line 686 "Parser.rl" currentCls->add(CLASS_SCRIPT_OL_CHIKI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5750 "Parser.cpp" +#line 5751 "Parser.cpp" break; } case 129: { { -#line 686 "Parser.rl" +#line 687 "Parser.rl" {te = p+1;{ -#line 686 "Parser.rl" +#line 687 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_ITALIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5761 "Parser.cpp" +#line 5762 "Parser.cpp" break; } case 130: { { -#line 687 "Parser.rl" +#line 688 "Parser.rl" {te = p+1;{ -#line 687 "Parser.rl" +#line 688 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_PERSIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5772 "Parser.cpp" +#line 5773 "Parser.cpp" break; } case 131: { { -#line 688 "Parser.rl" +#line 689 "Parser.rl" {te = p+1;{ -#line 688 "Parser.rl" +#line 689 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_SOUTH_ARABIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5783 "Parser.cpp" +#line 5784 "Parser.cpp" break; } case 132: { { -#line 689 "Parser.rl" +#line 690 "Parser.rl" {te = p+1;{ -#line 689 "Parser.rl" +#line 690 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_TURKIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5794 "Parser.cpp" +#line 5795 "Parser.cpp" break; } case 133: { { -#line 690 "Parser.rl" +#line 691 "Parser.rl" {te = p+1;{ -#line 690 "Parser.rl" +#line 691 "Parser.rl" currentCls->add(CLASS_SCRIPT_ORIYA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5805 "Parser.cpp" +#line 5806 "Parser.cpp" break; } case 134: { { -#line 691 "Parser.rl" +#line 692 "Parser.rl" {te = p+1;{ -#line 691 "Parser.rl" +#line 692 "Parser.rl" currentCls->add(CLASS_SCRIPT_OSMANYA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5816 "Parser.cpp" +#line 5817 "Parser.cpp" break; } case 135: { { -#line 692 "Parser.rl" +#line 693 "Parser.rl" {te = p+1;{ -#line 692 "Parser.rl" +#line 693 "Parser.rl" currentCls->add(CLASS_SCRIPT_PHAGS_PA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5827 "Parser.cpp" +#line 5828 "Parser.cpp" break; } case 136: { { -#line 693 "Parser.rl" +#line 694 "Parser.rl" {te = p+1;{ -#line 693 "Parser.rl" +#line 694 "Parser.rl" currentCls->add(CLASS_SCRIPT_PHOENICIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5838 "Parser.cpp" +#line 5839 "Parser.cpp" break; } case 137: { { -#line 694 "Parser.rl" +#line 695 "Parser.rl" {te = p+1;{ -#line 694 "Parser.rl" +#line 695 "Parser.rl" currentCls->add(CLASS_SCRIPT_REJANG, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5849 "Parser.cpp" +#line 5850 "Parser.cpp" break; } case 138: { { -#line 695 "Parser.rl" +#line 696 "Parser.rl" {te = p+1;{ -#line 695 "Parser.rl" +#line 696 "Parser.rl" currentCls->add(CLASS_SCRIPT_RUNIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5860 "Parser.cpp" +#line 5861 "Parser.cpp" break; } case 139: { { -#line 696 "Parser.rl" +#line 697 "Parser.rl" {te = p+1;{ -#line 696 "Parser.rl" +#line 697 "Parser.rl" currentCls->add(CLASS_SCRIPT_SAMARITAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5871 "Parser.cpp" +#line 5872 "Parser.cpp" break; } case 140: { { -#line 697 "Parser.rl" +#line 698 "Parser.rl" {te = p+1;{ -#line 697 "Parser.rl" +#line 698 "Parser.rl" currentCls->add(CLASS_SCRIPT_SAURASHTRA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5882 "Parser.cpp" +#line 5883 "Parser.cpp" break; } case 141: { { -#line 698 "Parser.rl" +#line 699 "Parser.rl" {te = p+1;{ -#line 698 "Parser.rl" +#line 699 "Parser.rl" currentCls->add(CLASS_SCRIPT_SHAVIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5893 "Parser.cpp" +#line 5894 "Parser.cpp" break; } case 142: { { -#line 699 "Parser.rl" +#line 700 "Parser.rl" {te = p+1;{ -#line 699 "Parser.rl" +#line 700 "Parser.rl" currentCls->add(CLASS_SCRIPT_SINHALA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5904 "Parser.cpp" +#line 5905 "Parser.cpp" break; } case 143: { { -#line 700 "Parser.rl" +#line 701 "Parser.rl" {te = p+1;{ -#line 700 "Parser.rl" +#line 701 "Parser.rl" currentCls->add(CLASS_SCRIPT_SUNDANESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5915 "Parser.cpp" +#line 5916 "Parser.cpp" break; } case 144: { { -#line 701 "Parser.rl" +#line 702 "Parser.rl" {te = p+1;{ -#line 701 "Parser.rl" +#line 702 "Parser.rl" currentCls->add(CLASS_SCRIPT_SYLOTI_NAGRI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5926 "Parser.cpp" +#line 5927 "Parser.cpp" break; } case 145: { { -#line 702 "Parser.rl" +#line 703 "Parser.rl" {te = p+1;{ -#line 702 "Parser.rl" +#line 703 "Parser.rl" currentCls->add(CLASS_SCRIPT_SYRIAC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5937 "Parser.cpp" +#line 5938 "Parser.cpp" break; } case 146: { { -#line 703 "Parser.rl" +#line 704 "Parser.rl" {te = p+1;{ -#line 703 "Parser.rl" +#line 704 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAGALOG, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5948 "Parser.cpp" +#line 5949 "Parser.cpp" break; } case 147: { { -#line 704 "Parser.rl" +#line 705 "Parser.rl" {te = p+1;{ -#line 704 "Parser.rl" +#line 705 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAGBANWA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5959 "Parser.cpp" +#line 5960 "Parser.cpp" break; } case 148: { { -#line 705 "Parser.rl" +#line 706 "Parser.rl" {te = p+1;{ -#line 705 "Parser.rl" +#line 706 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAI_LE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5970 "Parser.cpp" +#line 5971 "Parser.cpp" break; } case 149: { { -#line 706 "Parser.rl" +#line 707 "Parser.rl" {te = p+1;{ -#line 706 "Parser.rl" +#line 707 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAI_THAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5981 "Parser.cpp" +#line 5982 "Parser.cpp" break; } case 150: { { -#line 707 "Parser.rl" +#line 708 "Parser.rl" {te = p+1;{ -#line 707 "Parser.rl" +#line 708 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAI_VIET, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5992 "Parser.cpp" +#line 5993 "Parser.cpp" break; } case 151: { { -#line 708 "Parser.rl" +#line 709 "Parser.rl" {te = p+1;{ -#line 708 "Parser.rl" +#line 709 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAMIL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6003 "Parser.cpp" +#line 6004 "Parser.cpp" break; } case 152: { { -#line 709 "Parser.rl" +#line 710 "Parser.rl" {te = p+1;{ -#line 709 "Parser.rl" +#line 710 "Parser.rl" currentCls->add(CLASS_SCRIPT_TELUGU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6014 "Parser.cpp" +#line 6015 "Parser.cpp" break; } case 153: { { -#line 710 "Parser.rl" +#line 711 "Parser.rl" {te = p+1;{ -#line 710 "Parser.rl" +#line 711 "Parser.rl" currentCls->add(CLASS_SCRIPT_THAANA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6025 "Parser.cpp" +#line 6026 "Parser.cpp" break; } case 154: { { -#line 711 "Parser.rl" +#line 712 "Parser.rl" {te = p+1;{ -#line 711 "Parser.rl" +#line 712 "Parser.rl" currentCls->add(CLASS_SCRIPT_THAI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6036 "Parser.cpp" +#line 6037 "Parser.cpp" break; } case 155: { { -#line 712 "Parser.rl" +#line 713 "Parser.rl" {te = p+1;{ -#line 712 "Parser.rl" +#line 713 "Parser.rl" currentCls->add(CLASS_SCRIPT_TIBETAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6047 "Parser.cpp" +#line 6048 "Parser.cpp" break; } case 156: { { -#line 713 "Parser.rl" +#line 714 "Parser.rl" {te = p+1;{ -#line 713 "Parser.rl" +#line 714 "Parser.rl" currentCls->add(CLASS_SCRIPT_TIFINAGH, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6058 "Parser.cpp" +#line 6059 "Parser.cpp" break; } case 157: { { -#line 714 "Parser.rl" +#line 715 "Parser.rl" {te = p+1;{ -#line 714 "Parser.rl" +#line 715 "Parser.rl" currentCls->add(CLASS_SCRIPT_UGARITIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6069 "Parser.cpp" +#line 6070 "Parser.cpp" break; } case 158: { { -#line 715 "Parser.rl" +#line 716 "Parser.rl" {te = p+1;{ -#line 715 "Parser.rl" +#line 716 "Parser.rl" currentCls->add(CLASS_SCRIPT_VAI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6080 "Parser.cpp" +#line 6081 "Parser.cpp" break; } case 159: { { -#line 716 "Parser.rl" +#line 717 "Parser.rl" {te = p+1;{ -#line 716 "Parser.rl" +#line 717 "Parser.rl" currentCls->add(CLASS_SCRIPT_YI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6091 "Parser.cpp" +#line 6092 "Parser.cpp" break; } case 160: { { -#line 717 "Parser.rl" +#line 718 "Parser.rl" {te = p+1;{ -#line 717 "Parser.rl" +#line 718 "Parser.rl" currentCls->add(CLASS_UCP_ANY, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6102 "Parser.cpp" +#line 6103 "Parser.cpp" break; } case 161: { { -#line 718 "Parser.rl" +#line 719 "Parser.rl" {te = p+1;{ -#line 718 "Parser.rl" +#line 719 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6113 "Parser.cpp" +#line 6114 "Parser.cpp" break; } case 162: { { -#line 580 "Parser.rl" +#line 581 "Parser.rl" {te = p;p = p - 1;{ -#line 580 "Parser.rl" +#line 581 "Parser.rl" currentCls->add(CLASS_UCP_C, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6124 "Parser.cpp" +#line 6125 "Parser.cpp" break; } case 163: { { -#line 584 "Parser.rl" +#line 585 "Parser.rl" {te = p;p = p - 1;{ -#line 584 "Parser.rl" +#line 585 "Parser.rl" currentCls->add(CLASS_UCP_CO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6135 "Parser.cpp" +#line 6136 "Parser.cpp" break; } case 164: { { -#line 586 "Parser.rl" +#line 587 "Parser.rl" {te = p;p = p - 1;{ -#line 586 "Parser.rl" +#line 587 "Parser.rl" currentCls->add(CLASS_UCP_L, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6146 "Parser.cpp" +#line 6147 "Parser.cpp" break; } case 165: { { -#line 593 "Parser.rl" +#line 594 "Parser.rl" {te = p;p = p - 1;{ -#line 593 "Parser.rl" +#line 594 "Parser.rl" currentCls->add(CLASS_UCP_M, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6157 "Parser.cpp" +#line 6158 "Parser.cpp" break; } case 166: { { -#line 595 "Parser.rl" +#line 596 "Parser.rl" {te = p;p = p - 1;{ -#line 595 "Parser.rl" +#line 596 "Parser.rl" currentCls->add(CLASS_UCP_ME, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6168 "Parser.cpp" +#line 6169 "Parser.cpp" break; } case 167: { { -#line 597 "Parser.rl" +#line 598 "Parser.rl" {te = p;p = p - 1;{ -#line 597 "Parser.rl" +#line 598 "Parser.rl" currentCls->add(CLASS_UCP_N, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6179 "Parser.cpp" +#line 6180 "Parser.cpp" break; } case 168: { { -#line 601 "Parser.rl" +#line 602 "Parser.rl" {te = p;p = p - 1;{ -#line 601 "Parser.rl" +#line 602 "Parser.rl" currentCls->add(CLASS_UCP_P, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6190 "Parser.cpp" +#line 6191 "Parser.cpp" break; } case 169: { { -#line 609 "Parser.rl" +#line 610 "Parser.rl" {te = p;p = p - 1;{ -#line 609 "Parser.rl" +#line 610 "Parser.rl" currentCls->add(CLASS_UCP_S, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6201 "Parser.cpp" +#line 6202 "Parser.cpp" break; } case 170: { { -#line 614 "Parser.rl" +#line 615 "Parser.rl" {te = p;p = p - 1;{ -#line 614 "Parser.rl" +#line 615 "Parser.rl" currentCls->add(CLASS_UCP_Z, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6212 "Parser.cpp" +#line 6213 "Parser.cpp" break; } case 171: { { -#line 653 "Parser.rl" +#line 654 "Parser.rl" {te = p;p = p - 1;{ -#line 653 "Parser.rl" +#line 654 "Parser.rl" currentCls->add(CLASS_SCRIPT_HAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6223 "Parser.cpp" +#line 6224 "Parser.cpp" break; } case 172: { { -#line 718 "Parser.rl" +#line 719 "Parser.rl" {te = p;p = p - 1;{ -#line 718 "Parser.rl" +#line 719 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6234 "Parser.cpp" +#line 6235 "Parser.cpp" break; } case 173: { { -#line 580 "Parser.rl" +#line 581 "Parser.rl" {p = ((te))-1; { -#line 580 "Parser.rl" +#line 581 "Parser.rl" currentCls->add(CLASS_UCP_C, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6246 "Parser.cpp" +#line 6247 "Parser.cpp" break; } case 174: { { -#line 584 "Parser.rl" +#line 585 "Parser.rl" {p = ((te))-1; { -#line 584 "Parser.rl" +#line 585 "Parser.rl" currentCls->add(CLASS_UCP_CO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6258 "Parser.cpp" +#line 6259 "Parser.cpp" break; } case 175: { { -#line 586 "Parser.rl" +#line 587 "Parser.rl" {p = ((te))-1; { -#line 586 "Parser.rl" +#line 587 "Parser.rl" currentCls->add(CLASS_UCP_L, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6270 "Parser.cpp" +#line 6271 "Parser.cpp" break; } case 176: { { -#line 593 "Parser.rl" +#line 594 "Parser.rl" {p = ((te))-1; { -#line 593 "Parser.rl" +#line 594 "Parser.rl" currentCls->add(CLASS_UCP_M, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6282 "Parser.cpp" +#line 6283 "Parser.cpp" break; } case 177: { { -#line 595 "Parser.rl" +#line 596 "Parser.rl" {p = ((te))-1; { -#line 595 "Parser.rl" +#line 596 "Parser.rl" currentCls->add(CLASS_UCP_ME, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6294 "Parser.cpp" +#line 6295 "Parser.cpp" break; } case 178: { { -#line 597 "Parser.rl" +#line 598 "Parser.rl" {p = ((te))-1; { -#line 597 "Parser.rl" +#line 598 "Parser.rl" currentCls->add(CLASS_UCP_N, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6306 "Parser.cpp" +#line 6307 "Parser.cpp" break; } case 179: { { -#line 601 "Parser.rl" +#line 602 "Parser.rl" {p = ((te))-1; { -#line 601 "Parser.rl" +#line 602 "Parser.rl" currentCls->add(CLASS_UCP_P, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6318 "Parser.cpp" +#line 6319 "Parser.cpp" break; } case 180: { { -#line 609 "Parser.rl" +#line 610 "Parser.rl" {p = ((te))-1; { -#line 609 "Parser.rl" +#line 610 "Parser.rl" currentCls->add(CLASS_UCP_S, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6330 "Parser.cpp" +#line 6331 "Parser.cpp" break; } case 181: { { -#line 653 "Parser.rl" +#line 654 "Parser.rl" {p = ((te))-1; { -#line 653 "Parser.rl" +#line 654 "Parser.rl" currentCls->add(CLASS_SCRIPT_HAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6342 "Parser.cpp" +#line 6343 "Parser.cpp" break; } case 182: { { -#line 718 "Parser.rl" +#line 719 "Parser.rl" {p = ((te))-1; { -#line 718 "Parser.rl" +#line 719 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6354 "Parser.cpp" +#line 6355 "Parser.cpp" break; } case 183: { { -#line 733 "Parser.rl" +#line 734 "Parser.rl" {te = p+1;{ -#line 733 "Parser.rl" +#line 734 "Parser.rl" currentCls->add(CLASS_UCP_C, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6372 "Parser.cpp" +#line 6373 "Parser.cpp" break; } case 184: { { -#line 741 "Parser.rl" +#line 742 "Parser.rl" {te = p+1;{ -#line 741 "Parser.rl" +#line 742 "Parser.rl" currentCls->add(CLASS_UCP_L, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6390 "Parser.cpp" +#line 6391 "Parser.cpp" break; } case 185: { { -#line 749 "Parser.rl" +#line 750 "Parser.rl" {te = p+1;{ -#line 749 "Parser.rl" +#line 750 "Parser.rl" currentCls->add(CLASS_UCP_M, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6408 "Parser.cpp" +#line 6409 "Parser.cpp" break; } case 186: { { -#line 757 "Parser.rl" +#line 758 "Parser.rl" {te = p+1;{ -#line 757 "Parser.rl" +#line 758 "Parser.rl" currentCls->add(CLASS_UCP_N, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6426 "Parser.cpp" +#line 6427 "Parser.cpp" break; } case 187: { { -#line 765 "Parser.rl" +#line 766 "Parser.rl" {te = p+1;{ -#line 765 "Parser.rl" +#line 766 "Parser.rl" currentCls->add(CLASS_UCP_P, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6444 "Parser.cpp" +#line 6445 "Parser.cpp" break; } case 188: { { -#line 773 "Parser.rl" +#line 774 "Parser.rl" {te = p+1;{ -#line 773 "Parser.rl" +#line 774 "Parser.rl" currentCls->add(CLASS_UCP_S, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6462 "Parser.cpp" +#line 6463 "Parser.cpp" break; } case 189: { { -#line 781 "Parser.rl" +#line 782 "Parser.rl" {te = p+1;{ -#line 781 "Parser.rl" +#line 782 "Parser.rl" currentCls->add(CLASS_UCP_Z, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6480 "Parser.cpp" +#line 6481 "Parser.cpp" break; } case 190: { { -#line 790 "Parser.rl" +#line 791 "Parser.rl" {te = p+1;{ -#line 790 "Parser.rl" +#line 791 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6491 "Parser.cpp" +#line 6492 "Parser.cpp" break; } case 191: { { -#line 796 "Parser.rl" +#line 797 "Parser.rl" {te = p+1;{ -#line 796 "Parser.rl" +#line 797 "Parser.rl" throw LocatedParseError("Unsupported POSIX collating " "element"); } }} -#line 6505 "Parser.cpp" +#line 6506 "Parser.cpp" break; } case 192: { { -#line 803 "Parser.rl" +#line 804 "Parser.rl" {te = p+1;{ -#line 803 "Parser.rl" +#line 804 "Parser.rl" currentCls->add(CLASS_ALNUM, false); } }} -#line 6518 "Parser.cpp" +#line 6519 "Parser.cpp" break; } case 193: { { -#line 806 "Parser.rl" +#line 807 "Parser.rl" {te = p+1;{ -#line 806 "Parser.rl" +#line 807 "Parser.rl" currentCls->add(CLASS_ALNUM, true); } }} -#line 6531 "Parser.cpp" +#line 6532 "Parser.cpp" break; } case 194: { { -#line 809 "Parser.rl" +#line 810 "Parser.rl" {te = p+1;{ -#line 809 "Parser.rl" +#line 810 "Parser.rl" currentCls->add(CLASS_ALPHA, false); } }} -#line 6544 "Parser.cpp" +#line 6545 "Parser.cpp" break; } case 195: { { -#line 812 "Parser.rl" +#line 813 "Parser.rl" {te = p+1;{ -#line 812 "Parser.rl" +#line 813 "Parser.rl" currentCls->add(CLASS_ALPHA, true); } }} -#line 6557 "Parser.cpp" +#line 6558 "Parser.cpp" break; } case 196: { { -#line 815 "Parser.rl" +#line 816 "Parser.rl" {te = p+1;{ -#line 815 "Parser.rl" +#line 816 "Parser.rl" currentCls->add(CLASS_ASCII, false); } }} -#line 6570 "Parser.cpp" +#line 6571 "Parser.cpp" break; } case 197: { { -#line 818 "Parser.rl" +#line 819 "Parser.rl" {te = p+1;{ -#line 818 "Parser.rl" +#line 819 "Parser.rl" currentCls->add(CLASS_ASCII, true); } }} -#line 6583 "Parser.cpp" +#line 6584 "Parser.cpp" break; } case 198: { { -#line 821 "Parser.rl" +#line 822 "Parser.rl" {te = p+1;{ -#line 821 "Parser.rl" +#line 822 "Parser.rl" currentCls->add(CLASS_BLANK, false); } }} -#line 6596 "Parser.cpp" +#line 6597 "Parser.cpp" break; } case 199: { { -#line 824 "Parser.rl" +#line 825 "Parser.rl" {te = p+1;{ -#line 824 "Parser.rl" +#line 825 "Parser.rl" currentCls->add(CLASS_BLANK, true); } }} -#line 6609 "Parser.cpp" +#line 6610 "Parser.cpp" break; } case 200: { { -#line 827 "Parser.rl" +#line 828 "Parser.rl" {te = p+1;{ -#line 827 "Parser.rl" +#line 828 "Parser.rl" currentCls->add(CLASS_CNTRL, false); } }} -#line 6622 "Parser.cpp" +#line 6623 "Parser.cpp" break; } case 201: { { -#line 830 "Parser.rl" +#line 831 "Parser.rl" {te = p+1;{ -#line 830 "Parser.rl" +#line 831 "Parser.rl" currentCls->add(CLASS_CNTRL, true); } }} -#line 6635 "Parser.cpp" +#line 6636 "Parser.cpp" break; } case 202: { { -#line 833 "Parser.rl" +#line 834 "Parser.rl" {te = p+1;{ -#line 833 "Parser.rl" +#line 834 "Parser.rl" currentCls->add(CLASS_DIGIT, false); } }} -#line 6648 "Parser.cpp" +#line 6649 "Parser.cpp" break; } case 203: { { -#line 836 "Parser.rl" +#line 837 "Parser.rl" {te = p+1;{ -#line 836 "Parser.rl" +#line 837 "Parser.rl" currentCls->add(CLASS_DIGIT, true); } }} -#line 6661 "Parser.cpp" +#line 6662 "Parser.cpp" break; } case 204: { { -#line 839 "Parser.rl" +#line 840 "Parser.rl" {te = p+1;{ -#line 839 "Parser.rl" +#line 840 "Parser.rl" currentCls->add(CLASS_GRAPH, false); } }} -#line 6674 "Parser.cpp" +#line 6675 "Parser.cpp" break; } case 205: { { -#line 842 "Parser.rl" +#line 843 "Parser.rl" {te = p+1;{ -#line 842 "Parser.rl" +#line 843 "Parser.rl" currentCls->add(CLASS_GRAPH, true); } }} -#line 6687 "Parser.cpp" +#line 6688 "Parser.cpp" break; } case 206: { { -#line 845 "Parser.rl" +#line 846 "Parser.rl" {te = p+1;{ -#line 845 "Parser.rl" +#line 846 "Parser.rl" currentCls->add(CLASS_LOWER, false); } }} -#line 6700 "Parser.cpp" +#line 6701 "Parser.cpp" break; } case 207: { { -#line 848 "Parser.rl" +#line 849 "Parser.rl" {te = p+1;{ -#line 848 "Parser.rl" +#line 849 "Parser.rl" currentCls->add(CLASS_LOWER, true); } }} -#line 6713 "Parser.cpp" +#line 6714 "Parser.cpp" break; } case 208: { { -#line 851 "Parser.rl" +#line 852 "Parser.rl" {te = p+1;{ -#line 851 "Parser.rl" +#line 852 "Parser.rl" currentCls->add(CLASS_PRINT, false); } }} -#line 6726 "Parser.cpp" +#line 6727 "Parser.cpp" break; } case 209: { { -#line 854 "Parser.rl" +#line 855 "Parser.rl" {te = p+1;{ -#line 854 "Parser.rl" +#line 855 "Parser.rl" currentCls->add(CLASS_PRINT, true); } }} -#line 6739 "Parser.cpp" +#line 6740 "Parser.cpp" break; } case 210: { { -#line 857 "Parser.rl" +#line 858 "Parser.rl" {te = p+1;{ -#line 857 "Parser.rl" +#line 858 "Parser.rl" currentCls->add(CLASS_PUNCT, false); } }} -#line 6752 "Parser.cpp" +#line 6753 "Parser.cpp" break; } case 211: { { -#line 860 "Parser.rl" +#line 861 "Parser.rl" {te = p+1;{ -#line 860 "Parser.rl" +#line 861 "Parser.rl" currentCls->add(CLASS_PUNCT, true); } }} -#line 6765 "Parser.cpp" +#line 6766 "Parser.cpp" break; } case 212: { { -#line 864 "Parser.rl" +#line 865 "Parser.rl" {te = p+1;{ -#line 864 "Parser.rl" +#line 865 "Parser.rl" currentCls->add(CLASS_SPACE, false); } }} -#line 6778 "Parser.cpp" +#line 6779 "Parser.cpp" break; } case 213: { { -#line 867 "Parser.rl" +#line 868 "Parser.rl" {te = p+1;{ -#line 867 "Parser.rl" +#line 868 "Parser.rl" currentCls->add(CLASS_SPACE, true); } }} -#line 6791 "Parser.cpp" +#line 6792 "Parser.cpp" break; } case 214: { { -#line 870 "Parser.rl" +#line 871 "Parser.rl" {te = p+1;{ -#line 870 "Parser.rl" +#line 871 "Parser.rl" currentCls->add(CLASS_UPPER, false); } }} -#line 6804 "Parser.cpp" +#line 6805 "Parser.cpp" break; } case 215: { { -#line 873 "Parser.rl" +#line 874 "Parser.rl" {te = p+1;{ -#line 873 "Parser.rl" +#line 874 "Parser.rl" currentCls->add(CLASS_UPPER, true); } }} -#line 6817 "Parser.cpp" +#line 6818 "Parser.cpp" break; } case 216: { { -#line 876 "Parser.rl" +#line 877 "Parser.rl" {te = p+1;{ -#line 876 "Parser.rl" +#line 877 "Parser.rl" currentCls->add(CLASS_WORD, false); } }} -#line 6830 "Parser.cpp" +#line 6831 "Parser.cpp" break; } case 217: { { -#line 879 "Parser.rl" +#line 880 "Parser.rl" {te = p+1;{ -#line 879 "Parser.rl" +#line 880 "Parser.rl" currentCls->add(CLASS_WORD, true); } }} -#line 6843 "Parser.cpp" +#line 6844 "Parser.cpp" break; } case 218: { { -#line 882 "Parser.rl" +#line 883 "Parser.rl" {te = p+1;{ -#line 882 "Parser.rl" +#line 883 "Parser.rl" currentCls->add(CLASS_XDIGIT, false); } }} -#line 6856 "Parser.cpp" +#line 6857 "Parser.cpp" break; } case 219: { { -#line 885 "Parser.rl" +#line 886 "Parser.rl" {te = p+1;{ -#line 885 "Parser.rl" +#line 886 "Parser.rl" currentCls->add(CLASS_XDIGIT, true); } }} -#line 6869 "Parser.cpp" +#line 6870 "Parser.cpp" break; } case 220: { { -#line 890 "Parser.rl" +#line 891 "Parser.rl" {te = p+1;{ -#line 890 "Parser.rl" +#line 891 "Parser.rl" throw LocatedParseError("Invalid POSIX named class"); } }} -#line 6882 "Parser.cpp" +#line 6883 "Parser.cpp" break; } case 221: { { -#line 893 "Parser.rl" +#line 894 "Parser.rl" {te = p+1;{ -#line 893 "Parser.rl" +#line 894 "Parser.rl" {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { stack.resize(2 * (top + 1)); } } - stack[top] = cs; top += 1;cs = 843;goto _again;}} + stack[top] = cs; top += 1;cs = 844;goto _again;}} }} -#line 6902 "Parser.cpp" +#line 6903 "Parser.cpp" break; } case 222: { { -#line 896 "Parser.rl" +#line 897 "Parser.rl" {te = p+1;{ -#line 896 "Parser.rl" +#line 897 "Parser.rl" /*noop*/} }} -#line 6913 "Parser.cpp" +#line 6914 "Parser.cpp" break; } case 223: { { -#line 898 "Parser.rl" +#line 899 "Parser.rl" {te = p+1;{ -#line 898 "Parser.rl" +#line 899 "Parser.rl" currentCls->add('\x08'); } }} -#line 6926 "Parser.cpp" +#line 6927 "Parser.cpp" break; } case 224: { { -#line 902 "Parser.rl" +#line 903 "Parser.rl" {te = p+1;{ -#line 902 "Parser.rl" +#line 903 "Parser.rl" currentCls->add('\x09'); } }} -#line 6939 "Parser.cpp" +#line 6940 "Parser.cpp" break; } case 225: { { -#line 906 "Parser.rl" +#line 907 "Parser.rl" {te = p+1;{ -#line 906 "Parser.rl" +#line 907 "Parser.rl" currentCls->add('\x0a'); } }} -#line 6952 "Parser.cpp" +#line 6953 "Parser.cpp" break; } case 226: { { -#line 910 "Parser.rl" +#line 911 "Parser.rl" {te = p+1;{ -#line 910 "Parser.rl" +#line 911 "Parser.rl" currentCls->add('\x0d'); } }} -#line 6965 "Parser.cpp" +#line 6966 "Parser.cpp" break; } case 227: { { -#line 914 "Parser.rl" +#line 915 "Parser.rl" {te = p+1;{ -#line 914 "Parser.rl" +#line 915 "Parser.rl" currentCls->add('\x0c'); } }} -#line 6978 "Parser.cpp" +#line 6979 "Parser.cpp" break; } case 228: { { -#line 918 "Parser.rl" +#line 919 "Parser.rl" {te = p+1;{ -#line 918 "Parser.rl" +#line 919 "Parser.rl" currentCls->add('\x07'); } }} -#line 6991 "Parser.cpp" +#line 6992 "Parser.cpp" break; } case 229: { { -#line 922 "Parser.rl" +#line 923 "Parser.rl" {te = p+1;{ -#line 922 "Parser.rl" +#line 923 "Parser.rl" currentCls->add('\x1b'); } }} -#line 7004 "Parser.cpp" +#line 7005 "Parser.cpp" break; } case 230: { { -#line 926 "Parser.rl" +#line 927 "Parser.rl" {te = p+1;{ -#line 926 "Parser.rl" +#line 927 "Parser.rl" currentCls->add(CLASS_HORZ, false); } }} -#line 7017 "Parser.cpp" +#line 7018 "Parser.cpp" break; } case 231: { { -#line 930 "Parser.rl" +#line 931 "Parser.rl" {te = p+1;{ -#line 930 "Parser.rl" +#line 931 "Parser.rl" currentCls->add(CLASS_HORZ, true); } }} -#line 7030 "Parser.cpp" +#line 7031 "Parser.cpp" break; } case 232: { { -#line 934 "Parser.rl" +#line 935 "Parser.rl" {te = p+1;{ -#line 934 "Parser.rl" +#line 935 "Parser.rl" currentCls->add(CLASS_VERT, false); } }} -#line 7043 "Parser.cpp" +#line 7044 "Parser.cpp" break; } case 233: { { -#line 938 "Parser.rl" +#line 939 "Parser.rl" {te = p+1;{ -#line 938 "Parser.rl" +#line 939 "Parser.rl" currentCls->add(CLASS_VERT, true); } }} -#line 7056 "Parser.cpp" +#line 7057 "Parser.cpp" break; } case 234: { { -#line 942 "Parser.rl" +#line 943 "Parser.rl" {te = p+1;{ -#line 942 "Parser.rl" +#line 943 "Parser.rl" negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7320,20 +7321,20 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 7078 "Parser.cpp" +#line 7079 "Parser.cpp" break; } case 235: { { -#line 948 "Parser.rl" +#line 949 "Parser.rl" {te = p+1;{ -#line 948 "Parser.rl" +#line 949 "Parser.rl" negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7343,20 +7344,20 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 7100 "Parser.cpp" +#line 7101 "Parser.cpp" break; } case 236: { { -#line 954 "Parser.rl" +#line 955 "Parser.rl" {te = p+1;{ -#line 954 "Parser.rl" +#line 955 "Parser.rl" negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7366,20 +7367,20 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 7122 "Parser.cpp" +#line 7123 "Parser.cpp" break; } case 237: { { -#line 960 "Parser.rl" +#line 961 "Parser.rl" {te = p+1;{ -#line 960 "Parser.rl" +#line 961 "Parser.rl" negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7389,43 +7390,43 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 7144 "Parser.cpp" +#line 7145 "Parser.cpp" break; } case 238: { { -#line 970 "Parser.rl" +#line 971 "Parser.rl" {te = p+1;{ -#line 970 "Parser.rl" +#line 971 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7157 "Parser.cpp" +#line 7158 "Parser.cpp" break; } case 239: { { -#line 973 "Parser.rl" +#line 974 "Parser.rl" {te = p+1;{ -#line 973 "Parser.rl" +#line 974 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7170 "Parser.cpp" +#line 7171 "Parser.cpp" break; } case 240: { { -#line 977 "Parser.rl" +#line 978 "Parser.rl" {te = p+1;{ -#line 977 "Parser.rl" +#line 978 "Parser.rl" string oct(ts + 3, te - ts - 4); unsigned long val; @@ -7441,29 +7442,29 @@ static const short _regex_actions[] = { } }} -#line 7193 "Parser.cpp" +#line 7194 "Parser.cpp" break; } case 241: { { -#line 997 "Parser.rl" +#line 998 "Parser.rl" {te = p+1;{ -#line 997 "Parser.rl" +#line 998 "Parser.rl" currentCls->add(accumulator); } }} -#line 7206 "Parser.cpp" +#line 7207 "Parser.cpp" break; } case 242: { { -#line 1001 "Parser.rl" +#line 1002 "Parser.rl" {te = p+1;{ -#line 1001 "Parser.rl" +#line 1002 "Parser.rl" // whatever we found here currentCls->add(*(ts + 1)); @@ -7471,15 +7472,15 @@ static const short _regex_actions[] = { } }} -#line 7221 "Parser.cpp" +#line 7222 "Parser.cpp" break; } case 243: { { -#line 1007 "Parser.rl" +#line 1008 "Parser.rl" {te = p+1;{ -#line 1007 "Parser.rl" +#line 1008 "Parser.rl" string hex(ts + 3, te - ts - 4); unsigned long val; @@ -7495,15 +7496,15 @@ static const short _regex_actions[] = { } }} -#line 7244 "Parser.cpp" +#line 7245 "Parser.cpp" break; } case 244: { { -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" {te = p+1;{ -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -7515,113 +7516,113 @@ static const short _regex_actions[] = { } }} -#line 7263 "Parser.cpp" +#line 7264 "Parser.cpp" break; } case 245: { { -#line 1035 "Parser.rl" +#line 1036 "Parser.rl" {te = p+1;{ -#line 1035 "Parser.rl" +#line 1036 "Parser.rl" currentCls->add(CLASS_WORD, false); } }} -#line 7276 "Parser.cpp" +#line 7277 "Parser.cpp" break; } case 246: { { -#line 1039 "Parser.rl" +#line 1040 "Parser.rl" {te = p+1;{ -#line 1039 "Parser.rl" +#line 1040 "Parser.rl" currentCls->add(CLASS_WORD, true); } }} -#line 7289 "Parser.cpp" +#line 7290 "Parser.cpp" break; } case 247: { { -#line 1043 "Parser.rl" +#line 1044 "Parser.rl" {te = p+1;{ -#line 1043 "Parser.rl" +#line 1044 "Parser.rl" currentCls->add(CLASS_SPACE, false); } }} -#line 7302 "Parser.cpp" +#line 7303 "Parser.cpp" break; } case 248: { { -#line 1047 "Parser.rl" +#line 1048 "Parser.rl" {te = p+1;{ -#line 1047 "Parser.rl" +#line 1048 "Parser.rl" currentCls->add(CLASS_SPACE, true); } }} -#line 7315 "Parser.cpp" +#line 7316 "Parser.cpp" break; } case 249: { { -#line 1051 "Parser.rl" +#line 1052 "Parser.rl" {te = p+1;{ -#line 1051 "Parser.rl" +#line 1052 "Parser.rl" currentCls->add(CLASS_DIGIT, false); } }} -#line 7328 "Parser.cpp" +#line 7329 "Parser.cpp" break; } case 250: { { -#line 1055 "Parser.rl" +#line 1056 "Parser.rl" {te = p+1;{ -#line 1055 "Parser.rl" +#line 1056 "Parser.rl" currentCls->add(CLASS_DIGIT, true); } }} -#line 7341 "Parser.cpp" +#line 7342 "Parser.cpp" break; } case 251: { { -#line 1058 "Parser.rl" +#line 1059 "Parser.rl" {te = p+1;{ -#line 1058 "Parser.rl" +#line 1059 "Parser.rl" currentCls->addDash(); } }} -#line 7354 "Parser.cpp" +#line 7355 "Parser.cpp" break; } case 252: { { -#line 276 "Parser.rl" +#line 277 "Parser.rl" {te = p+1;{ -#line 276 "Parser.rl" +#line 277 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -7630,15 +7631,15 @@ static const short _regex_actions[] = { } }} -#line 7370 "Parser.cpp" +#line 7371 "Parser.cpp" break; } case 253: { { -#line 276 "Parser.rl" +#line 277 "Parser.rl" {te = p+1;{ -#line 276 "Parser.rl" +#line 277 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -7647,15 +7648,15 @@ static const short _regex_actions[] = { } }} -#line 7386 "Parser.cpp" +#line 7387 "Parser.cpp" break; } case 254: { { -#line 276 "Parser.rl" +#line 277 "Parser.rl" {te = p+1;{ -#line 276 "Parser.rl" +#line 277 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -7664,214 +7665,226 @@ static const short _regex_actions[] = { } }} -#line 7402 "Parser.cpp" +#line 7403 "Parser.cpp" break; } case 255: { { -#line 1075 "Parser.rl" +#line 1076 "Parser.rl" {te = p+1;{ -#line 1075 "Parser.rl" +#line 1076 "Parser.rl" // add the literal char currentCls->add(*(ts + 1)); } }} -#line 7416 "Parser.cpp" +#line 7417 "Parser.cpp" break; } case 256: { { -#line 1081 "Parser.rl" +#line 1082 "Parser.rl" {te = p+1;{ -#line 1081 "Parser.rl" +#line 1082 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint2c(ts)); } }} -#line 7430 "Parser.cpp" +#line 7431 "Parser.cpp" break; } case 257: { { -#line 1086 "Parser.rl" +#line 1087 "Parser.rl" {te = p+1;{ -#line 1086 "Parser.rl" +#line 1087 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint3c(ts)); } }} -#line 7444 "Parser.cpp" +#line 7445 "Parser.cpp" break; } case 258: { { -#line 1091 "Parser.rl" +#line 1092 "Parser.rl" {te = p+1;{ -#line 1091 "Parser.rl" +#line 1092 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint4c(ts)); } }} -#line 7458 "Parser.cpp" +#line 7459 "Parser.cpp" break; } case 259: { { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" {te = p+1;{ -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7472 "Parser.cpp" +#line 7473 "Parser.cpp" break; } case 260: { { -#line 1102 "Parser.rl" +#line 1106 "Parser.rl" + {te = p+1;{ +#line 1106 "Parser.rl" + throw LocatedParseError("Character class intersection/subtraction is not supported"); } + }} + +#line 7484 "Parser.cpp" + + break; + } + case 261: { + { +#line 1109 "Parser.rl" {te = p+1;{ -#line 1102 "Parser.rl" +#line 1109 "Parser.rl" currentCls->add((u8)*ts); } }} -#line 7485 "Parser.cpp" +#line 7497 "Parser.cpp" break; } - case 261: { + case 262: { { -#line 1106 "Parser.rl" +#line 1113 "Parser.rl" {te = p+1;{ -#line 1106 "Parser.rl" +#line 1113 "Parser.rl" currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); inCharClass = false; {cs = 746;goto _again;}} }} -#line 7500 "Parser.cpp" +#line 7512 "Parser.cpp" break; } - case 262: { + case 263: { { -#line 966 "Parser.rl" +#line 967 "Parser.rl" {te = p;p = p - 1;{ -#line 966 "Parser.rl" +#line 967 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 7511 "Parser.cpp" +#line 7523 "Parser.cpp" break; } - case 263: { + case 264: { { -#line 967 "Parser.rl" +#line 968 "Parser.rl" {te = p;p = p - 1;{ -#line 967 "Parser.rl" +#line 968 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 7522 "Parser.cpp" +#line 7534 "Parser.cpp" break; } - case 264: { + case 265: { { -#line 970 "Parser.rl" +#line 971 "Parser.rl" {te = p;p = p - 1;{ -#line 970 "Parser.rl" +#line 971 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7535 "Parser.cpp" +#line 7547 "Parser.cpp" break; } - case 265: { + case 266: { { -#line 973 "Parser.rl" +#line 974 "Parser.rl" {te = p;p = p - 1;{ -#line 973 "Parser.rl" +#line 974 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7548 "Parser.cpp" +#line 7560 "Parser.cpp" break; } - case 266: { + case 267: { { -#line 992 "Parser.rl" +#line 993 "Parser.rl" {te = p;p = p - 1;{ -#line 992 "Parser.rl" +#line 993 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 7561 "Parser.cpp" +#line 7573 "Parser.cpp" break; } - case 267: { + case 268: { { -#line 997 "Parser.rl" +#line 998 "Parser.rl" {te = p;p = p - 1;{ -#line 997 "Parser.rl" +#line 998 "Parser.rl" currentCls->add(accumulator); } }} -#line 7574 "Parser.cpp" +#line 7586 "Parser.cpp" break; } - case 268: { + case 269: { { -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" {te = p;p = p - 1;{ -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 7587 "Parser.cpp" +#line 7599 "Parser.cpp" break; } - case 269: { + case 270: { { -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" {te = p;p = p - 1;{ -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -7883,105 +7896,115 @@ static const short _regex_actions[] = { } }} -#line 7606 "Parser.cpp" +#line 7618 "Parser.cpp" break; } - case 270: { + case 271: { { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" {te = p;p = p - 1;{ -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7620 "Parser.cpp" +#line 7632 "Parser.cpp" break; } - case 271: { + case 272: { + { +#line 1103 "Parser.rl" + {te = p;p = p - 1;{ +#line 1103 "Parser.rl" + throw LocatedParseError("Nested character classes are not supported"); } + }} + +#line 7643 "Parser.cpp" + + break; + } + case 273: { { -#line 1102 "Parser.rl" +#line 1109 "Parser.rl" {te = p;p = p - 1;{ -#line 1102 "Parser.rl" +#line 1109 "Parser.rl" currentCls->add((u8)*ts); } }} -#line 7633 "Parser.cpp" +#line 7656 "Parser.cpp" break; } - case 272: { + case 274: { { -#line 992 "Parser.rl" +#line 993 "Parser.rl" {p = ((te))-1; { -#line 992 "Parser.rl" +#line 993 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 7647 "Parser.cpp" +#line 7670 "Parser.cpp" break; } - case 273: { + case 275: { { -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" {p = ((te))-1; { -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 7661 "Parser.cpp" +#line 7684 "Parser.cpp" break; } - case 274: { + case 276: { { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" {p = ((te))-1; { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7676 "Parser.cpp" +#line 7699 "Parser.cpp" break; } - case 275: { + case 277: { { -#line 1102 "Parser.rl" +#line 1103 "Parser.rl" {p = ((te))-1; { -#line 1102 "Parser.rl" - - currentCls->add((u8)*ts); - } +#line 1103 "Parser.rl" + throw LocatedParseError("Nested character classes are not supported"); } }} -#line 7690 "Parser.cpp" +#line 7711 "Parser.cpp" break; } - case 276: { + case 278: { { -#line 1120 "Parser.rl" +#line 1127 "Parser.rl" {te = p+1;{ -#line 1120 "Parser.rl" +#line 1127 "Parser.rl" if (currentCls->isNegated()) { // Already seen a caret; the second one is not a meta-character. @@ -7994,250 +8017,250 @@ static const short _regex_actions[] = { } }} -#line 7710 "Parser.cpp" +#line 7731 "Parser.cpp" break; } - case 277: { + case 279: { { -#line 1133 "Parser.rl" +#line 1140 "Parser.rl" {te = p+1;{ -#line 1133 "Parser.rl" +#line 1140 "Parser.rl" currentCls->add(']'); inCharClassEarly = false; } }} -#line 7724 "Parser.cpp" +#line 7745 "Parser.cpp" break; } - case 278: { + case 280: { { -#line 1138 "Parser.rl" +#line 1145 "Parser.rl" {te = p+1;{ -#line 1138 "Parser.rl" +#line 1145 "Parser.rl" {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { stack.resize(2 * (top + 1)); } } - stack[top] = cs; top += 1;cs = 843;goto _again;}} + stack[top] = cs; top += 1;cs = 844;goto _again;}} }} -#line 7743 "Parser.cpp" +#line 7764 "Parser.cpp" break; } - case 279: { + case 281: { { -#line 1139 "Parser.rl" +#line 1146 "Parser.rl" {te = p+1;{ -#line 1139 "Parser.rl" +#line 1146 "Parser.rl" /*noop*/} }} -#line 7754 "Parser.cpp" +#line 7775 "Parser.cpp" break; } - case 280: { + case 282: { { -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" {te = p+1;{ -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" inCharClassEarly = false; {p = p - 1; } {cs = 819;goto _again;}} }} -#line 7768 "Parser.cpp" +#line 7789 "Parser.cpp" break; } - case 281: { + case 283: { { -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" {te = p;p = p - 1;{ -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" inCharClassEarly = false; {p = p - 1; } {cs = 819;goto _again;}} }} -#line 7782 "Parser.cpp" +#line 7803 "Parser.cpp" break; } - case 282: { + case 284: { { -#line 1154 "Parser.rl" +#line 1161 "Parser.rl" {te = p+1;{ -#line 1154 "Parser.rl" +#line 1161 "Parser.rl" {cs = 746;goto _again;}} }} -#line 7794 "Parser.cpp" +#line 7815 "Parser.cpp" break; } - case 283: { + case 285: { { -#line 1159 "Parser.rl" +#line 1166 "Parser.rl" {te = p+1;{ -#line 1159 "Parser.rl" +#line 1166 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint2c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 7812 "Parser.cpp" +#line 7833 "Parser.cpp" break; } - case 284: { + case 286: { { -#line 1168 "Parser.rl" +#line 1175 "Parser.rl" {te = p+1;{ -#line 1168 "Parser.rl" +#line 1175 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint3c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 7830 "Parser.cpp" +#line 7851 "Parser.cpp" break; } - case 285: { + case 287: { { -#line 1177 "Parser.rl" +#line 1184 "Parser.rl" {te = p+1;{ -#line 1177 "Parser.rl" +#line 1184 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint4c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 7848 "Parser.cpp" +#line 7869 "Parser.cpp" break; } - case 286: { + case 288: { { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" {te = p+1;{ -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7862 "Parser.cpp" +#line 7883 "Parser.cpp" break; } - case 287: { + case 289: { { -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" {te = p+1;{ -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 7875 "Parser.cpp" +#line 7896 "Parser.cpp" break; } - case 288: { + case 290: { { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" {te = p;p = p - 1;{ -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7889 "Parser.cpp" +#line 7910 "Parser.cpp" break; } - case 289: { + case 291: { { -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" {te = p;p = p - 1;{ -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 7902 "Parser.cpp" +#line 7923 "Parser.cpp" break; } - case 290: { + case 292: { { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" {p = ((te))-1; { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7917 "Parser.cpp" +#line 7938 "Parser.cpp" break; } - case 291: { + case 293: { { -#line 1202 "Parser.rl" +#line 1209 "Parser.rl" {te = p+1;{ -#line 1202 "Parser.rl" +#line 1209 "Parser.rl" {top -= 1;cs = stack[top];goto _again;} } }} -#line 7930 "Parser.cpp" +#line 7951 "Parser.cpp" break; } - case 292: { + case 294: { { -#line 1207 "Parser.rl" +#line 1214 "Parser.rl" {te = p+1;{ -#line 1207 "Parser.rl" +#line 1214 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint2c(ts)); @@ -8245,15 +8268,15 @@ static const short _regex_actions[] = { } }} -#line 7945 "Parser.cpp" +#line 7966 "Parser.cpp" break; } - case 293: { + case 295: { { -#line 1213 "Parser.rl" +#line 1220 "Parser.rl" {te = p+1;{ -#line 1213 "Parser.rl" +#line 1220 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint3c(ts)); @@ -8261,15 +8284,15 @@ static const short _regex_actions[] = { } }} -#line 7960 "Parser.cpp" +#line 7981 "Parser.cpp" break; } - case 294: { + case 296: { { -#line 1219 "Parser.rl" +#line 1226 "Parser.rl" {te = p+1;{ -#line 1219 "Parser.rl" +#line 1226 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint4c(ts)); @@ -8277,160 +8300,160 @@ static const short _regex_actions[] = { } }} -#line 7975 "Parser.cpp" +#line 7996 "Parser.cpp" break; } - case 295: { + case 297: { { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" {te = p+1;{ -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7989 "Parser.cpp" +#line 8010 "Parser.cpp" break; } - case 296: { + case 298: { { -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" {te = p+1;{ -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" currentCls->add(*ts); inCharClassEarly = false; } }} -#line 8003 "Parser.cpp" +#line 8024 "Parser.cpp" break; } - case 297: { + case 299: { { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" {te = p;p = p - 1;{ -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 8017 "Parser.cpp" +#line 8038 "Parser.cpp" break; } - case 298: { + case 300: { { -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" {te = p;p = p - 1;{ -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" currentCls->add(*ts); inCharClassEarly = false; } }} -#line 8031 "Parser.cpp" +#line 8052 "Parser.cpp" break; } - case 299: { + case 301: { { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" {p = ((te))-1; { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 8046 "Parser.cpp" +#line 8067 "Parser.cpp" break; } - case 300: { + case 302: { { -#line 1243 "Parser.rl" +#line 1250 "Parser.rl" {te = p+1;{ -#line 1243 "Parser.rl" +#line 1250 "Parser.rl" inComment = false; {cs = 746;goto _again;}} }} -#line 8057 "Parser.cpp" +#line 8078 "Parser.cpp" break; } - case 301: { + case 303: { { #line 1 "-" {te = p+1;}} -#line 8065 "Parser.cpp" +#line 8086 "Parser.cpp" break; } - case 302: { + case 304: { { -#line 1255 "Parser.rl" +#line 1262 "Parser.rl" {te = p+1;{ -#line 1255 "Parser.rl" +#line 1262 "Parser.rl" inComment = false; {cs = 746;goto _again;}} }} -#line 8076 "Parser.cpp" +#line 8097 "Parser.cpp" break; } - case 303: { + case 305: { { #line 1 "-" {te = p+1;}} -#line 8084 "Parser.cpp" +#line 8105 "Parser.cpp" break; } - case 304: { + case 306: { { -#line 1491 "Parser.rl" - {act = 288;}} +#line 1498 "Parser.rl" + {act = 290;}} -#line 8092 "Parser.cpp" +#line 8113 "Parser.cpp" break; } - case 305: { + case 307: { { -#line 1508 "Parser.rl" - {act = 290;}} +#line 1515 "Parser.rl" + {act = 292;}} -#line 8100 "Parser.cpp" +#line 8121 "Parser.cpp" break; } - case 306: { + case 308: { { -#line 1737 "Parser.rl" - {act = 330;}} +#line 1744 "Parser.rl" + {act = 332;}} -#line 8108 "Parser.cpp" +#line 8129 "Parser.cpp" break; } - case 307: { + case 309: { { -#line 362 "Parser.rl" +#line 363 "Parser.rl" {te = p+1;{ -#line 362 "Parser.rl" +#line 363 "Parser.rl" if (sequences.empty()) { throw LocatedParseError("Unmatched parentheses"); @@ -8440,98 +8463,98 @@ static const short _regex_actions[] = { } }} -#line 8125 "Parser.cpp" +#line 8146 "Parser.cpp" break; } - case 308: { + case 310: { { -#line 1274 "Parser.rl" +#line 1281 "Parser.rl" {te = p+1;{ -#line 1274 "Parser.rl" +#line 1281 "Parser.rl" currentSeq->addAlternation(); } }} -#line 8138 "Parser.cpp" +#line 8159 "Parser.cpp" break; } - case 309: { + case 311: { { -#line 1279 "Parser.rl" +#line 1286 "Parser.rl" {te = p+1;{ -#line 1279 "Parser.rl" +#line 1286 "Parser.rl" throw LocatedParseError("POSIX named classes are only " "supported inside a class"); } }} -#line 8152 "Parser.cpp" +#line 8173 "Parser.cpp" break; } - case 310: { + case 312: { { -#line 1286 "Parser.rl" +#line 1293 "Parser.rl" {te = p+1;{ -#line 1286 "Parser.rl" +#line 1293 "Parser.rl" throw LocatedParseError("Unsupported POSIX collating " "element"); } }} -#line 8166 "Parser.cpp" +#line 8187 "Parser.cpp" break; } - case 311: { + case 313: { { -#line 1293 "Parser.rl" +#line 1300 "Parser.rl" {te = p+1;{ -#line 1293 "Parser.rl" +#line 1300 "Parser.rl" - {cs = 838;goto _again;}} + {cs = 839;goto _again;}} }} -#line 8178 "Parser.cpp" +#line 8199 "Parser.cpp" break; } - case 312: { + case 314: { { -#line 1297 "Parser.rl" +#line 1304 "Parser.rl" {te = p+1;{ -#line 1297 "Parser.rl" +#line 1304 "Parser.rl" /* noop */ } }} -#line 8189 "Parser.cpp" +#line 8210 "Parser.cpp" break; } - case 313: { + case 315: { { -#line 1299 "Parser.rl" +#line 1306 "Parser.rl" {te = p+1;{ -#line 1299 "Parser.rl" +#line 1306 "Parser.rl" currentSeq->addComponent(generateComponent(CLASS_ANY, false, mode)); } }} -#line 8202 "Parser.cpp" +#line 8223 "Parser.cpp" break; } - case 314: { + case 316: { { -#line 1303 "Parser.rl" +#line 1310 "Parser.rl" {te = p+1;{ -#line 1303 "Parser.rl" +#line 1310 "Parser.rl" if (mode.utf8) { throw LocatedParseError("\\C is unsupported in UTF8"); @@ -8540,15 +8563,15 @@ static const short _regex_actions[] = { } }} -#line 8218 "Parser.cpp" +#line 8239 "Parser.cpp" break; } - case 315: { + case 317: { { -#line 1317 "Parser.rl" +#line 1324 "Parser.rl" {te = p+1;{ -#line 1317 "Parser.rl" +#line 1324 "Parser.rl" if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_NONGREEDY)) { @@ -8557,15 +8580,15 @@ static const short _regex_actions[] = { } }} -#line 8234 "Parser.cpp" +#line 8255 "Parser.cpp" break; } - case 316: { + case 318: { { -#line 1324 "Parser.rl" +#line 1331 "Parser.rl" {te = p+1;{ -#line 1324 "Parser.rl" +#line 1331 "Parser.rl" if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_POSSESSIVE)) { @@ -8574,15 +8597,15 @@ static const short _regex_actions[] = { } }} -#line 8250 "Parser.cpp" +#line 8271 "Parser.cpp" break; } - case 317: { + case 319: { { -#line 1338 "Parser.rl" +#line 1345 "Parser.rl" {te = p+1;{ -#line 1338 "Parser.rl" +#line 1345 "Parser.rl" if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_NONGREEDY)) { @@ -8591,15 +8614,15 @@ static const short _regex_actions[] = { } }} -#line 8266 "Parser.cpp" +#line 8287 "Parser.cpp" break; } - case 318: { + case 320: { { -#line 1345 "Parser.rl" +#line 1352 "Parser.rl" {te = p+1;{ -#line 1345 "Parser.rl" +#line 1352 "Parser.rl" if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_POSSESSIVE)) { @@ -8608,15 +8631,15 @@ static const short _regex_actions[] = { } }} -#line 8282 "Parser.cpp" +#line 8303 "Parser.cpp" break; } - case 319: { + case 321: { { -#line 1359 "Parser.rl" +#line 1366 "Parser.rl" {te = p+1;{ -#line 1359 "Parser.rl" +#line 1366 "Parser.rl" if (!currentSeq->addRepeat( 0, 1, ComponentRepeat::REPEAT_NONGREEDY)) { @@ -8625,15 +8648,15 @@ static const short _regex_actions[] = { } }} -#line 8298 "Parser.cpp" +#line 8319 "Parser.cpp" break; } - case 320: { + case 322: { { -#line 1366 "Parser.rl" +#line 1373 "Parser.rl" {te = p+1;{ -#line 1366 "Parser.rl" +#line 1373 "Parser.rl" if (!currentSeq->addRepeat( 0, 1, ComponentRepeat::REPEAT_POSSESSIVE)) { @@ -8642,15 +8665,15 @@ static const short _regex_actions[] = { } }} -#line 8314 "Parser.cpp" +#line 8335 "Parser.cpp" break; } - case 321: { + case 323: { { -#line 1383 "Parser.rl" +#line 1390 "Parser.rl" {te = p+1;{ -#line 1383 "Parser.rl" +#line 1390 "Parser.rl" if (repeatN > repeatM || repeatM == 0) { throwInvalidRepeat(); @@ -8662,15 +8685,15 @@ static const short _regex_actions[] = { } }} -#line 8333 "Parser.cpp" +#line 8354 "Parser.cpp" break; } - case 322: { + case 324: { { -#line 1393 "Parser.rl" +#line 1400 "Parser.rl" {te = p+1;{ -#line 1393 "Parser.rl" +#line 1400 "Parser.rl" if (repeatN > repeatM || repeatM == 0) { throwInvalidRepeat(); @@ -8682,31 +8705,31 @@ static const short _regex_actions[] = { } }} -#line 8352 "Parser.cpp" +#line 8373 "Parser.cpp" break; } - case 323: { + case 325: { { -#line 322 "Parser.rl" +#line 323 "Parser.rl" {te = p+1;{ -#line 322 "Parser.rl" +#line 323 "Parser.rl" inComment = true; - {cs = 849;goto _again;}} + {cs = 850;goto _again;}} }} -#line 8365 "Parser.cpp" +#line 8386 "Parser.cpp" break; } - case 324: { + case 326: { { -#line 1410 "Parser.rl" +#line 1417 "Parser.rl" {te = p+1;{ -#line 1410 "Parser.rl" +#line 1417 "Parser.rl" {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -8716,27 +8739,27 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 787;goto _again;}} }} -#line 8384 "Parser.cpp" +#line 8405 "Parser.cpp" break; } - case 325: { + case 327: { { -#line 1414 "Parser.rl" +#line 1421 "Parser.rl" {te = p+1;{ -#line 1414 "Parser.rl" +#line 1421 "Parser.rl" assert(0); {p += 1; goto _out; } } }} -#line 8395 "Parser.cpp" +#line 8416 "Parser.cpp" break; } - case 326: { + case 328: { { -#line 1421 "Parser.rl" +#line 1428 "Parser.rl" {te = p+1;{ -#line 1421 "Parser.rl" +#line 1428 "Parser.rl" auto bound = mode.multiline ? ComponentBoundary::BEGIN_LINE : ComponentBoundary::BEGIN_STRING; @@ -8744,15 +8767,15 @@ static const short _regex_actions[] = { } }} -#line 8410 "Parser.cpp" +#line 8431 "Parser.cpp" break; } - case 327: { + case 329: { { -#line 1428 "Parser.rl" +#line 1435 "Parser.rl" {te = p+1;{ -#line 1428 "Parser.rl" +#line 1435 "Parser.rl" auto bound = mode.multiline ? ComponentBoundary::END_LINE : ComponentBoundary::END_STRING_OPTIONAL_LF; @@ -8760,188 +8783,188 @@ static const short _regex_actions[] = { } }} -#line 8425 "Parser.cpp" +#line 8446 "Parser.cpp" break; } - case 328: { + case 330: { { -#line 1434 "Parser.rl" +#line 1441 "Parser.rl" {te = p+1;{ -#line 1434 "Parser.rl" +#line 1441 "Parser.rl" auto bound = ComponentBoundary::BEGIN_STRING; currentSeq->addComponent(std::make_unique(bound)); } }} -#line 8439 "Parser.cpp" +#line 8460 "Parser.cpp" break; } - case 329: { + case 331: { { -#line 1439 "Parser.rl" +#line 1446 "Parser.rl" {te = p+1;{ -#line 1439 "Parser.rl" +#line 1446 "Parser.rl" auto bound = ComponentBoundary::END_STRING_OPTIONAL_LF; currentSeq->addComponent(std::make_unique(bound)); } }} -#line 8453 "Parser.cpp" +#line 8474 "Parser.cpp" break; } - case 330: { + case 332: { { -#line 1444 "Parser.rl" +#line 1451 "Parser.rl" {te = p+1;{ -#line 1444 "Parser.rl" +#line 1451 "Parser.rl" auto bound = ComponentBoundary::END_STRING; currentSeq->addComponent(std::make_unique(bound)); } }} -#line 8467 "Parser.cpp" +#line 8488 "Parser.cpp" break; } - case 331: { + case 333: { { -#line 1449 "Parser.rl" +#line 1456 "Parser.rl" {te = p+1;{ -#line 1449 "Parser.rl" +#line 1456 "Parser.rl" currentSeq->addComponent( std::make_unique(ts - ptr, false, mode)); } }} -#line 8481 "Parser.cpp" +#line 8502 "Parser.cpp" break; } - case 332: { + case 334: { { -#line 1454 "Parser.rl" +#line 1461 "Parser.rl" {te = p+1;{ -#line 1454 "Parser.rl" +#line 1461 "Parser.rl" currentSeq->addComponent( std::make_unique(ts - ptr, true, mode)); } }} -#line 8495 "Parser.cpp" +#line 8516 "Parser.cpp" break; } - case 333: { + case 335: { { -#line 1464 "Parser.rl" +#line 1471 "Parser.rl" {te = p+1;{ -#line 1464 "Parser.rl" +#line 1471 "Parser.rl" addLiteral(currentSeq, '\x09', mode); } }} -#line 8508 "Parser.cpp" +#line 8529 "Parser.cpp" break; } - case 334: { + case 336: { { -#line 1468 "Parser.rl" +#line 1475 "Parser.rl" {te = p+1;{ -#line 1468 "Parser.rl" +#line 1475 "Parser.rl" addLiteral(currentSeq, '\x0a', mode); } }} -#line 8521 "Parser.cpp" +#line 8542 "Parser.cpp" break; } - case 335: { + case 337: { { -#line 1472 "Parser.rl" +#line 1479 "Parser.rl" {te = p+1;{ -#line 1472 "Parser.rl" +#line 1479 "Parser.rl" addLiteral(currentSeq, '\x0d', mode); } }} -#line 8534 "Parser.cpp" +#line 8555 "Parser.cpp" break; } - case 336: { + case 338: { { -#line 1476 "Parser.rl" +#line 1483 "Parser.rl" {te = p+1;{ -#line 1476 "Parser.rl" +#line 1483 "Parser.rl" addLiteral(currentSeq, '\x0c', mode); } }} -#line 8547 "Parser.cpp" +#line 8568 "Parser.cpp" break; } - case 337: { + case 339: { { -#line 1480 "Parser.rl" +#line 1487 "Parser.rl" {te = p+1;{ -#line 1480 "Parser.rl" +#line 1487 "Parser.rl" addLiteral(currentSeq, '\x07', mode); } }} -#line 8560 "Parser.cpp" +#line 8581 "Parser.cpp" break; } - case 338: { + case 340: { { -#line 1484 "Parser.rl" +#line 1491 "Parser.rl" {te = p+1;{ -#line 1484 "Parser.rl" +#line 1491 "Parser.rl" addLiteral(currentSeq, '\x1b', mode); } }} -#line 8573 "Parser.cpp" +#line 8594 "Parser.cpp" break; } - case 339: { + case 341: { { -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" {te = p+1;{ -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" addLiteral(currentSeq, octAccumulator, mode); } }} -#line 8586 "Parser.cpp" +#line 8607 "Parser.cpp" break; } - case 340: { + case 342: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p+1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -8950,15 +8973,15 @@ static const short _regex_actions[] = { } }} -#line 8602 "Parser.cpp" +#line 8623 "Parser.cpp" break; } - case 341: { + case 343: { { -#line 486 "Parser.rl" +#line 487 "Parser.rl" {te = p+1;{ -#line 486 "Parser.rl" +#line 487 "Parser.rl" // Accumulator is a negative offset. if (accumulator == 0) { @@ -8972,15 +8995,15 @@ static const short _regex_actions[] = { } }} -#line 8623 "Parser.cpp" +#line 8644 "Parser.cpp" break; } - case 342: { + case 344: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p+1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -8989,15 +9012,15 @@ static const short _regex_actions[] = { } }} -#line 8639 "Parser.cpp" +#line 8660 "Parser.cpp" break; } - case 343: { + case 345: { { -#line 486 "Parser.rl" +#line 487 "Parser.rl" {te = p+1;{ -#line 486 "Parser.rl" +#line 487 "Parser.rl" // Accumulator is a negative offset. if (accumulator == 0) { @@ -9011,85 +9034,85 @@ static const short _regex_actions[] = { } }} -#line 8660 "Parser.cpp" +#line 8681 "Parser.cpp" break; } - case 344: { + case 346: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8673 "Parser.cpp" +#line 8694 "Parser.cpp" break; } - case 345: { + case 347: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8686 "Parser.cpp" +#line 8707 "Parser.cpp" break; } - case 346: { + case 348: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8699 "Parser.cpp" +#line 8720 "Parser.cpp" break; } - case 347: { + case 349: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8712 "Parser.cpp" +#line 8733 "Parser.cpp" break; } - case 348: { + case 350: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8725 "Parser.cpp" +#line 8746 "Parser.cpp" break; } - case 349: { + case 351: { { -#line 1549 "Parser.rl" +#line 1556 "Parser.rl" {te = p+1;{ -#line 1549 "Parser.rl" +#line 1556 "Parser.rl" ostringstream str; str << "Onigiruma subroutine call at index " << ts - ptr << @@ -9098,15 +9121,15 @@ static const short _regex_actions[] = { } }} -#line 8741 "Parser.cpp" +#line 8762 "Parser.cpp" break; } - case 350: { + case 352: { { -#line 1560 "Parser.rl" +#line 1567 "Parser.rl" {te = p+1;{ -#line 1560 "Parser.rl" +#line 1567 "Parser.rl" string oct(ts + 3, te - ts - 4); unsigned long val; @@ -9122,29 +9145,29 @@ static const short _regex_actions[] = { } }} -#line 8764 "Parser.cpp" +#line 8785 "Parser.cpp" break; } - case 351: { + case 353: { { -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" {te = p+1;{ -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" addEscapedHex(currentSeq, accumulator, mode); } }} -#line 8777 "Parser.cpp" +#line 8798 "Parser.cpp" break; } - case 352: { + case 354: { { -#line 1582 "Parser.rl" +#line 1589 "Parser.rl" {te = p+1;{ -#line 1582 "Parser.rl" +#line 1589 "Parser.rl" string hex(ts + 3, te - ts - 4); unsigned long val; @@ -9160,15 +9183,15 @@ static const short _regex_actions[] = { } }} -#line 8800 "Parser.cpp" +#line 8821 "Parser.cpp" break; } - case 353: { + case 355: { { -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" {te = p+1;{ -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -9180,15 +9203,15 @@ static const short _regex_actions[] = { } }} -#line 8819 "Parser.cpp" +#line 8840 "Parser.cpp" break; } - case 354: { + case 356: { { -#line 1610 "Parser.rl" +#line 1617 "Parser.rl" {te = p+1;{ -#line 1610 "Parser.rl" +#line 1617 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -9197,172 +9220,172 @@ static const short _regex_actions[] = { } }} -#line 8835 "Parser.cpp" +#line 8856 "Parser.cpp" break; } - case 355: { + case 357: { { -#line 1618 "Parser.rl" +#line 1625 "Parser.rl" {te = p+1;{ -#line 1618 "Parser.rl" +#line 1625 "Parser.rl" auto cc = generateComponent(CLASS_WORD, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8849 "Parser.cpp" +#line 8870 "Parser.cpp" break; } - case 356: { + case 358: { { -#line 1623 "Parser.rl" +#line 1630 "Parser.rl" {te = p+1;{ -#line 1623 "Parser.rl" +#line 1630 "Parser.rl" auto cc = generateComponent(CLASS_WORD, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8863 "Parser.cpp" +#line 8884 "Parser.cpp" break; } - case 357: { + case 359: { { -#line 1628 "Parser.rl" +#line 1635 "Parser.rl" {te = p+1;{ -#line 1628 "Parser.rl" +#line 1635 "Parser.rl" auto cc = generateComponent(CLASS_SPACE, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8877 "Parser.cpp" +#line 8898 "Parser.cpp" break; } - case 358: { + case 360: { { -#line 1633 "Parser.rl" +#line 1640 "Parser.rl" {te = p+1;{ -#line 1633 "Parser.rl" +#line 1640 "Parser.rl" auto cc = generateComponent(CLASS_SPACE, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8891 "Parser.cpp" +#line 8912 "Parser.cpp" break; } - case 359: { + case 361: { { -#line 1638 "Parser.rl" +#line 1645 "Parser.rl" {te = p+1;{ -#line 1638 "Parser.rl" +#line 1645 "Parser.rl" auto cc = generateComponent(CLASS_DIGIT, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8905 "Parser.cpp" +#line 8926 "Parser.cpp" break; } - case 360: { + case 362: { { -#line 1643 "Parser.rl" +#line 1650 "Parser.rl" {te = p+1;{ -#line 1643 "Parser.rl" +#line 1650 "Parser.rl" auto cc = generateComponent(CLASS_DIGIT, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8919 "Parser.cpp" +#line 8940 "Parser.cpp" break; } - case 361: { + case 363: { { -#line 1648 "Parser.rl" +#line 1655 "Parser.rl" {te = p+1;{ -#line 1648 "Parser.rl" +#line 1655 "Parser.rl" auto cc = generateComponent(CLASS_HORZ, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8933 "Parser.cpp" +#line 8954 "Parser.cpp" break; } - case 362: { + case 364: { { -#line 1653 "Parser.rl" +#line 1660 "Parser.rl" {te = p+1;{ -#line 1653 "Parser.rl" +#line 1660 "Parser.rl" auto cc = generateComponent(CLASS_HORZ, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8947 "Parser.cpp" +#line 8968 "Parser.cpp" break; } - case 363: { + case 365: { { -#line 1658 "Parser.rl" +#line 1665 "Parser.rl" {te = p+1;{ -#line 1658 "Parser.rl" +#line 1665 "Parser.rl" auto cc = generateComponent(CLASS_VERT, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8961 "Parser.cpp" +#line 8982 "Parser.cpp" break; } - case 364: { + case 366: { { -#line 1663 "Parser.rl" +#line 1670 "Parser.rl" {te = p+1;{ -#line 1663 "Parser.rl" +#line 1670 "Parser.rl" auto cc = generateComponent(CLASS_VERT, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8975 "Parser.cpp" +#line 8996 "Parser.cpp" break; } - case 365: { + case 367: { { -#line 1668 "Parser.rl" +#line 1675 "Parser.rl" {te = p+1;{ -#line 1668 "Parser.rl" +#line 1675 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9372,22 +9395,22 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 8999 "Parser.cpp" +#line 9020 "Parser.cpp" break; } - case 366: { + case 368: { { -#line 1676 "Parser.rl" +#line 1683 "Parser.rl" {te = p+1;{ -#line 1676 "Parser.rl" +#line 1683 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9397,22 +9420,22 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 9023 "Parser.cpp" +#line 9044 "Parser.cpp" break; } - case 367: { + case 369: { { -#line 1684 "Parser.rl" +#line 1691 "Parser.rl" {te = p+1;{ -#line 1684 "Parser.rl" +#line 1691 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9422,22 +9445,22 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 9047 "Parser.cpp" +#line 9068 "Parser.cpp" break; } - case 368: { + case 370: { { -#line 1692 "Parser.rl" +#line 1699 "Parser.rl" {te = p+1;{ -#line 1692 "Parser.rl" +#line 1699 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9447,15 +9470,15 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 9071 "Parser.cpp" +#line 9092 "Parser.cpp" break; } - case 369: { + case 371: { { -#line 1704 "Parser.rl" +#line 1711 "Parser.rl" {te = p+1;{ -#line 1704 "Parser.rl" +#line 1711 "Parser.rl" ostringstream str; str << "\\R at index " << ts - ptr << " not supported."; @@ -9463,15 +9486,15 @@ static const short _regex_actions[] = { } }} -#line 9086 "Parser.cpp" +#line 9107 "Parser.cpp" break; } - case 370: { + case 372: { { -#line 1711 "Parser.rl" +#line 1718 "Parser.rl" {te = p+1;{ -#line 1711 "Parser.rl" +#line 1718 "Parser.rl" ostringstream str; str << "\\K at index " << ts - ptr << " not supported."; @@ -9479,15 +9502,15 @@ static const short _regex_actions[] = { } }} -#line 9101 "Parser.cpp" +#line 9122 "Parser.cpp" break; } - case 371: { + case 373: { { -#line 1726 "Parser.rl" +#line 1733 "Parser.rl" {te = p+1;{ -#line 1726 "Parser.rl" +#line 1733 "Parser.rl" ostringstream str; str << "\\G at index " << ts - ptr << " not supported."; @@ -9495,72 +9518,72 @@ static const short _regex_actions[] = { } }} -#line 9116 "Parser.cpp" +#line 9137 "Parser.cpp" break; } - case 372: { + case 374: { { -#line 1732 "Parser.rl" +#line 1739 "Parser.rl" {te = p+1;{ -#line 1732 "Parser.rl" +#line 1739 "Parser.rl" currentSeq->addComponent(std::make_unique(ts - ptr, mode)); } }} -#line 9129 "Parser.cpp" +#line 9150 "Parser.cpp" break; } - case 373: { + case 375: { { -#line 1737 "Parser.rl" +#line 1744 "Parser.rl" {te = p+1;{ -#line 1737 "Parser.rl" +#line 1744 "Parser.rl" addLiteral(currentSeq, *(ts + 1), mode); } }} -#line 9142 "Parser.cpp" +#line 9163 "Parser.cpp" break; } - case 374: { + case 376: { { -#line 316 "Parser.rl" +#line 317 "Parser.rl" {te = p+1;{ -#line 316 "Parser.rl" +#line 317 "Parser.rl" inComment = true; - {cs = 848;goto _again;}} + {cs = 849;goto _again;}} }} -#line 9155 "Parser.cpp" +#line 9176 "Parser.cpp" break; } - case 375: { + case 377: { { -#line 433 "Parser.rl" +#line 434 "Parser.rl" {te = p+1;{ -#line 433 "Parser.rl" +#line 434 "Parser.rl" mode = newMode; currentSeq->addComponent(std::make_unique()); } }} -#line 9169 "Parser.cpp" +#line 9190 "Parser.cpp" break; } - case 376: { + case 378: { { -#line 355 "Parser.rl" +#line 356 "Parser.rl" {te = p+1;{ -#line 355 "Parser.rl" +#line 356 "Parser.rl" PUSH_SEQUENCE; mode = newMode; @@ -9569,15 +9592,15 @@ static const short _regex_actions[] = { } }} -#line 9185 "Parser.cpp" +#line 9206 "Parser.cpp" break; } - case 377: { + case 379: { { -#line 369 "Parser.rl" +#line 370 "Parser.rl" {te = p+1;{ -#line 369 "Parser.rl" +#line 370 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9586,15 +9609,15 @@ static const short _regex_actions[] = { } }} -#line 9201 "Parser.cpp" +#line 9222 "Parser.cpp" break; } - case 378: { + case 380: { { -#line 375 "Parser.rl" +#line 376 "Parser.rl" {te = p+1;{ -#line 375 "Parser.rl" +#line 376 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9603,15 +9626,15 @@ static const short _regex_actions[] = { } }} -#line 9217 "Parser.cpp" +#line 9238 "Parser.cpp" break; } - case 379: { + case 381: { { -#line 381 "Parser.rl" +#line 382 "Parser.rl" {te = p+1;{ -#line 381 "Parser.rl" +#line 382 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9620,15 +9643,15 @@ static const short _regex_actions[] = { } }} -#line 9233 "Parser.cpp" +#line 9254 "Parser.cpp" break; } - case 380: { + case 382: { { -#line 387 "Parser.rl" +#line 388 "Parser.rl" {te = p+1;{ -#line 387 "Parser.rl" +#line 388 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9637,43 +9660,43 @@ static const short _regex_actions[] = { } }} -#line 9249 "Parser.cpp" +#line 9270 "Parser.cpp" break; } - case 381: { + case 383: { { -#line 393 "Parser.rl" +#line 394 "Parser.rl" {te = p+1;{ -#line 393 "Parser.rl" +#line 394 "Parser.rl" throw LocatedParseError("Embedded code is not supported"); } }} -#line 9262 "Parser.cpp" +#line 9283 "Parser.cpp" break; } - case 382: { + case 384: { { -#line 393 "Parser.rl" +#line 394 "Parser.rl" {te = p+1;{ -#line 393 "Parser.rl" +#line 394 "Parser.rl" throw LocatedParseError("Embedded code is not supported"); } }} -#line 9275 "Parser.cpp" +#line 9296 "Parser.cpp" break; } - case 383: { + case 385: { { -#line 416 "Parser.rl" +#line 417 "Parser.rl" {te = p+1;{ -#line 416 "Parser.rl" +#line 417 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9681,15 +9704,15 @@ static const short _regex_actions[] = { } }} -#line 9290 "Parser.cpp" +#line 9311 "Parser.cpp" break; } - case 384: { + case 386: { { -#line 336 "Parser.rl" +#line 337 "Parser.rl" {te = p+1;{ -#line 336 "Parser.rl" +#line 337 "Parser.rl" assert(!label.empty()); // should be guaranteed by machine char c = *label.begin(); @@ -9703,145 +9726,145 @@ static const short _regex_actions[] = { auto seq = std::make_unique(); seq->setCaptureIndex(groupIndex++); seq->setCaptureName(label); - currentSeq = enterSequence(currentSeq, move(seq)); + currentSeq = enterSequence(currentSeq, std::move(seq)); } }} -#line 9315 "Parser.cpp" +#line 9336 "Parser.cpp" break; } - case 385: { + case 387: { { -#line 399 "Parser.rl" +#line 400 "Parser.rl" {te = p+1;{ -#line 399 "Parser.rl" +#line 400 "Parser.rl" throw LocatedParseError("Subpattern reference unsupported"); } }} -#line 9328 "Parser.cpp" +#line 9349 "Parser.cpp" break; } - case 386: { + case 388: { { -#line 399 "Parser.rl" +#line 400 "Parser.rl" {te = p+1;{ -#line 399 "Parser.rl" +#line 400 "Parser.rl" throw LocatedParseError("Subpattern reference unsupported"); } }} -#line 9341 "Parser.cpp" +#line 9362 "Parser.cpp" break; } - case 387: { + case 389: { { -#line 1783 "Parser.rl" +#line 1790 "Parser.rl" {te = p+1;{ -#line 1783 "Parser.rl" +#line 1790 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKAHEAD, ComponentAssertion::POS); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9361 "Parser.cpp" +#line 9382 "Parser.cpp" break; } - case 388: { + case 390: { { -#line 1794 "Parser.rl" +#line 1801 "Parser.rl" {te = p+1;{ -#line 1794 "Parser.rl" +#line 1801 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKAHEAD, ComponentAssertion::NEG); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9381 "Parser.cpp" +#line 9402 "Parser.cpp" break; } - case 389: { + case 391: { { -#line 1805 "Parser.rl" +#line 1812 "Parser.rl" {te = p+1;{ -#line 1805 "Parser.rl" +#line 1812 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKBEHIND, ComponentAssertion::POS); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9401 "Parser.cpp" +#line 9422 "Parser.cpp" break; } - case 390: { + case 392: { { -#line 1816 "Parser.rl" +#line 1823 "Parser.rl" {te = p+1;{ -#line 1816 "Parser.rl" +#line 1823 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKBEHIND, ComponentAssertion::NEG); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9421 "Parser.cpp" +#line 9442 "Parser.cpp" break; } - case 391: { + case 393: { { -#line 1828 "Parser.rl" +#line 1835 "Parser.rl" {te = p+1;{ -#line 1828 "Parser.rl" +#line 1835 "Parser.rl" throw LocatedParseError("Pattern recursion not supported"); } }} -#line 9434 "Parser.cpp" +#line 9455 "Parser.cpp" break; } - case 392: { + case 394: { { -#line 402 "Parser.rl" +#line 403 "Parser.rl" {te = p+1;{ -#line 402 "Parser.rl" +#line 403 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -9852,15 +9875,15 @@ static const short _regex_actions[] = { } }} -#line 9452 "Parser.cpp" +#line 9473 "Parser.cpp" break; } - case 393: { + case 395: { { -#line 410 "Parser.rl" +#line 411 "Parser.rl" {te = p+1;{ -#line 410 "Parser.rl" +#line 411 "Parser.rl" PUSH_SEQUENCE; assert(!label.empty()); @@ -9869,15 +9892,15 @@ static const short _regex_actions[] = { } }} -#line 9468 "Parser.cpp" +#line 9489 "Parser.cpp" break; } - case 394: { + case 396: { { -#line 1844 "Parser.rl" +#line 1851 "Parser.rl" {te = p+1;{ -#line 1844 "Parser.rl" +#line 1851 "Parser.rl" ostringstream str; str << "Callout at index " << ts - ptr << " not supported."; @@ -9885,101 +9908,101 @@ static const short _regex_actions[] = { } }} -#line 9483 "Parser.cpp" +#line 9504 "Parser.cpp" break; } - case 395: { + case 397: { { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" {te = p+1;{ -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" throw LocatedParseError("Unrecognised character after (?"); } }} -#line 9496 "Parser.cpp" +#line 9517 "Parser.cpp" break; } - case 396: { + case 398: { { -#line 1857 "Parser.rl" +#line 1864 "Parser.rl" {te = p+1;{ -#line 1857 "Parser.rl" +#line 1864 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint2c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 9514 "Parser.cpp" +#line 9535 "Parser.cpp" break; } - case 397: { + case 399: { { -#line 1866 "Parser.rl" +#line 1873 "Parser.rl" {te = p+1;{ -#line 1866 "Parser.rl" +#line 1873 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint3c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 9532 "Parser.cpp" +#line 9553 "Parser.cpp" break; } - case 398: { + case 400: { { -#line 1875 "Parser.rl" +#line 1882 "Parser.rl" {te = p+1;{ -#line 1875 "Parser.rl" +#line 1882 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint4c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 9550 "Parser.cpp" +#line 9571 "Parser.cpp" break; } - case 399: { + case 401: { { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" {te = p+1;{ -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 9564 "Parser.cpp" +#line 9585 "Parser.cpp" break; } - case 400: { + case 402: { { -#line 1893 "Parser.rl" +#line 1900 "Parser.rl" {te = p+1;{ -#line 1893 "Parser.rl" +#line 1900 "Parser.rl" if (mode.ignore_space == false) { addLiteral(currentSeq, *ts, mode); @@ -9987,46 +10010,46 @@ static const short _regex_actions[] = { } }} -#line 9579 "Parser.cpp" +#line 9600 "Parser.cpp" break; } - case 401: { + case 403: { { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" {te = p+1;{ -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 9592 "Parser.cpp" +#line 9613 "Parser.cpp" break; } - case 402: { + case 404: { { -#line 328 "Parser.rl" +#line 329 "Parser.rl" {te = p;p = p - 1;{ -#line 328 "Parser.rl" +#line 329 "Parser.rl" PUSH_SEQUENCE; auto seq = std::make_unique(); seq->setCaptureIndex(groupIndex++); - currentSeq = enterSequence(currentSeq, move(seq)); + currentSeq = enterSequence(currentSeq, std::move(seq)); } }} -#line 9608 "Parser.cpp" +#line 9629 "Parser.cpp" break; } - case 403: { + case 405: { { -#line 421 "Parser.rl" +#line 422 "Parser.rl" {te = p;p = p - 1;{ -#line 421 "Parser.rl" +#line 422 "Parser.rl" assert(!currentCls); assert(!inCharClass); // not reentrant @@ -10034,18 +10057,18 @@ static const short _regex_actions[] = { inCharClass = true; inCharClassEarly = true; currentClsBegin = ts; - {cs = 836;goto _again;}} + {cs = 837;goto _again;}} }} -#line 9626 "Parser.cpp" +#line 9647 "Parser.cpp" break; } - case 404: { + case 406: { { -#line 1310 "Parser.rl" +#line 1317 "Parser.rl" {te = p;p = p - 1;{ -#line 1310 "Parser.rl" +#line 1317 "Parser.rl" if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_GREEDY)) { @@ -10054,15 +10077,15 @@ static const short _regex_actions[] = { } }} -#line 9642 "Parser.cpp" +#line 9663 "Parser.cpp" break; } - case 405: { + case 407: { { -#line 1331 "Parser.rl" +#line 1338 "Parser.rl" {te = p;p = p - 1;{ -#line 1331 "Parser.rl" +#line 1338 "Parser.rl" if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_GREEDY)) { @@ -10071,15 +10094,15 @@ static const short _regex_actions[] = { } }} -#line 9658 "Parser.cpp" +#line 9679 "Parser.cpp" break; } - case 406: { + case 408: { { -#line 1352 "Parser.rl" +#line 1359 "Parser.rl" {te = p;p = p - 1;{ -#line 1352 "Parser.rl" +#line 1359 "Parser.rl" if (!currentSeq->addRepeat( 0, 1, ComponentRepeat::REPEAT_GREEDY)) { @@ -10088,15 +10111,15 @@ static const short _regex_actions[] = { } }} -#line 9674 "Parser.cpp" +#line 9695 "Parser.cpp" break; } - case 407: { + case 409: { { -#line 1373 "Parser.rl" +#line 1380 "Parser.rl" {te = p;p = p - 1;{ -#line 1373 "Parser.rl" +#line 1380 "Parser.rl" if (repeatN > repeatM || repeatM == 0) { throwInvalidRepeat(); @@ -10108,29 +10131,29 @@ static const short _regex_actions[] = { } }} -#line 9693 "Parser.cpp" +#line 9714 "Parser.cpp" break; } - case 408: { + case 410: { { -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" {te = p;p = p - 1;{ -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" addLiteral(currentSeq, octAccumulator, mode); } }} -#line 9706 "Parser.cpp" +#line 9727 "Parser.cpp" break; } - case 409: { + case 411: { { -#line 1491 "Parser.rl" +#line 1498 "Parser.rl" {te = p;p = p - 1;{ -#line 1491 "Parser.rl" +#line 1498 "Parser.rl" // If there are enough capturing sub expressions, this may be // a back reference @@ -10143,15 +10166,15 @@ static const short _regex_actions[] = { } }} -#line 9726 "Parser.cpp" +#line 9747 "Parser.cpp" break; } - case 410: { + case 412: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p;p = p - 1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -10160,15 +10183,15 @@ static const short _regex_actions[] = { } }} -#line 9742 "Parser.cpp" +#line 9763 "Parser.cpp" break; } - case 411: { + case 413: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p;p = p - 1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -10177,15 +10200,15 @@ static const short _regex_actions[] = { } }} -#line 9758 "Parser.cpp" +#line 9779 "Parser.cpp" break; } - case 412: { + case 414: { { -#line 486 "Parser.rl" +#line 487 "Parser.rl" {te = p;p = p - 1;{ -#line 486 "Parser.rl" +#line 487 "Parser.rl" // Accumulator is a negative offset. if (accumulator == 0) { @@ -10199,71 +10222,71 @@ static const short _regex_actions[] = { } }} -#line 9779 "Parser.cpp" +#line 9800 "Parser.cpp" break; } - case 413: { + case 415: { { -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" {te = p;p = p - 1;{ -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" throw LocatedParseError("Invalid reference after \\g"); } }} -#line 9792 "Parser.cpp" +#line 9813 "Parser.cpp" break; } - case 414: { + case 416: { { -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" {te = p;p = p - 1;{ -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 9805 "Parser.cpp" +#line 9826 "Parser.cpp" break; } - case 415: { + case 417: { { -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" {te = p;p = p - 1;{ -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" addEscapedHex(currentSeq, accumulator, mode); } }} -#line 9818 "Parser.cpp" +#line 9839 "Parser.cpp" break; } - case 416: { + case 418: { { -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" {te = p;p = p - 1;{ -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 9831 "Parser.cpp" +#line 9852 "Parser.cpp" break; } - case 417: { + case 419: { { -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" {te = p;p = p - 1;{ -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -10275,39 +10298,39 @@ static const short _regex_actions[] = { } }} -#line 9850 "Parser.cpp" +#line 9871 "Parser.cpp" break; } - case 418: { + case 420: { { -#line 1700 "Parser.rl" +#line 1707 "Parser.rl" {te = p;p = p - 1;{ -#line 1700 "Parser.rl" +#line 1707 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 9861 "Parser.cpp" +#line 9882 "Parser.cpp" break; } - case 419: { + case 421: { { -#line 1701 "Parser.rl" +#line 1708 "Parser.rl" {te = p;p = p - 1;{ -#line 1701 "Parser.rl" +#line 1708 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 9872 "Parser.cpp" +#line 9893 "Parser.cpp" break; } - case 420: { + case 422: { { -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" {te = p;p = p - 1;{ -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" ostringstream str; str << "\\k at index " << ts - ptr << " not supported."; @@ -10315,15 +10338,15 @@ static const short _regex_actions[] = { } }} -#line 9887 "Parser.cpp" +#line 9908 "Parser.cpp" break; } - case 421: { + case 423: { { -#line 1742 "Parser.rl" +#line 1749 "Parser.rl" {te = p;p = p - 1;{ -#line 1742 "Parser.rl" +#line 1749 "Parser.rl" assert(ts + 1 == pe); ostringstream str; @@ -10332,91 +10355,91 @@ static const short _regex_actions[] = { } }} -#line 9903 "Parser.cpp" +#line 9924 "Parser.cpp" break; } - case 422: { + case 424: { { -#line 396 "Parser.rl" +#line 397 "Parser.rl" {te = p;p = p - 1;{ -#line 396 "Parser.rl" +#line 397 "Parser.rl" throw LocatedParseError("Conditional subpattern unsupported"); } }} -#line 9916 "Parser.cpp" +#line 9937 "Parser.cpp" break; } - case 423: { + case 425: { { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" {te = p;p = p - 1;{ -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" throw LocatedParseError("Unrecognised character after (?"); } }} -#line 9929 "Parser.cpp" +#line 9950 "Parser.cpp" break; } - case 424: { + case 426: { { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" {te = p;p = p - 1;{ -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 9943 "Parser.cpp" +#line 9964 "Parser.cpp" break; } - case 425: { + case 427: { { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" {te = p;p = p - 1;{ -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 9956 "Parser.cpp" +#line 9977 "Parser.cpp" break; } - case 426: { + case 428: { { -#line 328 "Parser.rl" +#line 329 "Parser.rl" {p = ((te))-1; { -#line 328 "Parser.rl" +#line 329 "Parser.rl" PUSH_SEQUENCE; auto seq = std::make_unique(); seq->setCaptureIndex(groupIndex++); - currentSeq = enterSequence(currentSeq, move(seq)); + currentSeq = enterSequence(currentSeq, std::move(seq)); } }} -#line 9973 "Parser.cpp" +#line 9994 "Parser.cpp" break; } - case 427: { + case 429: { { -#line 421 "Parser.rl" +#line 422 "Parser.rl" {p = ((te))-1; { -#line 421 "Parser.rl" +#line 422 "Parser.rl" assert(!currentCls); assert(!inCharClass); // not reentrant @@ -10424,64 +10447,64 @@ static const short _regex_actions[] = { inCharClass = true; inCharClassEarly = true; currentClsBegin = ts; - {cs = 836;goto _again;}} + {cs = 837;goto _again;}} }} -#line 9992 "Parser.cpp" +#line 10013 "Parser.cpp" break; } - case 428: { + case 430: { { -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" {p = ((te))-1; { -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" throw LocatedParseError("Invalid reference after \\g"); } }} -#line 10006 "Parser.cpp" +#line 10027 "Parser.cpp" break; } - case 429: { + case 431: { { -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" {p = ((te))-1; { -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 10020 "Parser.cpp" +#line 10041 "Parser.cpp" break; } - case 430: { + case 432: { { -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" {p = ((te))-1; { -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 10034 "Parser.cpp" +#line 10055 "Parser.cpp" break; } - case 431: { + case 433: { { -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" {p = ((te))-1; { -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" ostringstream str; str << "\\k at index " << ts - ptr << " not supported."; @@ -10489,79 +10512,79 @@ static const short _regex_actions[] = { } }} -#line 10050 "Parser.cpp" +#line 10071 "Parser.cpp" break; } - case 432: { + case 434: { { -#line 396 "Parser.rl" +#line 397 "Parser.rl" {p = ((te))-1; { -#line 396 "Parser.rl" +#line 397 "Parser.rl" throw LocatedParseError("Conditional subpattern unsupported"); } }} -#line 10064 "Parser.cpp" +#line 10085 "Parser.cpp" break; } - case 433: { + case 435: { { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" {p = ((te))-1; { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" throw LocatedParseError("Unrecognised character after (?"); } }} -#line 10078 "Parser.cpp" +#line 10099 "Parser.cpp" break; } - case 434: { + case 436: { { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" {p = ((te))-1; { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 10093 "Parser.cpp" +#line 10114 "Parser.cpp" break; } - case 435: { + case 437: { { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" {p = ((te))-1; { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 10107 "Parser.cpp" +#line 10128 "Parser.cpp" break; } - case 436: { + case 438: { { #line 1 "NONE" {switch( act ) { - case 288: { + case 290: { p = ((te))-1; { -#line 1491 "Parser.rl" +#line 1498 "Parser.rl" // If there are enough capturing sub expressions, this may be // a back reference @@ -10574,10 +10597,10 @@ static const short _regex_actions[] = { } break; } - case 290: { + case 292: { p = ((te))-1; { -#line 1508 "Parser.rl" +#line 1515 "Parser.rl" // if there are enough left parens to this point, back ref if (accumulator < groupIndex) { @@ -10611,10 +10634,10 @@ static const short _regex_actions[] = { } break; } - case 330: { + case 332: { p = ((te))-1; { -#line 1737 "Parser.rl" +#line 1744 "Parser.rl" addLiteral(currentSeq, *(ts + 1), mode); } @@ -10623,7 +10646,7 @@ static const short _regex_actions[] = { }} } -#line 10179 "Parser.cpp" +#line 10200 "Parser.cpp" break; } @@ -10650,7 +10673,7 @@ else { #line 1 "NONE" {ts = 0;}} -#line 10205 "Parser.cpp" +#line 10226 "Parser.cpp" break; } @@ -10667,7 +10690,7 @@ else { _out: {} } -#line 1983 "Parser.rl" +#line 1990 "Parser.rl" if (p != pe && *p != '\0') { @@ -10711,7 +10734,7 @@ assert(currentSeq == rootSeq.get()); // Ensure that all references are valid. checkReferences(*rootSeq, groupIndex, groupNames); -return move(rootSeq); +return rootSeq; } catch (LocatedParseError &error) { if (ts >= ptr && ts <= pe) { error.locate(ts - ptr); diff --git a/contrib/vectorscan-cmake/rageled_files/aarch64/control_verbs.cpp b/contrib/vectorscan-cmake/rageled_files/aarch64/control_verbs.cpp index 6204fc0a31462..57cea09266693 100644 --- a/contrib/vectorscan-cmake/rageled_files/aarch64/control_verbs.cpp +++ b/contrib/vectorscan-cmake/rageled_files/aarch64/control_verbs.cpp @@ -74,7 +74,7 @@ static const signed char _ControlVerbs_actions[] = { 155, 157, 159, 160, 161, 0 }; - static const char _ControlVerbs_trans_keys[] = { + static const unsigned char _ControlVerbs_trans_keys[] = { 41u, 65u, 66u, 67u, 76u, 78u, 85u, 41u, 41u, 78u, 41u, 89u, 41u, 67u, 41u, 82u, 41u, 76u, 41u, 70u, 41u, 41u, 83u, 41u, @@ -276,7 +276,7 @@ static const signed char _ControlVerbs_actions[] = { te = 0; } -#line 105 "control_verbs.rl" +#line 106 "control_verbs.rl" try { @@ -285,7 +285,7 @@ static const signed char _ControlVerbs_actions[] = { { int _klen; unsigned int _trans = 0; - const char * _keys; + const unsigned char * _keys; const signed char * _acts; unsigned int _nacts; _resume: {} @@ -321,9 +321,9 @@ static const signed char _ControlVerbs_actions[] = { _klen = (int)_ControlVerbs_single_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + _klen - 1; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + _klen - 1; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _keys += _klen; @@ -345,9 +345,9 @@ static const signed char _ControlVerbs_actions[] = { _klen = (int)_ControlVerbs_range_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + (_klen<<1) - 2; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + (_klen<<1) - 2; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _trans += (unsigned int)_klen; @@ -389,9 +389,9 @@ static const signed char _ControlVerbs_actions[] = { } case 3: { { -#line 76 "control_verbs.rl" +#line 77 "control_verbs.rl" {te = p+1;{ -#line 76 "control_verbs.rl" +#line 77 "control_verbs.rl" mode.utf8 = true; } @@ -403,9 +403,9 @@ static const signed char _ControlVerbs_actions[] = { } case 4: { { -#line 80 "control_verbs.rl" +#line 81 "control_verbs.rl" {te = p+1;{ -#line 80 "control_verbs.rl" +#line 81 "control_verbs.rl" mode.ucp = true; } @@ -417,9 +417,9 @@ static const signed char _ControlVerbs_actions[] = { } case 5: { { -#line 84 "control_verbs.rl" +#line 85 "control_verbs.rl" {te = p+1;{ -#line 84 "control_verbs.rl" +#line 85 "control_verbs.rl" ostringstream str; str << "Unsupported control verb " << string(ts, te - ts); @@ -433,9 +433,9 @@ static const signed char _ControlVerbs_actions[] = { } case 6: { { -#line 90 "control_verbs.rl" +#line 91 "control_verbs.rl" {te = p+1;{ -#line 90 "control_verbs.rl" +#line 91 "control_verbs.rl" ostringstream str; str << "Unknown control verb " << string(ts, te - ts); @@ -449,9 +449,9 @@ static const signed char _ControlVerbs_actions[] = { } case 7: { { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {te = p+1;{ -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = p - 1; } {p += 1; goto _out; } @@ -464,9 +464,9 @@ static const signed char _ControlVerbs_actions[] = { } case 8: { { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {te = p;p = p - 1;{ -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = p - 1; } {p += 1; goto _out; } @@ -479,10 +479,10 @@ static const signed char _ControlVerbs_actions[] = { } case 9: { { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = ((te))-1; { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = p - 1; } {p += 1; goto _out; } @@ -530,9 +530,11 @@ static const signed char _ControlVerbs_actions[] = { _out: {} } -#line 108 "control_verbs.rl" +#line 109 "control_verbs.rl" } catch (LocatedParseError &error) { + // cppcheck-suppress pointerLessThanZero + // cppcheck-suppress pointerPositive if (ts >= ptr && ts <= pe) { error.locate(ts - ptr + start); } else { diff --git a/contrib/vectorscan-cmake/rageled_files/amd64/Parser.cpp b/contrib/vectorscan-cmake/rageled_files/amd64/Parser.cpp index ffda15155821d..05494b959c23a 100644 --- a/contrib/vectorscan-cmake/rageled_files/amd64/Parser.cpp +++ b/contrib/vectorscan-cmake/rageled_files/amd64/Parser.cpp @@ -164,8 +164,8 @@ namespace ue2 { assert(child); ComponentSequence *seq = child.get(); - parent->addComponent(move(child)); - return seq; + parent->addComponent(std::move(child)); + return seq; // cppcheck-suppress returnDanglingLifetime } static @@ -176,7 +176,7 @@ namespace ue2 { assert(cc); cc->add(c); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } else { currentSeq->addComponent(getLiteralComponentClass(c, mode.caseless)); } @@ -191,7 +191,7 @@ namespace ue2 { assert(cc); cc->add(accum); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } else { if (accum > 255) { throw LocatedParseError(err_msg); @@ -225,7 +225,7 @@ namespace ue2 { static unichar readUtf8CodePoint2c(const char *s) { - auto *ts = (const u8 *)s; + auto *ts = reinterpret_cast(s); assert(ts[0] >= 0xc0 && ts[0] < 0xe0); assert(ts[1] >= 0x80 && ts[1] < 0xc0); unichar val = ts[0] & 0x1f; @@ -238,7 +238,7 @@ namespace ue2 { static unichar readUtf8CodePoint3c(const char *s) { - auto *ts = (const u8 *)s; + auto *ts = reinterpret_cast(s); assert(ts[0] >= 0xe0 && ts[0] < 0xf0); assert(ts[1] >= 0x80 && ts[1] < 0xc0); assert(ts[2] >= 0x80 && ts[2] < 0xc0); @@ -254,7 +254,7 @@ namespace ue2 { static unichar readUtf8CodePoint4c(const char *s) { - auto *ts = (const u8 *)s; + auto *ts = reinterpret_cast(s); assert(ts[0] >= 0xf0 && ts[0] < 0xf8); assert(ts[1] >= 0x80 && ts[1] < 0xc0); assert(ts[2] >= 0x80 && ts[2] < 0xc0); @@ -272,7 +272,7 @@ namespace ue2 { } -#line 1909 "Parser.rl" +#line 1916 "Parser.rl" @@ -352,18 +352,18 @@ static const short _regex_actions[] = { 289, 1, 290, 1, 291, 1, 292, 1, 293, 1, 294, 1, 295, 1, 296, 1, 297, 1, 298, 1, 299, 1, 300, 1, - 301, 1, 302, 1, 303, 1, 307, 1, - 308, 1, 309, 1, 310, 1, 311, 1, + 301, 1, 302, 1, 303, 1, 304, 1, + 305, 1, 309, 1, 310, 1, 311, 1, 312, 1, 313, 1, 314, 1, 315, 1, 316, 1, 317, 1, 318, 1, 319, 1, 320, 1, 321, 1, 322, 1, 323, 1, 324, 1, 325, 1, 326, 1, 327, 1, 328, 1, 329, 1, 330, 1, 331, 1, 332, 1, 333, 1, 334, 1, 335, 1, - 336, 1, 337, 1, 338, 1, 342, 1, - 343, 1, 344, 1, 345, 1, 346, 1, + 336, 1, 337, 1, 338, 1, 339, 1, + 340, 1, 344, 1, 345, 1, 346, 1, 347, 1, 348, 1, 349, 1, 350, 1, - 352, 1, 353, 1, 354, 1, 355, 1, + 351, 1, 352, 1, 354, 1, 355, 1, 356, 1, 357, 1, 358, 1, 359, 1, 360, 1, 361, 1, 362, 1, 363, 1, 364, 1, 365, 1, 366, 1, 367, 1, @@ -384,29 +384,30 @@ static const short _regex_actions[] = { 424, 1, 425, 1, 426, 1, 427, 1, 428, 1, 429, 1, 430, 1, 431, 1, 432, 1, 433, 1, 434, 1, 435, 1, - 436, 2, 3, 0, 2, 4, 5, 2, - 5, 1, 2, 9, 10, 2, 9, 238, - 2, 9, 239, 2, 9, 339, 2, 10, - 1, 2, 10, 340, 2, 10, 341, 2, - 11, 241, 2, 11, 351, 2, 12, 241, - 2, 12, 351, 2, 13, 241, 2, 13, - 351, 2, 14, 375, 2, 14, 376, 2, - 25, 0, 2, 25, 3, 2, 25, 6, - 2, 25, 14, 3, 25, 5, 306, 3, - 25, 10, 305, 3, 25, 14, 15, 4, - 25, 9, 304, 10, 0 + 436, 1, 437, 1, 438, 2, 3, 0, + 2, 4, 5, 2, 5, 1, 2, 9, + 10, 2, 9, 238, 2, 9, 239, 2, + 9, 341, 2, 10, 1, 2, 10, 342, + 2, 10, 343, 2, 11, 241, 2, 11, + 353, 2, 12, 241, 2, 12, 353, 2, + 13, 241, 2, 13, 353, 2, 14, 377, + 2, 14, 378, 2, 25, 0, 2, 25, + 3, 2, 25, 6, 2, 25, 14, 3, + 25, 5, 308, 3, 25, 10, 307, 3, + 25, 14, 15, 4, 25, 9, 306, 10, + 0 }; static const short _regex_key_offsets[] = { - 0, 0, 2, 4, 6, 7, 29, 37, - 45, 52, 60, 61, 69, 77, 85, 92, - 100, 103, 105, 114, 121, 129, 137, 140, - 146, 154, 157, 164, 171, 179, 186, 190, - 197, 200, 203, 205, 208, 211, 213, 216, - 219, 221, 222, 224, 225, 233, 235, 238, - 241, 242, 250, 258, 266, 274, 281, 289, - 296, 304, 311, 319, 321, 324, 331, 335, - 338, 341, 342, 344, 345, 347, 349, 350, + 0, 0, 1, 23, 31, 39, 46, 54, + 55, 63, 71, 79, 86, 94, 97, 99, + 108, 115, 123, 131, 134, 140, 148, 151, + 158, 165, 173, 180, 184, 191, 194, 197, + 199, 202, 205, 207, 210, 213, 215, 216, + 218, 219, 227, 229, 232, 235, 236, 244, + 252, 260, 268, 275, 283, 290, 298, 305, + 313, 315, 318, 325, 329, 332, 335, 337, + 339, 341, 342, 344, 345, 347, 349, 350, 351, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 369, 370, 371, 372, 373, 374, 375, 376, 377, @@ -468,312 +469,312 @@ static const short _regex_actions[] = { 849, 850, 851, 853, 854, 855, 856, 857, 858, 859, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, 873, - 874, 875, 876, 877, 879, 881, 883, 886, - 889, 891, 906, 909, 912, 914, 928, 933, - 938, 942, 946, 949, 952, 956, 960, 963, - 966, 970, 974, 978, 981, 984, 988, 992, - 996, 1000, 1003, 1006, 1010, 1014, 1018, 1022, - 1025, 1028, 1032, 1036, 1040, 1044, 1047, 1050, - 1054, 1058, 1062, 1066, 1069, 1072, 1076, 1080, - 1084, 1088, 1091, 1094, 1099, 1103, 1107, 1111, - 1114, 1117, 1121, 1125, 1129, 1132, 1135, 1139, - 1143, 1147, 1151, 1154, 1157, 1161, 1165, 1169, - 1173, 1176, 1179, 1183, 1187, 1191, 1194, 1197, - 1201, 1205, 1209, 1213, 1217, 1220, 1223, 1228, - 1233, 1237, 1241, 1244, 1247, 1251, 1255, 1258, - 1261, 1265, 1269, 1273, 1276, 1279, 1283, 1287, - 1291, 1295, 1298, 1301, 1305, 1309, 1313, 1317, - 1320, 1323, 1327, 1331, 1335, 1339, 1342, 1345, - 1349, 1353, 1357, 1361, 1364, 1367, 1371, 1375, - 1379, 1383, 1386, 1389, 1394, 1398, 1402, 1406, - 1409, 1412, 1416, 1420, 1424, 1427, 1430, 1434, - 1438, 1442, 1446, 1449, 1452, 1456, 1460, 1464, - 1468, 1471, 1474, 1478, 1482, 1486, 1489, 1492, - 1496, 1500, 1504, 1508, 1512, 1515, 1518, 1521, - 1524, 1526, 1528, 1531, 1538, 1540, 1542, 1544, - 1546, 1548, 1550, 1577, 1579, 1581, 1583, 1585, - 1592, 1599, 1613, 1615, 1621, 1624, 1633, 1634, - 1637, 1640, 1647, 1649, 1651, 1653, 1656, 1701, - 1703, 1705, 1709, 1713, 1715, 1716, 1716, 1722, - 1724, 1726, 1728, 1730, 1733, 1734, 1735, 1742, - 1748, 1754, 1756, 1758, 1760, 1761, 1764, 1787, + 874, 875, 876, 877, 880, 883, 885, 900, + 903, 906, 908, 922, 927, 932, 936, 940, + 943, 946, 950, 954, 957, 960, 964, 968, + 972, 975, 978, 982, 986, 990, 994, 997, + 1000, 1004, 1008, 1012, 1016, 1019, 1022, 1026, + 1030, 1034, 1038, 1041, 1044, 1048, 1052, 1056, + 1060, 1063, 1066, 1070, 1074, 1078, 1082, 1085, + 1088, 1093, 1097, 1101, 1105, 1108, 1111, 1115, + 1119, 1123, 1126, 1129, 1133, 1137, 1141, 1145, + 1148, 1151, 1155, 1159, 1163, 1167, 1170, 1173, + 1177, 1181, 1185, 1188, 1191, 1195, 1199, 1203, + 1207, 1211, 1214, 1217, 1222, 1227, 1231, 1235, + 1238, 1241, 1245, 1249, 1252, 1255, 1259, 1263, + 1267, 1270, 1273, 1277, 1281, 1285, 1289, 1292, + 1295, 1299, 1303, 1307, 1311, 1314, 1317, 1321, + 1325, 1329, 1333, 1336, 1339, 1343, 1347, 1351, + 1355, 1358, 1361, 1365, 1369, 1373, 1377, 1380, + 1383, 1388, 1392, 1396, 1400, 1403, 1406, 1410, + 1414, 1418, 1421, 1424, 1428, 1432, 1436, 1440, + 1443, 1446, 1450, 1454, 1458, 1462, 1465, 1468, + 1472, 1476, 1480, 1483, 1486, 1490, 1494, 1498, + 1502, 1506, 1509, 1512, 1515, 1518, 1520, 1522, + 1525, 1532, 1534, 1536, 1538, 1540, 1542, 1544, + 1546, 1548, 1550, 1577, 1579, 1586, 1593, 1607, + 1609, 1615, 1618, 1627, 1628, 1631, 1634, 1641, + 1643, 1645, 1647, 1650, 1695, 1697, 1699, 1703, + 1707, 1709, 1710, 1710, 1716, 1718, 1720, 1722, + 1724, 1727, 1728, 1729, 1736, 1742, 1748, 1750, + 1752, 1754, 1756, 1758, 1760, 1761, 1764, 1787, 1790, 1795, 1804, 1806, 1807, 1809, 1814, 1817, 1819, 1821, 1822, 1824, 1834, 1840, 1841, 1846, 1850, 1858, 1860, 1869, 1873, 1874, 1875, 1879, - 1880, 1883, 1883, 1890, 1904, 1906, 1908, 1910, - 1913, 1952, 1954, 1956, 1958, 1960, 1961, 1961, - 1962, 1963, 1970, 1976, 1982, 1985, 1987, 1998, - 2000, 2002, 2004, 2005, 2016, 2018, 2020, 2022, - 2023, 2024, 0 + 1880, 1883, 1883, 1890, 1905, 1906, 1909, 1948, + 1950, 1952, 1954, 1956, 1957, 1957, 1958, 1959, + 1966, 1972, 1978, 1980, 1982, 1984, 1987, 1989, + 2000, 2001, 2003, 2005, 2007, 2018, 2019, 2021, + 2023, 2025, 2026, 0 }; - static const char _regex_trans_keys[] = { - -128, -65, -128, -65, -128, -65, 41, 33, - 35, 38, 39, 40, 41, 43, 45, 58, - 60, 61, 62, 63, 67, 80, 105, 109, - 115, 120, 123, 48, 57, 41, 95, 48, - 57, 65, 90, 97, 122, 39, 95, 48, - 57, 65, 90, 97, 122, 95, 48, 57, - 65, 90, 97, 122, 39, 95, 48, 57, - 65, 90, 97, 122, 41, 41, 95, 48, - 57, 65, 90, 97, 122, 41, 95, 48, - 57, 65, 90, 97, 122, 41, 95, 48, - 57, 65, 90, 97, 122, 95, 48, 57, - 65, 90, 97, 122, 62, 95, 48, 57, - 65, 90, 97, 122, 33, 60, 61, 33, - 61, 38, 41, 95, 48, 57, 65, 90, - 97, 122, 95, 48, 57, 65, 90, 97, - 122, 41, 95, 48, 57, 65, 90, 97, - 122, 41, 95, 48, 57, 65, 90, 97, - 122, 41, 48, 57, 41, 58, 105, 109, - 115, 120, 62, 95, 48, 57, 65, 90, - 97, 122, 41, 48, 57, 95, 48, 57, - 65, 90, 97, 122, 95, 48, 57, 65, - 90, 97, 122, 41, 95, 48, 57, 65, - 90, 97, 122, 95, 48, 57, 65, 90, - 97, 122, 105, 109, 115, 120, 41, 45, - 58, 105, 109, 115, 120, 46, 92, 93, - 46, 92, 93, 46, 92, 58, 92, 93, - 58, 92, 93, 58, 92, 61, 92, 93, - 61, 92, 93, 61, 92, 39, 48, 57, - 62, 45, 95, 48, 57, 65, 90, 97, - 122, 48, 57, 125, 48, 57, 125, 48, - 57, 125, 95, 125, 48, 57, 65, 90, - 97, 122, 95, 125, 48, 57, 65, 90, - 97, 122, 95, 125, 48, 57, 65, 90, - 97, 122, 95, 125, 48, 57, 65, 90, - 97, 122, 95, 48, 57, 65, 90, 97, - 122, 39, 95, 48, 57, 65, 90, 97, - 122, 95, 48, 57, 65, 90, 97, 122, - 62, 95, 48, 57, 65, 90, 97, 122, - 95, 48, 57, 65, 90, 97, 122, 95, - 125, 48, 57, 65, 90, 97, 122, 48, - 55, 125, 48, 55, 125, 48, 57, 65, - 70, 97, 102, 44, 125, 48, 57, 125, - 48, 57, 125, 48, 57, 41, 41, 80, - 41, 41, 70, 41, 56, 41, 121, 97, - 109, 98, 105, 99, 101, 110, 105, 97, - 110, 101, 115, 116, 97, 110, 108, 109, - 116, 105, 110, 101, 115, 101, 117, 109, - 97, 107, 110, 103, 97, 108, 105, 112, - 111, 109, 111, 102, 111, 97, 104, 105, - 109, 105, 108, 108, 101, 103, 104, 105, - 110, 101, 115, 101, 105, 100, 110, 114, - 97, 100, 105, 97, 110, 95, 65, 98, - 111, 114, 105, 103, 105, 110, 97, 108, - 105, 97, 110, 97, 101, 109, 114, 111, - 107, 101, 101, 109, 111, 110, 116, 105, - 99, 110, 101, 105, 102, 111, 114, 109, - 112, 114, 114, 105, 111, 116, 105, 108, - 108, 105, 99, 115, 118, 101, 114, 101, - 116, 97, 110, 97, 103, 97, 114, 105, - 121, 112, 116, 105, 97, 110, 95, 72, - 105, 101, 114, 111, 103, 108, 121, 112, - 104, 115, 104, 105, 111, 112, 105, 99, - 111, 114, 103, 105, 97, 110, 97, 103, - 111, 108, 105, 116, 105, 99, 116, 104, - 105, 99, 101, 101, 107, 106, 114, 97, - 114, 97, 116, 105, 109, 117, 107, 104, - 105, 110, 117, 108, 110, 111, 111, 98, - 114, 101, 119, 114, 97, 103, 97, 110, - 97, 112, 101, 114, 105, 97, 108, 95, - 65, 114, 97, 109, 97, 105, 99, 104, - 115, 101, 114, 105, 116, 101, 100, 99, - 114, 105, 112, 116, 105, 111, 110, 97, - 108, 95, 80, 97, 104, 114, 108, 97, - 118, 105, 116, 104, 105, 97, 110, 118, - 97, 110, 101, 115, 101, 105, 110, 116, - 121, 116, 104, 105, 110, 97, 100, 97, - 97, 107, 97, 110, 97, 97, 104, 95, - 76, 105, 97, 109, 114, 111, 115, 104, - 116, 104, 105, 101, 114, 111, 116, 105, - 110, 112, 99, 104, 97, 109, 110, 115, - 98, 117, 101, 97, 114, 95, 66, 117, - 99, 100, 105, 97, 110, 105, 97, 110, - 108, 110, 97, 121, 97, 108, 97, 109, - 100, 97, 105, 99, 116, 101, 105, 95, - 77, 97, 121, 101, 107, 110, 103, 111, - 108, 105, 97, 110, 97, 110, 109, 97, - 114, 119, 95, 84, 97, 105, 95, 76, - 117, 101, 111, 104, 97, 109, 95, 100, - 67, 104, 105, 107, 105, 95, 73, 80, - 83, 84, 116, 97, 108, 105, 99, 101, - 114, 115, 105, 97, 110, 111, 117, 116, - 104, 95, 65, 114, 97, 98, 105, 97, - 110, 117, 114, 107, 105, 99, 105, 121, - 97, 109, 97, 110, 121, 97, 97, 111, - 103, 115, 95, 80, 97, 101, 110, 105, - 99, 105, 97, 110, 106, 97, 110, 103, - 110, 105, 99, 109, 117, 97, 114, 105, - 116, 97, 110, 114, 97, 115, 104, 116, - 114, 97, 97, 118, 105, 97, 110, 110, - 104, 97, 108, 97, 110, 100, 97, 110, - 101, 115, 101, 108, 114, 111, 116, 105, - 95, 78, 97, 103, 114, 105, 105, 97, - 99, 103, 105, 109, 97, 98, 108, 111, - 103, 97, 110, 119, 97, 95, 76, 84, - 86, 101, 104, 97, 109, 105, 101, 116, - 105, 108, 108, 117, 103, 117, 97, 97, - 105, 110, 97, 98, 102, 101, 116, 97, - 110, 105, 110, 97, 103, 104, 97, 114, - 105, 116, 105, 99, 105, 110, 115, 112, - 100, 123, 94, 125, 94, -128, -65, -128, - -65, -128, -65, 46, 92, 93, 46, 92, - 93, 46, 92, 58, 92, 93, 94, 97, - 98, 99, 100, 103, 108, 112, 115, 117, - 119, 120, 58, 92, 93, 58, 92, 93, - 58, 92, 58, 92, 93, 97, 98, 99, - 100, 103, 108, 112, 115, 117, 119, 120, - 58, 92, 93, 108, 115, 58, 92, 93, - 110, 112, 58, 92, 93, 117, 58, 92, - 93, 109, 58, 92, 93, 58, 92, 93, - 58, 92, 93, 104, 58, 92, 93, 97, - 58, 92, 93, 58, 92, 93, 58, 92, - 93, 99, 58, 92, 93, 105, 58, 92, - 93, 105, 58, 92, 93, 58, 92, 93, - 58, 92, 93, 108, 58, 92, 93, 97, - 58, 92, 93, 110, 58, 92, 93, 107, - 58, 92, 93, 58, 92, 93, 58, 92, - 93, 110, 58, 92, 93, 116, 58, 92, - 93, 114, 58, 92, 93, 108, 58, 92, - 93, 58, 92, 93, 58, 92, 93, 105, - 58, 92, 93, 103, 58, 92, 93, 105, - 58, 92, 93, 116, 58, 92, 93, 58, - 92, 93, 58, 92, 93, 114, 58, 92, - 93, 97, 58, 92, 93, 112, 58, 92, - 93, 104, 58, 92, 93, 58, 92, 93, - 58, 92, 93, 111, 58, 92, 93, 119, - 58, 92, 93, 101, 58, 92, 93, 114, - 58, 92, 93, 58, 92, 93, 58, 92, - 93, 114, 117, 58, 92, 93, 105, 58, - 92, 93, 110, 58, 92, 93, 116, 58, - 92, 93, 58, 92, 93, 58, 92, 93, - 110, 58, 92, 93, 99, 58, 92, 93, - 116, 58, 92, 93, 58, 92, 93, 58, - 92, 93, 112, 58, 92, 93, 97, 58, - 92, 93, 99, 58, 92, 93, 101, 58, - 92, 93, 58, 92, 93, 58, 92, 93, - 112, 58, 92, 93, 112, 58, 92, 93, - 101, 58, 92, 93, 114, 58, 92, 93, - 58, 92, 93, 58, 92, 93, 111, 58, - 92, 93, 114, 58, 92, 93, 100, 58, - 92, 93, 58, 92, 93, 58, 92, 93, - 100, 58, 92, 93, 105, 58, 92, 93, - 103, 58, 92, 93, 105, 58, 92, 93, - 116, 58, 92, 93, 58, 92, 93, 58, - 92, 93, 108, 115, 58, 92, 93, 110, - 112, 58, 92, 93, 117, 58, 92, 93, - 109, 58, 92, 93, 58, 92, 93, 58, - 92, 93, 104, 58, 92, 93, 97, 58, - 92, 93, 58, 92, 93, 58, 92, 93, - 99, 58, 92, 93, 105, 58, 92, 93, - 105, 58, 92, 93, 58, 92, 93, 58, - 92, 93, 108, 58, 92, 93, 97, 58, - 92, 93, 110, 58, 92, 93, 107, 58, - 92, 93, 58, 92, 93, 58, 92, 93, - 110, 58, 92, 93, 116, 58, 92, 93, - 114, 58, 92, 93, 108, 58, 92, 93, - 58, 92, 93, 58, 92, 93, 105, 58, - 92, 93, 103, 58, 92, 93, 105, 58, - 92, 93, 116, 58, 92, 93, 58, 92, - 93, 58, 92, 93, 114, 58, 92, 93, - 97, 58, 92, 93, 112, 58, 92, 93, - 104, 58, 92, 93, 58, 92, 93, 58, - 92, 93, 111, 58, 92, 93, 119, 58, - 92, 93, 101, 58, 92, 93, 114, 58, - 92, 93, 58, 92, 93, 58, 92, 93, - 114, 117, 58, 92, 93, 105, 58, 92, - 93, 110, 58, 92, 93, 116, 58, 92, - 93, 58, 92, 93, 58, 92, 93, 110, - 58, 92, 93, 99, 58, 92, 93, 116, - 58, 92, 93, 58, 92, 93, 58, 92, - 93, 112, 58, 92, 93, 97, 58, 92, - 93, 99, 58, 92, 93, 101, 58, 92, - 93, 58, 92, 93, 58, 92, 93, 112, - 58, 92, 93, 112, 58, 92, 93, 101, - 58, 92, 93, 114, 58, 92, 93, 58, - 92, 93, 58, 92, 93, 111, 58, 92, - 93, 114, 58, 92, 93, 100, 58, 92, - 93, 58, 92, 93, 58, 92, 93, 100, - 58, 92, 93, 105, 58, 92, 93, 103, - 58, 92, 93, 105, 58, 92, 93, 116, - 58, 92, 93, 58, 92, 93, 61, 92, - 93, 61, 92, 93, 61, 92, 48, 55, - 125, 48, 55, 125, 48, 57, 65, 70, - 97, 102, -128, -65, -128, -65, -128, -65, - -128, -65, -128, -65, -128, -65, 0, 32, - 35, 36, 40, 41, 42, 43, 46, 63, - 91, 92, 94, 123, 124, -128, -65, -64, - -33, -32, -17, -16, -9, -8, -1, 9, - 13, -128, -65, -128, -65, -128, -65, 42, - 63, 95, 48, 57, 65, 90, 97, 122, - 95, 48, 57, 65, 90, 97, 122, 39, - 48, 60, 63, 82, 95, 49, 55, 56, - 57, 65, 90, 97, 122, 48, 57, 105, - 109, 115, 120, 48, 57, 41, 48, 57, - 33, 61, 95, 48, 57, 65, 90, 97, - 122, 123, 41, 48, 57, 60, 61, 62, - 41, 45, 58, 105, 109, 115, 120, 43, - 63, 43, 63, 43, 63, 46, 58, 61, - 48, 65, 66, 67, 68, 69, 71, 72, - 75, 76, 78, 80, 81, 82, 83, 85, - 86, 87, 88, 90, 97, 98, 99, 100, - 101, 102, 103, 104, 107, 108, 110, 111, - 112, 114, 115, 116, 117, 118, 119, 120, - 122, 49, 55, 56, 57, 48, 55, 48, - 55, 48, 55, 56, 57, 48, 55, 56, - 57, 48, 57, 123, 39, 45, 60, 123, - 48, 57, 48, 57, 48, 57, 48, 57, - 48, 57, 39, 60, 123, 123, 123, 123, - 48, 57, 65, 70, 97, 102, 48, 57, - 65, 70, 97, 102, 48, 57, 65, 70, - 97, 102, 48, 57, 43, 63, 41, 85, - 41, 41, 67, 84, 65, 66, 67, 68, - 69, 71, 72, 73, 74, 75, 76, 77, - 78, 79, 80, 82, 83, 84, 85, 86, - 88, 89, 90, 110, 114, 118, 97, 101, - 111, 114, 117, 97, 99, 102, 104, 110, - 111, 115, 117, 121, 109, 112, 101, 103, - 116, 101, 108, 111, 114, 117, 97, 101, - 105, 103, 117, 109, 110, 97, 97, 104, - 38, 97, 101, 105, 108, 109, 111, 116, - 117, 121, 97, 99, 101, 110, 111, 121, - 101, 100, 101, 107, 108, 111, 103, 108, - 114, 115, 99, 100, 101, 102, 104, 105, - 111, 115, 101, 117, 97, 99, 104, 105, - 107, 109, 111, 117, 121, 97, 101, 104, - 105, 103, 97, 97, 112, 115, 119, 105, - 108, 112, 115, 67, 76, 77, 78, 80, - 83, 90, 45, 91, 92, 93, -128, -65, - -64, -33, -32, -17, -16, -9, -8, -1, - -128, -65, -128, -65, -128, -65, 46, 58, - 61, 48, 68, 69, 72, 76, 78, 80, - 81, 83, 85, 86, 87, 97, 98, 99, - 100, 101, 102, 103, 104, 108, 110, 111, - 112, 114, 115, 116, 117, 118, 119, 120, - 49, 55, 56, 57, 65, 90, 105, 122, - 48, 55, 48, 55, 48, 55, 48, 55, - 123, 123, 123, 123, 48, 57, 65, 70, - 97, 102, 48, 57, 65, 70, 97, 102, - 48, 57, 65, 70, 97, 102, 92, 93, - 94, 69, 81, 92, -128, -65, -64, -33, - -32, -17, -16, -9, -8, -1, -128, -65, - -128, -65, -128, -65, 69, 92, -128, -65, - -64, -33, -32, -17, -16, -9, -8, -1, - -128, -65, -128, -65, -128, -65, 69, 41, - 10, 0 + static const unsigned char _regex_trans_keys[] = { + 41u, 33u, 35u, 38u, 39u, 40u, 41u, 43u, + 45u, 58u, 60u, 61u, 62u, 63u, 67u, 80u, + 105u, 109u, 115u, 120u, 123u, 48u, 57u, 41u, + 95u, 48u, 57u, 65u, 90u, 97u, 122u, 39u, + 95u, 48u, 57u, 65u, 90u, 97u, 122u, 95u, + 48u, 57u, 65u, 90u, 97u, 122u, 39u, 95u, + 48u, 57u, 65u, 90u, 97u, 122u, 41u, 41u, + 95u, 48u, 57u, 65u, 90u, 97u, 122u, 41u, + 95u, 48u, 57u, 65u, 90u, 97u, 122u, 41u, + 95u, 48u, 57u, 65u, 90u, 97u, 122u, 95u, + 48u, 57u, 65u, 90u, 97u, 122u, 62u, 95u, + 48u, 57u, 65u, 90u, 97u, 122u, 33u, 60u, + 61u, 33u, 61u, 38u, 41u, 95u, 48u, 57u, + 65u, 90u, 97u, 122u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 41u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 41u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 41u, 48u, 57u, 41u, 58u, + 105u, 109u, 115u, 120u, 62u, 95u, 48u, 57u, + 65u, 90u, 97u, 122u, 41u, 48u, 57u, 95u, + 48u, 57u, 65u, 90u, 97u, 122u, 95u, 48u, + 57u, 65u, 90u, 97u, 122u, 41u, 95u, 48u, + 57u, 65u, 90u, 97u, 122u, 95u, 48u, 57u, + 65u, 90u, 97u, 122u, 105u, 109u, 115u, 120u, + 41u, 45u, 58u, 105u, 109u, 115u, 120u, 46u, + 92u, 93u, 46u, 92u, 93u, 46u, 92u, 58u, + 92u, 93u, 58u, 92u, 93u, 58u, 92u, 61u, + 92u, 93u, 61u, 92u, 93u, 61u, 92u, 39u, + 48u, 57u, 62u, 45u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 48u, 57u, 125u, 48u, 57u, + 125u, 48u, 57u, 125u, 95u, 125u, 48u, 57u, + 65u, 90u, 97u, 122u, 95u, 125u, 48u, 57u, + 65u, 90u, 97u, 122u, 95u, 125u, 48u, 57u, + 65u, 90u, 97u, 122u, 95u, 125u, 48u, 57u, + 65u, 90u, 97u, 122u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 39u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 95u, 48u, 57u, 65u, 90u, + 97u, 122u, 62u, 95u, 48u, 57u, 65u, 90u, + 97u, 122u, 95u, 48u, 57u, 65u, 90u, 97u, + 122u, 95u, 125u, 48u, 57u, 65u, 90u, 97u, + 122u, 48u, 55u, 125u, 48u, 55u, 125u, 48u, + 57u, 65u, 70u, 97u, 102u, 44u, 125u, 48u, + 57u, 125u, 48u, 57u, 125u, 48u, 57u, 128u, + 191u, 128u, 191u, 128u, 191u, 41u, 41u, 80u, + 41u, 41u, 70u, 41u, 56u, 41u, 121u, 97u, + 109u, 98u, 105u, 99u, 101u, 110u, 105u, 97u, + 110u, 101u, 115u, 116u, 97u, 110u, 108u, 109u, + 116u, 105u, 110u, 101u, 115u, 101u, 117u, 109u, + 97u, 107u, 110u, 103u, 97u, 108u, 105u, 112u, + 111u, 109u, 111u, 102u, 111u, 97u, 104u, 105u, + 109u, 105u, 108u, 108u, 101u, 103u, 104u, 105u, + 110u, 101u, 115u, 101u, 105u, 100u, 110u, 114u, + 97u, 100u, 105u, 97u, 110u, 95u, 65u, 98u, + 111u, 114u, 105u, 103u, 105u, 110u, 97u, 108u, + 105u, 97u, 110u, 97u, 101u, 109u, 114u, 111u, + 107u, 101u, 101u, 109u, 111u, 110u, 116u, 105u, + 99u, 110u, 101u, 105u, 102u, 111u, 114u, 109u, + 112u, 114u, 114u, 105u, 111u, 116u, 105u, 108u, + 108u, 105u, 99u, 115u, 118u, 101u, 114u, 101u, + 116u, 97u, 110u, 97u, 103u, 97u, 114u, 105u, + 121u, 112u, 116u, 105u, 97u, 110u, 95u, 72u, + 105u, 101u, 114u, 111u, 103u, 108u, 121u, 112u, + 104u, 115u, 104u, 105u, 111u, 112u, 105u, 99u, + 111u, 114u, 103u, 105u, 97u, 110u, 97u, 103u, + 111u, 108u, 105u, 116u, 105u, 99u, 116u, 104u, + 105u, 99u, 101u, 101u, 107u, 106u, 114u, 97u, + 114u, 97u, 116u, 105u, 109u, 117u, 107u, 104u, + 105u, 110u, 117u, 108u, 110u, 111u, 111u, 98u, + 114u, 101u, 119u, 114u, 97u, 103u, 97u, 110u, + 97u, 112u, 101u, 114u, 105u, 97u, 108u, 95u, + 65u, 114u, 97u, 109u, 97u, 105u, 99u, 104u, + 115u, 101u, 114u, 105u, 116u, 101u, 100u, 99u, + 114u, 105u, 112u, 116u, 105u, 111u, 110u, 97u, + 108u, 95u, 80u, 97u, 104u, 114u, 108u, 97u, + 118u, 105u, 116u, 104u, 105u, 97u, 110u, 118u, + 97u, 110u, 101u, 115u, 101u, 105u, 110u, 116u, + 121u, 116u, 104u, 105u, 110u, 97u, 100u, 97u, + 97u, 107u, 97u, 110u, 97u, 97u, 104u, 95u, + 76u, 105u, 97u, 109u, 114u, 111u, 115u, 104u, + 116u, 104u, 105u, 101u, 114u, 111u, 116u, 105u, + 110u, 112u, 99u, 104u, 97u, 109u, 110u, 115u, + 98u, 117u, 101u, 97u, 114u, 95u, 66u, 117u, + 99u, 100u, 105u, 97u, 110u, 105u, 97u, 110u, + 108u, 110u, 97u, 121u, 97u, 108u, 97u, 109u, + 100u, 97u, 105u, 99u, 116u, 101u, 105u, 95u, + 77u, 97u, 121u, 101u, 107u, 110u, 103u, 111u, + 108u, 105u, 97u, 110u, 97u, 110u, 109u, 97u, + 114u, 119u, 95u, 84u, 97u, 105u, 95u, 76u, + 117u, 101u, 111u, 104u, 97u, 109u, 95u, 100u, + 67u, 104u, 105u, 107u, 105u, 95u, 73u, 80u, + 83u, 84u, 116u, 97u, 108u, 105u, 99u, 101u, + 114u, 115u, 105u, 97u, 110u, 111u, 117u, 116u, + 104u, 95u, 65u, 114u, 97u, 98u, 105u, 97u, + 110u, 117u, 114u, 107u, 105u, 99u, 105u, 121u, + 97u, 109u, 97u, 110u, 121u, 97u, 97u, 111u, + 103u, 115u, 95u, 80u, 97u, 101u, 110u, 105u, + 99u, 105u, 97u, 110u, 106u, 97u, 110u, 103u, + 110u, 105u, 99u, 109u, 117u, 97u, 114u, 105u, + 116u, 97u, 110u, 114u, 97u, 115u, 104u, 116u, + 114u, 97u, 97u, 118u, 105u, 97u, 110u, 110u, + 104u, 97u, 108u, 97u, 110u, 100u, 97u, 110u, + 101u, 115u, 101u, 108u, 114u, 111u, 116u, 105u, + 95u, 78u, 97u, 103u, 114u, 105u, 105u, 97u, + 99u, 103u, 105u, 109u, 97u, 98u, 108u, 111u, + 103u, 97u, 110u, 119u, 97u, 95u, 76u, 84u, + 86u, 101u, 104u, 97u, 109u, 105u, 101u, 116u, + 105u, 108u, 108u, 117u, 103u, 117u, 97u, 97u, + 105u, 110u, 97u, 98u, 102u, 101u, 116u, 97u, + 110u, 105u, 110u, 97u, 103u, 104u, 97u, 114u, + 105u, 116u, 105u, 99u, 105u, 110u, 115u, 112u, + 100u, 123u, 94u, 125u, 94u, 46u, 92u, 93u, + 46u, 92u, 93u, 46u, 92u, 58u, 92u, 93u, + 94u, 97u, 98u, 99u, 100u, 103u, 108u, 112u, + 115u, 117u, 119u, 120u, 58u, 92u, 93u, 58u, + 92u, 93u, 58u, 92u, 58u, 92u, 93u, 97u, + 98u, 99u, 100u, 103u, 108u, 112u, 115u, 117u, + 119u, 120u, 58u, 92u, 93u, 108u, 115u, 58u, + 92u, 93u, 110u, 112u, 58u, 92u, 93u, 117u, + 58u, 92u, 93u, 109u, 58u, 92u, 93u, 58u, + 92u, 93u, 58u, 92u, 93u, 104u, 58u, 92u, + 93u, 97u, 58u, 92u, 93u, 58u, 92u, 93u, + 58u, 92u, 93u, 99u, 58u, 92u, 93u, 105u, + 58u, 92u, 93u, 105u, 58u, 92u, 93u, 58u, + 92u, 93u, 58u, 92u, 93u, 108u, 58u, 92u, + 93u, 97u, 58u, 92u, 93u, 110u, 58u, 92u, + 93u, 107u, 58u, 92u, 93u, 58u, 92u, 93u, + 58u, 92u, 93u, 110u, 58u, 92u, 93u, 116u, + 58u, 92u, 93u, 114u, 58u, 92u, 93u, 108u, + 58u, 92u, 93u, 58u, 92u, 93u, 58u, 92u, + 93u, 105u, 58u, 92u, 93u, 103u, 58u, 92u, + 93u, 105u, 58u, 92u, 93u, 116u, 58u, 92u, + 93u, 58u, 92u, 93u, 58u, 92u, 93u, 114u, + 58u, 92u, 93u, 97u, 58u, 92u, 93u, 112u, + 58u, 92u, 93u, 104u, 58u, 92u, 93u, 58u, + 92u, 93u, 58u, 92u, 93u, 111u, 58u, 92u, + 93u, 119u, 58u, 92u, 93u, 101u, 58u, 92u, + 93u, 114u, 58u, 92u, 93u, 58u, 92u, 93u, + 58u, 92u, 93u, 114u, 117u, 58u, 92u, 93u, + 105u, 58u, 92u, 93u, 110u, 58u, 92u, 93u, + 116u, 58u, 92u, 93u, 58u, 92u, 93u, 58u, + 92u, 93u, 110u, 58u, 92u, 93u, 99u, 58u, + 92u, 93u, 116u, 58u, 92u, 93u, 58u, 92u, + 93u, 58u, 92u, 93u, 112u, 58u, 92u, 93u, + 97u, 58u, 92u, 93u, 99u, 58u, 92u, 93u, + 101u, 58u, 92u, 93u, 58u, 92u, 93u, 58u, + 92u, 93u, 112u, 58u, 92u, 93u, 112u, 58u, + 92u, 93u, 101u, 58u, 92u, 93u, 114u, 58u, + 92u, 93u, 58u, 92u, 93u, 58u, 92u, 93u, + 111u, 58u, 92u, 93u, 114u, 58u, 92u, 93u, + 100u, 58u, 92u, 93u, 58u, 92u, 93u, 58u, + 92u, 93u, 100u, 58u, 92u, 93u, 105u, 58u, + 92u, 93u, 103u, 58u, 92u, 93u, 105u, 58u, + 92u, 93u, 116u, 58u, 92u, 93u, 58u, 92u, + 93u, 58u, 92u, 93u, 108u, 115u, 58u, 92u, + 93u, 110u, 112u, 58u, 92u, 93u, 117u, 58u, + 92u, 93u, 109u, 58u, 92u, 93u, 58u, 92u, + 93u, 58u, 92u, 93u, 104u, 58u, 92u, 93u, + 97u, 58u, 92u, 93u, 58u, 92u, 93u, 58u, + 92u, 93u, 99u, 58u, 92u, 93u, 105u, 58u, + 92u, 93u, 105u, 58u, 92u, 93u, 58u, 92u, + 93u, 58u, 92u, 93u, 108u, 58u, 92u, 93u, + 97u, 58u, 92u, 93u, 110u, 58u, 92u, 93u, + 107u, 58u, 92u, 93u, 58u, 92u, 93u, 58u, + 92u, 93u, 110u, 58u, 92u, 93u, 116u, 58u, + 92u, 93u, 114u, 58u, 92u, 93u, 108u, 58u, + 92u, 93u, 58u, 92u, 93u, 58u, 92u, 93u, + 105u, 58u, 92u, 93u, 103u, 58u, 92u, 93u, + 105u, 58u, 92u, 93u, 116u, 58u, 92u, 93u, + 58u, 92u, 93u, 58u, 92u, 93u, 114u, 58u, + 92u, 93u, 97u, 58u, 92u, 93u, 112u, 58u, + 92u, 93u, 104u, 58u, 92u, 93u, 58u, 92u, + 93u, 58u, 92u, 93u, 111u, 58u, 92u, 93u, + 119u, 58u, 92u, 93u, 101u, 58u, 92u, 93u, + 114u, 58u, 92u, 93u, 58u, 92u, 93u, 58u, + 92u, 93u, 114u, 117u, 58u, 92u, 93u, 105u, + 58u, 92u, 93u, 110u, 58u, 92u, 93u, 116u, + 58u, 92u, 93u, 58u, 92u, 93u, 58u, 92u, + 93u, 110u, 58u, 92u, 93u, 99u, 58u, 92u, + 93u, 116u, 58u, 92u, 93u, 58u, 92u, 93u, + 58u, 92u, 93u, 112u, 58u, 92u, 93u, 97u, + 58u, 92u, 93u, 99u, 58u, 92u, 93u, 101u, + 58u, 92u, 93u, 58u, 92u, 93u, 58u, 92u, + 93u, 112u, 58u, 92u, 93u, 112u, 58u, 92u, + 93u, 101u, 58u, 92u, 93u, 114u, 58u, 92u, + 93u, 58u, 92u, 93u, 58u, 92u, 93u, 111u, + 58u, 92u, 93u, 114u, 58u, 92u, 93u, 100u, + 58u, 92u, 93u, 58u, 92u, 93u, 58u, 92u, + 93u, 100u, 58u, 92u, 93u, 105u, 58u, 92u, + 93u, 103u, 58u, 92u, 93u, 105u, 58u, 92u, + 93u, 116u, 58u, 92u, 93u, 58u, 92u, 93u, + 61u, 92u, 93u, 61u, 92u, 93u, 61u, 92u, + 48u, 55u, 125u, 48u, 55u, 125u, 48u, 57u, + 65u, 70u, 97u, 102u, 128u, 191u, 128u, 191u, + 128u, 191u, 128u, 191u, 128u, 191u, 128u, 191u, + 128u, 191u, 128u, 191u, 128u, 191u, 0u, 32u, + 35u, 36u, 40u, 41u, 42u, 43u, 46u, 63u, + 91u, 92u, 94u, 123u, 124u, 9u, 13u, 128u, + 191u, 192u, 223u, 224u, 239u, 240u, 247u, 248u, + 255u, 42u, 63u, 95u, 48u, 57u, 65u, 90u, + 97u, 122u, 95u, 48u, 57u, 65u, 90u, 97u, + 122u, 39u, 48u, 60u, 63u, 82u, 95u, 49u, + 55u, 56u, 57u, 65u, 90u, 97u, 122u, 48u, + 57u, 105u, 109u, 115u, 120u, 48u, 57u, 41u, + 48u, 57u, 33u, 61u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 123u, 41u, 48u, 57u, 60u, + 61u, 62u, 41u, 45u, 58u, 105u, 109u, 115u, + 120u, 43u, 63u, 43u, 63u, 43u, 63u, 46u, + 58u, 61u, 48u, 65u, 66u, 67u, 68u, 69u, + 71u, 72u, 75u, 76u, 78u, 80u, 81u, 82u, + 83u, 85u, 86u, 87u, 88u, 90u, 97u, 98u, + 99u, 100u, 101u, 102u, 103u, 104u, 107u, 108u, + 110u, 111u, 112u, 114u, 115u, 116u, 117u, 118u, + 119u, 120u, 122u, 49u, 55u, 56u, 57u, 48u, + 55u, 48u, 55u, 48u, 55u, 56u, 57u, 48u, + 55u, 56u, 57u, 48u, 57u, 123u, 39u, 45u, + 60u, 123u, 48u, 57u, 48u, 57u, 48u, 57u, + 48u, 57u, 48u, 57u, 39u, 60u, 123u, 123u, + 123u, 123u, 48u, 57u, 65u, 70u, 97u, 102u, + 48u, 57u, 65u, 70u, 97u, 102u, 48u, 57u, + 65u, 70u, 97u, 102u, 48u, 57u, 43u, 63u, + 128u, 191u, 128u, 191u, 128u, 191u, 41u, 85u, + 41u, 41u, 67u, 84u, 65u, 66u, 67u, 68u, + 69u, 71u, 72u, 73u, 74u, 75u, 76u, 77u, + 78u, 79u, 80u, 82u, 83u, 84u, 85u, 86u, + 88u, 89u, 90u, 110u, 114u, 118u, 97u, 101u, + 111u, 114u, 117u, 97u, 99u, 102u, 104u, 110u, + 111u, 115u, 117u, 121u, 109u, 112u, 101u, 103u, + 116u, 101u, 108u, 111u, 114u, 117u, 97u, 101u, + 105u, 103u, 117u, 109u, 110u, 97u, 97u, 104u, + 38u, 97u, 101u, 105u, 108u, 109u, 111u, 116u, + 117u, 121u, 97u, 99u, 101u, 110u, 111u, 121u, + 101u, 100u, 101u, 107u, 108u, 111u, 103u, 108u, + 114u, 115u, 99u, 100u, 101u, 102u, 104u, 105u, + 111u, 115u, 101u, 117u, 97u, 99u, 104u, 105u, + 107u, 109u, 111u, 117u, 121u, 97u, 101u, 104u, + 105u, 103u, 97u, 97u, 112u, 115u, 119u, 105u, + 108u, 112u, 115u, 67u, 76u, 77u, 78u, 80u, + 83u, 90u, 38u, 45u, 91u, 92u, 93u, 128u, + 191u, 192u, 223u, 224u, 239u, 240u, 247u, 248u, + 255u, 38u, 46u, 58u, 61u, 48u, 68u, 69u, + 72u, 76u, 78u, 80u, 81u, 83u, 85u, 86u, + 87u, 97u, 98u, 99u, 100u, 101u, 102u, 103u, + 104u, 108u, 110u, 111u, 112u, 114u, 115u, 116u, + 117u, 118u, 119u, 120u, 49u, 55u, 56u, 57u, + 65u, 90u, 105u, 122u, 48u, 55u, 48u, 55u, + 48u, 55u, 48u, 55u, 123u, 123u, 123u, 123u, + 48u, 57u, 65u, 70u, 97u, 102u, 48u, 57u, + 65u, 70u, 97u, 102u, 48u, 57u, 65u, 70u, + 97u, 102u, 128u, 191u, 128u, 191u, 128u, 191u, + 92u, 93u, 94u, 69u, 81u, 92u, 128u, 191u, + 192u, 223u, 224u, 239u, 240u, 247u, 248u, 255u, + 69u, 128u, 191u, 128u, 191u, 128u, 191u, 92u, + 128u, 191u, 192u, 223u, 224u, 239u, 240u, 247u, + 248u, 255u, 69u, 128u, 191u, 128u, 191u, 128u, + 191u, 41u, 10u, 0u }; static const signed char _regex_single_lengths[] = { - 0, 0, 0, 0, 1, 20, 2, 2, - 1, 2, 1, 2, 2, 2, 1, 2, - 3, 2, 3, 1, 2, 2, 1, 6, - 2, 1, 1, 1, 2, 1, 4, 7, - 3, 3, 2, 3, 3, 2, 3, 3, - 2, 1, 0, 1, 2, 0, 1, 1, - 1, 2, 2, 2, 2, 1, 2, 1, - 2, 1, 2, 0, 1, 1, 2, 1, - 1, 1, 2, 1, 2, 2, 1, 1, + 0, 1, 20, 2, 2, 1, 2, 1, + 2, 2, 2, 1, 2, 3, 2, 3, + 1, 2, 2, 1, 6, 2, 1, 1, + 1, 2, 1, 4, 7, 3, 3, 2, + 3, 3, 2, 3, 3, 2, 1, 0, + 1, 2, 0, 1, 1, 1, 2, 2, + 2, 2, 1, 2, 1, 2, 1, 2, + 0, 1, 1, 2, 1, 1, 0, 0, + 0, 1, 2, 1, 2, 2, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -835,54 +836,54 @@ static const short _regex_actions[] = { 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 0, 0, 0, 3, 3, - 2, 15, 3, 3, 2, 14, 5, 5, - 4, 4, 3, 3, 4, 4, 3, 3, - 4, 4, 4, 3, 3, 4, 4, 4, + 1, 1, 1, 3, 3, 2, 15, 3, + 3, 2, 14, 5, 5, 4, 4, 3, + 3, 4, 4, 3, 3, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 4, 4, 4, 4, 3, 3, 4, 4, + 4, 4, 3, 3, 4, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 5, 4, 4, 4, 3, 3, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, - 4, 4, 4, 3, 3, 4, 4, 4, - 4, 3, 3, 5, 4, 4, 4, 3, - 3, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, 4, 4, 4, - 3, 3, 4, 4, 4, 3, 3, 4, - 4, 4, 4, 4, 3, 3, 5, 5, - 4, 4, 3, 3, 4, 4, 3, 3, - 4, 4, 4, 3, 3, 4, 4, 4, + 4, 3, 3, 5, 5, 4, 4, 3, + 3, 4, 4, 3, 3, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 4, 4, 4, 4, 3, 3, 4, 4, + 4, 4, 3, 3, 4, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 5, 4, 4, 4, 3, 3, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, - 4, 4, 4, 3, 3, 4, 4, 4, - 4, 3, 3, 5, 4, 4, 4, 3, - 3, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, 4, 4, 4, - 3, 3, 4, 4, 4, 3, 3, 4, - 4, 4, 4, 4, 3, 3, 3, 3, - 2, 0, 1, 1, 0, 0, 0, 0, - 0, 0, 15, 0, 0, 0, 2, 1, - 1, 6, 0, 4, 1, 3, 1, 1, - 3, 7, 2, 2, 2, 3, 41, 0, - 0, 0, 0, 0, 1, 0, 4, 0, - 0, 0, 0, 3, 1, 1, 1, 0, - 0, 0, 2, 2, 1, 3, 23, 3, + 4, 3, 3, 3, 3, 2, 0, 1, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 15, 2, 1, 1, 6, 0, + 4, 1, 3, 1, 1, 3, 7, 2, + 2, 2, 3, 41, 0, 0, 0, 0, + 0, 1, 0, 4, 0, 0, 0, 0, + 3, 1, 1, 1, 0, 0, 0, 2, + 0, 0, 0, 2, 1, 3, 23, 3, 5, 9, 2, 1, 2, 5, 3, 2, 2, 1, 2, 10, 6, 1, 5, 4, 8, 2, 9, 4, 1, 1, 4, 1, - 3, 0, 7, 4, 0, 0, 0, 3, - 31, 0, 0, 0, 0, 1, 0, 1, - 1, 1, 0, 0, 3, 2, 1, 0, - 0, 0, 1, 1, 0, 0, 0, 1, - 1, 1, 0 + 3, 0, 7, 5, 1, 3, 31, 0, + 0, 0, 0, 1, 0, 1, 1, 1, + 0, 0, 0, 0, 0, 3, 2, 1, + 1, 0, 0, 0, 1, 1, 0, 0, + 0, 1, 1, 0 }; static const signed char _regex_range_lengths[] = { - 0, 1, 1, 1, 0, 1, 3, 3, - 3, 3, 0, 3, 3, 3, 3, 3, - 0, 0, 3, 3, 3, 3, 1, 0, - 3, 1, 3, 3, 3, 3, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 0, 3, 1, 1, 1, - 0, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 1, 1, 3, 1, 1, + 0, 0, 1, 3, 3, 3, 3, 0, + 3, 3, 3, 3, 3, 0, 0, 3, + 3, 3, 3, 1, 0, 3, 1, 3, + 3, 3, 3, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, + 0, 3, 1, 1, 1, 0, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 1, 1, 3, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -945,7 +946,6 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -967,33 +967,34 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 1, 3, 1, 1, 1, 1, - 1, 1, 6, 1, 1, 1, 0, 3, - 3, 4, 1, 1, 1, 3, 0, 1, - 0, 0, 0, 0, 0, 0, 2, 1, - 1, 2, 2, 1, 0, 0, 1, 1, - 1, 1, 1, 0, 0, 0, 3, 3, - 3, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 1, + 3, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 6, 0, 3, 3, 4, 1, + 1, 1, 3, 0, 1, 0, 0, 0, + 0, 0, 0, 2, 1, 1, 2, 2, + 1, 0, 0, 1, 1, 1, 1, 1, + 0, 0, 0, 3, 3, 3, 1, 0, + 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 5, 1, 1, 1, 0, - 4, 1, 1, 1, 1, 0, 0, 0, - 0, 3, 3, 3, 0, 0, 5, 1, - 1, 1, 0, 5, 1, 1, 1, 0, - 0, 0, 0 + 0, 0, 0, 5, 0, 0, 4, 1, + 1, 1, 1, 0, 0, 0, 0, 3, + 3, 3, 1, 1, 1, 0, 0, 5, + 0, 1, 1, 1, 5, 0, 1, 1, + 1, 0, 0, 0 }; static const short _regex_index_offsets[] = { - 0, 0, 2, 4, 6, 8, 30, 36, - 42, 47, 53, 55, 61, 67, 73, 78, - 84, 88, 91, 98, 103, 109, 115, 118, - 125, 131, 134, 139, 144, 150, 155, 160, - 168, 172, 176, 179, 183, 187, 190, 194, - 198, 201, 203, 205, 207, 213, 215, 218, - 221, 223, 229, 235, 241, 247, 252, 258, - 263, 269, 274, 280, 282, 285, 290, 294, - 297, 300, 302, 305, 307, 310, 313, 315, + 0, 0, 2, 24, 30, 36, 41, 47, + 49, 55, 61, 67, 72, 78, 82, 85, + 92, 97, 103, 109, 112, 119, 125, 128, + 133, 138, 144, 149, 154, 162, 166, 170, + 173, 177, 181, 184, 188, 192, 195, 197, + 199, 201, 207, 209, 212, 215, 217, 223, + 229, 235, 241, 246, 252, 257, 263, 268, + 274, 276, 279, 284, 288, 291, 294, 296, + 298, 300, 302, 305, 307, 310, 313, 315, 317, 320, 322, 324, 326, 328, 330, 332, 334, 336, 338, 340, 342, 344, 346, 350, 352, 354, 356, 358, 360, 362, 364, 366, @@ -1055,47 +1056,46 @@ static const short _regex_actions[] = { 1279, 1281, 1283, 1286, 1288, 1290, 1292, 1294, 1296, 1298, 1300, 1302, 1304, 1306, 1308, 1310, 1312, 1314, 1316, 1318, 1320, 1322, 1324, 1326, - 1328, 1330, 1332, 1334, 1336, 1338, 1340, 1344, - 1348, 1351, 1367, 1371, 1375, 1378, 1393, 1399, - 1405, 1410, 1415, 1419, 1423, 1428, 1433, 1437, - 1441, 1446, 1451, 1456, 1460, 1464, 1469, 1474, - 1479, 1484, 1488, 1492, 1497, 1502, 1507, 1512, - 1516, 1520, 1525, 1530, 1535, 1540, 1544, 1548, - 1553, 1558, 1563, 1568, 1572, 1576, 1581, 1586, - 1591, 1596, 1600, 1604, 1610, 1615, 1620, 1625, - 1629, 1633, 1638, 1643, 1648, 1652, 1656, 1661, - 1666, 1671, 1676, 1680, 1684, 1689, 1694, 1699, - 1704, 1708, 1712, 1717, 1722, 1727, 1731, 1735, - 1740, 1745, 1750, 1755, 1760, 1764, 1768, 1774, - 1780, 1785, 1790, 1794, 1798, 1803, 1808, 1812, - 1816, 1821, 1826, 1831, 1835, 1839, 1844, 1849, - 1854, 1859, 1863, 1867, 1872, 1877, 1882, 1887, - 1891, 1895, 1900, 1905, 1910, 1915, 1919, 1923, - 1928, 1933, 1938, 1943, 1947, 1951, 1956, 1961, - 1966, 1971, 1975, 1979, 1985, 1990, 1995, 2000, - 2004, 2008, 2013, 2018, 2023, 2027, 2031, 2036, - 2041, 2046, 2051, 2055, 2059, 2064, 2069, 2074, - 2079, 2083, 2087, 2092, 2097, 2102, 2106, 2110, - 2115, 2120, 2125, 2130, 2135, 2139, 2143, 2147, - 2151, 2154, 2156, 2159, 2164, 2166, 2168, 2170, - 2172, 2174, 2176, 2198, 2200, 2202, 2204, 2207, - 2212, 2217, 2228, 2230, 2236, 2239, 2246, 2248, - 2251, 2255, 2263, 2266, 2269, 2272, 2276, 2320, - 2322, 2324, 2327, 2330, 2332, 2334, 2335, 2341, - 2343, 2345, 2347, 2349, 2353, 2355, 2357, 2362, - 2366, 2370, 2372, 2375, 2378, 2380, 2384, 2408, + 1328, 1330, 1332, 1334, 1338, 1342, 1345, 1361, + 1365, 1369, 1372, 1387, 1393, 1399, 1404, 1409, + 1413, 1417, 1422, 1427, 1431, 1435, 1440, 1445, + 1450, 1454, 1458, 1463, 1468, 1473, 1478, 1482, + 1486, 1491, 1496, 1501, 1506, 1510, 1514, 1519, + 1524, 1529, 1534, 1538, 1542, 1547, 1552, 1557, + 1562, 1566, 1570, 1575, 1580, 1585, 1590, 1594, + 1598, 1604, 1609, 1614, 1619, 1623, 1627, 1632, + 1637, 1642, 1646, 1650, 1655, 1660, 1665, 1670, + 1674, 1678, 1683, 1688, 1693, 1698, 1702, 1706, + 1711, 1716, 1721, 1725, 1729, 1734, 1739, 1744, + 1749, 1754, 1758, 1762, 1768, 1774, 1779, 1784, + 1788, 1792, 1797, 1802, 1806, 1810, 1815, 1820, + 1825, 1829, 1833, 1838, 1843, 1848, 1853, 1857, + 1861, 1866, 1871, 1876, 1881, 1885, 1889, 1894, + 1899, 1904, 1909, 1913, 1917, 1922, 1927, 1932, + 1937, 1941, 1945, 1950, 1955, 1960, 1965, 1969, + 1973, 1979, 1984, 1989, 1994, 1998, 2002, 2007, + 2012, 2017, 2021, 2025, 2030, 2035, 2040, 2045, + 2049, 2053, 2058, 2063, 2068, 2073, 2077, 2081, + 2086, 2091, 2096, 2100, 2104, 2109, 2114, 2119, + 2124, 2129, 2133, 2137, 2141, 2145, 2148, 2150, + 2153, 2158, 2160, 2162, 2164, 2166, 2168, 2170, + 2172, 2174, 2176, 2198, 2201, 2206, 2211, 2222, + 2224, 2230, 2233, 2240, 2242, 2245, 2249, 2257, + 2260, 2263, 2266, 2270, 2314, 2316, 2318, 2321, + 2324, 2326, 2328, 2329, 2335, 2337, 2339, 2341, + 2343, 2347, 2349, 2351, 2356, 2360, 2364, 2366, + 2369, 2371, 2373, 2375, 2378, 2380, 2384, 2408, 2412, 2418, 2428, 2431, 2433, 2436, 2442, 2446, 2449, 2452, 2454, 2457, 2468, 2475, 2477, 2483, 2488, 2497, 2500, 2510, 2515, 2517, 2519, 2524, - 2526, 2530, 2531, 2539, 2549, 2551, 2553, 2555, - 2559, 2595, 2597, 2599, 2601, 2603, 2605, 2606, - 2608, 2610, 2615, 2619, 2623, 2627, 2630, 2637, - 2639, 2641, 2643, 2645, 2652, 2654, 2656, 2658, - 2660, 2662, 0 + 2526, 2530, 2531, 2539, 2550, 2552, 2556, 2592, + 2594, 2596, 2598, 2600, 2602, 2603, 2605, 2607, + 2612, 2616, 2620, 2622, 2624, 2626, 2630, 2633, + 2640, 2642, 2644, 2646, 2648, 2655, 2657, 2659, + 2661, 2663, 2665, 0 }; static const signed char _regex_trans_cond_spaces[] = { - 0, -1, 0, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1132,6 +1132,8 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 0, -1, + 0, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1261,8 +1263,6 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 0, -1, - 0, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1365,13 +1365,12 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 0, -1, 0, -1, + -1, -1, -1, -1, -1, -1, 0, -1, + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 0, - 0, 0, 0, 0, -1, -1, 0, -1, - 0, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1393,6 +1392,7 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 0, -1, 0, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1412,24 +1412,24 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 0, - 0, 0, 0, 0, -1, 0, -1, 0, - -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 2, 2, -1, -1, -1, -1, -1, 0, - 0, 0, 0, 0, -1, 0, -1, 0, - -1, 0, -1, -1, -1, -1, 0, 0, - 0, 0, 0, -1, 0, -1, 0, -1, - 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 0, -1, 0, -1, + 0, -1, -1, 2, 2, -1, -1, -1, + -1, -1, 0, 0, 0, 0, 0, -1, + -1, -1, 0, -1, 0, -1, 0, -1, + -1, 0, 0, 0, 0, 0, -1, -1, + -1, 0, -1, 0, -1, 0, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1534,48 +1534,49 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 0 + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 0 }; static const short _regex_trans_offsets[] = { - 0, 2, 3, 5, 6, 8, 9, 10, - 11, 12, 13, 14, 15, 16, 17, 18, - 19, 20, 21, 22, 23, 24, 25, 26, - 27, 28, 29, 30, 31, 32, 33, 34, - 35, 36, 37, 38, 39, 40, 41, 42, - 43, 44, 45, 46, 47, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, - 59, 60, 61, 62, 63, 64, 65, 66, - 67, 68, 69, 70, 71, 72, 73, 74, - 75, 76, 77, 78, 79, 80, 81, 82, - 83, 84, 85, 86, 87, 88, 89, 90, - 91, 92, 93, 94, 95, 96, 97, 98, - 99, 100, 101, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, - 115, 116, 117, 118, 119, 120, 121, 122, - 123, 124, 125, 126, 127, 128, 129, 130, - 131, 132, 133, 134, 135, 136, 137, 138, - 139, 140, 141, 142, 143, 144, 145, 146, - 147, 148, 149, 150, 151, 152, 153, 154, - 155, 156, 157, 158, 159, 160, 161, 162, - 163, 164, 165, 166, 167, 168, 169, 170, - 171, 172, 173, 174, 175, 176, 177, 178, - 179, 180, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 194, - 195, 196, 197, 198, 199, 200, 201, 202, - 203, 204, 205, 206, 207, 208, 209, 210, - 211, 212, 213, 214, 215, 216, 217, 218, - 219, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, 231, 232, 233, 234, - 235, 236, 237, 238, 239, 240, 241, 242, - 243, 244, 245, 246, 247, 248, 249, 250, - 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, 273, 274, - 275, 276, 277, 278, 279, 280, 281, 282, - 283, 284, 285, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 298, - 299, 300, 301, 302, 303, 304, 305, 306, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 74, 75, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 87, + 88, 89, 90, 91, 92, 93, 94, 95, + 96, 97, 98, 99, 100, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, + 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, + 128, 129, 130, 131, 132, 133, 134, 135, + 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, + 152, 153, 154, 155, 156, 157, 158, 159, + 160, 161, 162, 163, 164, 165, 166, 167, + 168, 169, 170, 171, 172, 173, 174, 175, + 176, 177, 178, 179, 180, 181, 182, 183, + 184, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 198, 199, + 200, 201, 202, 203, 204, 205, 206, 207, + 208, 209, 210, 211, 212, 213, 214, 215, + 216, 217, 218, 219, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, + 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, + 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, + 272, 273, 274, 275, 276, 277, 278, 279, + 280, 281, 282, 283, 284, 285, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 296, + 297, 299, 300, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, @@ -1704,137 +1705,137 @@ static const short _regex_actions[] = { 1307, 1308, 1309, 1310, 1311, 1312, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1321, 1322, 1323, 1324, 1325, 1326, 1327, 1328, 1329, 1330, - 1331, 1332, 1333, 1334, 1335, 1336, 1337, 1339, - 1340, 1342, 1343, 1345, 1346, 1347, 1348, 1349, - 1350, 1351, 1352, 1353, 1354, 1355, 1356, 1357, - 1358, 1359, 1360, 1361, 1362, 1363, 1364, 1365, - 1366, 1367, 1368, 1369, 1370, 1371, 1372, 1373, - 1374, 1375, 1376, 1377, 1378, 1379, 1380, 1381, - 1382, 1383, 1384, 1385, 1386, 1387, 1388, 1389, - 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1397, - 1398, 1399, 1400, 1401, 1402, 1403, 1404, 1405, - 1406, 1407, 1408, 1409, 1410, 1411, 1412, 1413, - 1414, 1415, 1416, 1417, 1418, 1419, 1420, 1421, - 1422, 1423, 1424, 1425, 1426, 1427, 1428, 1429, - 1430, 1431, 1432, 1433, 1434, 1435, 1436, 1437, - 1438, 1439, 1440, 1441, 1442, 1443, 1444, 1445, - 1446, 1447, 1448, 1449, 1450, 1451, 1452, 1453, - 1454, 1455, 1456, 1457, 1458, 1459, 1460, 1461, - 1462, 1463, 1464, 1465, 1466, 1467, 1468, 1469, - 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, - 1478, 1479, 1480, 1481, 1482, 1483, 1484, 1485, - 1486, 1487, 1488, 1489, 1490, 1491, 1492, 1493, - 1494, 1495, 1496, 1497, 1498, 1499, 1500, 1501, - 1502, 1503, 1504, 1505, 1506, 1507, 1508, 1509, - 1510, 1511, 1512, 1513, 1514, 1515, 1516, 1517, - 1518, 1519, 1520, 1521, 1522, 1523, 1524, 1525, - 1526, 1527, 1528, 1529, 1530, 1531, 1532, 1533, - 1534, 1535, 1536, 1537, 1538, 1539, 1540, 1541, - 1542, 1543, 1544, 1545, 1546, 1547, 1548, 1549, - 1550, 1551, 1552, 1553, 1554, 1555, 1556, 1557, - 1558, 1559, 1560, 1561, 1562, 1563, 1564, 1565, - 1566, 1567, 1568, 1569, 1570, 1571, 1572, 1573, - 1574, 1575, 1576, 1577, 1578, 1579, 1580, 1581, - 1582, 1583, 1584, 1585, 1586, 1587, 1588, 1589, - 1590, 1591, 1592, 1593, 1594, 1595, 1596, 1597, - 1598, 1599, 1600, 1601, 1602, 1603, 1604, 1605, - 1606, 1607, 1608, 1609, 1610, 1611, 1612, 1613, - 1614, 1615, 1616, 1617, 1618, 1619, 1620, 1621, - 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, - 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637, - 1638, 1639, 1640, 1641, 1642, 1643, 1644, 1645, - 1646, 1647, 1648, 1649, 1650, 1651, 1652, 1653, - 1654, 1655, 1656, 1657, 1658, 1659, 1660, 1661, - 1662, 1663, 1664, 1665, 1666, 1667, 1668, 1669, - 1670, 1671, 1672, 1673, 1674, 1675, 1676, 1677, - 1678, 1679, 1680, 1681, 1682, 1683, 1684, 1685, - 1686, 1687, 1688, 1689, 1690, 1691, 1692, 1693, - 1694, 1695, 1696, 1697, 1698, 1699, 1700, 1701, - 1702, 1703, 1704, 1705, 1706, 1707, 1708, 1709, - 1710, 1711, 1712, 1713, 1714, 1715, 1716, 1717, - 1718, 1719, 1720, 1721, 1722, 1723, 1724, 1725, - 1726, 1727, 1728, 1729, 1730, 1731, 1732, 1733, - 1734, 1735, 1736, 1737, 1738, 1739, 1740, 1741, - 1742, 1743, 1744, 1745, 1746, 1747, 1748, 1749, - 1750, 1751, 1752, 1753, 1754, 1755, 1756, 1757, - 1758, 1759, 1760, 1761, 1762, 1763, 1764, 1765, - 1766, 1767, 1768, 1769, 1770, 1771, 1772, 1773, - 1774, 1775, 1776, 1777, 1778, 1779, 1780, 1781, - 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, - 1790, 1791, 1792, 1793, 1794, 1795, 1796, 1797, - 1798, 1799, 1800, 1801, 1802, 1803, 1804, 1805, - 1806, 1807, 1808, 1809, 1810, 1811, 1812, 1813, - 1814, 1815, 1816, 1817, 1818, 1819, 1820, 1821, - 1822, 1823, 1824, 1825, 1826, 1827, 1828, 1829, - 1830, 1831, 1832, 1833, 1834, 1835, 1836, 1837, - 1838, 1839, 1840, 1841, 1842, 1843, 1844, 1845, - 1846, 1847, 1848, 1849, 1850, 1851, 1852, 1853, - 1854, 1855, 1856, 1857, 1858, 1859, 1860, 1861, - 1862, 1863, 1864, 1865, 1866, 1867, 1868, 1869, - 1870, 1871, 1872, 1873, 1874, 1875, 1876, 1877, - 1878, 1879, 1880, 1881, 1882, 1883, 1884, 1885, - 1886, 1887, 1888, 1889, 1890, 1891, 1892, 1893, - 1894, 1895, 1896, 1897, 1898, 1899, 1900, 1901, - 1902, 1903, 1904, 1905, 1906, 1907, 1908, 1909, - 1910, 1911, 1912, 1913, 1914, 1915, 1916, 1917, - 1918, 1919, 1920, 1921, 1922, 1923, 1924, 1925, - 1926, 1927, 1928, 1929, 1930, 1931, 1932, 1933, - 1934, 1935, 1936, 1937, 1938, 1939, 1940, 1941, - 1942, 1943, 1944, 1945, 1946, 1947, 1948, 1949, - 1950, 1951, 1952, 1953, 1954, 1955, 1956, 1957, - 1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, - 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, - 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, - 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, - 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, - 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, - 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, - 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, - 2022, 2023, 2024, 2025, 2026, 2027, 2028, 2029, - 2030, 2031, 2032, 2033, 2034, 2035, 2036, 2037, - 2038, 2039, 2040, 2041, 2042, 2043, 2044, 2045, - 2046, 2047, 2048, 2049, 2050, 2051, 2052, 2053, - 2054, 2055, 2056, 2057, 2058, 2059, 2060, 2061, - 2062, 2063, 2064, 2065, 2066, 2067, 2068, 2069, - 2070, 2071, 2072, 2073, 2074, 2075, 2076, 2077, - 2078, 2079, 2080, 2081, 2082, 2083, 2084, 2085, - 2086, 2087, 2088, 2089, 2090, 2091, 2092, 2093, - 2094, 2095, 2096, 2097, 2098, 2099, 2100, 2101, - 2102, 2103, 2104, 2105, 2106, 2107, 2108, 2109, - 2110, 2111, 2112, 2113, 2114, 2115, 2116, 2117, - 2118, 2119, 2120, 2121, 2122, 2123, 2124, 2125, - 2126, 2127, 2128, 2129, 2130, 2131, 2132, 2133, - 2134, 2135, 2136, 2137, 2138, 2139, 2140, 2141, - 2142, 2143, 2144, 2145, 2146, 2147, 2148, 2149, - 2150, 2151, 2152, 2153, 2154, 2155, 2156, 2157, - 2158, 2159, 2160, 2161, 2162, 2163, 2164, 2165, - 2166, 2167, 2168, 2169, 2170, 2172, 2173, 2175, + 1331, 1332, 1333, 1334, 1335, 1336, 1337, 1338, + 1339, 1340, 1341, 1342, 1343, 1344, 1345, 1346, + 1347, 1348, 1349, 1350, 1351, 1352, 1353, 1354, + 1355, 1356, 1357, 1358, 1359, 1360, 1361, 1362, + 1363, 1364, 1365, 1366, 1367, 1368, 1369, 1370, + 1371, 1372, 1373, 1374, 1375, 1376, 1377, 1378, + 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1386, + 1387, 1388, 1389, 1390, 1391, 1392, 1393, 1394, + 1395, 1396, 1397, 1398, 1399, 1400, 1401, 1402, + 1403, 1404, 1405, 1406, 1407, 1408, 1409, 1410, + 1411, 1412, 1413, 1414, 1415, 1416, 1417, 1418, + 1419, 1420, 1421, 1422, 1423, 1424, 1425, 1426, + 1427, 1428, 1429, 1430, 1431, 1432, 1433, 1434, + 1435, 1436, 1437, 1438, 1439, 1440, 1441, 1442, + 1443, 1444, 1445, 1446, 1447, 1448, 1449, 1450, + 1451, 1452, 1453, 1454, 1455, 1456, 1457, 1458, + 1459, 1460, 1461, 1462, 1463, 1464, 1465, 1466, + 1467, 1468, 1469, 1470, 1471, 1472, 1473, 1474, + 1475, 1476, 1477, 1478, 1479, 1480, 1481, 1482, + 1483, 1484, 1485, 1486, 1487, 1488, 1489, 1490, + 1491, 1492, 1493, 1494, 1495, 1496, 1497, 1498, + 1499, 1500, 1501, 1502, 1503, 1504, 1505, 1506, + 1507, 1508, 1509, 1510, 1511, 1512, 1513, 1514, + 1515, 1516, 1517, 1518, 1519, 1520, 1521, 1522, + 1523, 1524, 1525, 1526, 1527, 1528, 1529, 1530, + 1531, 1532, 1533, 1534, 1535, 1536, 1537, 1538, + 1539, 1540, 1541, 1542, 1543, 1544, 1545, 1546, + 1547, 1548, 1549, 1550, 1551, 1552, 1553, 1554, + 1555, 1556, 1557, 1558, 1559, 1560, 1561, 1562, + 1563, 1564, 1565, 1566, 1567, 1568, 1569, 1570, + 1571, 1572, 1573, 1574, 1575, 1576, 1577, 1578, + 1579, 1580, 1581, 1582, 1583, 1584, 1585, 1586, + 1587, 1588, 1589, 1590, 1591, 1592, 1593, 1594, + 1595, 1596, 1597, 1598, 1599, 1600, 1601, 1602, + 1603, 1604, 1605, 1606, 1607, 1608, 1609, 1610, + 1611, 1612, 1613, 1614, 1615, 1616, 1617, 1618, + 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1626, + 1627, 1628, 1629, 1630, 1631, 1632, 1633, 1634, + 1635, 1636, 1637, 1638, 1639, 1640, 1641, 1642, + 1643, 1644, 1645, 1646, 1647, 1648, 1649, 1650, + 1651, 1652, 1653, 1654, 1655, 1656, 1657, 1658, + 1659, 1660, 1661, 1662, 1663, 1664, 1665, 1666, + 1667, 1668, 1669, 1670, 1671, 1672, 1673, 1674, + 1675, 1676, 1677, 1678, 1679, 1680, 1681, 1682, + 1683, 1684, 1685, 1686, 1687, 1688, 1689, 1690, + 1691, 1692, 1693, 1694, 1695, 1696, 1697, 1698, + 1699, 1700, 1701, 1702, 1703, 1704, 1705, 1706, + 1707, 1708, 1709, 1710, 1711, 1712, 1713, 1714, + 1715, 1716, 1717, 1718, 1719, 1720, 1721, 1722, + 1723, 1724, 1725, 1726, 1727, 1728, 1729, 1730, + 1731, 1732, 1733, 1734, 1735, 1736, 1737, 1738, + 1739, 1740, 1741, 1742, 1743, 1744, 1745, 1746, + 1747, 1748, 1749, 1750, 1751, 1752, 1753, 1754, + 1755, 1756, 1757, 1758, 1759, 1760, 1761, 1762, + 1763, 1764, 1765, 1766, 1767, 1768, 1769, 1770, + 1771, 1772, 1773, 1774, 1775, 1776, 1777, 1778, + 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786, + 1787, 1788, 1789, 1790, 1791, 1792, 1793, 1794, + 1795, 1796, 1797, 1798, 1799, 1800, 1801, 1802, + 1803, 1804, 1805, 1806, 1807, 1808, 1809, 1810, + 1811, 1812, 1813, 1814, 1815, 1816, 1817, 1818, + 1819, 1820, 1821, 1822, 1823, 1824, 1825, 1826, + 1827, 1828, 1829, 1830, 1831, 1832, 1833, 1834, + 1835, 1836, 1837, 1838, 1839, 1840, 1841, 1842, + 1843, 1844, 1845, 1846, 1847, 1848, 1849, 1850, + 1851, 1852, 1853, 1854, 1855, 1856, 1857, 1858, + 1859, 1860, 1861, 1862, 1863, 1864, 1865, 1866, + 1867, 1868, 1869, 1870, 1871, 1872, 1873, 1874, + 1875, 1876, 1877, 1878, 1879, 1880, 1881, 1882, + 1883, 1884, 1885, 1886, 1887, 1888, 1889, 1890, + 1891, 1892, 1893, 1894, 1895, 1896, 1897, 1898, + 1899, 1900, 1901, 1902, 1903, 1904, 1905, 1906, + 1907, 1908, 1909, 1910, 1911, 1912, 1913, 1914, + 1915, 1916, 1917, 1918, 1919, 1920, 1921, 1922, + 1923, 1924, 1925, 1926, 1927, 1928, 1929, 1930, + 1931, 1932, 1933, 1934, 1935, 1936, 1937, 1938, + 1939, 1940, 1941, 1942, 1943, 1944, 1945, 1946, + 1947, 1948, 1949, 1950, 1951, 1952, 1953, 1954, + 1955, 1956, 1957, 1958, 1959, 1960, 1961, 1962, + 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, + 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, + 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, + 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, + 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, + 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, + 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, + 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026, + 2027, 2028, 2029, 2030, 2031, 2032, 2033, 2034, + 2035, 2036, 2037, 2038, 2039, 2040, 2041, 2042, + 2043, 2044, 2045, 2046, 2047, 2048, 2049, 2050, + 2051, 2052, 2053, 2054, 2055, 2056, 2057, 2058, + 2059, 2060, 2061, 2062, 2063, 2064, 2065, 2066, + 2067, 2068, 2069, 2070, 2071, 2072, 2073, 2074, + 2075, 2076, 2077, 2078, 2079, 2080, 2081, 2082, + 2083, 2084, 2085, 2086, 2087, 2088, 2089, 2090, + 2091, 2092, 2093, 2094, 2095, 2096, 2097, 2098, + 2099, 2100, 2101, 2102, 2103, 2104, 2105, 2106, + 2107, 2108, 2109, 2110, 2111, 2112, 2113, 2114, + 2115, 2116, 2117, 2118, 2119, 2120, 2121, 2122, + 2123, 2124, 2125, 2126, 2127, 2128, 2129, 2130, + 2131, 2132, 2133, 2134, 2135, 2136, 2137, 2138, + 2139, 2140, 2141, 2142, 2143, 2144, 2145, 2146, + 2147, 2148, 2149, 2150, 2151, 2152, 2153, 2154, + 2155, 2156, 2157, 2158, 2159, 2160, 2161, 2163, + 2164, 2166, 2167, 2169, 2170, 2172, 2173, 2175, 2176, 2178, 2179, 2181, 2182, 2184, 2185, 2187, 2188, 2189, 2190, 2192, 2193, 2194, 2195, 2196, 2197, 2198, 2199, 2200, 2201, 2202, 2203, 2204, - 2206, 2208, 2210, 2212, 2214, 2215, 2216, 2218, - 2219, 2221, 2222, 2224, 2225, 2226, 2227, 2228, - 2229, 2230, 2231, 2232, 2233, 2234, 2235, 2236, - 2237, 2238, 2239, 2240, 2241, 2242, 2243, 2244, - 2245, 2246, 2247, 2248, 2249, 2250, 2251, 2252, - 2253, 2254, 2255, 2256, 2257, 2258, 2259, 2260, - 2261, 2262, 2263, 2264, 2265, 2266, 2267, 2268, - 2269, 2270, 2271, 2272, 2273, 2274, 2275, 2276, - 2277, 2278, 2279, 2280, 2281, 2282, 2283, 2284, - 2285, 2286, 2287, 2288, 2289, 2290, 2291, 2292, - 2293, 2294, 2295, 2296, 2297, 2298, 2299, 2300, - 2301, 2302, 2303, 2304, 2305, 2306, 2307, 2308, - 2309, 2310, 2311, 2312, 2313, 2314, 2315, 2316, - 2317, 2318, 2319, 2320, 2321, 2322, 2323, 2324, - 2325, 2326, 2327, 2328, 2329, 2330, 2331, 2332, - 2333, 2334, 2335, 2336, 2337, 2338, 2339, 2340, - 2341, 2342, 2343, 2344, 2345, 2346, 2347, 2348, - 2349, 2350, 2351, 2352, 2353, 2354, 2355, 2356, - 2357, 2358, 2359, 2360, 2361, 2362, 2363, 2364, - 2365, 2366, 2367, 2368, 2369, 2370, 2371, 2372, - 2373, 2374, 2375, 2376, 2377, 2378, 2379, 2380, - 2381, 2382, 2383, 2384, 2385, 2386, 2387, 2388, - 2389, 2390, 2391, 2392, 2393, 2394, 2395, 2396, + 2205, 2207, 2209, 2211, 2213, 2215, 2216, 2217, + 2218, 2219, 2220, 2221, 2222, 2223, 2224, 2225, + 2226, 2227, 2228, 2229, 2230, 2231, 2232, 2233, + 2234, 2235, 2236, 2237, 2238, 2239, 2240, 2241, + 2242, 2243, 2244, 2245, 2246, 2247, 2248, 2249, + 2250, 2251, 2252, 2253, 2254, 2255, 2256, 2257, + 2258, 2259, 2260, 2261, 2262, 2263, 2264, 2265, + 2266, 2267, 2268, 2269, 2270, 2271, 2272, 2273, + 2274, 2275, 2276, 2277, 2278, 2279, 2280, 2281, + 2282, 2283, 2284, 2285, 2286, 2287, 2288, 2289, + 2290, 2291, 2292, 2293, 2294, 2295, 2296, 2297, + 2298, 2299, 2300, 2301, 2302, 2303, 2304, 2305, + 2306, 2307, 2308, 2309, 2310, 2311, 2312, 2313, + 2314, 2315, 2316, 2317, 2318, 2319, 2320, 2321, + 2322, 2323, 2324, 2325, 2326, 2327, 2328, 2329, + 2330, 2331, 2332, 2333, 2334, 2335, 2336, 2337, + 2338, 2339, 2340, 2341, 2342, 2343, 2344, 2345, + 2346, 2347, 2348, 2349, 2350, 2351, 2352, 2353, + 2354, 2355, 2356, 2357, 2358, 2359, 2360, 2361, + 2362, 2363, 2364, 2365, 2366, 2367, 2368, 2369, + 2370, 2371, 2372, 2373, 2374, 2375, 2376, 2377, + 2378, 2379, 2380, 2381, 2382, 2383, 2384, 2385, + 2386, 2387, 2389, 2390, 2392, 2393, 2395, 2396, 2397, 2398, 2399, 2400, 2401, 2402, 2403, 2404, 2405, 2406, 2407, 2408, 2409, 2410, 2411, 2412, 2413, 2414, 2415, 2416, 2417, 2418, 2419, 2420, @@ -1856,21 +1857,21 @@ static const short _regex_actions[] = { 2541, 2542, 2543, 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, 2553, 2554, 2555, 2556, 2557, 2558, 2559, 2560, 2561, 2562, 2563, 2564, - 2566, 2568, 2570, 2572, 2574, 2575, 2577, 2578, - 2580, 2581, 2583, 2584, 2585, 2586, 2587, 2588, - 2589, 2590, 2591, 2592, 2593, 2594, 2595, 2596, - 2597, 2598, 2599, 2600, 2601, 2602, 2603, 2604, - 2605, 2606, 2607, 2608, 2609, 2610, 2611, 2612, - 2613, 2614, 2615, 2616, 2617, 2618, 2619, 2620, - 2621, 2622, 2623, 2624, 2625, 2626, 2627, 2628, - 2629, 2630, 2631, 2632, 2633, 2634, 2635, 2636, - 2637, 2638, 2639, 2640, 2641, 2642, 2643, 2644, - 2645, 2646, 2647, 2648, 2649, 2650, 2651, 2652, - 2653, 2655, 2657, 2658, 2659, 2660, 2661, 2662, - 2664, 2666, 2668, 2670, 2672, 2673, 2675, 2676, - 2678, 2679, 2681, 2682, 2683, 2684, 2685, 2687, - 2689, 2691, 2693, 2695, 2696, 2698, 2699, 2701, - 2702, 2704, 2705, 2706, 2707, 2708, 2709, 2710, + 2565, 2567, 2569, 2571, 2573, 2575, 2576, 2577, + 2578, 2579, 2580, 2581, 2582, 2583, 2584, 2585, + 2586, 2587, 2588, 2589, 2590, 2591, 2592, 2593, + 2594, 2595, 2596, 2597, 2598, 2599, 2600, 2601, + 2602, 2603, 2604, 2605, 2606, 2607, 2608, 2609, + 2610, 2611, 2612, 2613, 2614, 2615, 2616, 2617, + 2618, 2619, 2620, 2621, 2622, 2623, 2624, 2625, + 2626, 2627, 2628, 2629, 2630, 2631, 2632, 2633, + 2634, 2635, 2636, 2637, 2638, 2639, 2640, 2641, + 2642, 2643, 2644, 2645, 2646, 2648, 2649, 2651, + 2652, 2654, 2655, 2656, 2658, 2660, 2661, 2662, + 2663, 2664, 2665, 2667, 2669, 2671, 2673, 2675, + 2676, 2677, 2678, 2680, 2681, 2683, 2684, 2686, + 2687, 2688, 2690, 2692, 2694, 2696, 2698, 2699, + 2700, 2701, 2703, 2704, 2706, 2707, 2709, 2710, 2711, 2712, 2713, 2714, 2715, 2716, 2717, 2718, 2719, 2720, 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, 2729, 2730, 2731, 2732, 2733, 2734, @@ -1977,11 +1978,10 @@ static const short _regex_actions[] = { 3535, 3536, 3537, 3538, 3539, 3540, 3541, 3542, 3543, 3544, 3545, 3546, 3547, 3548, 3549, 3550, 3551, 3552, 3553, 3554, 3555, 3556, 3557, 3558, - 3559, 3560, 0 + 3559, 3560, 3561, 3562, 3563, 3564, 0 }; static const signed char _regex_trans_lengths[] = { - 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2018,6 +2018,8 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 2, 1, + 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2147,8 +2149,6 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 2, 1, - 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2251,13 +2251,12 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 2, 1, 2, 1, + 1, 1, 1, 1, 1, 1, 2, 1, + 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 1, 1, 2, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 2, - 2, 2, 2, 2, 1, 1, 2, 1, - 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2279,6 +2278,7 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2298,22 +2298,23 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 2, - 2, 2, 2, 2, 1, 2, 1, 2, - 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 2, 2, 1, 1, 1, 1, 1, 2, - 2, 2, 2, 2, 1, 2, 1, 2, - 1, 2, 1, 1, 1, 1, 2, 2, - 2, 2, 2, 1, 2, 1, 2, 1, - 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 2, 1, 2, 1, + 2, 1, 1, 2, 2, 1, 1, 1, + 1, 1, 2, 2, 2, 2, 2, 1, + 1, 1, 2, 1, 2, 1, 2, 1, + 1, 2, 2, 2, 2, 2, 1, 1, + 1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2420,11 +2421,10 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 0 + 1, 1, 1, 1, 1, 1, 0 }; static const signed char _regex_cond_keys[] = { - 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2461,6 +2461,8 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, + 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2591,8 +2593,6 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 0, 0, 1, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2695,14 +2695,13 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 0, 0, 1, 0, + 0, 0, 1, 0, 0, 1, 0, 0, + 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, - 0, 1, 0, 1, 0, 1, 0, 0, - 0, 1, 0, 0, 1, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2724,6 +2723,8 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 1, + 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2744,9 +2745,8 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, - 0, 1, 0, 1, 0, 1, 0, 0, - 1, 0, 0, 1, 0, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2755,14 +2755,15 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 0, - 1, 0, 0, 0, 0, 0, 0, 1, - 0, 1, 0, 1, 0, 1, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, - 1, 0, 0, 0, 0, 0, 1, 0, - 1, 0, 1, 0, 1, 0, 1, 0, - 0, 1, 0, 0, 1, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 1, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 0, 0, 0, 1, + 0, 0, 1, 0, 0, 1, 0, 0, + 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 0, 0, 0, 1, 0, + 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2869,48 +2870,48 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0 + 0, 0, 0, 0, 0, 0 }; static const short _regex_cond_targs[] = { - 746, 746, 746, 746, 3, 746, 746, 746, - 746, 746, 746, 746, 746, 751, 752, 753, - 746, 754, 755, 746, 757, 746, 746, 758, - 759, 760, 761, 761, 761, 761, 746, 756, - 746, 746, 6, 6, 6, 6, 746, 746, - 7, 7, 7, 7, 746, 9, 9, 9, - 9, 746, 10, 9, 9, 9, 9, 746, - 746, 746, 746, 11, 11, 11, 11, 746, - 746, 11, 12, 11, 11, 746, 746, 11, - 12, 11, 11, 746, 15, 15, 15, 15, - 746, 10, 15, 15, 15, 15, 746, 746, - 17, 746, 746, 746, 746, 746, 19, 746, - 11, 21, 11, 11, 746, 20, 20, 20, - 20, 746, 746, 20, 20, 20, 20, 746, - 746, 11, 21, 11, 11, 746, 746, 22, - 746, 746, 746, 23, 23, 23, 23, 746, - 746, 24, 24, 24, 24, 746, 746, 25, - 746, 24, 24, 24, 24, 746, 28, 28, - 28, 28, 746, 746, 28, 28, 28, 28, - 746, 6, 6, 6, 6, 746, 23, 23, - 23, 23, 746, 746, 30, 746, 31, 31, - 31, 31, 746, 33, 34, 746, 32, 33, - 34, 746, 32, 33, 34, 32, 36, 37, - 746, 35, 36, 37, 746, 35, 36, 37, - 35, 39, 40, 746, 38, 39, 40, 746, - 38, 39, 40, 38, 746, 41, 775, 746, - 746, 43, 45, 52, 49, 52, 52, 746, - 46, 746, 746, 47, 746, 746, 48, 746, - 746, 746, 52, 746, 50, 52, 52, 746, - 52, 746, 51, 52, 52, 746, 52, 746, - 52, 52, 52, 746, 52, 746, 52, 52, - 52, 746, 54, 54, 54, 54, 746, 746, - 54, 54, 54, 54, 746, 56, 56, 56, - 56, 746, 746, 56, 56, 56, 56, 746, - 58, 58, 58, 58, 746, 58, 746, 58, - 58, 58, 746, 60, 746, 746, 60, 746, - 746, 61, 61, 61, 746, 63, 786, 62, - 746, 786, 64, 746, 786, 64, 746, 787, + 746, 746, 746, 746, 748, 749, 750, 746, + 751, 752, 746, 754, 746, 746, 755, 756, + 757, 758, 758, 758, 758, 746, 753, 746, + 746, 3, 3, 3, 3, 746, 746, 4, + 4, 4, 4, 746, 6, 6, 6, 6, + 746, 7, 6, 6, 6, 6, 746, 746, + 746, 746, 8, 8, 8, 8, 746, 746, + 8, 9, 8, 8, 746, 746, 8, 9, + 8, 8, 746, 12, 12, 12, 12, 746, + 7, 12, 12, 12, 12, 746, 746, 14, + 746, 746, 746, 746, 746, 16, 746, 8, + 18, 8, 8, 746, 17, 17, 17, 17, + 746, 746, 17, 17, 17, 17, 746, 746, + 8, 18, 8, 8, 746, 746, 19, 746, + 746, 746, 20, 20, 20, 20, 746, 746, + 21, 21, 21, 21, 746, 746, 22, 746, + 21, 21, 21, 21, 746, 25, 25, 25, + 25, 746, 746, 25, 25, 25, 25, 746, + 3, 3, 3, 3, 746, 20, 20, 20, + 20, 746, 746, 27, 746, 28, 28, 28, + 28, 746, 30, 31, 746, 29, 30, 31, + 746, 29, 30, 31, 29, 33, 34, 746, + 32, 33, 34, 746, 32, 33, 34, 32, + 36, 37, 746, 35, 36, 37, 746, 35, + 36, 37, 35, 746, 38, 772, 746, 746, + 40, 42, 49, 46, 49, 49, 746, 43, + 746, 746, 44, 746, 746, 45, 746, 746, + 746, 49, 746, 47, 49, 49, 746, 49, + 746, 48, 49, 49, 746, 49, 746, 49, + 49, 49, 746, 49, 746, 49, 49, 49, + 746, 51, 51, 51, 51, 746, 746, 51, + 51, 51, 51, 746, 53, 53, 53, 53, + 746, 746, 53, 53, 53, 53, 746, 55, + 55, 55, 55, 746, 55, 746, 55, 55, + 55, 746, 57, 746, 746, 57, 746, 746, + 58, 58, 58, 746, 60, 783, 59, 746, + 783, 61, 746, 783, 61, 746, 746, 746, + 746, 746, 64, 746, 746, 746, 746, 787, 65, 787, 67, 65, 787, 65, 787, 69, 65, 787, 70, 65, 787, 65, 790, 790, 73, 76, 790, 74, 790, 75, 790, 790, @@ -3040,139 +3041,139 @@ static const short _regex_actions[] = { 790, 552, 790, 553, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 560, 0, 562, 561, 817, 0, 0, - 561, 819, 819, 819, 819, 565, 819, 819, - 819, 819, 567, 568, 819, 566, 567, 568, - 819, 566, 567, 568, 566, 571, 572, 819, - 573, 654, 669, 675, 681, 687, 693, 699, - 710, 716, 722, 727, 570, 571, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 570, - 571, 572, 819, 574, 589, 595, 601, 607, - 613, 619, 630, 636, 642, 647, 570, 571, - 572, 819, 575, 584, 570, 571, 572, 819, - 576, 580, 570, 571, 572, 819, 577, 570, - 571, 572, 819, 578, 570, 579, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 581, 570, 571, 572, 819, 582, 570, 583, - 572, 819, 570, 571, 572, 819, 570, 571, - 572, 819, 585, 570, 571, 572, 819, 586, - 570, 571, 572, 819, 587, 570, 588, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 590, 570, 571, 572, 819, 591, 570, - 571, 572, 819, 592, 570, 571, 572, 819, - 593, 570, 594, 572, 819, 570, 571, 572, - 819, 570, 571, 572, 819, 596, 570, 571, - 572, 819, 597, 570, 571, 572, 819, 598, - 570, 571, 572, 819, 599, 570, 600, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 602, 570, 571, 572, 819, 603, 570, - 571, 572, 819, 604, 570, 571, 572, 819, - 605, 570, 606, 572, 819, 570, 571, 572, - 819, 570, 571, 572, 819, 608, 570, 571, - 572, 819, 609, 570, 571, 572, 819, 610, - 570, 571, 572, 819, 611, 570, 612, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 614, 570, 571, 572, 819, 615, 570, - 571, 572, 819, 616, 570, 571, 572, 819, - 617, 570, 618, 572, 819, 570, 571, 572, - 819, 570, 571, 572, 819, 620, 625, 570, - 571, 572, 819, 621, 570, 571, 572, 819, - 622, 570, 571, 572, 819, 623, 570, 624, - 572, 819, 570, 571, 572, 819, 570, 571, - 572, 819, 626, 570, 571, 572, 819, 627, - 570, 571, 572, 819, 628, 570, 629, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 631, 570, 571, 572, 819, 632, 570, - 571, 572, 819, 633, 570, 571, 572, 819, - 634, 570, 635, 572, 819, 570, 571, 572, - 819, 570, 571, 572, 819, 637, 570, 571, - 572, 819, 638, 570, 571, 572, 819, 639, - 570, 571, 572, 819, 640, 570, 641, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 643, 570, 571, 572, 819, 644, 570, - 571, 572, 819, 645, 570, 646, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 648, 570, 571, 572, 819, 649, 570, 571, - 572, 819, 650, 570, 571, 572, 819, 651, - 570, 571, 572, 819, 652, 570, 653, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 655, 664, 570, 571, 572, 819, 656, - 660, 570, 571, 572, 819, 657, 570, 571, - 572, 819, 658, 570, 659, 572, 819, 570, - 571, 572, 819, 570, 571, 572, 819, 661, - 570, 571, 572, 819, 662, 570, 663, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 665, 570, 571, 572, 819, 666, 570, - 571, 572, 819, 667, 570, 668, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 670, 570, 571, 572, 819, 671, 570, 571, - 572, 819, 672, 570, 571, 572, 819, 673, - 570, 674, 572, 819, 570, 571, 572, 819, - 570, 571, 572, 819, 676, 570, 571, 572, - 819, 677, 570, 571, 572, 819, 678, 570, - 571, 572, 819, 679, 570, 680, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 682, 570, 571, 572, 819, 683, 570, 571, - 572, 819, 684, 570, 571, 572, 819, 685, - 570, 686, 572, 819, 570, 571, 572, 819, - 570, 571, 572, 819, 688, 570, 571, 572, - 819, 689, 570, 571, 572, 819, 690, 570, - 571, 572, 819, 691, 570, 692, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 694, 570, 571, 572, 819, 695, 570, 571, - 572, 819, 696, 570, 571, 572, 819, 697, - 570, 698, 572, 819, 570, 571, 572, 819, - 570, 571, 572, 819, 700, 705, 570, 571, - 572, 819, 701, 570, 571, 572, 819, 702, - 570, 571, 572, 819, 703, 570, 704, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 706, 570, 571, 572, 819, 707, 570, - 571, 572, 819, 708, 570, 709, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 711, 570, 571, 572, 819, 712, 570, 571, - 572, 819, 713, 570, 571, 572, 819, 714, - 570, 715, 572, 819, 570, 571, 572, 819, - 570, 571, 572, 819, 717, 570, 571, 572, - 819, 718, 570, 571, 572, 819, 719, 570, - 571, 572, 819, 720, 570, 721, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 723, 570, 571, 572, 819, 724, 570, 571, - 572, 819, 725, 570, 726, 572, 819, 570, - 571, 572, 819, 570, 571, 572, 819, 728, - 570, 571, 572, 819, 729, 570, 571, 572, - 819, 730, 570, 571, 572, 819, 731, 570, - 571, 572, 819, 732, 570, 733, 572, 819, - 570, 571, 572, 819, 570, 735, 736, 819, - 734, 735, 736, 819, 734, 735, 736, 734, - 738, 819, 819, 738, 819, 819, 739, 739, - 739, 819, 838, 838, 838, 838, 742, 838, - 838, 838, 838, 843, 843, 843, 843, 745, - 843, 843, 843, 843, 746, 746, 746, 746, - 746, 750, 746, 762, 763, 746, 764, 765, - 766, 746, 785, 746, 746, 746, 746, 747, - 746, 748, 746, 749, 746, 746, 746, 746, - 746, 746, 746, 746, 1, 746, 746, 2, - 746, 4, 5, 746, 6, 6, 6, 6, - 746, 7, 7, 7, 7, 746, 8, 11, - 14, 16, 18, 11, 12, 13, 11, 11, - 746, 22, 746, 23, 23, 23, 23, 22, - 746, 746, 22, 746, 746, 746, 24, 24, - 24, 24, 746, 746, 746, 746, 25, 746, - 26, 27, 29, 746, 746, 30, 746, 31, - 31, 31, 31, 746, 746, 746, 746, 746, - 746, 746, 746, 746, 746, 32, 35, 38, - 746, 767, 746, 746, 746, 746, 746, 746, - 746, 746, 746, 746, 772, 746, 746, 746, - 746, 746, 746, 746, 746, 746, 746, 773, - 746, 746, 746, 774, 746, 779, 746, 746, - 780, 781, 746, 746, 746, 746, 746, 746, - 782, 746, 769, 771, 746, 768, 746, 746, - 746, 770, 771, 746, 771, 771, 746, 771, - 746, 746, 746, 746, 41, 42, 43, 44, - 777, 746, 776, 746, 746, 746, 778, 746, - 746, 746, 53, 55, 57, 746, 59, 746, - 746, 746, 784, 783, 783, 783, 746, 746, - 746, 746, 746, 61, 61, 61, 746, 62, - 746, 746, 746, 746, 787, 789, 788, 787, + 561, 564, 565, 819, 563, 564, 565, 819, + 563, 564, 565, 563, 568, 569, 819, 570, + 651, 666, 672, 678, 684, 690, 696, 707, + 713, 719, 724, 567, 568, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 567, 568, + 569, 819, 571, 586, 592, 598, 604, 610, + 616, 627, 633, 639, 644, 567, 568, 569, + 819, 572, 581, 567, 568, 569, 819, 573, + 577, 567, 568, 569, 819, 574, 567, 568, + 569, 819, 575, 567, 576, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 578, + 567, 568, 569, 819, 579, 567, 580, 569, + 819, 567, 568, 569, 819, 567, 568, 569, + 819, 582, 567, 568, 569, 819, 583, 567, + 568, 569, 819, 584, 567, 585, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 587, 567, 568, 569, 819, 588, 567, 568, + 569, 819, 589, 567, 568, 569, 819, 590, + 567, 591, 569, 819, 567, 568, 569, 819, + 567, 568, 569, 819, 593, 567, 568, 569, + 819, 594, 567, 568, 569, 819, 595, 567, + 568, 569, 819, 596, 567, 597, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 599, 567, 568, 569, 819, 600, 567, 568, + 569, 819, 601, 567, 568, 569, 819, 602, + 567, 603, 569, 819, 567, 568, 569, 819, + 567, 568, 569, 819, 605, 567, 568, 569, + 819, 606, 567, 568, 569, 819, 607, 567, + 568, 569, 819, 608, 567, 609, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 611, 567, 568, 569, 819, 612, 567, 568, + 569, 819, 613, 567, 568, 569, 819, 614, + 567, 615, 569, 819, 567, 568, 569, 819, + 567, 568, 569, 819, 617, 622, 567, 568, + 569, 819, 618, 567, 568, 569, 819, 619, + 567, 568, 569, 819, 620, 567, 621, 569, + 819, 567, 568, 569, 819, 567, 568, 569, + 819, 623, 567, 568, 569, 819, 624, 567, + 568, 569, 819, 625, 567, 626, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 628, 567, 568, 569, 819, 629, 567, 568, + 569, 819, 630, 567, 568, 569, 819, 631, + 567, 632, 569, 819, 567, 568, 569, 819, + 567, 568, 569, 819, 634, 567, 568, 569, + 819, 635, 567, 568, 569, 819, 636, 567, + 568, 569, 819, 637, 567, 638, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 640, 567, 568, 569, 819, 641, 567, 568, + 569, 819, 642, 567, 643, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 645, + 567, 568, 569, 819, 646, 567, 568, 569, + 819, 647, 567, 568, 569, 819, 648, 567, + 568, 569, 819, 649, 567, 650, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 652, 661, 567, 568, 569, 819, 653, 657, + 567, 568, 569, 819, 654, 567, 568, 569, + 819, 655, 567, 656, 569, 819, 567, 568, + 569, 819, 567, 568, 569, 819, 658, 567, + 568, 569, 819, 659, 567, 660, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 662, 567, 568, 569, 819, 663, 567, 568, + 569, 819, 664, 567, 665, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 667, + 567, 568, 569, 819, 668, 567, 568, 569, + 819, 669, 567, 568, 569, 819, 670, 567, + 671, 569, 819, 567, 568, 569, 819, 567, + 568, 569, 819, 673, 567, 568, 569, 819, + 674, 567, 568, 569, 819, 675, 567, 568, + 569, 819, 676, 567, 677, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 679, + 567, 568, 569, 819, 680, 567, 568, 569, + 819, 681, 567, 568, 569, 819, 682, 567, + 683, 569, 819, 567, 568, 569, 819, 567, + 568, 569, 819, 685, 567, 568, 569, 819, + 686, 567, 568, 569, 819, 687, 567, 568, + 569, 819, 688, 567, 689, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 691, + 567, 568, 569, 819, 692, 567, 568, 569, + 819, 693, 567, 568, 569, 819, 694, 567, + 695, 569, 819, 567, 568, 569, 819, 567, + 568, 569, 819, 697, 702, 567, 568, 569, + 819, 698, 567, 568, 569, 819, 699, 567, + 568, 569, 819, 700, 567, 701, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 703, 567, 568, 569, 819, 704, 567, 568, + 569, 819, 705, 567, 706, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 708, + 567, 568, 569, 819, 709, 567, 568, 569, + 819, 710, 567, 568, 569, 819, 711, 567, + 712, 569, 819, 567, 568, 569, 819, 567, + 568, 569, 819, 714, 567, 568, 569, 819, + 715, 567, 568, 569, 819, 716, 567, 568, + 569, 819, 717, 567, 718, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 720, + 567, 568, 569, 819, 721, 567, 568, 569, + 819, 722, 567, 723, 569, 819, 567, 568, + 569, 819, 567, 568, 569, 819, 725, 567, + 568, 569, 819, 726, 567, 568, 569, 819, + 727, 567, 568, 569, 819, 728, 567, 568, + 569, 819, 729, 567, 730, 569, 819, 567, + 568, 569, 819, 567, 732, 733, 819, 731, + 732, 733, 819, 731, 732, 733, 731, 735, + 819, 819, 735, 819, 819, 736, 736, 736, + 819, 819, 819, 819, 819, 739, 819, 819, + 819, 819, 839, 839, 839, 839, 742, 839, + 839, 839, 839, 844, 844, 844, 844, 745, + 844, 844, 844, 844, 746, 746, 746, 746, + 746, 747, 746, 759, 760, 746, 761, 762, + 763, 746, 782, 746, 746, 746, 746, 746, + 784, 746, 785, 746, 786, 746, 746, 746, + 1, 2, 746, 3, 3, 3, 3, 746, + 4, 4, 4, 4, 746, 5, 8, 11, + 13, 15, 8, 9, 10, 8, 8, 746, + 19, 746, 20, 20, 20, 20, 19, 746, + 746, 19, 746, 746, 746, 21, 21, 21, + 21, 746, 746, 746, 746, 22, 746, 23, + 24, 26, 746, 746, 27, 746, 28, 28, + 28, 28, 746, 746, 746, 746, 746, 746, + 746, 746, 746, 746, 29, 32, 35, 746, + 764, 746, 746, 746, 746, 746, 746, 746, + 746, 746, 746, 769, 746, 746, 746, 746, + 746, 746, 746, 746, 746, 746, 770, 746, + 746, 746, 771, 746, 776, 746, 746, 777, + 778, 746, 746, 746, 746, 746, 746, 779, + 746, 766, 768, 746, 765, 746, 746, 746, + 767, 768, 746, 768, 768, 746, 768, 746, + 746, 746, 746, 38, 39, 40, 41, 774, + 746, 773, 746, 746, 746, 775, 746, 746, + 746, 50, 52, 54, 746, 56, 746, 746, + 746, 781, 780, 780, 780, 746, 746, 746, + 746, 746, 58, 58, 58, 746, 59, 746, + 746, 746, 746, 746, 746, 746, 746, 62, + 746, 746, 63, 746, 787, 789, 788, 787, 65, 787, 66, 68, 65, 791, 792, 793, 795, 796, 797, 798, 800, 801, 802, 803, 804, 806, 807, 808, 809, 810, 811, 812, @@ -3193,25 +3194,26 @@ static const short _regex_actions[] = { 548, 790, 554, 790, 555, 556, 557, 558, 790, 790, 790, 790, 790, 790, 790, 0, 818, 818, 818, 818, 818, 818, 818, 818, - 819, 823, 824, 819, 819, 819, 819, 820, - 819, 821, 819, 822, 819, 819, 819, 819, - 819, 819, 819, 563, 819, 819, 564, 819, - 566, 569, 734, 819, 825, 819, 819, 819, - 819, 819, 829, 819, 819, 819, 819, 819, - 819, 819, 830, 819, 819, 819, 819, 819, - 819, 819, 831, 832, 819, 819, 819, 819, - 819, 819, 833, 827, 819, 819, 819, 819, - 826, 819, 819, 819, 828, 819, 819, 819, - 819, 819, 819, 737, 819, 819, 819, 835, - 834, 834, 834, 819, 819, 819, 819, 819, - 739, 739, 739, 819, 837, 836, 836, 836, - 836, 836, 836, 836, 836, 842, 838, 838, - 838, 839, 838, 840, 838, 841, 838, 838, - 838, 838, 838, 838, 838, 740, 838, 838, - 741, 838, 838, 838, 847, 843, 843, 843, - 844, 843, 845, 843, 846, 843, 843, 843, - 843, 843, 843, 843, 743, 843, 843, 744, - 843, 843, 843, 848, 848, 849, 849, 0, + 820, 819, 821, 822, 819, 819, 819, 819, + 834, 819, 835, 819, 836, 819, 819, 819, + 819, 819, 563, 566, 731, 819, 823, 819, + 819, 819, 819, 819, 827, 819, 819, 819, + 819, 819, 819, 819, 828, 819, 819, 819, + 819, 819, 819, 819, 829, 830, 819, 819, + 819, 819, 819, 819, 831, 825, 819, 819, + 819, 819, 824, 819, 819, 819, 826, 819, + 819, 819, 819, 819, 819, 734, 819, 819, + 819, 833, 832, 832, 832, 819, 819, 819, + 819, 819, 736, 736, 736, 819, 819, 819, + 819, 819, 737, 819, 819, 738, 819, 838, + 837, 837, 837, 837, 837, 837, 837, 837, + 840, 839, 839, 839, 841, 839, 842, 839, + 843, 839, 839, 839, 839, 839, 839, 839, + 839, 839, 740, 839, 839, 741, 839, 845, + 844, 844, 844, 846, 844, 847, 844, 848, + 844, 844, 844, 844, 844, 844, 844, 844, + 844, 743, 844, 844, 744, 844, 849, 849, + 850, 850, 0, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, @@ -3219,8 +3221,8 @@ static const short _regex_actions[] = { 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, - 746, 746, 746, 746, 746, 746, 746, 746, - 787, 787, 787, 787, 787, 787, 790, 790, + 746, 746, 746, 787, 787, 787, 787, 787, + 787, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, @@ -3281,8 +3283,7 @@ static const short _regex_actions[] = { 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, - 790, 790, 790, 790, 790, 790, 559, 560, - 561, 562, 819, 819, 819, 819, 819, 819, + 790, 559, 560, 561, 562, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, @@ -3304,62 +3305,62 @@ static const short _regex_actions[] = { 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, - 819, 819, 819, 838, 838, 838, 843, 843, - 843, 746, 746, 746, 746, 746, 746, 746, + 819, 819, 819, 819, 819, 819, 839, 839, + 839, 844, 844, 844, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, - 746, 746, 787, 787, 787, 790, 790, 790, + 746, 746, 746, 746, 746, 787, 787, 787, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, - 817, 818, 819, 819, 819, 819, 819, 819, + 790, 790, 790, 817, 818, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, - 819, 819, 819, 836, 836, 838, 838, 838, - 838, 838, 843, 843, 843, 843, 843, 848, - 849, 0 + 819, 819, 819, 819, 819, 819, 819, 837, + 837, 839, 839, 839, 839, 839, 844, 844, + 844, 844, 844, 849, 850, 0 }; static const short _regex_cond_actions[] = { - 843, 769, 843, 843, 0, 843, 843, 771, - 843, 827, 631, 731, 723, 45, 903, 903, - 897, 45, 912, 900, 903, 729, 741, 0, - 45, 45, 923, 923, 923, 923, 737, 45, - 765, 747, 0, 0, 0, 0, 841, 743, - 3, 3, 3, 3, 841, 3, 3, 3, - 3, 839, 0, 3, 3, 3, 3, 839, - 761, 839, 761, 3, 3, 3, 3, 839, - 759, 3, 870, 3, 3, 839, 761, 3, - 870, 3, 3, 839, 3, 3, 3, 3, - 839, 0, 3, 3, 3, 3, 839, 751, - 0, 749, 839, 755, 753, 839, 0, 757, - 3, 3, 3, 3, 839, 0, 0, 0, - 0, 839, 757, 0, 0, 0, 0, 839, - 757, 3, 3, 3, 3, 839, 745, 0, - 841, 725, 727, 27, 27, 27, 27, 841, - 743, 3, 3, 3, 3, 841, 763, 0, - 841, 3, 3, 3, 3, 841, 3, 3, - 3, 3, 841, 673, 3, 3, 3, 3, - 841, 0, 0, 0, 0, 841, 27, 27, - 27, 27, 841, 725, 0, 727, 25, 25, - 25, 25, 841, 0, 0, 829, 0, 0, - 0, 603, 0, 0, 0, 0, 0, 0, - 829, 0, 0, 0, 601, 0, 0, 0, - 0, 0, 0, 829, 0, 0, 0, 603, - 0, 0, 0, 0, 675, 0, 17, 831, - 675, 0, 7, 3, 870, 3, 3, 831, - 17, 831, 663, 17, 831, 663, 17, 831, - 663, 831, 3, 661, 870, 3, 3, 831, - 3, 661, 870, 3, 3, 831, 3, 661, - 3, 3, 3, 831, 3, 665, 3, 3, - 3, 831, 3, 3, 3, 3, 837, 671, - 3, 3, 3, 3, 837, 3, 3, 3, - 3, 837, 669, 3, 3, 3, 3, 837, - 3, 3, 3, 3, 837, 3, 667, 3, - 3, 3, 837, 0, 833, 677, 0, 833, - 679, 0, 0, 0, 835, 0, 29, 11, - 845, 31, 13, 845, 0, 13, 845, 53, + 831, 635, 735, 727, 45, 907, 907, 901, + 45, 916, 904, 907, 733, 745, 0, 45, + 45, 927, 927, 927, 927, 741, 45, 769, + 751, 0, 0, 0, 0, 845, 747, 3, + 3, 3, 3, 845, 3, 3, 3, 3, + 843, 0, 3, 3, 3, 3, 843, 765, + 843, 765, 3, 3, 3, 3, 843, 763, + 3, 874, 3, 3, 843, 765, 3, 874, + 3, 3, 843, 3, 3, 3, 3, 843, + 0, 3, 3, 3, 3, 843, 755, 0, + 753, 843, 759, 757, 843, 0, 761, 3, + 3, 3, 3, 843, 0, 0, 0, 0, + 843, 761, 0, 0, 0, 0, 843, 761, + 3, 3, 3, 3, 843, 749, 0, 845, + 729, 731, 27, 27, 27, 27, 845, 747, + 3, 3, 3, 3, 845, 767, 0, 845, + 3, 3, 3, 3, 845, 3, 3, 3, + 3, 845, 677, 3, 3, 3, 3, 845, + 0, 0, 0, 0, 845, 27, 27, 27, + 27, 845, 729, 0, 731, 25, 25, 25, + 25, 845, 0, 0, 833, 0, 0, 0, + 607, 0, 0, 0, 0, 0, 0, 833, + 0, 0, 0, 605, 0, 0, 0, 0, + 0, 0, 833, 0, 0, 0, 607, 0, + 0, 0, 0, 679, 0, 17, 835, 679, + 0, 7, 3, 874, 3, 3, 835, 17, + 835, 667, 17, 835, 667, 17, 835, 667, + 835, 3, 665, 874, 3, 3, 835, 3, + 665, 874, 3, 3, 835, 3, 665, 3, + 3, 3, 835, 3, 669, 3, 3, 3, + 835, 3, 3, 3, 3, 841, 675, 3, + 3, 3, 3, 841, 3, 3, 3, 3, + 841, 673, 3, 3, 3, 3, 841, 3, + 3, 3, 3, 841, 3, 671, 3, 3, + 3, 841, 0, 837, 681, 0, 837, 683, + 0, 0, 0, 839, 0, 29, 11, 849, + 31, 13, 849, 0, 13, 849, 847, 773, + 847, 847, 0, 847, 847, 775, 847, 53, 0, 53, 0, 0, 51, 0, 53, 0, 0, 49, 0, 0, 47, 0, 315, 359, 0, 0, 359, 0, 359, 0, 359, 127, @@ -3489,139 +3490,139 @@ static const short _regex_actions[] = { 359, 0, 359, 0, 359, 309, 359, 311, 359, 119, 359, 121, 359, 123, 359, 125, 359, 0, 39, 33, 35, 37, 39, 39, - 35, 537, 503, 537, 537, 0, 537, 537, - 505, 537, 0, 0, 539, 0, 0, 0, - 377, 0, 0, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 539, - 0, 0, 0, 435, 0, 0, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 381, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 385, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 389, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 393, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 397, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 401, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 405, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 409, 0, 0, 0, 539, 0, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 413, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 417, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 421, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 425, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 429, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 433, 0, 0, 0, - 539, 0, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 379, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 383, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 387, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 391, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 395, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 399, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 403, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 407, - 0, 0, 0, 539, 0, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 411, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 415, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 419, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 423, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 427, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 431, 0, 0, 0, 539, - 0, 0, 0, 377, 0, 0, 0, 0, - 0, 533, 471, 0, 533, 475, 0, 0, - 0, 535, 569, 557, 569, 569, 0, 569, - 569, 559, 569, 587, 575, 587, 587, 0, - 587, 587, 577, 587, 633, 775, 777, 629, - 637, 45, 597, 0, 0, 609, 0, 45, - 0, 635, 909, 599, 777, 773, 777, 0, - 777, 45, 777, 45, 777, 773, 775, 777, - 823, 767, 823, 823, 0, 823, 823, 0, - 823, 0, 0, 779, 0, 0, 0, 0, - 821, 3, 3, 3, 3, 821, 1, 3, - 1, 0, 3, 3, 855, 855, 3, 3, - 819, 0, 821, 27, 27, 27, 27, 0, - 821, 745, 0, 821, 735, 733, 3, 3, - 3, 3, 821, 739, 821, 763, 0, 821, - 1, 1, 0, 821, 725, 0, 727, 25, - 25, 25, 25, 821, 615, 613, 783, 619, - 617, 785, 623, 621, 787, 0, 0, 0, - 781, 5, 639, 647, 611, 695, 607, 717, - 699, 715, 683, 683, 0, 605, 713, 691, - 683, 703, 687, 719, 641, 657, 645, 0, - 693, 659, 655, 906, 697, 45, 683, 651, - 45, 0, 653, 689, 649, 683, 701, 685, - 7, 643, 852, 915, 721, 15, 791, 867, - 791, 858, 919, 795, 927, 919, 793, 919, - 847, 709, 711, 681, 0, 7, 0, 849, - 17, 801, 17, 799, 876, 799, 17, 797, - 873, 797, 1, 1, 1, 815, 0, 803, - 705, 707, 45, 19, 23, 21, 805, 882, - 894, 888, 805, 0, 0, 0, 807, 11, - 825, 627, 625, 789, 55, 45, 45, 53, + 35, 0, 0, 543, 0, 0, 0, 377, + 0, 0, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 543, 0, + 0, 0, 435, 0, 0, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 381, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 385, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 389, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 393, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 397, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 401, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 405, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 409, + 0, 0, 0, 543, 0, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 413, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 417, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 421, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 425, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 429, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 433, 0, 0, 0, 543, + 0, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 379, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 383, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 387, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 391, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 395, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 399, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 403, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 407, 0, + 0, 0, 543, 0, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 411, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 415, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 419, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 423, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 427, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 431, 0, 0, 0, 543, 0, + 0, 0, 377, 0, 0, 0, 0, 0, + 537, 471, 0, 537, 475, 0, 0, 0, + 539, 541, 503, 541, 541, 0, 541, 541, + 505, 541, 573, 561, 573, 573, 0, 573, + 573, 563, 573, 591, 579, 591, 591, 0, + 591, 591, 581, 591, 637, 779, 781, 633, + 641, 45, 601, 0, 0, 613, 0, 45, + 0, 639, 913, 603, 779, 781, 777, 781, + 0, 781, 45, 781, 45, 781, 777, 781, + 0, 0, 783, 0, 0, 0, 0, 825, + 3, 3, 3, 3, 825, 1, 3, 1, + 0, 3, 3, 859, 859, 3, 3, 823, + 0, 825, 27, 27, 27, 27, 0, 825, + 749, 0, 825, 739, 737, 3, 3, 3, + 3, 825, 743, 825, 767, 0, 825, 1, + 1, 0, 825, 729, 0, 731, 25, 25, + 25, 25, 825, 619, 617, 787, 623, 621, + 789, 627, 625, 791, 0, 0, 0, 785, + 5, 643, 651, 615, 699, 611, 721, 703, + 719, 687, 687, 0, 609, 717, 695, 687, + 707, 691, 723, 645, 661, 649, 0, 697, + 663, 659, 910, 701, 45, 687, 655, 45, + 0, 657, 693, 653, 687, 705, 689, 7, + 647, 856, 919, 725, 15, 795, 871, 795, + 862, 923, 799, 931, 923, 797, 923, 851, + 713, 715, 685, 0, 7, 0, 853, 17, + 805, 17, 803, 880, 803, 17, 801, 877, + 801, 1, 1, 1, 819, 0, 807, 709, + 711, 45, 19, 23, 21, 809, 886, 898, + 892, 809, 0, 0, 0, 811, 11, 829, + 631, 629, 793, 827, 771, 827, 827, 0, + 827, 827, 0, 827, 55, 45, 45, 53, 0, 53, 0, 0, 0, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, @@ -3642,47 +3643,47 @@ static const short _regex_actions[] = { 0, 339, 0, 339, 0, 0, 0, 0, 339, 313, 339, 113, 115, 117, 335, 39, 361, 363, 365, 367, 369, 371, 373, 375, - 491, 45, 0, 511, 509, 507, 509, 0, - 509, 45, 509, 45, 509, 507, 509, 529, - 501, 529, 529, 0, 529, 529, 0, 529, - 0, 0, 0, 531, 5, 489, 439, 457, - 493, 493, 0, 437, 485, 493, 461, 481, - 451, 441, 0, 487, 453, 449, 495, 455, - 493, 445, 45, 0, 447, 483, 443, 493, - 459, 479, 7, 9, 473, 497, 497, 499, - 15, 517, 861, 517, 15, 519, 864, 519, - 467, 469, 477, 0, 521, 463, 465, 45, - 19, 23, 21, 523, 879, 891, 885, 523, - 0, 0, 0, 525, 0, 549, 543, 549, - 541, 549, 547, 545, 551, 0, 563, 561, - 563, 0, 563, 45, 563, 45, 563, 561, - 563, 565, 555, 565, 565, 0, 565, 565, - 0, 565, 553, 567, 0, 581, 579, 581, - 0, 581, 45, 581, 45, 581, 579, 581, - 583, 573, 583, 583, 0, 583, 583, 0, - 583, 571, 585, 589, 591, 593, 595, 0, - 843, 843, 843, 827, 827, 841, 841, 839, - 839, 839, 839, 839, 839, 839, 839, 839, - 839, 839, 839, 839, 839, 841, 841, 841, - 841, 841, 841, 841, 841, 841, 841, 829, - 829, 829, 829, 829, 829, 829, 829, 829, - 831, 831, 831, 831, 831, 831, 831, 831, - 831, 831, 831, 831, 837, 837, 837, 837, - 837, 837, 833, 833, 835, 845, 845, 845, - 59, 59, 59, 59, 59, 59, 359, 359, - 359, 359, 359, 359, 359, 359, 359, 359, + 0, 491, 45, 0, 513, 511, 507, 511, + 0, 511, 45, 511, 45, 511, 507, 511, + 509, 535, 0, 0, 0, 533, 5, 489, + 439, 457, 493, 493, 0, 437, 485, 493, + 461, 481, 451, 441, 0, 487, 453, 449, + 495, 455, 493, 445, 45, 0, 447, 483, + 443, 493, 459, 479, 7, 9, 473, 497, + 497, 499, 15, 519, 865, 519, 15, 521, + 868, 521, 467, 469, 477, 0, 523, 463, + 465, 45, 19, 23, 21, 525, 883, 895, + 889, 525, 0, 0, 0, 527, 531, 501, + 531, 531, 0, 531, 531, 0, 531, 0, + 553, 547, 553, 545, 553, 551, 549, 555, + 0, 567, 565, 567, 0, 567, 45, 567, + 45, 567, 565, 567, 557, 571, 569, 559, + 569, 569, 0, 569, 569, 0, 569, 0, + 585, 583, 585, 0, 585, 45, 585, 45, + 585, 583, 585, 575, 589, 587, 577, 587, + 587, 0, 587, 587, 0, 587, 593, 595, + 597, 599, 0, 831, 831, 845, 845, 843, + 843, 843, 843, 843, 843, 843, 843, 843, + 843, 843, 843, 843, 843, 845, 845, 845, + 845, 845, 845, 845, 845, 845, 845, 833, + 833, 833, 833, 833, 833, 833, 833, 833, + 835, 835, 835, 835, 835, 835, 835, 835, + 835, 835, 835, 835, 841, 841, 841, 841, + 841, 841, 837, 837, 839, 849, 849, 849, + 847, 847, 847, 59, 59, 59, 59, 59, + 59, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 341, 341, 341, 341, 341, 341, 341, + 359, 359, 359, 359, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, - 341, 341, 341, 341, 343, 343, 343, 343, - 343, 343, 341, 341, 341, 341, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 343, + 343, 343, 343, 343, 343, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, - 341, 341, 341, 359, 359, 359, 359, 359, + 341, 341, 341, 341, 341, 341, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, @@ -3691,7 +3692,8 @@ static const short _regex_actions[] = { 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 357, 357, 357, 357, 357, 359, 359, 359, + 359, 359, 359, 357, 357, 357, 357, 357, + 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, @@ -3702,72 +3704,71 @@ static const short _regex_actions[] = { 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, - 345, 345, 345, 345, 347, 347, 347, 347, - 347, 347, 347, 347, 347, 347, 347, 349, - 349, 349, 349, 349, 349, 349, 349, 349, + 345, 345, 345, 345, 345, 345, 345, 347, 347, 347, 347, 347, 347, 347, 347, 347, - 347, 347, 347, 347, 351, 351, 351, 351, - 351, 351, 351, 351, 351, 351, 359, 359, + 347, 347, 349, 349, 349, 349, 349, 349, + 349, 349, 349, 347, 347, 347, 347, 347, + 347, 347, 347, 347, 347, 347, 347, 351, + 351, 351, 351, 351, 351, 351, 351, 351, + 351, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 353, 353, 353, 353, 353, 353, 353, 353, 353, 353, 353, - 353, 353, 359, 359, 359, 359, 359, 359, - 359, 355, 355, 355, 355, 355, 355, 355, + 353, 353, 353, 353, 353, 359, 359, 359, + 359, 359, 359, 359, 355, 355, 355, 355, + 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, - 355, 355, 355, 355, 355, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 359, 39, 39, - 39, 39, 537, 537, 537, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 533, 533, 535, 569, 569, 569, 587, 587, - 587, 0, 823, 823, 823, 779, 821, 821, - 819, 821, 821, 821, 821, 821, 821, 821, - 821, 783, 785, 787, 781, 817, 791, 791, - 795, 793, 847, 811, 809, 801, 799, 799, - 797, 797, 815, 803, 813, 805, 805, 807, - 825, 789, 0, 57, 57, 0, 339, 339, - 319, 321, 339, 339, 339, 339, 337, 339, - 339, 339, 323, 325, 327, 329, 339, 331, - 339, 333, 339, 339, 339, 339, 339, 335, - 0, 0, 0, 529, 529, 529, 531, 531, - 517, 517, 519, 519, 513, 527, 521, 515, - 523, 523, 525, 0, 551, 0, 565, 565, - 565, 567, 0, 583, 583, 583, 585, 0, - 0, 0 + 359, 359, 359, 359, 359, 359, 359, 359, + 359, 39, 39, 39, 39, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 537, 537, 539, 541, 541, 541, 573, 573, + 573, 591, 591, 591, 0, 783, 825, 825, + 823, 825, 825, 825, 825, 825, 825, 825, + 825, 787, 789, 791, 785, 821, 795, 795, + 799, 797, 851, 815, 813, 805, 803, 803, + 801, 801, 819, 807, 817, 809, 809, 811, + 829, 793, 827, 827, 827, 0, 57, 57, + 0, 339, 339, 319, 321, 339, 339, 339, + 339, 337, 339, 339, 339, 323, 325, 327, + 329, 339, 331, 339, 333, 339, 339, 339, + 339, 339, 335, 0, 0, 0, 535, 533, + 535, 519, 519, 521, 521, 515, 529, 523, + 517, 525, 525, 527, 531, 531, 531, 0, + 555, 0, 571, 569, 569, 569, 0, 589, + 587, 587, 587, 0, 0, 0 }; static const signed char _regex_to_state_actions[] = { @@ -3875,9 +3876,9 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 41, 0, 41, 0, - 0, 0, 0, 41, 0, 0, 0, 0, - 41, 41, 0 + 0, 0, 0, 0, 0, 41, 0, 41, + 0, 0, 0, 0, 41, 0, 0, 0, + 0, 41, 41, 0 }; static const signed char _regex_from_state_actions[] = { @@ -3985,119 +3986,119 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 43, 0, 43, 0, - 0, 0, 0, 43, 0, 0, 0, 0, - 43, 43, 0 + 0, 0, 0, 0, 0, 43, 0, 43, + 0, 0, 0, 0, 43, 0, 0, 0, + 0, 43, 43, 0 }; static const short _regex_eof_trans[] = { - 2665, 2666, 2667, 2668, 2669, 2670, 2671, 2672, - 2673, 2674, 2675, 2676, 2677, 2678, 2679, 2680, - 2681, 2682, 2683, 2684, 2685, 2686, 2687, 2688, - 2689, 2690, 2691, 2692, 2693, 2694, 2695, 2696, - 2697, 2698, 2699, 2700, 2701, 2702, 2703, 2704, - 2705, 2706, 2707, 2708, 2709, 2710, 2711, 2712, - 2713, 2714, 2715, 2716, 2717, 2718, 2719, 2720, - 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, - 2729, 2730, 2731, 2732, 2733, 2734, 2735, 2736, - 2737, 2738, 2739, 2740, 2741, 2742, 2743, 2744, - 2745, 2746, 2747, 2748, 2749, 2750, 2751, 2752, - 2753, 2754, 2755, 2756, 2757, 2758, 2759, 2760, - 2761, 2762, 2763, 2764, 2765, 2766, 2767, 2768, - 2769, 2770, 2771, 2772, 2773, 2774, 2775, 2776, - 2777, 2778, 2779, 2780, 2781, 2782, 2783, 2784, - 2785, 2786, 2787, 2788, 2789, 2790, 2791, 2792, - 2793, 2794, 2795, 2796, 2797, 2798, 2799, 2800, - 2801, 2802, 2803, 2804, 2805, 2806, 2807, 2808, - 2809, 2810, 2811, 2812, 2813, 2814, 2815, 2816, - 2817, 2818, 2819, 2820, 2821, 2822, 2823, 2824, - 2825, 2826, 2827, 2828, 2829, 2830, 2831, 2832, - 2833, 2834, 2835, 2836, 2837, 2838, 2839, 2840, - 2841, 2842, 2843, 2844, 2845, 2846, 2847, 2848, - 2849, 2850, 2851, 2852, 2853, 2854, 2855, 2856, - 2857, 2858, 2859, 2860, 2861, 2862, 2863, 2864, - 2865, 2866, 2867, 2868, 2869, 2870, 2871, 2872, - 2873, 2874, 2875, 2876, 2877, 2878, 2879, 2880, - 2881, 2882, 2883, 2884, 2885, 2886, 2887, 2888, - 2889, 2890, 2891, 2892, 2893, 2894, 2895, 2896, - 2897, 2898, 2899, 2900, 2901, 2902, 2903, 2904, - 2905, 2906, 2907, 2908, 2909, 2910, 2911, 2912, - 2913, 2914, 2915, 2916, 2917, 2918, 2919, 2920, - 2921, 2922, 2923, 2924, 2925, 2926, 2927, 2928, - 2929, 2930, 2931, 2932, 2933, 2934, 2935, 2936, - 2937, 2938, 2939, 2940, 2941, 2942, 2943, 2944, - 2945, 2946, 2947, 2948, 2949, 2950, 2951, 2952, - 2953, 2954, 2955, 2956, 2957, 2958, 2959, 2960, - 2961, 2962, 2963, 2964, 2965, 2966, 2967, 2968, - 2969, 2970, 2971, 2972, 2973, 2974, 2975, 2976, - 2977, 2978, 2979, 2980, 2981, 2982, 2983, 2984, - 2985, 2986, 2987, 2988, 2989, 2990, 2991, 2992, - 2993, 2994, 2995, 2996, 2997, 2998, 2999, 3000, - 3001, 3002, 3003, 3004, 3005, 3006, 3007, 3008, - 3009, 3010, 3011, 3012, 3013, 3014, 3015, 3016, - 3017, 3018, 3019, 3020, 3021, 3022, 3023, 3024, - 3025, 3026, 3027, 3028, 3029, 3030, 3031, 3032, - 3033, 3034, 3035, 3036, 3037, 3038, 3039, 3040, - 3041, 3042, 3043, 3044, 3045, 3046, 3047, 3048, - 3049, 3050, 3051, 3052, 3053, 3054, 3055, 3056, - 3057, 3058, 3059, 3060, 3061, 3062, 3063, 3064, - 3065, 3066, 3067, 3068, 3069, 3070, 3071, 3072, - 3073, 3074, 3075, 3076, 3077, 3078, 3079, 3080, - 3081, 3082, 3083, 3084, 3085, 3086, 3087, 3088, - 3089, 3090, 3091, 3092, 3093, 3094, 3095, 3096, - 3097, 3098, 3099, 3100, 3101, 3102, 3103, 3104, - 3105, 3106, 3107, 3108, 3109, 3110, 3111, 3112, - 3113, 3114, 3115, 3116, 3117, 3118, 3119, 3120, - 3121, 3122, 3123, 3124, 3125, 3126, 3127, 3128, - 3129, 3130, 3131, 3132, 3133, 3134, 3135, 3136, - 3137, 3138, 3139, 3140, 3141, 3142, 3143, 3144, - 3145, 3146, 3147, 3148, 3149, 3150, 3151, 3152, - 3153, 3154, 3155, 3156, 3157, 3158, 3159, 3160, - 3161, 3162, 3163, 3164, 3165, 3166, 3167, 3168, - 3169, 3170, 3171, 3172, 3173, 3174, 3175, 3176, - 3177, 3178, 3179, 3180, 3181, 3182, 3183, 3184, - 3185, 3186, 3187, 3188, 3189, 3190, 3191, 3192, - 3193, 3194, 3195, 3196, 3197, 3198, 3199, 3200, - 3201, 3202, 3203, 3204, 3205, 3206, 3207, 3208, - 3209, 3210, 3211, 3212, 3213, 3214, 3215, 3216, - 3217, 3218, 3219, 3220, 3221, 3222, 3223, 3224, - 3225, 3226, 3227, 3228, 3229, 3230, 3231, 3232, - 3233, 3234, 3235, 3236, 3237, 3238, 3239, 3240, - 3241, 3242, 3243, 3244, 3245, 3246, 3247, 3248, - 3249, 3250, 3251, 3252, 3253, 3254, 3255, 3256, - 3257, 3258, 3259, 3260, 3261, 3262, 3263, 3264, - 3265, 3266, 3267, 3268, 3269, 3270, 3271, 3272, - 3273, 3274, 3275, 3276, 3277, 3278, 3279, 3280, - 3281, 3282, 3283, 3284, 3285, 3286, 3287, 3288, - 3289, 3290, 3291, 3292, 3293, 3294, 3295, 3296, - 3297, 3298, 3299, 3300, 3301, 3302, 3303, 3304, - 3305, 3306, 3307, 3308, 3309, 3310, 3311, 3312, - 3313, 3314, 3315, 3316, 3317, 3318, 3319, 3320, - 3321, 3322, 3323, 3324, 3325, 3326, 3327, 3328, - 3329, 3330, 3331, 3332, 3333, 3334, 3335, 3336, - 3337, 3338, 3339, 3340, 3341, 3342, 3343, 3344, - 3345, 3346, 3347, 3348, 3349, 3350, 3351, 3352, - 3353, 3354, 3355, 3356, 3357, 3358, 3359, 3360, - 3361, 3362, 3363, 3364, 3365, 3366, 3367, 3368, - 3369, 3370, 3371, 3372, 3373, 3374, 3375, 3376, - 3377, 3378, 3379, 3380, 3381, 3382, 3383, 3384, - 3385, 3386, 3387, 3388, 3389, 3390, 3391, 3392, - 3393, 3394, 3395, 3396, 3397, 3398, 3399, 3400, - 3401, 3402, 3403, 3404, 3405, 3406, 3407, 3408, - 3409, 3410, 3411, 3412, 3413, 3414, 3415, 3416, - 3417, 3418, 3419, 3420, 3421, 3422, 3423, 3424, - 3425, 3426, 3427, 3428, 3429, 3430, 3431, 3432, - 3433, 3434, 3435, 3436, 3437, 3438, 3439, 3440, - 3441, 3442, 3443, 3444, 3445, 3446, 3447, 3448, - 3449, 3450, 3451, 3452, 3453, 3454, 3455, 3456, - 3457, 3458, 3459, 3460, 3461, 3462, 3463, 3464, - 3465, 3466, 3467, 3468, 3469, 3470, 3471, 3472, - 3473, 3474, 3475, 3476, 3477, 3478, 3479, 3480, - 3481, 3482, 3483, 3484, 3485, 3486, 3487, 3488, - 3489, 3490, 3491, 3492, 3493, 3494, 3495, 3496, - 3497, 3498, 3499, 3500, 3501, 3502, 3503, 3504, - 3505, 3506, 3507, 3508, 3509, 3510, 3511, 3512, - 3513, 3514, 0 + 2668, 2669, 2670, 2671, 2672, 2673, 2674, 2675, + 2676, 2677, 2678, 2679, 2680, 2681, 2682, 2683, + 2684, 2685, 2686, 2687, 2688, 2689, 2690, 2691, + 2692, 2693, 2694, 2695, 2696, 2697, 2698, 2699, + 2700, 2701, 2702, 2703, 2704, 2705, 2706, 2707, + 2708, 2709, 2710, 2711, 2712, 2713, 2714, 2715, + 2716, 2717, 2718, 2719, 2720, 2721, 2722, 2723, + 2724, 2725, 2726, 2727, 2728, 2729, 2730, 2731, + 2732, 2733, 2734, 2735, 2736, 2737, 2738, 2739, + 2740, 2741, 2742, 2743, 2744, 2745, 2746, 2747, + 2748, 2749, 2750, 2751, 2752, 2753, 2754, 2755, + 2756, 2757, 2758, 2759, 2760, 2761, 2762, 2763, + 2764, 2765, 2766, 2767, 2768, 2769, 2770, 2771, + 2772, 2773, 2774, 2775, 2776, 2777, 2778, 2779, + 2780, 2781, 2782, 2783, 2784, 2785, 2786, 2787, + 2788, 2789, 2790, 2791, 2792, 2793, 2794, 2795, + 2796, 2797, 2798, 2799, 2800, 2801, 2802, 2803, + 2804, 2805, 2806, 2807, 2808, 2809, 2810, 2811, + 2812, 2813, 2814, 2815, 2816, 2817, 2818, 2819, + 2820, 2821, 2822, 2823, 2824, 2825, 2826, 2827, + 2828, 2829, 2830, 2831, 2832, 2833, 2834, 2835, + 2836, 2837, 2838, 2839, 2840, 2841, 2842, 2843, + 2844, 2845, 2846, 2847, 2848, 2849, 2850, 2851, + 2852, 2853, 2854, 2855, 2856, 2857, 2858, 2859, + 2860, 2861, 2862, 2863, 2864, 2865, 2866, 2867, + 2868, 2869, 2870, 2871, 2872, 2873, 2874, 2875, + 2876, 2877, 2878, 2879, 2880, 2881, 2882, 2883, + 2884, 2885, 2886, 2887, 2888, 2889, 2890, 2891, + 2892, 2893, 2894, 2895, 2896, 2897, 2898, 2899, + 2900, 2901, 2902, 2903, 2904, 2905, 2906, 2907, + 2908, 2909, 2910, 2911, 2912, 2913, 2914, 2915, + 2916, 2917, 2918, 2919, 2920, 2921, 2922, 2923, + 2924, 2925, 2926, 2927, 2928, 2929, 2930, 2931, + 2932, 2933, 2934, 2935, 2936, 2937, 2938, 2939, + 2940, 2941, 2942, 2943, 2944, 2945, 2946, 2947, + 2948, 2949, 2950, 2951, 2952, 2953, 2954, 2955, + 2956, 2957, 2958, 2959, 2960, 2961, 2962, 2963, + 2964, 2965, 2966, 2967, 2968, 2969, 2970, 2971, + 2972, 2973, 2974, 2975, 2976, 2977, 2978, 2979, + 2980, 2981, 2982, 2983, 2984, 2985, 2986, 2987, + 2988, 2989, 2990, 2991, 2992, 2993, 2994, 2995, + 2996, 2997, 2998, 2999, 3000, 3001, 3002, 3003, + 3004, 3005, 3006, 3007, 3008, 3009, 3010, 3011, + 3012, 3013, 3014, 3015, 3016, 3017, 3018, 3019, + 3020, 3021, 3022, 3023, 3024, 3025, 3026, 3027, + 3028, 3029, 3030, 3031, 3032, 3033, 3034, 3035, + 3036, 3037, 3038, 3039, 3040, 3041, 3042, 3043, + 3044, 3045, 3046, 3047, 3048, 3049, 3050, 3051, + 3052, 3053, 3054, 3055, 3056, 3057, 3058, 3059, + 3060, 3061, 3062, 3063, 3064, 3065, 3066, 3067, + 3068, 3069, 3070, 3071, 3072, 3073, 3074, 3075, + 3076, 3077, 3078, 3079, 3080, 3081, 3082, 3083, + 3084, 3085, 3086, 3087, 3088, 3089, 3090, 3091, + 3092, 3093, 3094, 3095, 3096, 3097, 3098, 3099, + 3100, 3101, 3102, 3103, 3104, 3105, 3106, 3107, + 3108, 3109, 3110, 3111, 3112, 3113, 3114, 3115, + 3116, 3117, 3118, 3119, 3120, 3121, 3122, 3123, + 3124, 3125, 3126, 3127, 3128, 3129, 3130, 3131, + 3132, 3133, 3134, 3135, 3136, 3137, 3138, 3139, + 3140, 3141, 3142, 3143, 3144, 3145, 3146, 3147, + 3148, 3149, 3150, 3151, 3152, 3153, 3154, 3155, + 3156, 3157, 3158, 3159, 3160, 3161, 3162, 3163, + 3164, 3165, 3166, 3167, 3168, 3169, 3170, 3171, + 3172, 3173, 3174, 3175, 3176, 3177, 3178, 3179, + 3180, 3181, 3182, 3183, 3184, 3185, 3186, 3187, + 3188, 3189, 3190, 3191, 3192, 3193, 3194, 3195, + 3196, 3197, 3198, 3199, 3200, 3201, 3202, 3203, + 3204, 3205, 3206, 3207, 3208, 3209, 3210, 3211, + 3212, 3213, 3214, 3215, 3216, 3217, 3218, 3219, + 3220, 3221, 3222, 3223, 3224, 3225, 3226, 3227, + 3228, 3229, 3230, 3231, 3232, 3233, 3234, 3235, + 3236, 3237, 3238, 3239, 3240, 3241, 3242, 3243, + 3244, 3245, 3246, 3247, 3248, 3249, 3250, 3251, + 3252, 3253, 3254, 3255, 3256, 3257, 3258, 3259, + 3260, 3261, 3262, 3263, 3264, 3265, 3266, 3267, + 3268, 3269, 3270, 3271, 3272, 3273, 3274, 3275, + 3276, 3277, 3278, 3279, 3280, 3281, 3282, 3283, + 3284, 3285, 3286, 3287, 3288, 3289, 3290, 3291, + 3292, 3293, 3294, 3295, 3296, 3297, 3298, 3299, + 3300, 3301, 3302, 3303, 3304, 3305, 3306, 3307, + 3308, 3309, 3310, 3311, 3312, 3313, 3314, 3315, + 3316, 3317, 3318, 3319, 3320, 3321, 3322, 3323, + 3324, 3325, 3326, 3327, 3328, 3329, 3330, 3331, + 3332, 3333, 3334, 3335, 3336, 3337, 3338, 3339, + 3340, 3341, 3342, 3343, 3344, 3345, 3346, 3347, + 3348, 3349, 3350, 3351, 3352, 3353, 3354, 3355, + 3356, 3357, 3358, 3359, 3360, 3361, 3362, 3363, + 3364, 3365, 3366, 3367, 3368, 3369, 3370, 3371, + 3372, 3373, 3374, 3375, 3376, 3377, 3378, 3379, + 3380, 3381, 3382, 3383, 3384, 3385, 3386, 3387, + 3388, 3389, 3390, 3391, 3392, 3393, 3394, 3395, + 3396, 3397, 3398, 3399, 3400, 3401, 3402, 3403, + 3404, 3405, 3406, 3407, 3408, 3409, 3410, 3411, + 3412, 3413, 3414, 3415, 3416, 3417, 3418, 3419, + 3420, 3421, 3422, 3423, 3424, 3425, 3426, 3427, + 3428, 3429, 3430, 3431, 3432, 3433, 3434, 3435, + 3436, 3437, 3438, 3439, 3440, 3441, 3442, 3443, + 3444, 3445, 3446, 3447, 3448, 3449, 3450, 3451, + 3452, 3453, 3454, 3455, 3456, 3457, 3458, 3459, + 3460, 3461, 3462, 3463, 3464, 3465, 3466, 3467, + 3468, 3469, 3470, 3471, 3472, 3473, 3474, 3475, + 3476, 3477, 3478, 3479, 3480, 3481, 3482, 3483, + 3484, 3485, 3486, 3487, 3488, 3489, 3490, 3491, + 3492, 3493, 3494, 3495, 3496, 3497, 3498, 3499, + 3500, 3501, 3502, 3503, 3504, 3505, 3506, 3507, + 3508, 3509, 3510, 3511, 3512, 3513, 3514, 3515, + 3516, 3517, 3518, 0 }; static const int regex_start = 746; @@ -4108,15 +4109,15 @@ static const short _regex_actions[] = { static const int regex_en_readBracedUCP = 559; static const int regex_en_readUCPSingle = 818; static const int regex_en_charClassGuts = 819; - static const int regex_en_readClass = 836; - static const int regex_en_readQuotedLiteral = 838; - static const int regex_en_readQuotedClass = 843; - static const int regex_en_readComment = 848; - static const int regex_en_readNewlineTerminatedComment = 849; + static const int regex_en_readClass = 837; + static const int regex_en_readQuotedLiteral = 839; + static const int regex_en_readQuotedClass = 844; + static const int regex_en_readComment = 849; + static const int regex_en_readNewlineTerminatedComment = 850; static const int regex_en_main = 746; -#line 1911 "Parser.rl" +#line 1918 "Parser.rl" /** \brief Main parser call, returns root Component or nullptr. */ @@ -4189,7 +4190,7 @@ static const short _regex_actions[] = { try { // Embed the Ragel machine here -#line 4187 "Parser.cpp" +#line 4188 "Parser.cpp" { cs = (int)regex_start; top = 0; @@ -4198,17 +4199,17 @@ static const short _regex_actions[] = { act = 0; } -#line 1982 "Parser.rl" +#line 1989 "Parser.rl" -#line 4196 "Parser.cpp" +#line 4197 "Parser.cpp" { int _cpc; int _klen; const signed char * _ckeys; unsigned int _trans = 0; unsigned int _cond = 0; - const char * _keys; + const unsigned char * _keys; const short * _acts; unsigned int _nacts; _resume: {} @@ -4224,7 +4225,7 @@ static const short _regex_actions[] = { #line 1 "NONE" {ts = p;}} -#line 4218 "Parser.cpp" +#line 4219 "Parser.cpp" break; } @@ -4244,9 +4245,9 @@ static const short _regex_actions[] = { _klen = (int)_regex_single_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + _klen - 1; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + _klen - 1; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _keys += _klen; @@ -4268,9 +4269,9 @@ static const short _regex_actions[] = { _klen = (int)_regex_range_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + (_klen<<1) - 2; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + (_klen<<1) - 2; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _trans += (unsigned int)_klen; @@ -4300,19 +4301,19 @@ static const short _regex_actions[] = { case 0: { if ( ( mode.utf8 ) -#line 4293 "Parser.cpp" +#line 4294 "Parser.cpp" ) _cpc += 1; break; } case 1: { if ( ( mode.ignore_space ) -#line 4298 "Parser.cpp" +#line 4299 "Parser.cpp" ) _cpc += 1; break; } case 2: { if ( ( inCharClassEarly ) -#line 4303 "Parser.cpp" +#line 4304 "Parser.cpp" ) _cpc += 1; break; } @@ -4323,7 +4324,7 @@ static const short _regex_actions[] = { const signed char *_mid; while ( 1 ) { if ( _upper < _lower ) { - _cond = 3561; + _cond = 3565; break; } @@ -4352,159 +4353,159 @@ static const short _regex_actions[] = { { case 0: { { -#line 285 "Parser.rl" +#line 286 "Parser.rl" label.clear();} -#line 4345 "Parser.cpp" +#line 4346 "Parser.cpp" break; } case 1: { { -#line 286 "Parser.rl" +#line 287 "Parser.rl" label.push_back((( (*( p)))));} -#line 4353 "Parser.cpp" +#line 4354 "Parser.cpp" break; } case 2: { { -#line 287 "Parser.rl" +#line 288 "Parser.rl" octAccumulator = 0;} -#line 4361 "Parser.cpp" +#line 4362 "Parser.cpp" break; } case 3: { { -#line 288 "Parser.rl" +#line 289 "Parser.rl" accumulator = 0;} -#line 4369 "Parser.cpp" +#line 4370 "Parser.cpp" break; } case 4: { { -#line 289 "Parser.rl" +#line 290 "Parser.rl" octAccumulator = 0; pushOct(&octAccumulator, (( (*( p))))); } -#line 4380 "Parser.cpp" +#line 4381 "Parser.cpp" break; } case 5: { { -#line 293 "Parser.rl" +#line 294 "Parser.rl" accumulator = 0; pushDec(&accumulator, (( (*( p))))); } -#line 4391 "Parser.cpp" +#line 4392 "Parser.cpp" break; } case 6: { { -#line 297 "Parser.rl" +#line 298 "Parser.rl" repeatN = 0; repeatM = 0; } -#line 4399 "Parser.cpp" +#line 4400 "Parser.cpp" break; } case 7: { { -#line 298 "Parser.rl" +#line 299 "Parser.rl" pushDec(&repeatN, (( (*( p))))); } -#line 4407 "Parser.cpp" +#line 4408 "Parser.cpp" break; } case 8: { { -#line 299 "Parser.rl" +#line 300 "Parser.rl" pushDec(&repeatM, (( (*( p))))); } -#line 4415 "Parser.cpp" +#line 4416 "Parser.cpp" break; } case 9: { { -#line 300 "Parser.rl" +#line 301 "Parser.rl" pushOct(&octAccumulator, (( (*( p))))); } -#line 4423 "Parser.cpp" +#line 4424 "Parser.cpp" break; } case 10: { { -#line 301 "Parser.rl" +#line 302 "Parser.rl" pushDec(&accumulator, (( (*( p))))); } -#line 4431 "Parser.cpp" +#line 4432 "Parser.cpp" break; } case 11: { { -#line 302 "Parser.rl" +#line 303 "Parser.rl" accumulator *= 16; accumulator += (( (*( p)))) - '0'; } -#line 4442 "Parser.cpp" +#line 4443 "Parser.cpp" break; } case 12: { { -#line 306 "Parser.rl" +#line 307 "Parser.rl" accumulator *= 16; accumulator += 10 + (( (*( p)))) - 'a'; } -#line 4453 "Parser.cpp" +#line 4454 "Parser.cpp" break; } case 13: { { -#line 310 "Parser.rl" +#line 311 "Parser.rl" accumulator *= 16; accumulator += 10 + (( (*( p)))) - 'A'; } -#line 4464 "Parser.cpp" +#line 4465 "Parser.cpp" break; } case 14: { { -#line 430 "Parser.rl" +#line 431 "Parser.rl" newMode = mode; } -#line 4474 "Parser.cpp" +#line 4475 "Parser.cpp" break; } case 15: { { -#line 437 "Parser.rl" +#line 438 "Parser.rl" switch ((( (*( p))))) { case 'i': @@ -4525,13 +4526,13 @@ static const short _regex_actions[] = { } } -#line 4500 "Parser.cpp" +#line 4501 "Parser.cpp" break; } case 16: { { -#line 456 "Parser.rl" +#line 457 "Parser.rl" switch ((( (*( p))))) { case 'i': @@ -4552,42 +4553,42 @@ static const short _regex_actions[] = { } } -#line 4526 "Parser.cpp" +#line 4527 "Parser.cpp" break; } case 17: { { -#line 510 "Parser.rl" +#line 511 "Parser.rl" repeatM = repeatN;} -#line 4534 "Parser.cpp" +#line 4535 "Parser.cpp" break; } case 18: { { -#line 510 "Parser.rl" +#line 511 "Parser.rl" repeatM = ComponentRepeat::NoLimit;} -#line 4542 "Parser.cpp" +#line 4543 "Parser.cpp" break; } case 19: { { -#line 722 "Parser.rl" +#line 723 "Parser.rl" negated = !negated; } -#line 4550 "Parser.cpp" +#line 4551 "Parser.cpp" break; } case 20: { { -#line 723 "Parser.rl" +#line 724 "Parser.rl" {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -4596,30 +4597,30 @@ static const short _regex_actions[] = { } stack[top] = cs; top += 1;cs = 790;goto _again;}} -#line 4566 "Parser.cpp" +#line 4567 "Parser.cpp" break; } case 21: { { -#line 724 "Parser.rl" +#line 725 "Parser.rl" if (!inCharClass) { // not inside [..] currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } -#line 4579 "Parser.cpp" +#line 4580 "Parser.cpp" break; } case 22: { { -#line 730 "Parser.rl" +#line 731 "Parser.rl" throw LocatedParseError("Malformed property"); } -#line 4587 "Parser.cpp" +#line 4588 "Parser.cpp" break; } @@ -4628,60 +4629,60 @@ static const short _regex_actions[] = { #line 1 "NONE" {te = p+1;}} -#line 4595 "Parser.cpp" +#line 4596 "Parser.cpp" break; } case 26: { { -#line 550 "Parser.rl" +#line 551 "Parser.rl" {te = p+1;{ -#line 550 "Parser.rl" +#line 551 "Parser.rl" throw LocatedParseError("(*UTF8) must be at start of " "expression, encountered"); } }} -#line 4609 "Parser.cpp" +#line 4610 "Parser.cpp" break; } case 27: { { -#line 554 "Parser.rl" +#line 555 "Parser.rl" {te = p+1;{ -#line 554 "Parser.rl" +#line 555 "Parser.rl" throw LocatedParseError("(*UTF) must be at start of " "expression, encountered"); } }} -#line 4623 "Parser.cpp" +#line 4624 "Parser.cpp" break; } case 28: { { -#line 558 "Parser.rl" +#line 559 "Parser.rl" {te = p+1;{ -#line 558 "Parser.rl" +#line 559 "Parser.rl" throw LocatedParseError("(*UCP) must be at start of " "expression, encountered"); } }} -#line 4637 "Parser.cpp" +#line 4638 "Parser.cpp" break; } case 29: { { -#line 564 "Parser.rl" +#line 565 "Parser.rl" {te = p+1;{ -#line 564 "Parser.rl" +#line 565 "Parser.rl" ParseMode temp_mode; assert(ts - 2 >= ptr); // parser needs the '(*' at the start too. @@ -4691,2626 +4692,2626 @@ static const short _regex_actions[] = { } }} -#line 4654 "Parser.cpp" +#line 4655 "Parser.cpp" break; } case 30: { { -#line 571 "Parser.rl" +#line 572 "Parser.rl" {te = p+1;{ -#line 571 "Parser.rl" +#line 572 "Parser.rl" throw LocatedParseError("Unknown control verb"); } }} -#line 4667 "Parser.cpp" +#line 4668 "Parser.cpp" break; } case 31: { { -#line 571 "Parser.rl" +#line 572 "Parser.rl" {te = p;p = p - 1;{ -#line 571 "Parser.rl" +#line 572 "Parser.rl" throw LocatedParseError("Unknown control verb"); } }} -#line 4680 "Parser.cpp" +#line 4681 "Parser.cpp" break; } case 32: { { -#line 571 "Parser.rl" +#line 572 "Parser.rl" {p = ((te))-1; { -#line 571 "Parser.rl" +#line 572 "Parser.rl" throw LocatedParseError("Unknown control verb"); } }} -#line 4694 "Parser.cpp" +#line 4695 "Parser.cpp" break; } case 33: { { -#line 581 "Parser.rl" +#line 582 "Parser.rl" {te = p+1;{ -#line 581 "Parser.rl" +#line 582 "Parser.rl" currentCls->add(CLASS_UCP_CC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4705 "Parser.cpp" +#line 4706 "Parser.cpp" break; } case 34: { { -#line 582 "Parser.rl" +#line 583 "Parser.rl" {te = p+1;{ -#line 582 "Parser.rl" +#line 583 "Parser.rl" currentCls->add(CLASS_UCP_CF, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4716 "Parser.cpp" +#line 4717 "Parser.cpp" break; } case 35: { { -#line 583 "Parser.rl" +#line 584 "Parser.rl" {te = p+1;{ -#line 583 "Parser.rl" +#line 584 "Parser.rl" currentCls->add(CLASS_UCP_CN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4727 "Parser.cpp" +#line 4728 "Parser.cpp" break; } case 36: { { -#line 585 "Parser.rl" +#line 586 "Parser.rl" {te = p+1;{ -#line 585 "Parser.rl" +#line 586 "Parser.rl" currentCls->add(CLASS_UCP_CS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4738 "Parser.cpp" +#line 4739 "Parser.cpp" break; } case 37: { { -#line 587 "Parser.rl" +#line 588 "Parser.rl" {te = p+1;{ -#line 587 "Parser.rl" +#line 588 "Parser.rl" currentCls->add(CLASS_UCP_LL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4749 "Parser.cpp" +#line 4750 "Parser.cpp" break; } case 38: { { -#line 588 "Parser.rl" +#line 589 "Parser.rl" {te = p+1;{ -#line 588 "Parser.rl" +#line 589 "Parser.rl" currentCls->add(CLASS_UCP_LM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4760 "Parser.cpp" +#line 4761 "Parser.cpp" break; } case 39: { { -#line 589 "Parser.rl" +#line 590 "Parser.rl" {te = p+1;{ -#line 589 "Parser.rl" +#line 590 "Parser.rl" currentCls->add(CLASS_UCP_LO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4771 "Parser.cpp" +#line 4772 "Parser.cpp" break; } case 40: { { -#line 590 "Parser.rl" +#line 591 "Parser.rl" {te = p+1;{ -#line 590 "Parser.rl" +#line 591 "Parser.rl" currentCls->add(CLASS_UCP_LT, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4782 "Parser.cpp" +#line 4783 "Parser.cpp" break; } case 41: { { -#line 591 "Parser.rl" +#line 592 "Parser.rl" {te = p+1;{ -#line 591 "Parser.rl" +#line 592 "Parser.rl" currentCls->add(CLASS_UCP_LU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4793 "Parser.cpp" +#line 4794 "Parser.cpp" break; } case 42: { { -#line 592 "Parser.rl" +#line 593 "Parser.rl" {te = p+1;{ -#line 592 "Parser.rl" +#line 593 "Parser.rl" currentCls->add(CLASS_UCP_L_AND, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4804 "Parser.cpp" +#line 4805 "Parser.cpp" break; } case 43: { { -#line 594 "Parser.rl" +#line 595 "Parser.rl" {te = p+1;{ -#line 594 "Parser.rl" +#line 595 "Parser.rl" currentCls->add(CLASS_UCP_MC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4815 "Parser.cpp" +#line 4816 "Parser.cpp" break; } case 44: { { -#line 596 "Parser.rl" +#line 597 "Parser.rl" {te = p+1;{ -#line 596 "Parser.rl" +#line 597 "Parser.rl" currentCls->add(CLASS_UCP_MN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4826 "Parser.cpp" +#line 4827 "Parser.cpp" break; } case 45: { { -#line 598 "Parser.rl" +#line 599 "Parser.rl" {te = p+1;{ -#line 598 "Parser.rl" +#line 599 "Parser.rl" currentCls->add(CLASS_UCP_ND, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4837 "Parser.cpp" +#line 4838 "Parser.cpp" break; } case 46: { { -#line 599 "Parser.rl" +#line 600 "Parser.rl" {te = p+1;{ -#line 599 "Parser.rl" +#line 600 "Parser.rl" currentCls->add(CLASS_UCP_NL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4848 "Parser.cpp" +#line 4849 "Parser.cpp" break; } case 47: { { -#line 600 "Parser.rl" +#line 601 "Parser.rl" {te = p+1;{ -#line 600 "Parser.rl" +#line 601 "Parser.rl" currentCls->add(CLASS_UCP_NO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4859 "Parser.cpp" +#line 4860 "Parser.cpp" break; } case 48: { { -#line 602 "Parser.rl" +#line 603 "Parser.rl" {te = p+1;{ -#line 602 "Parser.rl" +#line 603 "Parser.rl" currentCls->add(CLASS_UCP_PC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4870 "Parser.cpp" +#line 4871 "Parser.cpp" break; } case 49: { { -#line 603 "Parser.rl" +#line 604 "Parser.rl" {te = p+1;{ -#line 603 "Parser.rl" +#line 604 "Parser.rl" currentCls->add(CLASS_UCP_PD, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4881 "Parser.cpp" +#line 4882 "Parser.cpp" break; } case 50: { { -#line 604 "Parser.rl" +#line 605 "Parser.rl" {te = p+1;{ -#line 604 "Parser.rl" +#line 605 "Parser.rl" currentCls->add(CLASS_UCP_PE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4892 "Parser.cpp" +#line 4893 "Parser.cpp" break; } case 51: { { -#line 605 "Parser.rl" +#line 606 "Parser.rl" {te = p+1;{ -#line 605 "Parser.rl" +#line 606 "Parser.rl" currentCls->add(CLASS_UCP_PF, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4903 "Parser.cpp" +#line 4904 "Parser.cpp" break; } case 52: { { -#line 606 "Parser.rl" +#line 607 "Parser.rl" {te = p+1;{ -#line 606 "Parser.rl" +#line 607 "Parser.rl" currentCls->add(CLASS_UCP_PI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4914 "Parser.cpp" +#line 4915 "Parser.cpp" break; } case 53: { { -#line 607 "Parser.rl" +#line 608 "Parser.rl" {te = p+1;{ -#line 607 "Parser.rl" +#line 608 "Parser.rl" currentCls->add(CLASS_UCP_PO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4925 "Parser.cpp" +#line 4926 "Parser.cpp" break; } case 54: { { -#line 608 "Parser.rl" +#line 609 "Parser.rl" {te = p+1;{ -#line 608 "Parser.rl" +#line 609 "Parser.rl" currentCls->add(CLASS_UCP_PS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4936 "Parser.cpp" +#line 4937 "Parser.cpp" break; } case 55: { { -#line 610 "Parser.rl" +#line 611 "Parser.rl" {te = p+1;{ -#line 610 "Parser.rl" +#line 611 "Parser.rl" currentCls->add(CLASS_UCP_SC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4947 "Parser.cpp" +#line 4948 "Parser.cpp" break; } case 56: { { -#line 611 "Parser.rl" +#line 612 "Parser.rl" {te = p+1;{ -#line 611 "Parser.rl" +#line 612 "Parser.rl" currentCls->add(CLASS_UCP_SK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4958 "Parser.cpp" +#line 4959 "Parser.cpp" break; } case 57: { { -#line 612 "Parser.rl" +#line 613 "Parser.rl" {te = p+1;{ -#line 612 "Parser.rl" +#line 613 "Parser.rl" currentCls->add(CLASS_UCP_SM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4969 "Parser.cpp" +#line 4970 "Parser.cpp" break; } case 58: { { -#line 613 "Parser.rl" +#line 614 "Parser.rl" {te = p+1;{ -#line 613 "Parser.rl" +#line 614 "Parser.rl" currentCls->add(CLASS_UCP_SO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4980 "Parser.cpp" +#line 4981 "Parser.cpp" break; } case 59: { { -#line 615 "Parser.rl" +#line 616 "Parser.rl" {te = p+1;{ -#line 615 "Parser.rl" +#line 616 "Parser.rl" currentCls->add(CLASS_UCP_ZL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4991 "Parser.cpp" +#line 4992 "Parser.cpp" break; } case 60: { { -#line 616 "Parser.rl" +#line 617 "Parser.rl" {te = p+1;{ -#line 616 "Parser.rl" +#line 617 "Parser.rl" currentCls->add(CLASS_UCP_ZP, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5002 "Parser.cpp" +#line 5003 "Parser.cpp" break; } case 61: { { -#line 617 "Parser.rl" +#line 618 "Parser.rl" {te = p+1;{ -#line 617 "Parser.rl" +#line 618 "Parser.rl" currentCls->add(CLASS_UCP_ZS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5013 "Parser.cpp" +#line 5014 "Parser.cpp" break; } case 62: { { -#line 618 "Parser.rl" +#line 619 "Parser.rl" {te = p+1;{ -#line 618 "Parser.rl" +#line 619 "Parser.rl" currentCls->add(CLASS_UCP_XAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5024 "Parser.cpp" +#line 5025 "Parser.cpp" break; } case 63: { { -#line 619 "Parser.rl" +#line 620 "Parser.rl" {te = p+1;{ -#line 619 "Parser.rl" +#line 620 "Parser.rl" currentCls->add(CLASS_UCP_XPS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5035 "Parser.cpp" +#line 5036 "Parser.cpp" break; } case 64: { { -#line 620 "Parser.rl" +#line 621 "Parser.rl" {te = p+1;{ -#line 620 "Parser.rl" +#line 621 "Parser.rl" currentCls->add(CLASS_UCP_XSP, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5046 "Parser.cpp" +#line 5047 "Parser.cpp" break; } case 65: { { -#line 621 "Parser.rl" +#line 622 "Parser.rl" {te = p+1;{ -#line 621 "Parser.rl" +#line 622 "Parser.rl" currentCls->add(CLASS_UCP_XWD, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5057 "Parser.cpp" +#line 5058 "Parser.cpp" break; } case 66: { { -#line 622 "Parser.rl" +#line 623 "Parser.rl" {te = p+1;{ -#line 622 "Parser.rl" +#line 623 "Parser.rl" currentCls->add(CLASS_SCRIPT_ARABIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5068 "Parser.cpp" +#line 5069 "Parser.cpp" break; } case 67: { { -#line 623 "Parser.rl" +#line 624 "Parser.rl" {te = p+1;{ -#line 623 "Parser.rl" +#line 624 "Parser.rl" currentCls->add(CLASS_SCRIPT_ARMENIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5079 "Parser.cpp" +#line 5080 "Parser.cpp" break; } case 68: { { -#line 624 "Parser.rl" +#line 625 "Parser.rl" {te = p+1;{ -#line 624 "Parser.rl" +#line 625 "Parser.rl" currentCls->add(CLASS_SCRIPT_AVESTAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5090 "Parser.cpp" +#line 5091 "Parser.cpp" break; } case 69: { { -#line 625 "Parser.rl" +#line 626 "Parser.rl" {te = p+1;{ -#line 625 "Parser.rl" +#line 626 "Parser.rl" currentCls->add(CLASS_SCRIPT_BALINESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5101 "Parser.cpp" +#line 5102 "Parser.cpp" break; } case 70: { { -#line 626 "Parser.rl" +#line 627 "Parser.rl" {te = p+1;{ -#line 626 "Parser.rl" +#line 627 "Parser.rl" currentCls->add(CLASS_SCRIPT_BAMUM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5112 "Parser.cpp" +#line 5113 "Parser.cpp" break; } case 71: { { -#line 627 "Parser.rl" +#line 628 "Parser.rl" {te = p+1;{ -#line 627 "Parser.rl" +#line 628 "Parser.rl" currentCls->add(CLASS_SCRIPT_BATAK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5123 "Parser.cpp" +#line 5124 "Parser.cpp" break; } case 72: { { -#line 628 "Parser.rl" +#line 629 "Parser.rl" {te = p+1;{ -#line 628 "Parser.rl" +#line 629 "Parser.rl" currentCls->add(CLASS_SCRIPT_BENGALI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5134 "Parser.cpp" +#line 5135 "Parser.cpp" break; } case 73: { { -#line 629 "Parser.rl" +#line 630 "Parser.rl" {te = p+1;{ -#line 629 "Parser.rl" +#line 630 "Parser.rl" currentCls->add(CLASS_SCRIPT_BOPOMOFO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5145 "Parser.cpp" +#line 5146 "Parser.cpp" break; } case 74: { { -#line 630 "Parser.rl" +#line 631 "Parser.rl" {te = p+1;{ -#line 630 "Parser.rl" +#line 631 "Parser.rl" currentCls->add(CLASS_SCRIPT_BRAHMI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5156 "Parser.cpp" +#line 5157 "Parser.cpp" break; } case 75: { { -#line 631 "Parser.rl" +#line 632 "Parser.rl" {te = p+1;{ -#line 631 "Parser.rl" +#line 632 "Parser.rl" currentCls->add(CLASS_SCRIPT_BRAILLE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5167 "Parser.cpp" +#line 5168 "Parser.cpp" break; } case 76: { { -#line 632 "Parser.rl" +#line 633 "Parser.rl" {te = p+1;{ -#line 632 "Parser.rl" +#line 633 "Parser.rl" currentCls->add(CLASS_SCRIPT_BUGINESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5178 "Parser.cpp" +#line 5179 "Parser.cpp" break; } case 77: { { -#line 633 "Parser.rl" +#line 634 "Parser.rl" {te = p+1;{ -#line 633 "Parser.rl" +#line 634 "Parser.rl" currentCls->add(CLASS_SCRIPT_BUHID, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5189 "Parser.cpp" +#line 5190 "Parser.cpp" break; } case 78: { { -#line 634 "Parser.rl" +#line 635 "Parser.rl" {te = p+1;{ -#line 634 "Parser.rl" +#line 635 "Parser.rl" currentCls->add(CLASS_SCRIPT_CANADIAN_ABORIGINAL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5200 "Parser.cpp" +#line 5201 "Parser.cpp" break; } case 79: { { -#line 635 "Parser.rl" +#line 636 "Parser.rl" {te = p+1;{ -#line 635 "Parser.rl" +#line 636 "Parser.rl" currentCls->add(CLASS_SCRIPT_CARIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5211 "Parser.cpp" +#line 5212 "Parser.cpp" break; } case 80: { { -#line 636 "Parser.rl" +#line 637 "Parser.rl" {te = p+1;{ -#line 636 "Parser.rl" +#line 637 "Parser.rl" currentCls->add(CLASS_SCRIPT_CHAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5222 "Parser.cpp" +#line 5223 "Parser.cpp" break; } case 81: { { -#line 637 "Parser.rl" +#line 638 "Parser.rl" {te = p+1;{ -#line 637 "Parser.rl" +#line 638 "Parser.rl" currentCls->add(CLASS_SCRIPT_CHEROKEE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5233 "Parser.cpp" +#line 5234 "Parser.cpp" break; } case 82: { { -#line 638 "Parser.rl" +#line 639 "Parser.rl" {te = p+1;{ -#line 638 "Parser.rl" +#line 639 "Parser.rl" currentCls->add(CLASS_SCRIPT_COMMON, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5244 "Parser.cpp" +#line 5245 "Parser.cpp" break; } case 83: { { -#line 639 "Parser.rl" +#line 640 "Parser.rl" {te = p+1;{ -#line 639 "Parser.rl" +#line 640 "Parser.rl" currentCls->add(CLASS_SCRIPT_COPTIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5255 "Parser.cpp" +#line 5256 "Parser.cpp" break; } case 84: { { -#line 640 "Parser.rl" +#line 641 "Parser.rl" {te = p+1;{ -#line 640 "Parser.rl" +#line 641 "Parser.rl" currentCls->add(CLASS_SCRIPT_CUNEIFORM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5266 "Parser.cpp" +#line 5267 "Parser.cpp" break; } case 85: { { -#line 641 "Parser.rl" +#line 642 "Parser.rl" {te = p+1;{ -#line 641 "Parser.rl" +#line 642 "Parser.rl" currentCls->add(CLASS_SCRIPT_CYPRIOT, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5277 "Parser.cpp" +#line 5278 "Parser.cpp" break; } case 86: { { -#line 642 "Parser.rl" +#line 643 "Parser.rl" {te = p+1;{ -#line 642 "Parser.rl" +#line 643 "Parser.rl" currentCls->add(CLASS_SCRIPT_CYRILLIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5288 "Parser.cpp" +#line 5289 "Parser.cpp" break; } case 87: { { -#line 643 "Parser.rl" +#line 644 "Parser.rl" {te = p+1;{ -#line 643 "Parser.rl" +#line 644 "Parser.rl" currentCls->add(CLASS_SCRIPT_DESERET, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5299 "Parser.cpp" +#line 5300 "Parser.cpp" break; } case 88: { { -#line 644 "Parser.rl" +#line 645 "Parser.rl" {te = p+1;{ -#line 644 "Parser.rl" +#line 645 "Parser.rl" currentCls->add(CLASS_SCRIPT_DEVANAGARI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5310 "Parser.cpp" +#line 5311 "Parser.cpp" break; } case 89: { { -#line 645 "Parser.rl" +#line 646 "Parser.rl" {te = p+1;{ -#line 645 "Parser.rl" +#line 646 "Parser.rl" currentCls->add(CLASS_SCRIPT_EGYPTIAN_HIEROGLYPHS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5321 "Parser.cpp" +#line 5322 "Parser.cpp" break; } case 90: { { -#line 646 "Parser.rl" +#line 647 "Parser.rl" {te = p+1;{ -#line 646 "Parser.rl" +#line 647 "Parser.rl" currentCls->add(CLASS_SCRIPT_ETHIOPIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5332 "Parser.cpp" +#line 5333 "Parser.cpp" break; } case 91: { { -#line 647 "Parser.rl" +#line 648 "Parser.rl" {te = p+1;{ -#line 647 "Parser.rl" +#line 648 "Parser.rl" currentCls->add(CLASS_SCRIPT_GEORGIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5343 "Parser.cpp" +#line 5344 "Parser.cpp" break; } case 92: { { -#line 648 "Parser.rl" +#line 649 "Parser.rl" {te = p+1;{ -#line 648 "Parser.rl" +#line 649 "Parser.rl" currentCls->add(CLASS_SCRIPT_GLAGOLITIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5354 "Parser.cpp" +#line 5355 "Parser.cpp" break; } case 93: { { -#line 649 "Parser.rl" +#line 650 "Parser.rl" {te = p+1;{ -#line 649 "Parser.rl" +#line 650 "Parser.rl" currentCls->add(CLASS_SCRIPT_GOTHIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5365 "Parser.cpp" +#line 5366 "Parser.cpp" break; } case 94: { { -#line 650 "Parser.rl" +#line 651 "Parser.rl" {te = p+1;{ -#line 650 "Parser.rl" +#line 651 "Parser.rl" currentCls->add(CLASS_SCRIPT_GREEK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5376 "Parser.cpp" +#line 5377 "Parser.cpp" break; } case 95: { { -#line 651 "Parser.rl" +#line 652 "Parser.rl" {te = p+1;{ -#line 651 "Parser.rl" +#line 652 "Parser.rl" currentCls->add(CLASS_SCRIPT_GUJARATI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5387 "Parser.cpp" +#line 5388 "Parser.cpp" break; } case 96: { { -#line 652 "Parser.rl" +#line 653 "Parser.rl" {te = p+1;{ -#line 652 "Parser.rl" +#line 653 "Parser.rl" currentCls->add(CLASS_SCRIPT_GURMUKHI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5398 "Parser.cpp" +#line 5399 "Parser.cpp" break; } case 97: { { -#line 654 "Parser.rl" +#line 655 "Parser.rl" {te = p+1;{ -#line 654 "Parser.rl" +#line 655 "Parser.rl" currentCls->add(CLASS_SCRIPT_HANGUL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5409 "Parser.cpp" +#line 5410 "Parser.cpp" break; } case 98: { { -#line 655 "Parser.rl" +#line 656 "Parser.rl" {te = p+1;{ -#line 655 "Parser.rl" +#line 656 "Parser.rl" currentCls->add(CLASS_SCRIPT_HANUNOO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5420 "Parser.cpp" +#line 5421 "Parser.cpp" break; } case 99: { { -#line 656 "Parser.rl" +#line 657 "Parser.rl" {te = p+1;{ -#line 656 "Parser.rl" +#line 657 "Parser.rl" currentCls->add(CLASS_SCRIPT_HEBREW, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5431 "Parser.cpp" +#line 5432 "Parser.cpp" break; } case 100: { { -#line 657 "Parser.rl" +#line 658 "Parser.rl" {te = p+1;{ -#line 657 "Parser.rl" +#line 658 "Parser.rl" currentCls->add(CLASS_SCRIPT_HIRAGANA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5442 "Parser.cpp" +#line 5443 "Parser.cpp" break; } case 101: { { -#line 658 "Parser.rl" +#line 659 "Parser.rl" {te = p+1;{ -#line 658 "Parser.rl" +#line 659 "Parser.rl" currentCls->add(CLASS_SCRIPT_IMPERIAL_ARAMAIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5453 "Parser.cpp" +#line 5454 "Parser.cpp" break; } case 102: { { -#line 659 "Parser.rl" +#line 660 "Parser.rl" {te = p+1;{ -#line 659 "Parser.rl" +#line 660 "Parser.rl" currentCls->add(CLASS_SCRIPT_INHERITED, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5464 "Parser.cpp" +#line 5465 "Parser.cpp" break; } case 103: { { -#line 660 "Parser.rl" +#line 661 "Parser.rl" {te = p+1;{ -#line 660 "Parser.rl" +#line 661 "Parser.rl" currentCls->add(CLASS_SCRIPT_INSCRIPTIONAL_PAHLAVI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5475 "Parser.cpp" +#line 5476 "Parser.cpp" break; } case 104: { { -#line 661 "Parser.rl" +#line 662 "Parser.rl" {te = p+1;{ -#line 661 "Parser.rl" +#line 662 "Parser.rl" currentCls->add(CLASS_SCRIPT_INSCRIPTIONAL_PARTHIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5486 "Parser.cpp" +#line 5487 "Parser.cpp" break; } case 105: { { -#line 662 "Parser.rl" +#line 663 "Parser.rl" {te = p+1;{ -#line 662 "Parser.rl" +#line 663 "Parser.rl" currentCls->add(CLASS_SCRIPT_JAVANESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5497 "Parser.cpp" +#line 5498 "Parser.cpp" break; } case 106: { { -#line 663 "Parser.rl" +#line 664 "Parser.rl" {te = p+1;{ -#line 663 "Parser.rl" +#line 664 "Parser.rl" currentCls->add(CLASS_SCRIPT_KAITHI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5508 "Parser.cpp" +#line 5509 "Parser.cpp" break; } case 107: { { -#line 664 "Parser.rl" +#line 665 "Parser.rl" {te = p+1;{ -#line 664 "Parser.rl" +#line 665 "Parser.rl" currentCls->add(CLASS_SCRIPT_KANNADA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5519 "Parser.cpp" +#line 5520 "Parser.cpp" break; } case 108: { { -#line 665 "Parser.rl" +#line 666 "Parser.rl" {te = p+1;{ -#line 665 "Parser.rl" +#line 666 "Parser.rl" currentCls->add(CLASS_SCRIPT_KATAKANA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5530 "Parser.cpp" +#line 5531 "Parser.cpp" break; } case 109: { { -#line 666 "Parser.rl" +#line 667 "Parser.rl" {te = p+1;{ -#line 666 "Parser.rl" +#line 667 "Parser.rl" currentCls->add(CLASS_SCRIPT_KAYAH_LI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5541 "Parser.cpp" +#line 5542 "Parser.cpp" break; } case 110: { { -#line 667 "Parser.rl" +#line 668 "Parser.rl" {te = p+1;{ -#line 667 "Parser.rl" +#line 668 "Parser.rl" currentCls->add(CLASS_SCRIPT_KHAROSHTHI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5552 "Parser.cpp" +#line 5553 "Parser.cpp" break; } case 111: { { -#line 668 "Parser.rl" +#line 669 "Parser.rl" {te = p+1;{ -#line 668 "Parser.rl" +#line 669 "Parser.rl" currentCls->add(CLASS_SCRIPT_KHMER, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5563 "Parser.cpp" +#line 5564 "Parser.cpp" break; } case 112: { { -#line 669 "Parser.rl" +#line 670 "Parser.rl" {te = p+1;{ -#line 669 "Parser.rl" +#line 670 "Parser.rl" currentCls->add(CLASS_SCRIPT_LAO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5574 "Parser.cpp" +#line 5575 "Parser.cpp" break; } case 113: { { -#line 670 "Parser.rl" +#line 671 "Parser.rl" {te = p+1;{ -#line 670 "Parser.rl" +#line 671 "Parser.rl" currentCls->add(CLASS_SCRIPT_LATIN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5585 "Parser.cpp" +#line 5586 "Parser.cpp" break; } case 114: { { -#line 671 "Parser.rl" +#line 672 "Parser.rl" {te = p+1;{ -#line 671 "Parser.rl" +#line 672 "Parser.rl" currentCls->add(CLASS_SCRIPT_LEPCHA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5596 "Parser.cpp" +#line 5597 "Parser.cpp" break; } case 115: { { -#line 672 "Parser.rl" +#line 673 "Parser.rl" {te = p+1;{ -#line 672 "Parser.rl" +#line 673 "Parser.rl" currentCls->add(CLASS_SCRIPT_LIMBU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5607 "Parser.cpp" +#line 5608 "Parser.cpp" break; } case 116: { { -#line 673 "Parser.rl" +#line 674 "Parser.rl" {te = p+1;{ -#line 673 "Parser.rl" +#line 674 "Parser.rl" currentCls->add(CLASS_SCRIPT_LINEAR_B, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5618 "Parser.cpp" +#line 5619 "Parser.cpp" break; } case 117: { { -#line 674 "Parser.rl" +#line 675 "Parser.rl" {te = p+1;{ -#line 674 "Parser.rl" +#line 675 "Parser.rl" currentCls->add(CLASS_SCRIPT_LISU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5629 "Parser.cpp" +#line 5630 "Parser.cpp" break; } case 118: { { -#line 675 "Parser.rl" +#line 676 "Parser.rl" {te = p+1;{ -#line 675 "Parser.rl" +#line 676 "Parser.rl" currentCls->add(CLASS_SCRIPT_LYCIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5640 "Parser.cpp" +#line 5641 "Parser.cpp" break; } case 119: { { -#line 676 "Parser.rl" +#line 677 "Parser.rl" {te = p+1;{ -#line 676 "Parser.rl" +#line 677 "Parser.rl" currentCls->add(CLASS_SCRIPT_LYDIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5651 "Parser.cpp" +#line 5652 "Parser.cpp" break; } case 120: { { -#line 677 "Parser.rl" +#line 678 "Parser.rl" {te = p+1;{ -#line 677 "Parser.rl" +#line 678 "Parser.rl" currentCls->add(CLASS_SCRIPT_MALAYALAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5662 "Parser.cpp" +#line 5663 "Parser.cpp" break; } case 121: { { -#line 678 "Parser.rl" +#line 679 "Parser.rl" {te = p+1;{ -#line 678 "Parser.rl" +#line 679 "Parser.rl" currentCls->add(CLASS_SCRIPT_MANDAIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5673 "Parser.cpp" +#line 5674 "Parser.cpp" break; } case 122: { { -#line 679 "Parser.rl" +#line 680 "Parser.rl" {te = p+1;{ -#line 679 "Parser.rl" +#line 680 "Parser.rl" currentCls->add(CLASS_SCRIPT_MEETEI_MAYEK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5684 "Parser.cpp" +#line 5685 "Parser.cpp" break; } case 123: { { -#line 680 "Parser.rl" +#line 681 "Parser.rl" {te = p+1;{ -#line 680 "Parser.rl" +#line 681 "Parser.rl" currentCls->add(CLASS_SCRIPT_MONGOLIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5695 "Parser.cpp" +#line 5696 "Parser.cpp" break; } case 124: { { -#line 681 "Parser.rl" +#line 682 "Parser.rl" {te = p+1;{ -#line 681 "Parser.rl" +#line 682 "Parser.rl" currentCls->add(CLASS_SCRIPT_MYANMAR, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5706 "Parser.cpp" +#line 5707 "Parser.cpp" break; } case 125: { { -#line 682 "Parser.rl" +#line 683 "Parser.rl" {te = p+1;{ -#line 682 "Parser.rl" +#line 683 "Parser.rl" currentCls->add(CLASS_SCRIPT_NEW_TAI_LUE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5717 "Parser.cpp" +#line 5718 "Parser.cpp" break; } case 126: { { -#line 683 "Parser.rl" +#line 684 "Parser.rl" {te = p+1;{ -#line 683 "Parser.rl" +#line 684 "Parser.rl" currentCls->add(CLASS_SCRIPT_NKO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5728 "Parser.cpp" +#line 5729 "Parser.cpp" break; } case 127: { { -#line 684 "Parser.rl" +#line 685 "Parser.rl" {te = p+1;{ -#line 684 "Parser.rl" +#line 685 "Parser.rl" currentCls->add(CLASS_SCRIPT_OGHAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5739 "Parser.cpp" +#line 5740 "Parser.cpp" break; } case 128: { { -#line 685 "Parser.rl" +#line 686 "Parser.rl" {te = p+1;{ -#line 685 "Parser.rl" +#line 686 "Parser.rl" currentCls->add(CLASS_SCRIPT_OL_CHIKI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5750 "Parser.cpp" +#line 5751 "Parser.cpp" break; } case 129: { { -#line 686 "Parser.rl" +#line 687 "Parser.rl" {te = p+1;{ -#line 686 "Parser.rl" +#line 687 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_ITALIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5761 "Parser.cpp" +#line 5762 "Parser.cpp" break; } case 130: { { -#line 687 "Parser.rl" +#line 688 "Parser.rl" {te = p+1;{ -#line 687 "Parser.rl" +#line 688 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_PERSIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5772 "Parser.cpp" +#line 5773 "Parser.cpp" break; } case 131: { { -#line 688 "Parser.rl" +#line 689 "Parser.rl" {te = p+1;{ -#line 688 "Parser.rl" +#line 689 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_SOUTH_ARABIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5783 "Parser.cpp" +#line 5784 "Parser.cpp" break; } case 132: { { -#line 689 "Parser.rl" +#line 690 "Parser.rl" {te = p+1;{ -#line 689 "Parser.rl" +#line 690 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_TURKIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5794 "Parser.cpp" +#line 5795 "Parser.cpp" break; } case 133: { { -#line 690 "Parser.rl" +#line 691 "Parser.rl" {te = p+1;{ -#line 690 "Parser.rl" +#line 691 "Parser.rl" currentCls->add(CLASS_SCRIPT_ORIYA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5805 "Parser.cpp" +#line 5806 "Parser.cpp" break; } case 134: { { -#line 691 "Parser.rl" +#line 692 "Parser.rl" {te = p+1;{ -#line 691 "Parser.rl" +#line 692 "Parser.rl" currentCls->add(CLASS_SCRIPT_OSMANYA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5816 "Parser.cpp" +#line 5817 "Parser.cpp" break; } case 135: { { -#line 692 "Parser.rl" +#line 693 "Parser.rl" {te = p+1;{ -#line 692 "Parser.rl" +#line 693 "Parser.rl" currentCls->add(CLASS_SCRIPT_PHAGS_PA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5827 "Parser.cpp" +#line 5828 "Parser.cpp" break; } case 136: { { -#line 693 "Parser.rl" +#line 694 "Parser.rl" {te = p+1;{ -#line 693 "Parser.rl" +#line 694 "Parser.rl" currentCls->add(CLASS_SCRIPT_PHOENICIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5838 "Parser.cpp" +#line 5839 "Parser.cpp" break; } case 137: { { -#line 694 "Parser.rl" +#line 695 "Parser.rl" {te = p+1;{ -#line 694 "Parser.rl" +#line 695 "Parser.rl" currentCls->add(CLASS_SCRIPT_REJANG, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5849 "Parser.cpp" +#line 5850 "Parser.cpp" break; } case 138: { { -#line 695 "Parser.rl" +#line 696 "Parser.rl" {te = p+1;{ -#line 695 "Parser.rl" +#line 696 "Parser.rl" currentCls->add(CLASS_SCRIPT_RUNIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5860 "Parser.cpp" +#line 5861 "Parser.cpp" break; } case 139: { { -#line 696 "Parser.rl" +#line 697 "Parser.rl" {te = p+1;{ -#line 696 "Parser.rl" +#line 697 "Parser.rl" currentCls->add(CLASS_SCRIPT_SAMARITAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5871 "Parser.cpp" +#line 5872 "Parser.cpp" break; } case 140: { { -#line 697 "Parser.rl" +#line 698 "Parser.rl" {te = p+1;{ -#line 697 "Parser.rl" +#line 698 "Parser.rl" currentCls->add(CLASS_SCRIPT_SAURASHTRA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5882 "Parser.cpp" +#line 5883 "Parser.cpp" break; } case 141: { { -#line 698 "Parser.rl" +#line 699 "Parser.rl" {te = p+1;{ -#line 698 "Parser.rl" +#line 699 "Parser.rl" currentCls->add(CLASS_SCRIPT_SHAVIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5893 "Parser.cpp" +#line 5894 "Parser.cpp" break; } case 142: { { -#line 699 "Parser.rl" +#line 700 "Parser.rl" {te = p+1;{ -#line 699 "Parser.rl" +#line 700 "Parser.rl" currentCls->add(CLASS_SCRIPT_SINHALA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5904 "Parser.cpp" +#line 5905 "Parser.cpp" break; } case 143: { { -#line 700 "Parser.rl" +#line 701 "Parser.rl" {te = p+1;{ -#line 700 "Parser.rl" +#line 701 "Parser.rl" currentCls->add(CLASS_SCRIPT_SUNDANESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5915 "Parser.cpp" +#line 5916 "Parser.cpp" break; } case 144: { { -#line 701 "Parser.rl" +#line 702 "Parser.rl" {te = p+1;{ -#line 701 "Parser.rl" +#line 702 "Parser.rl" currentCls->add(CLASS_SCRIPT_SYLOTI_NAGRI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5926 "Parser.cpp" +#line 5927 "Parser.cpp" break; } case 145: { { -#line 702 "Parser.rl" +#line 703 "Parser.rl" {te = p+1;{ -#line 702 "Parser.rl" +#line 703 "Parser.rl" currentCls->add(CLASS_SCRIPT_SYRIAC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5937 "Parser.cpp" +#line 5938 "Parser.cpp" break; } case 146: { { -#line 703 "Parser.rl" +#line 704 "Parser.rl" {te = p+1;{ -#line 703 "Parser.rl" +#line 704 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAGALOG, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5948 "Parser.cpp" +#line 5949 "Parser.cpp" break; } case 147: { { -#line 704 "Parser.rl" +#line 705 "Parser.rl" {te = p+1;{ -#line 704 "Parser.rl" +#line 705 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAGBANWA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5959 "Parser.cpp" +#line 5960 "Parser.cpp" break; } case 148: { { -#line 705 "Parser.rl" +#line 706 "Parser.rl" {te = p+1;{ -#line 705 "Parser.rl" +#line 706 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAI_LE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5970 "Parser.cpp" +#line 5971 "Parser.cpp" break; } case 149: { { -#line 706 "Parser.rl" +#line 707 "Parser.rl" {te = p+1;{ -#line 706 "Parser.rl" +#line 707 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAI_THAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5981 "Parser.cpp" +#line 5982 "Parser.cpp" break; } case 150: { { -#line 707 "Parser.rl" +#line 708 "Parser.rl" {te = p+1;{ -#line 707 "Parser.rl" +#line 708 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAI_VIET, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5992 "Parser.cpp" +#line 5993 "Parser.cpp" break; } case 151: { { -#line 708 "Parser.rl" +#line 709 "Parser.rl" {te = p+1;{ -#line 708 "Parser.rl" +#line 709 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAMIL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6003 "Parser.cpp" +#line 6004 "Parser.cpp" break; } case 152: { { -#line 709 "Parser.rl" +#line 710 "Parser.rl" {te = p+1;{ -#line 709 "Parser.rl" +#line 710 "Parser.rl" currentCls->add(CLASS_SCRIPT_TELUGU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6014 "Parser.cpp" +#line 6015 "Parser.cpp" break; } case 153: { { -#line 710 "Parser.rl" +#line 711 "Parser.rl" {te = p+1;{ -#line 710 "Parser.rl" +#line 711 "Parser.rl" currentCls->add(CLASS_SCRIPT_THAANA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6025 "Parser.cpp" +#line 6026 "Parser.cpp" break; } case 154: { { -#line 711 "Parser.rl" +#line 712 "Parser.rl" {te = p+1;{ -#line 711 "Parser.rl" +#line 712 "Parser.rl" currentCls->add(CLASS_SCRIPT_THAI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6036 "Parser.cpp" +#line 6037 "Parser.cpp" break; } case 155: { { -#line 712 "Parser.rl" +#line 713 "Parser.rl" {te = p+1;{ -#line 712 "Parser.rl" +#line 713 "Parser.rl" currentCls->add(CLASS_SCRIPT_TIBETAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6047 "Parser.cpp" +#line 6048 "Parser.cpp" break; } case 156: { { -#line 713 "Parser.rl" +#line 714 "Parser.rl" {te = p+1;{ -#line 713 "Parser.rl" +#line 714 "Parser.rl" currentCls->add(CLASS_SCRIPT_TIFINAGH, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6058 "Parser.cpp" +#line 6059 "Parser.cpp" break; } case 157: { { -#line 714 "Parser.rl" +#line 715 "Parser.rl" {te = p+1;{ -#line 714 "Parser.rl" +#line 715 "Parser.rl" currentCls->add(CLASS_SCRIPT_UGARITIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6069 "Parser.cpp" +#line 6070 "Parser.cpp" break; } case 158: { { -#line 715 "Parser.rl" +#line 716 "Parser.rl" {te = p+1;{ -#line 715 "Parser.rl" +#line 716 "Parser.rl" currentCls->add(CLASS_SCRIPT_VAI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6080 "Parser.cpp" +#line 6081 "Parser.cpp" break; } case 159: { { -#line 716 "Parser.rl" +#line 717 "Parser.rl" {te = p+1;{ -#line 716 "Parser.rl" +#line 717 "Parser.rl" currentCls->add(CLASS_SCRIPT_YI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6091 "Parser.cpp" +#line 6092 "Parser.cpp" break; } case 160: { { -#line 717 "Parser.rl" +#line 718 "Parser.rl" {te = p+1;{ -#line 717 "Parser.rl" +#line 718 "Parser.rl" currentCls->add(CLASS_UCP_ANY, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6102 "Parser.cpp" +#line 6103 "Parser.cpp" break; } case 161: { { -#line 718 "Parser.rl" +#line 719 "Parser.rl" {te = p+1;{ -#line 718 "Parser.rl" +#line 719 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6113 "Parser.cpp" +#line 6114 "Parser.cpp" break; } case 162: { { -#line 580 "Parser.rl" +#line 581 "Parser.rl" {te = p;p = p - 1;{ -#line 580 "Parser.rl" +#line 581 "Parser.rl" currentCls->add(CLASS_UCP_C, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6124 "Parser.cpp" +#line 6125 "Parser.cpp" break; } case 163: { { -#line 584 "Parser.rl" +#line 585 "Parser.rl" {te = p;p = p - 1;{ -#line 584 "Parser.rl" +#line 585 "Parser.rl" currentCls->add(CLASS_UCP_CO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6135 "Parser.cpp" +#line 6136 "Parser.cpp" break; } case 164: { { -#line 586 "Parser.rl" +#line 587 "Parser.rl" {te = p;p = p - 1;{ -#line 586 "Parser.rl" +#line 587 "Parser.rl" currentCls->add(CLASS_UCP_L, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6146 "Parser.cpp" +#line 6147 "Parser.cpp" break; } case 165: { { -#line 593 "Parser.rl" +#line 594 "Parser.rl" {te = p;p = p - 1;{ -#line 593 "Parser.rl" +#line 594 "Parser.rl" currentCls->add(CLASS_UCP_M, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6157 "Parser.cpp" +#line 6158 "Parser.cpp" break; } case 166: { { -#line 595 "Parser.rl" +#line 596 "Parser.rl" {te = p;p = p - 1;{ -#line 595 "Parser.rl" +#line 596 "Parser.rl" currentCls->add(CLASS_UCP_ME, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6168 "Parser.cpp" +#line 6169 "Parser.cpp" break; } case 167: { { -#line 597 "Parser.rl" +#line 598 "Parser.rl" {te = p;p = p - 1;{ -#line 597 "Parser.rl" +#line 598 "Parser.rl" currentCls->add(CLASS_UCP_N, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6179 "Parser.cpp" +#line 6180 "Parser.cpp" break; } case 168: { { -#line 601 "Parser.rl" +#line 602 "Parser.rl" {te = p;p = p - 1;{ -#line 601 "Parser.rl" +#line 602 "Parser.rl" currentCls->add(CLASS_UCP_P, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6190 "Parser.cpp" +#line 6191 "Parser.cpp" break; } case 169: { { -#line 609 "Parser.rl" +#line 610 "Parser.rl" {te = p;p = p - 1;{ -#line 609 "Parser.rl" +#line 610 "Parser.rl" currentCls->add(CLASS_UCP_S, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6201 "Parser.cpp" +#line 6202 "Parser.cpp" break; } case 170: { { -#line 614 "Parser.rl" +#line 615 "Parser.rl" {te = p;p = p - 1;{ -#line 614 "Parser.rl" +#line 615 "Parser.rl" currentCls->add(CLASS_UCP_Z, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6212 "Parser.cpp" +#line 6213 "Parser.cpp" break; } case 171: { { -#line 653 "Parser.rl" +#line 654 "Parser.rl" {te = p;p = p - 1;{ -#line 653 "Parser.rl" +#line 654 "Parser.rl" currentCls->add(CLASS_SCRIPT_HAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6223 "Parser.cpp" +#line 6224 "Parser.cpp" break; } case 172: { { -#line 718 "Parser.rl" +#line 719 "Parser.rl" {te = p;p = p - 1;{ -#line 718 "Parser.rl" +#line 719 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6234 "Parser.cpp" +#line 6235 "Parser.cpp" break; } case 173: { { -#line 580 "Parser.rl" +#line 581 "Parser.rl" {p = ((te))-1; { -#line 580 "Parser.rl" +#line 581 "Parser.rl" currentCls->add(CLASS_UCP_C, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6246 "Parser.cpp" +#line 6247 "Parser.cpp" break; } case 174: { { -#line 584 "Parser.rl" +#line 585 "Parser.rl" {p = ((te))-1; { -#line 584 "Parser.rl" +#line 585 "Parser.rl" currentCls->add(CLASS_UCP_CO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6258 "Parser.cpp" +#line 6259 "Parser.cpp" break; } case 175: { { -#line 586 "Parser.rl" +#line 587 "Parser.rl" {p = ((te))-1; { -#line 586 "Parser.rl" +#line 587 "Parser.rl" currentCls->add(CLASS_UCP_L, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6270 "Parser.cpp" +#line 6271 "Parser.cpp" break; } case 176: { { -#line 593 "Parser.rl" +#line 594 "Parser.rl" {p = ((te))-1; { -#line 593 "Parser.rl" +#line 594 "Parser.rl" currentCls->add(CLASS_UCP_M, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6282 "Parser.cpp" +#line 6283 "Parser.cpp" break; } case 177: { { -#line 595 "Parser.rl" +#line 596 "Parser.rl" {p = ((te))-1; { -#line 595 "Parser.rl" +#line 596 "Parser.rl" currentCls->add(CLASS_UCP_ME, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6294 "Parser.cpp" +#line 6295 "Parser.cpp" break; } case 178: { { -#line 597 "Parser.rl" +#line 598 "Parser.rl" {p = ((te))-1; { -#line 597 "Parser.rl" +#line 598 "Parser.rl" currentCls->add(CLASS_UCP_N, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6306 "Parser.cpp" +#line 6307 "Parser.cpp" break; } case 179: { { -#line 601 "Parser.rl" +#line 602 "Parser.rl" {p = ((te))-1; { -#line 601 "Parser.rl" +#line 602 "Parser.rl" currentCls->add(CLASS_UCP_P, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6318 "Parser.cpp" +#line 6319 "Parser.cpp" break; } case 180: { { -#line 609 "Parser.rl" +#line 610 "Parser.rl" {p = ((te))-1; { -#line 609 "Parser.rl" +#line 610 "Parser.rl" currentCls->add(CLASS_UCP_S, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6330 "Parser.cpp" +#line 6331 "Parser.cpp" break; } case 181: { { -#line 653 "Parser.rl" +#line 654 "Parser.rl" {p = ((te))-1; { -#line 653 "Parser.rl" +#line 654 "Parser.rl" currentCls->add(CLASS_SCRIPT_HAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6342 "Parser.cpp" +#line 6343 "Parser.cpp" break; } case 182: { { -#line 718 "Parser.rl" +#line 719 "Parser.rl" {p = ((te))-1; { -#line 718 "Parser.rl" +#line 719 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6354 "Parser.cpp" +#line 6355 "Parser.cpp" break; } case 183: { { -#line 733 "Parser.rl" +#line 734 "Parser.rl" {te = p+1;{ -#line 733 "Parser.rl" +#line 734 "Parser.rl" currentCls->add(CLASS_UCP_C, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6372 "Parser.cpp" +#line 6373 "Parser.cpp" break; } case 184: { { -#line 741 "Parser.rl" +#line 742 "Parser.rl" {te = p+1;{ -#line 741 "Parser.rl" +#line 742 "Parser.rl" currentCls->add(CLASS_UCP_L, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6390 "Parser.cpp" +#line 6391 "Parser.cpp" break; } case 185: { { -#line 749 "Parser.rl" +#line 750 "Parser.rl" {te = p+1;{ -#line 749 "Parser.rl" +#line 750 "Parser.rl" currentCls->add(CLASS_UCP_M, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6408 "Parser.cpp" +#line 6409 "Parser.cpp" break; } case 186: { { -#line 757 "Parser.rl" +#line 758 "Parser.rl" {te = p+1;{ -#line 757 "Parser.rl" +#line 758 "Parser.rl" currentCls->add(CLASS_UCP_N, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6426 "Parser.cpp" +#line 6427 "Parser.cpp" break; } case 187: { { -#line 765 "Parser.rl" +#line 766 "Parser.rl" {te = p+1;{ -#line 765 "Parser.rl" +#line 766 "Parser.rl" currentCls->add(CLASS_UCP_P, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6444 "Parser.cpp" +#line 6445 "Parser.cpp" break; } case 188: { { -#line 773 "Parser.rl" +#line 774 "Parser.rl" {te = p+1;{ -#line 773 "Parser.rl" +#line 774 "Parser.rl" currentCls->add(CLASS_UCP_S, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6462 "Parser.cpp" +#line 6463 "Parser.cpp" break; } case 189: { { -#line 781 "Parser.rl" +#line 782 "Parser.rl" {te = p+1;{ -#line 781 "Parser.rl" +#line 782 "Parser.rl" currentCls->add(CLASS_UCP_Z, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6480 "Parser.cpp" +#line 6481 "Parser.cpp" break; } case 190: { { -#line 790 "Parser.rl" +#line 791 "Parser.rl" {te = p+1;{ -#line 790 "Parser.rl" +#line 791 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6491 "Parser.cpp" +#line 6492 "Parser.cpp" break; } case 191: { { -#line 796 "Parser.rl" +#line 797 "Parser.rl" {te = p+1;{ -#line 796 "Parser.rl" +#line 797 "Parser.rl" throw LocatedParseError("Unsupported POSIX collating " "element"); } }} -#line 6505 "Parser.cpp" +#line 6506 "Parser.cpp" break; } case 192: { { -#line 803 "Parser.rl" +#line 804 "Parser.rl" {te = p+1;{ -#line 803 "Parser.rl" +#line 804 "Parser.rl" currentCls->add(CLASS_ALNUM, false); } }} -#line 6518 "Parser.cpp" +#line 6519 "Parser.cpp" break; } case 193: { { -#line 806 "Parser.rl" +#line 807 "Parser.rl" {te = p+1;{ -#line 806 "Parser.rl" +#line 807 "Parser.rl" currentCls->add(CLASS_ALNUM, true); } }} -#line 6531 "Parser.cpp" +#line 6532 "Parser.cpp" break; } case 194: { { -#line 809 "Parser.rl" +#line 810 "Parser.rl" {te = p+1;{ -#line 809 "Parser.rl" +#line 810 "Parser.rl" currentCls->add(CLASS_ALPHA, false); } }} -#line 6544 "Parser.cpp" +#line 6545 "Parser.cpp" break; } case 195: { { -#line 812 "Parser.rl" +#line 813 "Parser.rl" {te = p+1;{ -#line 812 "Parser.rl" +#line 813 "Parser.rl" currentCls->add(CLASS_ALPHA, true); } }} -#line 6557 "Parser.cpp" +#line 6558 "Parser.cpp" break; } case 196: { { -#line 815 "Parser.rl" +#line 816 "Parser.rl" {te = p+1;{ -#line 815 "Parser.rl" +#line 816 "Parser.rl" currentCls->add(CLASS_ASCII, false); } }} -#line 6570 "Parser.cpp" +#line 6571 "Parser.cpp" break; } case 197: { { -#line 818 "Parser.rl" +#line 819 "Parser.rl" {te = p+1;{ -#line 818 "Parser.rl" +#line 819 "Parser.rl" currentCls->add(CLASS_ASCII, true); } }} -#line 6583 "Parser.cpp" +#line 6584 "Parser.cpp" break; } case 198: { { -#line 821 "Parser.rl" +#line 822 "Parser.rl" {te = p+1;{ -#line 821 "Parser.rl" +#line 822 "Parser.rl" currentCls->add(CLASS_BLANK, false); } }} -#line 6596 "Parser.cpp" +#line 6597 "Parser.cpp" break; } case 199: { { -#line 824 "Parser.rl" +#line 825 "Parser.rl" {te = p+1;{ -#line 824 "Parser.rl" +#line 825 "Parser.rl" currentCls->add(CLASS_BLANK, true); } }} -#line 6609 "Parser.cpp" +#line 6610 "Parser.cpp" break; } case 200: { { -#line 827 "Parser.rl" +#line 828 "Parser.rl" {te = p+1;{ -#line 827 "Parser.rl" +#line 828 "Parser.rl" currentCls->add(CLASS_CNTRL, false); } }} -#line 6622 "Parser.cpp" +#line 6623 "Parser.cpp" break; } case 201: { { -#line 830 "Parser.rl" +#line 831 "Parser.rl" {te = p+1;{ -#line 830 "Parser.rl" +#line 831 "Parser.rl" currentCls->add(CLASS_CNTRL, true); } }} -#line 6635 "Parser.cpp" +#line 6636 "Parser.cpp" break; } case 202: { { -#line 833 "Parser.rl" +#line 834 "Parser.rl" {te = p+1;{ -#line 833 "Parser.rl" +#line 834 "Parser.rl" currentCls->add(CLASS_DIGIT, false); } }} -#line 6648 "Parser.cpp" +#line 6649 "Parser.cpp" break; } case 203: { { -#line 836 "Parser.rl" +#line 837 "Parser.rl" {te = p+1;{ -#line 836 "Parser.rl" +#line 837 "Parser.rl" currentCls->add(CLASS_DIGIT, true); } }} -#line 6661 "Parser.cpp" +#line 6662 "Parser.cpp" break; } case 204: { { -#line 839 "Parser.rl" +#line 840 "Parser.rl" {te = p+1;{ -#line 839 "Parser.rl" +#line 840 "Parser.rl" currentCls->add(CLASS_GRAPH, false); } }} -#line 6674 "Parser.cpp" +#line 6675 "Parser.cpp" break; } case 205: { { -#line 842 "Parser.rl" +#line 843 "Parser.rl" {te = p+1;{ -#line 842 "Parser.rl" +#line 843 "Parser.rl" currentCls->add(CLASS_GRAPH, true); } }} -#line 6687 "Parser.cpp" +#line 6688 "Parser.cpp" break; } case 206: { { -#line 845 "Parser.rl" +#line 846 "Parser.rl" {te = p+1;{ -#line 845 "Parser.rl" +#line 846 "Parser.rl" currentCls->add(CLASS_LOWER, false); } }} -#line 6700 "Parser.cpp" +#line 6701 "Parser.cpp" break; } case 207: { { -#line 848 "Parser.rl" +#line 849 "Parser.rl" {te = p+1;{ -#line 848 "Parser.rl" +#line 849 "Parser.rl" currentCls->add(CLASS_LOWER, true); } }} -#line 6713 "Parser.cpp" +#line 6714 "Parser.cpp" break; } case 208: { { -#line 851 "Parser.rl" +#line 852 "Parser.rl" {te = p+1;{ -#line 851 "Parser.rl" +#line 852 "Parser.rl" currentCls->add(CLASS_PRINT, false); } }} -#line 6726 "Parser.cpp" +#line 6727 "Parser.cpp" break; } case 209: { { -#line 854 "Parser.rl" +#line 855 "Parser.rl" {te = p+1;{ -#line 854 "Parser.rl" +#line 855 "Parser.rl" currentCls->add(CLASS_PRINT, true); } }} -#line 6739 "Parser.cpp" +#line 6740 "Parser.cpp" break; } case 210: { { -#line 857 "Parser.rl" +#line 858 "Parser.rl" {te = p+1;{ -#line 857 "Parser.rl" +#line 858 "Parser.rl" currentCls->add(CLASS_PUNCT, false); } }} -#line 6752 "Parser.cpp" +#line 6753 "Parser.cpp" break; } case 211: { { -#line 860 "Parser.rl" +#line 861 "Parser.rl" {te = p+1;{ -#line 860 "Parser.rl" +#line 861 "Parser.rl" currentCls->add(CLASS_PUNCT, true); } }} -#line 6765 "Parser.cpp" +#line 6766 "Parser.cpp" break; } case 212: { { -#line 864 "Parser.rl" +#line 865 "Parser.rl" {te = p+1;{ -#line 864 "Parser.rl" +#line 865 "Parser.rl" currentCls->add(CLASS_SPACE, false); } }} -#line 6778 "Parser.cpp" +#line 6779 "Parser.cpp" break; } case 213: { { -#line 867 "Parser.rl" +#line 868 "Parser.rl" {te = p+1;{ -#line 867 "Parser.rl" +#line 868 "Parser.rl" currentCls->add(CLASS_SPACE, true); } }} -#line 6791 "Parser.cpp" +#line 6792 "Parser.cpp" break; } case 214: { { -#line 870 "Parser.rl" +#line 871 "Parser.rl" {te = p+1;{ -#line 870 "Parser.rl" +#line 871 "Parser.rl" currentCls->add(CLASS_UPPER, false); } }} -#line 6804 "Parser.cpp" +#line 6805 "Parser.cpp" break; } case 215: { { -#line 873 "Parser.rl" +#line 874 "Parser.rl" {te = p+1;{ -#line 873 "Parser.rl" +#line 874 "Parser.rl" currentCls->add(CLASS_UPPER, true); } }} -#line 6817 "Parser.cpp" +#line 6818 "Parser.cpp" break; } case 216: { { -#line 876 "Parser.rl" +#line 877 "Parser.rl" {te = p+1;{ -#line 876 "Parser.rl" +#line 877 "Parser.rl" currentCls->add(CLASS_WORD, false); } }} -#line 6830 "Parser.cpp" +#line 6831 "Parser.cpp" break; } case 217: { { -#line 879 "Parser.rl" +#line 880 "Parser.rl" {te = p+1;{ -#line 879 "Parser.rl" +#line 880 "Parser.rl" currentCls->add(CLASS_WORD, true); } }} -#line 6843 "Parser.cpp" +#line 6844 "Parser.cpp" break; } case 218: { { -#line 882 "Parser.rl" +#line 883 "Parser.rl" {te = p+1;{ -#line 882 "Parser.rl" +#line 883 "Parser.rl" currentCls->add(CLASS_XDIGIT, false); } }} -#line 6856 "Parser.cpp" +#line 6857 "Parser.cpp" break; } case 219: { { -#line 885 "Parser.rl" +#line 886 "Parser.rl" {te = p+1;{ -#line 885 "Parser.rl" +#line 886 "Parser.rl" currentCls->add(CLASS_XDIGIT, true); } }} -#line 6869 "Parser.cpp" +#line 6870 "Parser.cpp" break; } case 220: { { -#line 890 "Parser.rl" +#line 891 "Parser.rl" {te = p+1;{ -#line 890 "Parser.rl" +#line 891 "Parser.rl" throw LocatedParseError("Invalid POSIX named class"); } }} -#line 6882 "Parser.cpp" +#line 6883 "Parser.cpp" break; } case 221: { { -#line 893 "Parser.rl" +#line 894 "Parser.rl" {te = p+1;{ -#line 893 "Parser.rl" +#line 894 "Parser.rl" {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { stack.resize(2 * (top + 1)); } } - stack[top] = cs; top += 1;cs = 843;goto _again;}} + stack[top] = cs; top += 1;cs = 844;goto _again;}} }} -#line 6902 "Parser.cpp" +#line 6903 "Parser.cpp" break; } case 222: { { -#line 896 "Parser.rl" +#line 897 "Parser.rl" {te = p+1;{ -#line 896 "Parser.rl" +#line 897 "Parser.rl" /*noop*/} }} -#line 6913 "Parser.cpp" +#line 6914 "Parser.cpp" break; } case 223: { { -#line 898 "Parser.rl" +#line 899 "Parser.rl" {te = p+1;{ -#line 898 "Parser.rl" +#line 899 "Parser.rl" currentCls->add('\x08'); } }} -#line 6926 "Parser.cpp" +#line 6927 "Parser.cpp" break; } case 224: { { -#line 902 "Parser.rl" +#line 903 "Parser.rl" {te = p+1;{ -#line 902 "Parser.rl" +#line 903 "Parser.rl" currentCls->add('\x09'); } }} -#line 6939 "Parser.cpp" +#line 6940 "Parser.cpp" break; } case 225: { { -#line 906 "Parser.rl" +#line 907 "Parser.rl" {te = p+1;{ -#line 906 "Parser.rl" +#line 907 "Parser.rl" currentCls->add('\x0a'); } }} -#line 6952 "Parser.cpp" +#line 6953 "Parser.cpp" break; } case 226: { { -#line 910 "Parser.rl" +#line 911 "Parser.rl" {te = p+1;{ -#line 910 "Parser.rl" +#line 911 "Parser.rl" currentCls->add('\x0d'); } }} -#line 6965 "Parser.cpp" +#line 6966 "Parser.cpp" break; } case 227: { { -#line 914 "Parser.rl" +#line 915 "Parser.rl" {te = p+1;{ -#line 914 "Parser.rl" +#line 915 "Parser.rl" currentCls->add('\x0c'); } }} -#line 6978 "Parser.cpp" +#line 6979 "Parser.cpp" break; } case 228: { { -#line 918 "Parser.rl" +#line 919 "Parser.rl" {te = p+1;{ -#line 918 "Parser.rl" +#line 919 "Parser.rl" currentCls->add('\x07'); } }} -#line 6991 "Parser.cpp" +#line 6992 "Parser.cpp" break; } case 229: { { -#line 922 "Parser.rl" +#line 923 "Parser.rl" {te = p+1;{ -#line 922 "Parser.rl" +#line 923 "Parser.rl" currentCls->add('\x1b'); } }} -#line 7004 "Parser.cpp" +#line 7005 "Parser.cpp" break; } case 230: { { -#line 926 "Parser.rl" +#line 927 "Parser.rl" {te = p+1;{ -#line 926 "Parser.rl" +#line 927 "Parser.rl" currentCls->add(CLASS_HORZ, false); } }} -#line 7017 "Parser.cpp" +#line 7018 "Parser.cpp" break; } case 231: { { -#line 930 "Parser.rl" +#line 931 "Parser.rl" {te = p+1;{ -#line 930 "Parser.rl" +#line 931 "Parser.rl" currentCls->add(CLASS_HORZ, true); } }} -#line 7030 "Parser.cpp" +#line 7031 "Parser.cpp" break; } case 232: { { -#line 934 "Parser.rl" +#line 935 "Parser.rl" {te = p+1;{ -#line 934 "Parser.rl" +#line 935 "Parser.rl" currentCls->add(CLASS_VERT, false); } }} -#line 7043 "Parser.cpp" +#line 7044 "Parser.cpp" break; } case 233: { { -#line 938 "Parser.rl" +#line 939 "Parser.rl" {te = p+1;{ -#line 938 "Parser.rl" +#line 939 "Parser.rl" currentCls->add(CLASS_VERT, true); } }} -#line 7056 "Parser.cpp" +#line 7057 "Parser.cpp" break; } case 234: { { -#line 942 "Parser.rl" +#line 943 "Parser.rl" {te = p+1;{ -#line 942 "Parser.rl" +#line 943 "Parser.rl" negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7320,20 +7321,20 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 7078 "Parser.cpp" +#line 7079 "Parser.cpp" break; } case 235: { { -#line 948 "Parser.rl" +#line 949 "Parser.rl" {te = p+1;{ -#line 948 "Parser.rl" +#line 949 "Parser.rl" negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7343,20 +7344,20 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 7100 "Parser.cpp" +#line 7101 "Parser.cpp" break; } case 236: { { -#line 954 "Parser.rl" +#line 955 "Parser.rl" {te = p+1;{ -#line 954 "Parser.rl" +#line 955 "Parser.rl" negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7366,20 +7367,20 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 7122 "Parser.cpp" +#line 7123 "Parser.cpp" break; } case 237: { { -#line 960 "Parser.rl" +#line 961 "Parser.rl" {te = p+1;{ -#line 960 "Parser.rl" +#line 961 "Parser.rl" negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7389,43 +7390,43 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 7144 "Parser.cpp" +#line 7145 "Parser.cpp" break; } case 238: { { -#line 970 "Parser.rl" +#line 971 "Parser.rl" {te = p+1;{ -#line 970 "Parser.rl" +#line 971 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7157 "Parser.cpp" +#line 7158 "Parser.cpp" break; } case 239: { { -#line 973 "Parser.rl" +#line 974 "Parser.rl" {te = p+1;{ -#line 973 "Parser.rl" +#line 974 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7170 "Parser.cpp" +#line 7171 "Parser.cpp" break; } case 240: { { -#line 977 "Parser.rl" +#line 978 "Parser.rl" {te = p+1;{ -#line 977 "Parser.rl" +#line 978 "Parser.rl" string oct(ts + 3, te - ts - 4); unsigned long val; @@ -7441,29 +7442,29 @@ static const short _regex_actions[] = { } }} -#line 7193 "Parser.cpp" +#line 7194 "Parser.cpp" break; } case 241: { { -#line 997 "Parser.rl" +#line 998 "Parser.rl" {te = p+1;{ -#line 997 "Parser.rl" +#line 998 "Parser.rl" currentCls->add(accumulator); } }} -#line 7206 "Parser.cpp" +#line 7207 "Parser.cpp" break; } case 242: { { -#line 1001 "Parser.rl" +#line 1002 "Parser.rl" {te = p+1;{ -#line 1001 "Parser.rl" +#line 1002 "Parser.rl" // whatever we found here currentCls->add(*(ts + 1)); @@ -7471,15 +7472,15 @@ static const short _regex_actions[] = { } }} -#line 7221 "Parser.cpp" +#line 7222 "Parser.cpp" break; } case 243: { { -#line 1007 "Parser.rl" +#line 1008 "Parser.rl" {te = p+1;{ -#line 1007 "Parser.rl" +#line 1008 "Parser.rl" string hex(ts + 3, te - ts - 4); unsigned long val; @@ -7495,15 +7496,15 @@ static const short _regex_actions[] = { } }} -#line 7244 "Parser.cpp" +#line 7245 "Parser.cpp" break; } case 244: { { -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" {te = p+1;{ -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -7515,113 +7516,113 @@ static const short _regex_actions[] = { } }} -#line 7263 "Parser.cpp" +#line 7264 "Parser.cpp" break; } case 245: { { -#line 1035 "Parser.rl" +#line 1036 "Parser.rl" {te = p+1;{ -#line 1035 "Parser.rl" +#line 1036 "Parser.rl" currentCls->add(CLASS_WORD, false); } }} -#line 7276 "Parser.cpp" +#line 7277 "Parser.cpp" break; } case 246: { { -#line 1039 "Parser.rl" +#line 1040 "Parser.rl" {te = p+1;{ -#line 1039 "Parser.rl" +#line 1040 "Parser.rl" currentCls->add(CLASS_WORD, true); } }} -#line 7289 "Parser.cpp" +#line 7290 "Parser.cpp" break; } case 247: { { -#line 1043 "Parser.rl" +#line 1044 "Parser.rl" {te = p+1;{ -#line 1043 "Parser.rl" +#line 1044 "Parser.rl" currentCls->add(CLASS_SPACE, false); } }} -#line 7302 "Parser.cpp" +#line 7303 "Parser.cpp" break; } case 248: { { -#line 1047 "Parser.rl" +#line 1048 "Parser.rl" {te = p+1;{ -#line 1047 "Parser.rl" +#line 1048 "Parser.rl" currentCls->add(CLASS_SPACE, true); } }} -#line 7315 "Parser.cpp" +#line 7316 "Parser.cpp" break; } case 249: { { -#line 1051 "Parser.rl" +#line 1052 "Parser.rl" {te = p+1;{ -#line 1051 "Parser.rl" +#line 1052 "Parser.rl" currentCls->add(CLASS_DIGIT, false); } }} -#line 7328 "Parser.cpp" +#line 7329 "Parser.cpp" break; } case 250: { { -#line 1055 "Parser.rl" +#line 1056 "Parser.rl" {te = p+1;{ -#line 1055 "Parser.rl" +#line 1056 "Parser.rl" currentCls->add(CLASS_DIGIT, true); } }} -#line 7341 "Parser.cpp" +#line 7342 "Parser.cpp" break; } case 251: { { -#line 1058 "Parser.rl" +#line 1059 "Parser.rl" {te = p+1;{ -#line 1058 "Parser.rl" +#line 1059 "Parser.rl" currentCls->addDash(); } }} -#line 7354 "Parser.cpp" +#line 7355 "Parser.cpp" break; } case 252: { { -#line 276 "Parser.rl" +#line 277 "Parser.rl" {te = p+1;{ -#line 276 "Parser.rl" +#line 277 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -7630,15 +7631,15 @@ static const short _regex_actions[] = { } }} -#line 7370 "Parser.cpp" +#line 7371 "Parser.cpp" break; } case 253: { { -#line 276 "Parser.rl" +#line 277 "Parser.rl" {te = p+1;{ -#line 276 "Parser.rl" +#line 277 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -7647,15 +7648,15 @@ static const short _regex_actions[] = { } }} -#line 7386 "Parser.cpp" +#line 7387 "Parser.cpp" break; } case 254: { { -#line 276 "Parser.rl" +#line 277 "Parser.rl" {te = p+1;{ -#line 276 "Parser.rl" +#line 277 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -7664,214 +7665,226 @@ static const short _regex_actions[] = { } }} -#line 7402 "Parser.cpp" +#line 7403 "Parser.cpp" break; } case 255: { { -#line 1075 "Parser.rl" +#line 1076 "Parser.rl" {te = p+1;{ -#line 1075 "Parser.rl" +#line 1076 "Parser.rl" // add the literal char currentCls->add(*(ts + 1)); } }} -#line 7416 "Parser.cpp" +#line 7417 "Parser.cpp" break; } case 256: { { -#line 1081 "Parser.rl" +#line 1082 "Parser.rl" {te = p+1;{ -#line 1081 "Parser.rl" +#line 1082 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint2c(ts)); } }} -#line 7430 "Parser.cpp" +#line 7431 "Parser.cpp" break; } case 257: { { -#line 1086 "Parser.rl" +#line 1087 "Parser.rl" {te = p+1;{ -#line 1086 "Parser.rl" +#line 1087 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint3c(ts)); } }} -#line 7444 "Parser.cpp" +#line 7445 "Parser.cpp" break; } case 258: { { -#line 1091 "Parser.rl" +#line 1092 "Parser.rl" {te = p+1;{ -#line 1091 "Parser.rl" +#line 1092 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint4c(ts)); } }} -#line 7458 "Parser.cpp" +#line 7459 "Parser.cpp" break; } case 259: { { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" {te = p+1;{ -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7472 "Parser.cpp" +#line 7473 "Parser.cpp" break; } case 260: { { -#line 1102 "Parser.rl" +#line 1106 "Parser.rl" + {te = p+1;{ +#line 1106 "Parser.rl" + throw LocatedParseError("Character class intersection/subtraction is not supported"); } + }} + +#line 7484 "Parser.cpp" + + break; + } + case 261: { + { +#line 1109 "Parser.rl" {te = p+1;{ -#line 1102 "Parser.rl" +#line 1109 "Parser.rl" currentCls->add((u8)*ts); } }} -#line 7485 "Parser.cpp" +#line 7497 "Parser.cpp" break; } - case 261: { + case 262: { { -#line 1106 "Parser.rl" +#line 1113 "Parser.rl" {te = p+1;{ -#line 1106 "Parser.rl" +#line 1113 "Parser.rl" currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); inCharClass = false; {cs = 746;goto _again;}} }} -#line 7500 "Parser.cpp" +#line 7512 "Parser.cpp" break; } - case 262: { + case 263: { { -#line 966 "Parser.rl" +#line 967 "Parser.rl" {te = p;p = p - 1;{ -#line 966 "Parser.rl" +#line 967 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 7511 "Parser.cpp" +#line 7523 "Parser.cpp" break; } - case 263: { + case 264: { { -#line 967 "Parser.rl" +#line 968 "Parser.rl" {te = p;p = p - 1;{ -#line 967 "Parser.rl" +#line 968 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 7522 "Parser.cpp" +#line 7534 "Parser.cpp" break; } - case 264: { + case 265: { { -#line 970 "Parser.rl" +#line 971 "Parser.rl" {te = p;p = p - 1;{ -#line 970 "Parser.rl" +#line 971 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7535 "Parser.cpp" +#line 7547 "Parser.cpp" break; } - case 265: { + case 266: { { -#line 973 "Parser.rl" +#line 974 "Parser.rl" {te = p;p = p - 1;{ -#line 973 "Parser.rl" +#line 974 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7548 "Parser.cpp" +#line 7560 "Parser.cpp" break; } - case 266: { + case 267: { { -#line 992 "Parser.rl" +#line 993 "Parser.rl" {te = p;p = p - 1;{ -#line 992 "Parser.rl" +#line 993 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 7561 "Parser.cpp" +#line 7573 "Parser.cpp" break; } - case 267: { + case 268: { { -#line 997 "Parser.rl" +#line 998 "Parser.rl" {te = p;p = p - 1;{ -#line 997 "Parser.rl" +#line 998 "Parser.rl" currentCls->add(accumulator); } }} -#line 7574 "Parser.cpp" +#line 7586 "Parser.cpp" break; } - case 268: { + case 269: { { -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" {te = p;p = p - 1;{ -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 7587 "Parser.cpp" +#line 7599 "Parser.cpp" break; } - case 269: { + case 270: { { -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" {te = p;p = p - 1;{ -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -7883,105 +7896,115 @@ static const short _regex_actions[] = { } }} -#line 7606 "Parser.cpp" +#line 7618 "Parser.cpp" break; } - case 270: { + case 271: { { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" {te = p;p = p - 1;{ -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7620 "Parser.cpp" +#line 7632 "Parser.cpp" break; } - case 271: { + case 272: { + { +#line 1103 "Parser.rl" + {te = p;p = p - 1;{ +#line 1103 "Parser.rl" + throw LocatedParseError("Nested character classes are not supported"); } + }} + +#line 7643 "Parser.cpp" + + break; + } + case 273: { { -#line 1102 "Parser.rl" +#line 1109 "Parser.rl" {te = p;p = p - 1;{ -#line 1102 "Parser.rl" +#line 1109 "Parser.rl" currentCls->add((u8)*ts); } }} -#line 7633 "Parser.cpp" +#line 7656 "Parser.cpp" break; } - case 272: { + case 274: { { -#line 992 "Parser.rl" +#line 993 "Parser.rl" {p = ((te))-1; { -#line 992 "Parser.rl" +#line 993 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 7647 "Parser.cpp" +#line 7670 "Parser.cpp" break; } - case 273: { + case 275: { { -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" {p = ((te))-1; { -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 7661 "Parser.cpp" +#line 7684 "Parser.cpp" break; } - case 274: { + case 276: { { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" {p = ((te))-1; { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7676 "Parser.cpp" +#line 7699 "Parser.cpp" break; } - case 275: { + case 277: { { -#line 1102 "Parser.rl" +#line 1103 "Parser.rl" {p = ((te))-1; { -#line 1102 "Parser.rl" - - currentCls->add((u8)*ts); - } +#line 1103 "Parser.rl" + throw LocatedParseError("Nested character classes are not supported"); } }} -#line 7690 "Parser.cpp" +#line 7711 "Parser.cpp" break; } - case 276: { + case 278: { { -#line 1120 "Parser.rl" +#line 1127 "Parser.rl" {te = p+1;{ -#line 1120 "Parser.rl" +#line 1127 "Parser.rl" if (currentCls->isNegated()) { // Already seen a caret; the second one is not a meta-character. @@ -7994,250 +8017,250 @@ static const short _regex_actions[] = { } }} -#line 7710 "Parser.cpp" +#line 7731 "Parser.cpp" break; } - case 277: { + case 279: { { -#line 1133 "Parser.rl" +#line 1140 "Parser.rl" {te = p+1;{ -#line 1133 "Parser.rl" +#line 1140 "Parser.rl" currentCls->add(']'); inCharClassEarly = false; } }} -#line 7724 "Parser.cpp" +#line 7745 "Parser.cpp" break; } - case 278: { + case 280: { { -#line 1138 "Parser.rl" +#line 1145 "Parser.rl" {te = p+1;{ -#line 1138 "Parser.rl" +#line 1145 "Parser.rl" {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { stack.resize(2 * (top + 1)); } } - stack[top] = cs; top += 1;cs = 843;goto _again;}} + stack[top] = cs; top += 1;cs = 844;goto _again;}} }} -#line 7743 "Parser.cpp" +#line 7764 "Parser.cpp" break; } - case 279: { + case 281: { { -#line 1139 "Parser.rl" +#line 1146 "Parser.rl" {te = p+1;{ -#line 1139 "Parser.rl" +#line 1146 "Parser.rl" /*noop*/} }} -#line 7754 "Parser.cpp" +#line 7775 "Parser.cpp" break; } - case 280: { + case 282: { { -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" {te = p+1;{ -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" inCharClassEarly = false; {p = p - 1; } {cs = 819;goto _again;}} }} -#line 7768 "Parser.cpp" +#line 7789 "Parser.cpp" break; } - case 281: { + case 283: { { -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" {te = p;p = p - 1;{ -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" inCharClassEarly = false; {p = p - 1; } {cs = 819;goto _again;}} }} -#line 7782 "Parser.cpp" +#line 7803 "Parser.cpp" break; } - case 282: { + case 284: { { -#line 1154 "Parser.rl" +#line 1161 "Parser.rl" {te = p+1;{ -#line 1154 "Parser.rl" +#line 1161 "Parser.rl" {cs = 746;goto _again;}} }} -#line 7794 "Parser.cpp" +#line 7815 "Parser.cpp" break; } - case 283: { + case 285: { { -#line 1159 "Parser.rl" +#line 1166 "Parser.rl" {te = p+1;{ -#line 1159 "Parser.rl" +#line 1166 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint2c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 7812 "Parser.cpp" +#line 7833 "Parser.cpp" break; } - case 284: { + case 286: { { -#line 1168 "Parser.rl" +#line 1175 "Parser.rl" {te = p+1;{ -#line 1168 "Parser.rl" +#line 1175 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint3c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 7830 "Parser.cpp" +#line 7851 "Parser.cpp" break; } - case 285: { + case 287: { { -#line 1177 "Parser.rl" +#line 1184 "Parser.rl" {te = p+1;{ -#line 1177 "Parser.rl" +#line 1184 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint4c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 7848 "Parser.cpp" +#line 7869 "Parser.cpp" break; } - case 286: { + case 288: { { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" {te = p+1;{ -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7862 "Parser.cpp" +#line 7883 "Parser.cpp" break; } - case 287: { + case 289: { { -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" {te = p+1;{ -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 7875 "Parser.cpp" +#line 7896 "Parser.cpp" break; } - case 288: { + case 290: { { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" {te = p;p = p - 1;{ -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7889 "Parser.cpp" +#line 7910 "Parser.cpp" break; } - case 289: { + case 291: { { -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" {te = p;p = p - 1;{ -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 7902 "Parser.cpp" +#line 7923 "Parser.cpp" break; } - case 290: { + case 292: { { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" {p = ((te))-1; { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7917 "Parser.cpp" +#line 7938 "Parser.cpp" break; } - case 291: { + case 293: { { -#line 1202 "Parser.rl" +#line 1209 "Parser.rl" {te = p+1;{ -#line 1202 "Parser.rl" +#line 1209 "Parser.rl" {top -= 1;cs = stack[top];goto _again;} } }} -#line 7930 "Parser.cpp" +#line 7951 "Parser.cpp" break; } - case 292: { + case 294: { { -#line 1207 "Parser.rl" +#line 1214 "Parser.rl" {te = p+1;{ -#line 1207 "Parser.rl" +#line 1214 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint2c(ts)); @@ -8245,15 +8268,15 @@ static const short _regex_actions[] = { } }} -#line 7945 "Parser.cpp" +#line 7966 "Parser.cpp" break; } - case 293: { + case 295: { { -#line 1213 "Parser.rl" +#line 1220 "Parser.rl" {te = p+1;{ -#line 1213 "Parser.rl" +#line 1220 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint3c(ts)); @@ -8261,15 +8284,15 @@ static const short _regex_actions[] = { } }} -#line 7960 "Parser.cpp" +#line 7981 "Parser.cpp" break; } - case 294: { + case 296: { { -#line 1219 "Parser.rl" +#line 1226 "Parser.rl" {te = p+1;{ -#line 1219 "Parser.rl" +#line 1226 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint4c(ts)); @@ -8277,160 +8300,160 @@ static const short _regex_actions[] = { } }} -#line 7975 "Parser.cpp" +#line 7996 "Parser.cpp" break; } - case 295: { + case 297: { { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" {te = p+1;{ -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7989 "Parser.cpp" +#line 8010 "Parser.cpp" break; } - case 296: { + case 298: { { -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" {te = p+1;{ -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" currentCls->add(*ts); inCharClassEarly = false; } }} -#line 8003 "Parser.cpp" +#line 8024 "Parser.cpp" break; } - case 297: { + case 299: { { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" {te = p;p = p - 1;{ -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 8017 "Parser.cpp" +#line 8038 "Parser.cpp" break; } - case 298: { + case 300: { { -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" {te = p;p = p - 1;{ -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" currentCls->add(*ts); inCharClassEarly = false; } }} -#line 8031 "Parser.cpp" +#line 8052 "Parser.cpp" break; } - case 299: { + case 301: { { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" {p = ((te))-1; { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 8046 "Parser.cpp" +#line 8067 "Parser.cpp" break; } - case 300: { + case 302: { { -#line 1243 "Parser.rl" +#line 1250 "Parser.rl" {te = p+1;{ -#line 1243 "Parser.rl" +#line 1250 "Parser.rl" inComment = false; {cs = 746;goto _again;}} }} -#line 8057 "Parser.cpp" +#line 8078 "Parser.cpp" break; } - case 301: { + case 303: { { #line 1 "-" {te = p+1;}} -#line 8065 "Parser.cpp" +#line 8086 "Parser.cpp" break; } - case 302: { + case 304: { { -#line 1255 "Parser.rl" +#line 1262 "Parser.rl" {te = p+1;{ -#line 1255 "Parser.rl" +#line 1262 "Parser.rl" inComment = false; {cs = 746;goto _again;}} }} -#line 8076 "Parser.cpp" +#line 8097 "Parser.cpp" break; } - case 303: { + case 305: { { #line 1 "-" {te = p+1;}} -#line 8084 "Parser.cpp" +#line 8105 "Parser.cpp" break; } - case 304: { + case 306: { { -#line 1491 "Parser.rl" - {act = 288;}} +#line 1498 "Parser.rl" + {act = 290;}} -#line 8092 "Parser.cpp" +#line 8113 "Parser.cpp" break; } - case 305: { + case 307: { { -#line 1508 "Parser.rl" - {act = 290;}} +#line 1515 "Parser.rl" + {act = 292;}} -#line 8100 "Parser.cpp" +#line 8121 "Parser.cpp" break; } - case 306: { + case 308: { { -#line 1737 "Parser.rl" - {act = 330;}} +#line 1744 "Parser.rl" + {act = 332;}} -#line 8108 "Parser.cpp" +#line 8129 "Parser.cpp" break; } - case 307: { + case 309: { { -#line 362 "Parser.rl" +#line 363 "Parser.rl" {te = p+1;{ -#line 362 "Parser.rl" +#line 363 "Parser.rl" if (sequences.empty()) { throw LocatedParseError("Unmatched parentheses"); @@ -8440,98 +8463,98 @@ static const short _regex_actions[] = { } }} -#line 8125 "Parser.cpp" +#line 8146 "Parser.cpp" break; } - case 308: { + case 310: { { -#line 1274 "Parser.rl" +#line 1281 "Parser.rl" {te = p+1;{ -#line 1274 "Parser.rl" +#line 1281 "Parser.rl" currentSeq->addAlternation(); } }} -#line 8138 "Parser.cpp" +#line 8159 "Parser.cpp" break; } - case 309: { + case 311: { { -#line 1279 "Parser.rl" +#line 1286 "Parser.rl" {te = p+1;{ -#line 1279 "Parser.rl" +#line 1286 "Parser.rl" throw LocatedParseError("POSIX named classes are only " "supported inside a class"); } }} -#line 8152 "Parser.cpp" +#line 8173 "Parser.cpp" break; } - case 310: { + case 312: { { -#line 1286 "Parser.rl" +#line 1293 "Parser.rl" {te = p+1;{ -#line 1286 "Parser.rl" +#line 1293 "Parser.rl" throw LocatedParseError("Unsupported POSIX collating " "element"); } }} -#line 8166 "Parser.cpp" +#line 8187 "Parser.cpp" break; } - case 311: { + case 313: { { -#line 1293 "Parser.rl" +#line 1300 "Parser.rl" {te = p+1;{ -#line 1293 "Parser.rl" +#line 1300 "Parser.rl" - {cs = 838;goto _again;}} + {cs = 839;goto _again;}} }} -#line 8178 "Parser.cpp" +#line 8199 "Parser.cpp" break; } - case 312: { + case 314: { { -#line 1297 "Parser.rl" +#line 1304 "Parser.rl" {te = p+1;{ -#line 1297 "Parser.rl" +#line 1304 "Parser.rl" /* noop */ } }} -#line 8189 "Parser.cpp" +#line 8210 "Parser.cpp" break; } - case 313: { + case 315: { { -#line 1299 "Parser.rl" +#line 1306 "Parser.rl" {te = p+1;{ -#line 1299 "Parser.rl" +#line 1306 "Parser.rl" currentSeq->addComponent(generateComponent(CLASS_ANY, false, mode)); } }} -#line 8202 "Parser.cpp" +#line 8223 "Parser.cpp" break; } - case 314: { + case 316: { { -#line 1303 "Parser.rl" +#line 1310 "Parser.rl" {te = p+1;{ -#line 1303 "Parser.rl" +#line 1310 "Parser.rl" if (mode.utf8) { throw LocatedParseError("\\C is unsupported in UTF8"); @@ -8540,15 +8563,15 @@ static const short _regex_actions[] = { } }} -#line 8218 "Parser.cpp" +#line 8239 "Parser.cpp" break; } - case 315: { + case 317: { { -#line 1317 "Parser.rl" +#line 1324 "Parser.rl" {te = p+1;{ -#line 1317 "Parser.rl" +#line 1324 "Parser.rl" if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_NONGREEDY)) { @@ -8557,15 +8580,15 @@ static const short _regex_actions[] = { } }} -#line 8234 "Parser.cpp" +#line 8255 "Parser.cpp" break; } - case 316: { + case 318: { { -#line 1324 "Parser.rl" +#line 1331 "Parser.rl" {te = p+1;{ -#line 1324 "Parser.rl" +#line 1331 "Parser.rl" if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_POSSESSIVE)) { @@ -8574,15 +8597,15 @@ static const short _regex_actions[] = { } }} -#line 8250 "Parser.cpp" +#line 8271 "Parser.cpp" break; } - case 317: { + case 319: { { -#line 1338 "Parser.rl" +#line 1345 "Parser.rl" {te = p+1;{ -#line 1338 "Parser.rl" +#line 1345 "Parser.rl" if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_NONGREEDY)) { @@ -8591,15 +8614,15 @@ static const short _regex_actions[] = { } }} -#line 8266 "Parser.cpp" +#line 8287 "Parser.cpp" break; } - case 318: { + case 320: { { -#line 1345 "Parser.rl" +#line 1352 "Parser.rl" {te = p+1;{ -#line 1345 "Parser.rl" +#line 1352 "Parser.rl" if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_POSSESSIVE)) { @@ -8608,15 +8631,15 @@ static const short _regex_actions[] = { } }} -#line 8282 "Parser.cpp" +#line 8303 "Parser.cpp" break; } - case 319: { + case 321: { { -#line 1359 "Parser.rl" +#line 1366 "Parser.rl" {te = p+1;{ -#line 1359 "Parser.rl" +#line 1366 "Parser.rl" if (!currentSeq->addRepeat( 0, 1, ComponentRepeat::REPEAT_NONGREEDY)) { @@ -8625,15 +8648,15 @@ static const short _regex_actions[] = { } }} -#line 8298 "Parser.cpp" +#line 8319 "Parser.cpp" break; } - case 320: { + case 322: { { -#line 1366 "Parser.rl" +#line 1373 "Parser.rl" {te = p+1;{ -#line 1366 "Parser.rl" +#line 1373 "Parser.rl" if (!currentSeq->addRepeat( 0, 1, ComponentRepeat::REPEAT_POSSESSIVE)) { @@ -8642,15 +8665,15 @@ static const short _regex_actions[] = { } }} -#line 8314 "Parser.cpp" +#line 8335 "Parser.cpp" break; } - case 321: { + case 323: { { -#line 1383 "Parser.rl" +#line 1390 "Parser.rl" {te = p+1;{ -#line 1383 "Parser.rl" +#line 1390 "Parser.rl" if (repeatN > repeatM || repeatM == 0) { throwInvalidRepeat(); @@ -8662,15 +8685,15 @@ static const short _regex_actions[] = { } }} -#line 8333 "Parser.cpp" +#line 8354 "Parser.cpp" break; } - case 322: { + case 324: { { -#line 1393 "Parser.rl" +#line 1400 "Parser.rl" {te = p+1;{ -#line 1393 "Parser.rl" +#line 1400 "Parser.rl" if (repeatN > repeatM || repeatM == 0) { throwInvalidRepeat(); @@ -8682,31 +8705,31 @@ static const short _regex_actions[] = { } }} -#line 8352 "Parser.cpp" +#line 8373 "Parser.cpp" break; } - case 323: { + case 325: { { -#line 322 "Parser.rl" +#line 323 "Parser.rl" {te = p+1;{ -#line 322 "Parser.rl" +#line 323 "Parser.rl" inComment = true; - {cs = 849;goto _again;}} + {cs = 850;goto _again;}} }} -#line 8365 "Parser.cpp" +#line 8386 "Parser.cpp" break; } - case 324: { + case 326: { { -#line 1410 "Parser.rl" +#line 1417 "Parser.rl" {te = p+1;{ -#line 1410 "Parser.rl" +#line 1417 "Parser.rl" {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -8716,27 +8739,27 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 787;goto _again;}} }} -#line 8384 "Parser.cpp" +#line 8405 "Parser.cpp" break; } - case 325: { + case 327: { { -#line 1414 "Parser.rl" +#line 1421 "Parser.rl" {te = p+1;{ -#line 1414 "Parser.rl" +#line 1421 "Parser.rl" assert(0); {p += 1; goto _out; } } }} -#line 8395 "Parser.cpp" +#line 8416 "Parser.cpp" break; } - case 326: { + case 328: { { -#line 1421 "Parser.rl" +#line 1428 "Parser.rl" {te = p+1;{ -#line 1421 "Parser.rl" +#line 1428 "Parser.rl" auto bound = mode.multiline ? ComponentBoundary::BEGIN_LINE : ComponentBoundary::BEGIN_STRING; @@ -8744,15 +8767,15 @@ static const short _regex_actions[] = { } }} -#line 8410 "Parser.cpp" +#line 8431 "Parser.cpp" break; } - case 327: { + case 329: { { -#line 1428 "Parser.rl" +#line 1435 "Parser.rl" {te = p+1;{ -#line 1428 "Parser.rl" +#line 1435 "Parser.rl" auto bound = mode.multiline ? ComponentBoundary::END_LINE : ComponentBoundary::END_STRING_OPTIONAL_LF; @@ -8760,188 +8783,188 @@ static const short _regex_actions[] = { } }} -#line 8425 "Parser.cpp" +#line 8446 "Parser.cpp" break; } - case 328: { + case 330: { { -#line 1434 "Parser.rl" +#line 1441 "Parser.rl" {te = p+1;{ -#line 1434 "Parser.rl" +#line 1441 "Parser.rl" auto bound = ComponentBoundary::BEGIN_STRING; currentSeq->addComponent(std::make_unique(bound)); } }} -#line 8439 "Parser.cpp" +#line 8460 "Parser.cpp" break; } - case 329: { + case 331: { { -#line 1439 "Parser.rl" +#line 1446 "Parser.rl" {te = p+1;{ -#line 1439 "Parser.rl" +#line 1446 "Parser.rl" auto bound = ComponentBoundary::END_STRING_OPTIONAL_LF; currentSeq->addComponent(std::make_unique(bound)); } }} -#line 8453 "Parser.cpp" +#line 8474 "Parser.cpp" break; } - case 330: { + case 332: { { -#line 1444 "Parser.rl" +#line 1451 "Parser.rl" {te = p+1;{ -#line 1444 "Parser.rl" +#line 1451 "Parser.rl" auto bound = ComponentBoundary::END_STRING; currentSeq->addComponent(std::make_unique(bound)); } }} -#line 8467 "Parser.cpp" +#line 8488 "Parser.cpp" break; } - case 331: { + case 333: { { -#line 1449 "Parser.rl" +#line 1456 "Parser.rl" {te = p+1;{ -#line 1449 "Parser.rl" +#line 1456 "Parser.rl" currentSeq->addComponent( std::make_unique(ts - ptr, false, mode)); } }} -#line 8481 "Parser.cpp" +#line 8502 "Parser.cpp" break; } - case 332: { + case 334: { { -#line 1454 "Parser.rl" +#line 1461 "Parser.rl" {te = p+1;{ -#line 1454 "Parser.rl" +#line 1461 "Parser.rl" currentSeq->addComponent( std::make_unique(ts - ptr, true, mode)); } }} -#line 8495 "Parser.cpp" +#line 8516 "Parser.cpp" break; } - case 333: { + case 335: { { -#line 1464 "Parser.rl" +#line 1471 "Parser.rl" {te = p+1;{ -#line 1464 "Parser.rl" +#line 1471 "Parser.rl" addLiteral(currentSeq, '\x09', mode); } }} -#line 8508 "Parser.cpp" +#line 8529 "Parser.cpp" break; } - case 334: { + case 336: { { -#line 1468 "Parser.rl" +#line 1475 "Parser.rl" {te = p+1;{ -#line 1468 "Parser.rl" +#line 1475 "Parser.rl" addLiteral(currentSeq, '\x0a', mode); } }} -#line 8521 "Parser.cpp" +#line 8542 "Parser.cpp" break; } - case 335: { + case 337: { { -#line 1472 "Parser.rl" +#line 1479 "Parser.rl" {te = p+1;{ -#line 1472 "Parser.rl" +#line 1479 "Parser.rl" addLiteral(currentSeq, '\x0d', mode); } }} -#line 8534 "Parser.cpp" +#line 8555 "Parser.cpp" break; } - case 336: { + case 338: { { -#line 1476 "Parser.rl" +#line 1483 "Parser.rl" {te = p+1;{ -#line 1476 "Parser.rl" +#line 1483 "Parser.rl" addLiteral(currentSeq, '\x0c', mode); } }} -#line 8547 "Parser.cpp" +#line 8568 "Parser.cpp" break; } - case 337: { + case 339: { { -#line 1480 "Parser.rl" +#line 1487 "Parser.rl" {te = p+1;{ -#line 1480 "Parser.rl" +#line 1487 "Parser.rl" addLiteral(currentSeq, '\x07', mode); } }} -#line 8560 "Parser.cpp" +#line 8581 "Parser.cpp" break; } - case 338: { + case 340: { { -#line 1484 "Parser.rl" +#line 1491 "Parser.rl" {te = p+1;{ -#line 1484 "Parser.rl" +#line 1491 "Parser.rl" addLiteral(currentSeq, '\x1b', mode); } }} -#line 8573 "Parser.cpp" +#line 8594 "Parser.cpp" break; } - case 339: { + case 341: { { -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" {te = p+1;{ -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" addLiteral(currentSeq, octAccumulator, mode); } }} -#line 8586 "Parser.cpp" +#line 8607 "Parser.cpp" break; } - case 340: { + case 342: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p+1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -8950,15 +8973,15 @@ static const short _regex_actions[] = { } }} -#line 8602 "Parser.cpp" +#line 8623 "Parser.cpp" break; } - case 341: { + case 343: { { -#line 486 "Parser.rl" +#line 487 "Parser.rl" {te = p+1;{ -#line 486 "Parser.rl" +#line 487 "Parser.rl" // Accumulator is a negative offset. if (accumulator == 0) { @@ -8972,15 +8995,15 @@ static const short _regex_actions[] = { } }} -#line 8623 "Parser.cpp" +#line 8644 "Parser.cpp" break; } - case 342: { + case 344: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p+1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -8989,15 +9012,15 @@ static const short _regex_actions[] = { } }} -#line 8639 "Parser.cpp" +#line 8660 "Parser.cpp" break; } - case 343: { + case 345: { { -#line 486 "Parser.rl" +#line 487 "Parser.rl" {te = p+1;{ -#line 486 "Parser.rl" +#line 487 "Parser.rl" // Accumulator is a negative offset. if (accumulator == 0) { @@ -9011,85 +9034,85 @@ static const short _regex_actions[] = { } }} -#line 8660 "Parser.cpp" +#line 8681 "Parser.cpp" break; } - case 344: { + case 346: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8673 "Parser.cpp" +#line 8694 "Parser.cpp" break; } - case 345: { + case 347: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8686 "Parser.cpp" +#line 8707 "Parser.cpp" break; } - case 346: { + case 348: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8699 "Parser.cpp" +#line 8720 "Parser.cpp" break; } - case 347: { + case 349: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8712 "Parser.cpp" +#line 8733 "Parser.cpp" break; } - case 348: { + case 350: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8725 "Parser.cpp" +#line 8746 "Parser.cpp" break; } - case 349: { + case 351: { { -#line 1549 "Parser.rl" +#line 1556 "Parser.rl" {te = p+1;{ -#line 1549 "Parser.rl" +#line 1556 "Parser.rl" ostringstream str; str << "Onigiruma subroutine call at index " << ts - ptr << @@ -9098,15 +9121,15 @@ static const short _regex_actions[] = { } }} -#line 8741 "Parser.cpp" +#line 8762 "Parser.cpp" break; } - case 350: { + case 352: { { -#line 1560 "Parser.rl" +#line 1567 "Parser.rl" {te = p+1;{ -#line 1560 "Parser.rl" +#line 1567 "Parser.rl" string oct(ts + 3, te - ts - 4); unsigned long val; @@ -9122,29 +9145,29 @@ static const short _regex_actions[] = { } }} -#line 8764 "Parser.cpp" +#line 8785 "Parser.cpp" break; } - case 351: { + case 353: { { -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" {te = p+1;{ -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" addEscapedHex(currentSeq, accumulator, mode); } }} -#line 8777 "Parser.cpp" +#line 8798 "Parser.cpp" break; } - case 352: { + case 354: { { -#line 1582 "Parser.rl" +#line 1589 "Parser.rl" {te = p+1;{ -#line 1582 "Parser.rl" +#line 1589 "Parser.rl" string hex(ts + 3, te - ts - 4); unsigned long val; @@ -9160,15 +9183,15 @@ static const short _regex_actions[] = { } }} -#line 8800 "Parser.cpp" +#line 8821 "Parser.cpp" break; } - case 353: { + case 355: { { -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" {te = p+1;{ -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -9180,15 +9203,15 @@ static const short _regex_actions[] = { } }} -#line 8819 "Parser.cpp" +#line 8840 "Parser.cpp" break; } - case 354: { + case 356: { { -#line 1610 "Parser.rl" +#line 1617 "Parser.rl" {te = p+1;{ -#line 1610 "Parser.rl" +#line 1617 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -9197,172 +9220,172 @@ static const short _regex_actions[] = { } }} -#line 8835 "Parser.cpp" +#line 8856 "Parser.cpp" break; } - case 355: { + case 357: { { -#line 1618 "Parser.rl" +#line 1625 "Parser.rl" {te = p+1;{ -#line 1618 "Parser.rl" +#line 1625 "Parser.rl" auto cc = generateComponent(CLASS_WORD, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8849 "Parser.cpp" +#line 8870 "Parser.cpp" break; } - case 356: { + case 358: { { -#line 1623 "Parser.rl" +#line 1630 "Parser.rl" {te = p+1;{ -#line 1623 "Parser.rl" +#line 1630 "Parser.rl" auto cc = generateComponent(CLASS_WORD, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8863 "Parser.cpp" +#line 8884 "Parser.cpp" break; } - case 357: { + case 359: { { -#line 1628 "Parser.rl" +#line 1635 "Parser.rl" {te = p+1;{ -#line 1628 "Parser.rl" +#line 1635 "Parser.rl" auto cc = generateComponent(CLASS_SPACE, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8877 "Parser.cpp" +#line 8898 "Parser.cpp" break; } - case 358: { + case 360: { { -#line 1633 "Parser.rl" +#line 1640 "Parser.rl" {te = p+1;{ -#line 1633 "Parser.rl" +#line 1640 "Parser.rl" auto cc = generateComponent(CLASS_SPACE, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8891 "Parser.cpp" +#line 8912 "Parser.cpp" break; } - case 359: { + case 361: { { -#line 1638 "Parser.rl" +#line 1645 "Parser.rl" {te = p+1;{ -#line 1638 "Parser.rl" +#line 1645 "Parser.rl" auto cc = generateComponent(CLASS_DIGIT, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8905 "Parser.cpp" +#line 8926 "Parser.cpp" break; } - case 360: { + case 362: { { -#line 1643 "Parser.rl" +#line 1650 "Parser.rl" {te = p+1;{ -#line 1643 "Parser.rl" +#line 1650 "Parser.rl" auto cc = generateComponent(CLASS_DIGIT, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8919 "Parser.cpp" +#line 8940 "Parser.cpp" break; } - case 361: { + case 363: { { -#line 1648 "Parser.rl" +#line 1655 "Parser.rl" {te = p+1;{ -#line 1648 "Parser.rl" +#line 1655 "Parser.rl" auto cc = generateComponent(CLASS_HORZ, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8933 "Parser.cpp" +#line 8954 "Parser.cpp" break; } - case 362: { + case 364: { { -#line 1653 "Parser.rl" +#line 1660 "Parser.rl" {te = p+1;{ -#line 1653 "Parser.rl" +#line 1660 "Parser.rl" auto cc = generateComponent(CLASS_HORZ, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8947 "Parser.cpp" +#line 8968 "Parser.cpp" break; } - case 363: { + case 365: { { -#line 1658 "Parser.rl" +#line 1665 "Parser.rl" {te = p+1;{ -#line 1658 "Parser.rl" +#line 1665 "Parser.rl" auto cc = generateComponent(CLASS_VERT, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8961 "Parser.cpp" +#line 8982 "Parser.cpp" break; } - case 364: { + case 366: { { -#line 1663 "Parser.rl" +#line 1670 "Parser.rl" {te = p+1;{ -#line 1663 "Parser.rl" +#line 1670 "Parser.rl" auto cc = generateComponent(CLASS_VERT, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8975 "Parser.cpp" +#line 8996 "Parser.cpp" break; } - case 365: { + case 367: { { -#line 1668 "Parser.rl" +#line 1675 "Parser.rl" {te = p+1;{ -#line 1668 "Parser.rl" +#line 1675 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9372,22 +9395,22 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 8999 "Parser.cpp" +#line 9020 "Parser.cpp" break; } - case 366: { + case 368: { { -#line 1676 "Parser.rl" +#line 1683 "Parser.rl" {te = p+1;{ -#line 1676 "Parser.rl" +#line 1683 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9397,22 +9420,22 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 9023 "Parser.cpp" +#line 9044 "Parser.cpp" break; } - case 367: { + case 369: { { -#line 1684 "Parser.rl" +#line 1691 "Parser.rl" {te = p+1;{ -#line 1684 "Parser.rl" +#line 1691 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9422,22 +9445,22 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 9047 "Parser.cpp" +#line 9068 "Parser.cpp" break; } - case 368: { + case 370: { { -#line 1692 "Parser.rl" +#line 1699 "Parser.rl" {te = p+1;{ -#line 1692 "Parser.rl" +#line 1699 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9447,15 +9470,15 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 9071 "Parser.cpp" +#line 9092 "Parser.cpp" break; } - case 369: { + case 371: { { -#line 1704 "Parser.rl" +#line 1711 "Parser.rl" {te = p+1;{ -#line 1704 "Parser.rl" +#line 1711 "Parser.rl" ostringstream str; str << "\\R at index " << ts - ptr << " not supported."; @@ -9463,15 +9486,15 @@ static const short _regex_actions[] = { } }} -#line 9086 "Parser.cpp" +#line 9107 "Parser.cpp" break; } - case 370: { + case 372: { { -#line 1711 "Parser.rl" +#line 1718 "Parser.rl" {te = p+1;{ -#line 1711 "Parser.rl" +#line 1718 "Parser.rl" ostringstream str; str << "\\K at index " << ts - ptr << " not supported."; @@ -9479,15 +9502,15 @@ static const short _regex_actions[] = { } }} -#line 9101 "Parser.cpp" +#line 9122 "Parser.cpp" break; } - case 371: { + case 373: { { -#line 1726 "Parser.rl" +#line 1733 "Parser.rl" {te = p+1;{ -#line 1726 "Parser.rl" +#line 1733 "Parser.rl" ostringstream str; str << "\\G at index " << ts - ptr << " not supported."; @@ -9495,72 +9518,72 @@ static const short _regex_actions[] = { } }} -#line 9116 "Parser.cpp" +#line 9137 "Parser.cpp" break; } - case 372: { + case 374: { { -#line 1732 "Parser.rl" +#line 1739 "Parser.rl" {te = p+1;{ -#line 1732 "Parser.rl" +#line 1739 "Parser.rl" currentSeq->addComponent(std::make_unique(ts - ptr, mode)); } }} -#line 9129 "Parser.cpp" +#line 9150 "Parser.cpp" break; } - case 373: { + case 375: { { -#line 1737 "Parser.rl" +#line 1744 "Parser.rl" {te = p+1;{ -#line 1737 "Parser.rl" +#line 1744 "Parser.rl" addLiteral(currentSeq, *(ts + 1), mode); } }} -#line 9142 "Parser.cpp" +#line 9163 "Parser.cpp" break; } - case 374: { + case 376: { { -#line 316 "Parser.rl" +#line 317 "Parser.rl" {te = p+1;{ -#line 316 "Parser.rl" +#line 317 "Parser.rl" inComment = true; - {cs = 848;goto _again;}} + {cs = 849;goto _again;}} }} -#line 9155 "Parser.cpp" +#line 9176 "Parser.cpp" break; } - case 375: { + case 377: { { -#line 433 "Parser.rl" +#line 434 "Parser.rl" {te = p+1;{ -#line 433 "Parser.rl" +#line 434 "Parser.rl" mode = newMode; currentSeq->addComponent(std::make_unique()); } }} -#line 9169 "Parser.cpp" +#line 9190 "Parser.cpp" break; } - case 376: { + case 378: { { -#line 355 "Parser.rl" +#line 356 "Parser.rl" {te = p+1;{ -#line 355 "Parser.rl" +#line 356 "Parser.rl" PUSH_SEQUENCE; mode = newMode; @@ -9569,15 +9592,15 @@ static const short _regex_actions[] = { } }} -#line 9185 "Parser.cpp" +#line 9206 "Parser.cpp" break; } - case 377: { + case 379: { { -#line 369 "Parser.rl" +#line 370 "Parser.rl" {te = p+1;{ -#line 369 "Parser.rl" +#line 370 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9586,15 +9609,15 @@ static const short _regex_actions[] = { } }} -#line 9201 "Parser.cpp" +#line 9222 "Parser.cpp" break; } - case 378: { + case 380: { { -#line 375 "Parser.rl" +#line 376 "Parser.rl" {te = p+1;{ -#line 375 "Parser.rl" +#line 376 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9603,15 +9626,15 @@ static const short _regex_actions[] = { } }} -#line 9217 "Parser.cpp" +#line 9238 "Parser.cpp" break; } - case 379: { + case 381: { { -#line 381 "Parser.rl" +#line 382 "Parser.rl" {te = p+1;{ -#line 381 "Parser.rl" +#line 382 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9620,15 +9643,15 @@ static const short _regex_actions[] = { } }} -#line 9233 "Parser.cpp" +#line 9254 "Parser.cpp" break; } - case 380: { + case 382: { { -#line 387 "Parser.rl" +#line 388 "Parser.rl" {te = p+1;{ -#line 387 "Parser.rl" +#line 388 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9637,43 +9660,43 @@ static const short _regex_actions[] = { } }} -#line 9249 "Parser.cpp" +#line 9270 "Parser.cpp" break; } - case 381: { + case 383: { { -#line 393 "Parser.rl" +#line 394 "Parser.rl" {te = p+1;{ -#line 393 "Parser.rl" +#line 394 "Parser.rl" throw LocatedParseError("Embedded code is not supported"); } }} -#line 9262 "Parser.cpp" +#line 9283 "Parser.cpp" break; } - case 382: { + case 384: { { -#line 393 "Parser.rl" +#line 394 "Parser.rl" {te = p+1;{ -#line 393 "Parser.rl" +#line 394 "Parser.rl" throw LocatedParseError("Embedded code is not supported"); } }} -#line 9275 "Parser.cpp" +#line 9296 "Parser.cpp" break; } - case 383: { + case 385: { { -#line 416 "Parser.rl" +#line 417 "Parser.rl" {te = p+1;{ -#line 416 "Parser.rl" +#line 417 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9681,15 +9704,15 @@ static const short _regex_actions[] = { } }} -#line 9290 "Parser.cpp" +#line 9311 "Parser.cpp" break; } - case 384: { + case 386: { { -#line 336 "Parser.rl" +#line 337 "Parser.rl" {te = p+1;{ -#line 336 "Parser.rl" +#line 337 "Parser.rl" assert(!label.empty()); // should be guaranteed by machine char c = *label.begin(); @@ -9703,145 +9726,145 @@ static const short _regex_actions[] = { auto seq = std::make_unique(); seq->setCaptureIndex(groupIndex++); seq->setCaptureName(label); - currentSeq = enterSequence(currentSeq, move(seq)); + currentSeq = enterSequence(currentSeq, std::move(seq)); } }} -#line 9315 "Parser.cpp" +#line 9336 "Parser.cpp" break; } - case 385: { + case 387: { { -#line 399 "Parser.rl" +#line 400 "Parser.rl" {te = p+1;{ -#line 399 "Parser.rl" +#line 400 "Parser.rl" throw LocatedParseError("Subpattern reference unsupported"); } }} -#line 9328 "Parser.cpp" +#line 9349 "Parser.cpp" break; } - case 386: { + case 388: { { -#line 399 "Parser.rl" +#line 400 "Parser.rl" {te = p+1;{ -#line 399 "Parser.rl" +#line 400 "Parser.rl" throw LocatedParseError("Subpattern reference unsupported"); } }} -#line 9341 "Parser.cpp" +#line 9362 "Parser.cpp" break; } - case 387: { + case 389: { { -#line 1783 "Parser.rl" +#line 1790 "Parser.rl" {te = p+1;{ -#line 1783 "Parser.rl" +#line 1790 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKAHEAD, ComponentAssertion::POS); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9361 "Parser.cpp" +#line 9382 "Parser.cpp" break; } - case 388: { + case 390: { { -#line 1794 "Parser.rl" +#line 1801 "Parser.rl" {te = p+1;{ -#line 1794 "Parser.rl" +#line 1801 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKAHEAD, ComponentAssertion::NEG); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9381 "Parser.cpp" +#line 9402 "Parser.cpp" break; } - case 389: { + case 391: { { -#line 1805 "Parser.rl" +#line 1812 "Parser.rl" {te = p+1;{ -#line 1805 "Parser.rl" +#line 1812 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKBEHIND, ComponentAssertion::POS); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9401 "Parser.cpp" +#line 9422 "Parser.cpp" break; } - case 390: { + case 392: { { -#line 1816 "Parser.rl" +#line 1823 "Parser.rl" {te = p+1;{ -#line 1816 "Parser.rl" +#line 1823 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKBEHIND, ComponentAssertion::NEG); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9421 "Parser.cpp" +#line 9442 "Parser.cpp" break; } - case 391: { + case 393: { { -#line 1828 "Parser.rl" +#line 1835 "Parser.rl" {te = p+1;{ -#line 1828 "Parser.rl" +#line 1835 "Parser.rl" throw LocatedParseError("Pattern recursion not supported"); } }} -#line 9434 "Parser.cpp" +#line 9455 "Parser.cpp" break; } - case 392: { + case 394: { { -#line 402 "Parser.rl" +#line 403 "Parser.rl" {te = p+1;{ -#line 402 "Parser.rl" +#line 403 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -9852,15 +9875,15 @@ static const short _regex_actions[] = { } }} -#line 9452 "Parser.cpp" +#line 9473 "Parser.cpp" break; } - case 393: { + case 395: { { -#line 410 "Parser.rl" +#line 411 "Parser.rl" {te = p+1;{ -#line 410 "Parser.rl" +#line 411 "Parser.rl" PUSH_SEQUENCE; assert(!label.empty()); @@ -9869,15 +9892,15 @@ static const short _regex_actions[] = { } }} -#line 9468 "Parser.cpp" +#line 9489 "Parser.cpp" break; } - case 394: { + case 396: { { -#line 1844 "Parser.rl" +#line 1851 "Parser.rl" {te = p+1;{ -#line 1844 "Parser.rl" +#line 1851 "Parser.rl" ostringstream str; str << "Callout at index " << ts - ptr << " not supported."; @@ -9885,101 +9908,101 @@ static const short _regex_actions[] = { } }} -#line 9483 "Parser.cpp" +#line 9504 "Parser.cpp" break; } - case 395: { + case 397: { { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" {te = p+1;{ -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" throw LocatedParseError("Unrecognised character after (?"); } }} -#line 9496 "Parser.cpp" +#line 9517 "Parser.cpp" break; } - case 396: { + case 398: { { -#line 1857 "Parser.rl" +#line 1864 "Parser.rl" {te = p+1;{ -#line 1857 "Parser.rl" +#line 1864 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint2c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 9514 "Parser.cpp" +#line 9535 "Parser.cpp" break; } - case 397: { + case 399: { { -#line 1866 "Parser.rl" +#line 1873 "Parser.rl" {te = p+1;{ -#line 1866 "Parser.rl" +#line 1873 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint3c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 9532 "Parser.cpp" +#line 9553 "Parser.cpp" break; } - case 398: { + case 400: { { -#line 1875 "Parser.rl" +#line 1882 "Parser.rl" {te = p+1;{ -#line 1875 "Parser.rl" +#line 1882 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint4c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 9550 "Parser.cpp" +#line 9571 "Parser.cpp" break; } - case 399: { + case 401: { { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" {te = p+1;{ -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 9564 "Parser.cpp" +#line 9585 "Parser.cpp" break; } - case 400: { + case 402: { { -#line 1893 "Parser.rl" +#line 1900 "Parser.rl" {te = p+1;{ -#line 1893 "Parser.rl" +#line 1900 "Parser.rl" if (mode.ignore_space == false) { addLiteral(currentSeq, *ts, mode); @@ -9987,46 +10010,46 @@ static const short _regex_actions[] = { } }} -#line 9579 "Parser.cpp" +#line 9600 "Parser.cpp" break; } - case 401: { + case 403: { { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" {te = p+1;{ -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 9592 "Parser.cpp" +#line 9613 "Parser.cpp" break; } - case 402: { + case 404: { { -#line 328 "Parser.rl" +#line 329 "Parser.rl" {te = p;p = p - 1;{ -#line 328 "Parser.rl" +#line 329 "Parser.rl" PUSH_SEQUENCE; auto seq = std::make_unique(); seq->setCaptureIndex(groupIndex++); - currentSeq = enterSequence(currentSeq, move(seq)); + currentSeq = enterSequence(currentSeq, std::move(seq)); } }} -#line 9608 "Parser.cpp" +#line 9629 "Parser.cpp" break; } - case 403: { + case 405: { { -#line 421 "Parser.rl" +#line 422 "Parser.rl" {te = p;p = p - 1;{ -#line 421 "Parser.rl" +#line 422 "Parser.rl" assert(!currentCls); assert(!inCharClass); // not reentrant @@ -10034,18 +10057,18 @@ static const short _regex_actions[] = { inCharClass = true; inCharClassEarly = true; currentClsBegin = ts; - {cs = 836;goto _again;}} + {cs = 837;goto _again;}} }} -#line 9626 "Parser.cpp" +#line 9647 "Parser.cpp" break; } - case 404: { + case 406: { { -#line 1310 "Parser.rl" +#line 1317 "Parser.rl" {te = p;p = p - 1;{ -#line 1310 "Parser.rl" +#line 1317 "Parser.rl" if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_GREEDY)) { @@ -10054,15 +10077,15 @@ static const short _regex_actions[] = { } }} -#line 9642 "Parser.cpp" +#line 9663 "Parser.cpp" break; } - case 405: { + case 407: { { -#line 1331 "Parser.rl" +#line 1338 "Parser.rl" {te = p;p = p - 1;{ -#line 1331 "Parser.rl" +#line 1338 "Parser.rl" if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_GREEDY)) { @@ -10071,15 +10094,15 @@ static const short _regex_actions[] = { } }} -#line 9658 "Parser.cpp" +#line 9679 "Parser.cpp" break; } - case 406: { + case 408: { { -#line 1352 "Parser.rl" +#line 1359 "Parser.rl" {te = p;p = p - 1;{ -#line 1352 "Parser.rl" +#line 1359 "Parser.rl" if (!currentSeq->addRepeat( 0, 1, ComponentRepeat::REPEAT_GREEDY)) { @@ -10088,15 +10111,15 @@ static const short _regex_actions[] = { } }} -#line 9674 "Parser.cpp" +#line 9695 "Parser.cpp" break; } - case 407: { + case 409: { { -#line 1373 "Parser.rl" +#line 1380 "Parser.rl" {te = p;p = p - 1;{ -#line 1373 "Parser.rl" +#line 1380 "Parser.rl" if (repeatN > repeatM || repeatM == 0) { throwInvalidRepeat(); @@ -10108,29 +10131,29 @@ static const short _regex_actions[] = { } }} -#line 9693 "Parser.cpp" +#line 9714 "Parser.cpp" break; } - case 408: { + case 410: { { -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" {te = p;p = p - 1;{ -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" addLiteral(currentSeq, octAccumulator, mode); } }} -#line 9706 "Parser.cpp" +#line 9727 "Parser.cpp" break; } - case 409: { + case 411: { { -#line 1491 "Parser.rl" +#line 1498 "Parser.rl" {te = p;p = p - 1;{ -#line 1491 "Parser.rl" +#line 1498 "Parser.rl" // If there are enough capturing sub expressions, this may be // a back reference @@ -10143,15 +10166,15 @@ static const short _regex_actions[] = { } }} -#line 9726 "Parser.cpp" +#line 9747 "Parser.cpp" break; } - case 410: { + case 412: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p;p = p - 1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -10160,15 +10183,15 @@ static const short _regex_actions[] = { } }} -#line 9742 "Parser.cpp" +#line 9763 "Parser.cpp" break; } - case 411: { + case 413: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p;p = p - 1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -10177,15 +10200,15 @@ static const short _regex_actions[] = { } }} -#line 9758 "Parser.cpp" +#line 9779 "Parser.cpp" break; } - case 412: { + case 414: { { -#line 486 "Parser.rl" +#line 487 "Parser.rl" {te = p;p = p - 1;{ -#line 486 "Parser.rl" +#line 487 "Parser.rl" // Accumulator is a negative offset. if (accumulator == 0) { @@ -10199,71 +10222,71 @@ static const short _regex_actions[] = { } }} -#line 9779 "Parser.cpp" +#line 9800 "Parser.cpp" break; } - case 413: { + case 415: { { -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" {te = p;p = p - 1;{ -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" throw LocatedParseError("Invalid reference after \\g"); } }} -#line 9792 "Parser.cpp" +#line 9813 "Parser.cpp" break; } - case 414: { + case 416: { { -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" {te = p;p = p - 1;{ -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 9805 "Parser.cpp" +#line 9826 "Parser.cpp" break; } - case 415: { + case 417: { { -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" {te = p;p = p - 1;{ -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" addEscapedHex(currentSeq, accumulator, mode); } }} -#line 9818 "Parser.cpp" +#line 9839 "Parser.cpp" break; } - case 416: { + case 418: { { -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" {te = p;p = p - 1;{ -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 9831 "Parser.cpp" +#line 9852 "Parser.cpp" break; } - case 417: { + case 419: { { -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" {te = p;p = p - 1;{ -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -10275,39 +10298,39 @@ static const short _regex_actions[] = { } }} -#line 9850 "Parser.cpp" +#line 9871 "Parser.cpp" break; } - case 418: { + case 420: { { -#line 1700 "Parser.rl" +#line 1707 "Parser.rl" {te = p;p = p - 1;{ -#line 1700 "Parser.rl" +#line 1707 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 9861 "Parser.cpp" +#line 9882 "Parser.cpp" break; } - case 419: { + case 421: { { -#line 1701 "Parser.rl" +#line 1708 "Parser.rl" {te = p;p = p - 1;{ -#line 1701 "Parser.rl" +#line 1708 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 9872 "Parser.cpp" +#line 9893 "Parser.cpp" break; } - case 420: { + case 422: { { -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" {te = p;p = p - 1;{ -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" ostringstream str; str << "\\k at index " << ts - ptr << " not supported."; @@ -10315,15 +10338,15 @@ static const short _regex_actions[] = { } }} -#line 9887 "Parser.cpp" +#line 9908 "Parser.cpp" break; } - case 421: { + case 423: { { -#line 1742 "Parser.rl" +#line 1749 "Parser.rl" {te = p;p = p - 1;{ -#line 1742 "Parser.rl" +#line 1749 "Parser.rl" assert(ts + 1 == pe); ostringstream str; @@ -10332,91 +10355,91 @@ static const short _regex_actions[] = { } }} -#line 9903 "Parser.cpp" +#line 9924 "Parser.cpp" break; } - case 422: { + case 424: { { -#line 396 "Parser.rl" +#line 397 "Parser.rl" {te = p;p = p - 1;{ -#line 396 "Parser.rl" +#line 397 "Parser.rl" throw LocatedParseError("Conditional subpattern unsupported"); } }} -#line 9916 "Parser.cpp" +#line 9937 "Parser.cpp" break; } - case 423: { + case 425: { { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" {te = p;p = p - 1;{ -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" throw LocatedParseError("Unrecognised character after (?"); } }} -#line 9929 "Parser.cpp" +#line 9950 "Parser.cpp" break; } - case 424: { + case 426: { { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" {te = p;p = p - 1;{ -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 9943 "Parser.cpp" +#line 9964 "Parser.cpp" break; } - case 425: { + case 427: { { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" {te = p;p = p - 1;{ -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 9956 "Parser.cpp" +#line 9977 "Parser.cpp" break; } - case 426: { + case 428: { { -#line 328 "Parser.rl" +#line 329 "Parser.rl" {p = ((te))-1; { -#line 328 "Parser.rl" +#line 329 "Parser.rl" PUSH_SEQUENCE; auto seq = std::make_unique(); seq->setCaptureIndex(groupIndex++); - currentSeq = enterSequence(currentSeq, move(seq)); + currentSeq = enterSequence(currentSeq, std::move(seq)); } }} -#line 9973 "Parser.cpp" +#line 9994 "Parser.cpp" break; } - case 427: { + case 429: { { -#line 421 "Parser.rl" +#line 422 "Parser.rl" {p = ((te))-1; { -#line 421 "Parser.rl" +#line 422 "Parser.rl" assert(!currentCls); assert(!inCharClass); // not reentrant @@ -10424,64 +10447,64 @@ static const short _regex_actions[] = { inCharClass = true; inCharClassEarly = true; currentClsBegin = ts; - {cs = 836;goto _again;}} + {cs = 837;goto _again;}} }} -#line 9992 "Parser.cpp" +#line 10013 "Parser.cpp" break; } - case 428: { + case 430: { { -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" {p = ((te))-1; { -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" throw LocatedParseError("Invalid reference after \\g"); } }} -#line 10006 "Parser.cpp" +#line 10027 "Parser.cpp" break; } - case 429: { + case 431: { { -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" {p = ((te))-1; { -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 10020 "Parser.cpp" +#line 10041 "Parser.cpp" break; } - case 430: { + case 432: { { -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" {p = ((te))-1; { -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 10034 "Parser.cpp" +#line 10055 "Parser.cpp" break; } - case 431: { + case 433: { { -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" {p = ((te))-1; { -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" ostringstream str; str << "\\k at index " << ts - ptr << " not supported."; @@ -10489,79 +10512,79 @@ static const short _regex_actions[] = { } }} -#line 10050 "Parser.cpp" +#line 10071 "Parser.cpp" break; } - case 432: { + case 434: { { -#line 396 "Parser.rl" +#line 397 "Parser.rl" {p = ((te))-1; { -#line 396 "Parser.rl" +#line 397 "Parser.rl" throw LocatedParseError("Conditional subpattern unsupported"); } }} -#line 10064 "Parser.cpp" +#line 10085 "Parser.cpp" break; } - case 433: { + case 435: { { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" {p = ((te))-1; { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" throw LocatedParseError("Unrecognised character after (?"); } }} -#line 10078 "Parser.cpp" +#line 10099 "Parser.cpp" break; } - case 434: { + case 436: { { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" {p = ((te))-1; { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 10093 "Parser.cpp" +#line 10114 "Parser.cpp" break; } - case 435: { + case 437: { { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" {p = ((te))-1; { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 10107 "Parser.cpp" +#line 10128 "Parser.cpp" break; } - case 436: { + case 438: { { #line 1 "NONE" {switch( act ) { - case 288: { + case 290: { p = ((te))-1; { -#line 1491 "Parser.rl" +#line 1498 "Parser.rl" // If there are enough capturing sub expressions, this may be // a back reference @@ -10574,10 +10597,10 @@ static const short _regex_actions[] = { } break; } - case 290: { + case 292: { p = ((te))-1; { -#line 1508 "Parser.rl" +#line 1515 "Parser.rl" // if there are enough left parens to this point, back ref if (accumulator < groupIndex) { @@ -10611,10 +10634,10 @@ static const short _regex_actions[] = { } break; } - case 330: { + case 332: { p = ((te))-1; { -#line 1737 "Parser.rl" +#line 1744 "Parser.rl" addLiteral(currentSeq, *(ts + 1), mode); } @@ -10623,7 +10646,7 @@ static const short _regex_actions[] = { }} } -#line 10179 "Parser.cpp" +#line 10200 "Parser.cpp" break; } @@ -10650,7 +10673,7 @@ else { #line 1 "NONE" {ts = 0;}} -#line 10205 "Parser.cpp" +#line 10226 "Parser.cpp" break; } @@ -10667,7 +10690,7 @@ else { _out: {} } -#line 1983 "Parser.rl" +#line 1990 "Parser.rl" if (p != pe && *p != '\0') { @@ -10711,7 +10734,7 @@ assert(currentSeq == rootSeq.get()); // Ensure that all references are valid. checkReferences(*rootSeq, groupIndex, groupNames); -return move(rootSeq); +return rootSeq; } catch (LocatedParseError &error) { if (ts >= ptr && ts <= pe) { error.locate(ts - ptr); diff --git a/contrib/vectorscan-cmake/rageled_files/amd64/control_verbs.cpp b/contrib/vectorscan-cmake/rageled_files/amd64/control_verbs.cpp index e0393e4935561..57cea09266693 100644 --- a/contrib/vectorscan-cmake/rageled_files/amd64/control_verbs.cpp +++ b/contrib/vectorscan-cmake/rageled_files/amd64/control_verbs.cpp @@ -74,28 +74,28 @@ static const signed char _ControlVerbs_actions[] = { 155, 157, 159, 160, 161, 0 }; - static const char _ControlVerbs_trans_keys[] = { - 41, 65, 66, 67, 76, 78, 85, 41, - 41, 78, 41, 89, 41, 67, 41, 82, - 41, 76, 41, 70, 41, 41, 83, 41, - 82, 41, 95, 41, 65, 85, 41, 78, - 41, 89, 41, 67, 41, 78, 41, 73, - 41, 67, 41, 79, 41, 68, 41, 69, - 41, 82, 41, 76, 41, 70, 73, 41, - 77, 41, 73, 41, 84, 41, 95, 41, - 77, 82, 41, 65, 41, 84, 41, 67, - 41, 72, 41, 61, 41, 48, 57, 41, - 48, 57, 41, 69, 41, 67, 41, 85, - 41, 82, 41, 83, 41, 73, 41, 79, - 41, 78, 41, 79, 41, 95, 41, 65, - 83, 41, 85, 41, 84, 41, 79, 41, - 95, 41, 80, 41, 79, 41, 83, 41, - 83, 41, 69, 41, 83, 41, 83, 41, - 84, 41, 65, 41, 82, 41, 84, 41, - 95, 41, 79, 41, 80, 41, 84, 41, - 67, 84, 41, 80, 41, 41, 70, 41, - 49, 51, 56, 41, 54, 41, 50, 41, - 40, 42, 0 + static const unsigned char _ControlVerbs_trans_keys[] = { + 41u, 65u, 66u, 67u, 76u, 78u, 85u, 41u, + 41u, 78u, 41u, 89u, 41u, 67u, 41u, 82u, + 41u, 76u, 41u, 70u, 41u, 41u, 83u, 41u, + 82u, 41u, 95u, 41u, 65u, 85u, 41u, 78u, + 41u, 89u, 41u, 67u, 41u, 78u, 41u, 73u, + 41u, 67u, 41u, 79u, 41u, 68u, 41u, 69u, + 41u, 82u, 41u, 76u, 41u, 70u, 73u, 41u, + 77u, 41u, 73u, 41u, 84u, 41u, 95u, 41u, + 77u, 82u, 41u, 65u, 41u, 84u, 41u, 67u, + 41u, 72u, 41u, 61u, 41u, 48u, 57u, 41u, + 48u, 57u, 41u, 69u, 41u, 67u, 41u, 85u, + 41u, 82u, 41u, 83u, 41u, 73u, 41u, 79u, + 41u, 78u, 41u, 79u, 41u, 95u, 41u, 65u, + 83u, 41u, 85u, 41u, 84u, 41u, 79u, 41u, + 95u, 41u, 80u, 41u, 79u, 41u, 83u, 41u, + 83u, 41u, 69u, 41u, 83u, 41u, 83u, 41u, + 84u, 41u, 65u, 41u, 82u, 41u, 84u, 41u, + 95u, 41u, 79u, 41u, 80u, 41u, 84u, 41u, + 67u, 84u, 41u, 80u, 41u, 41u, 70u, 41u, + 49u, 51u, 56u, 41u, 54u, 41u, 50u, 41u, + 40u, 42u, 0u }; static const signed char _ControlVerbs_single_lengths[] = { @@ -276,7 +276,7 @@ static const signed char _ControlVerbs_actions[] = { te = 0; } -#line 105 "control_verbs.rl" +#line 106 "control_verbs.rl" try { @@ -285,7 +285,7 @@ static const signed char _ControlVerbs_actions[] = { { int _klen; unsigned int _trans = 0; - const char * _keys; + const unsigned char * _keys; const signed char * _acts; unsigned int _nacts; _resume: {} @@ -321,9 +321,9 @@ static const signed char _ControlVerbs_actions[] = { _klen = (int)_ControlVerbs_single_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + _klen - 1; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + _klen - 1; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _keys += _klen; @@ -345,9 +345,9 @@ static const signed char _ControlVerbs_actions[] = { _klen = (int)_ControlVerbs_range_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + (_klen<<1) - 2; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + (_klen<<1) - 2; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _trans += (unsigned int)_klen; @@ -389,9 +389,9 @@ static const signed char _ControlVerbs_actions[] = { } case 3: { { -#line 76 "control_verbs.rl" +#line 77 "control_verbs.rl" {te = p+1;{ -#line 76 "control_verbs.rl" +#line 77 "control_verbs.rl" mode.utf8 = true; } @@ -403,9 +403,9 @@ static const signed char _ControlVerbs_actions[] = { } case 4: { { -#line 80 "control_verbs.rl" +#line 81 "control_verbs.rl" {te = p+1;{ -#line 80 "control_verbs.rl" +#line 81 "control_verbs.rl" mode.ucp = true; } @@ -417,9 +417,9 @@ static const signed char _ControlVerbs_actions[] = { } case 5: { { -#line 84 "control_verbs.rl" +#line 85 "control_verbs.rl" {te = p+1;{ -#line 84 "control_verbs.rl" +#line 85 "control_verbs.rl" ostringstream str; str << "Unsupported control verb " << string(ts, te - ts); @@ -433,9 +433,9 @@ static const signed char _ControlVerbs_actions[] = { } case 6: { { -#line 90 "control_verbs.rl" +#line 91 "control_verbs.rl" {te = p+1;{ -#line 90 "control_verbs.rl" +#line 91 "control_verbs.rl" ostringstream str; str << "Unknown control verb " << string(ts, te - ts); @@ -449,9 +449,9 @@ static const signed char _ControlVerbs_actions[] = { } case 7: { { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {te = p+1;{ -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = p - 1; } {p += 1; goto _out; } @@ -464,9 +464,9 @@ static const signed char _ControlVerbs_actions[] = { } case 8: { { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {te = p;p = p - 1;{ -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = p - 1; } {p += 1; goto _out; } @@ -479,10 +479,10 @@ static const signed char _ControlVerbs_actions[] = { } case 9: { { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = ((te))-1; { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = p - 1; } {p += 1; goto _out; } @@ -530,9 +530,11 @@ static const signed char _ControlVerbs_actions[] = { _out: {} } -#line 108 "control_verbs.rl" +#line 109 "control_verbs.rl" } catch (LocatedParseError &error) { + // cppcheck-suppress pointerLessThanZero + // cppcheck-suppress pointerPositive if (ts >= ptr && ts <= pe) { error.locate(ts - ptr + start); } else { diff --git a/contrib/zxc b/contrib/zxc index b9890cfe3466b..e568d1967309f 160000 --- a/contrib/zxc +++ b/contrib/zxc @@ -1 +1 @@ -Subproject commit b9890cfe3466b19d0eb5005874ce81728efb9565 +Subproject commit e568d1967309fd5d160f45f2fdabcbff6ab3f058 diff --git a/contrib/zxc-cmake/CMakeLists.txt b/contrib/zxc-cmake/CMakeLists.txt index 98c52cb1111bb..70451492f30dc 100644 --- a/contrib/zxc-cmake/CMakeLists.txt +++ b/contrib/zxc-cmake/CMakeLists.txt @@ -33,9 +33,14 @@ set(ZXC_CORE_SOURCES # emitted symbols (e.g. zxc_compress_chunk_wrapper_avx2). zxc_dispatch.c declares # these suffixed symbols unconditionally per architecture, so the exact set of # variants below must match what the dispatcher expects for each target: -# x86-64 -> _default, _sse2, _avx2, _avx512 -# aarch64 -> _default, _neon +# x86-64 -> _default, _avx2, _avx512 +# aarch64 -> _default only (NEON is the AArch64 baseline) # other -> _default only +# SSE2 is the x86-64 baseline and NEON the AArch64 one, so `_default` already +# compiles those code paths; zxc dropped the dedicated `_sse2` and `_neon` +# variants in v0.14.0 and the dispatcher no longer references them. The +# remaining `_neon32` variant is for 32-bit ARM, which ClickHouse does not +# target. set(ZXC_VARIANT_OBJECTS "") macro(zxc_add_variant suffix) @@ -54,11 +59,14 @@ macro(zxc_add_variant suffix) endmacro() if (OS_DARWIN AND ARCH_AMD64) - # zxc's x86-64 runtime dispatch calls `__builtin_cpu_init` / + # Historically zxc's x86-64 runtime dispatch called `__builtin_cpu_init` / # `__builtin_cpu_supports`, which reference compiler-rt's `__cpu_model`. That - # symbol is not linked into ClickHouse's macOS cross-build, so linking fails - # with "undefined ___cpu_model". macOS builds are for local development only, - # so restrict zxc to its portable scalar core there. + # symbol is not linked into ClickHouse's macOS cross-build, so linking failed + # with "undefined ___cpu_model". Since v0.14.0 the dispatcher issues `CPUID` + # and `XGETBV` directly and no longer needs `__cpu_model`, so this is now a + # conservative restriction rather than a required one: macOS builds are for + # local development only and are never test-run in CI, so there is nothing + # to gain from the x86 SIMD variants there. Lifting it is a separate change. set(ZXC_FORCE_SCALAR TRUE) elseif (SANITIZE STREQUAL "memory") # clang's MemorySanitizer has no precise shadow model for the x86 byte-shuffle @@ -67,7 +75,7 @@ elseif (SANITIZE STREQUAL "memory") # that the shuffle discards still poisons the logical result. zxc's SIMD # kernels deliberately do speculative 16/32-byte loads whose tail lanes fall # into malloc'ed scratch buffers or not-yet-written output and are then - # discarded by such shuffles (e.g. `zxc_decode_copy_overlap_run`, + # discarded by such shuffles (e.g. `zxc_decode_copy_overlap_run32`, # `zxc_pivco_merge`), which triggers false `use-of-uninitialized-value` # reports. The scalar core branches only on initialized bytes, so build only # that under MSan. (AArch64 NEON `tbl` gets a precise shadow, but keep all @@ -90,16 +98,16 @@ if (ZXC_FORCE_SCALAR) target_compile_definitions(${_tgt} PRIVATE ZXC_DISABLE_SIMD) endforeach() elseif (ARCH_AMD64) - # SSE2/AVX2 flags are redundant with the x86-64-v3 baseline but harmless; + # The AVX2 flags are redundant with the x86-64-v3 baseline but harmless; # AVX512 is NOT in the baseline, so those flags are required for that variant - # (only ever entered at runtime on AVX-512 capable CPUs). - zxc_add_variant(_sse2 -msse2) - zxc_add_variant(_avx2 -mavx2 -mfma -mbmi -mbmi2 -mlzcnt) + # (only ever entered at runtime on AVX-512 capable CPUs). The flag sets must + # stay in sync with `zxc_detect_cpu_features`, which admits a CPU to the AVX2 + # and AVX-512 tiers only after proving BMI1, BMI2 and LZCNT as well. Note + # that it does not probe FMA, so `-mfma` must not be added here: the + # compiler would be free to emit FMA instructions into a variant that runs + # on any AVX2 CPU. + zxc_add_variant(_avx2 -mavx2 -mbmi -mbmi2 -mlzcnt) zxc_add_variant(_avx512 -mavx512f -mavx512bw -mavx512vbmi -mavx512vbmi2 -mbmi -mbmi2 -mlzcnt) -elseif (ARCH_AARCH64) - # NEON is guaranteed by ClickHouse's armv8.2-a+simd baseline; no extra flags - # (adding -march=armv8-a would downgrade below the baseline). - zxc_add_variant(_neon) endif () add_library(_zxc ${ZXC_CORE_SOURCES} ${ZXC_VARIANT_OBJECTS}) diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index b8c0961ea8bbd..85b7712ef4acd 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -16,7 +16,7 @@ RUN arch=${TARGETARCH:-amd64} \ esac -FROM alpine:3.21 +FROM alpine ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ @@ -121,6 +121,8 @@ RUN clickhouse-keeper --version \ && chown root:clickhouse "${DEFAULT_LOG_DIR}" \ && chmod ugo+Xrw -R "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}" +RUN apk update && apk upgrade --no-cache + # /var/lib/clickhouse is necessary due to the current default configuration for Keeper VOLUME "${DEFAULT_DATA_DIR}" /var/lib/clickhouse EXPOSE 2181 10181 44444 9181 diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index 11082d291d566..3aaf63a034b48 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -123,8 +123,7 @@ RUN mkdir -p \ && chown root:clickhouse "${DEFAULT_LOG_DIR}" \ && chmod ugo+Xrw -R "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CLIENT_CONFIG_DIR}" "${DEFAULT_SERVER_CONFIG_DIR}" -RUN apk update -RUN apk add --upgrade libssl3 libcrypto3 +RUN apk update && apk upgrade --no-cache VOLUME "${DEFAULT_DATA_DIR}" EXPOSE 9000 8123 9009 diff --git a/docs/concepts/features/interfaces/http.mdx b/docs/concepts/features/interfaces/http.mdx index ff580401ef9c2..ee87cf7216b68 100644 --- a/docs/concepts/features/interfaces/http.mdx +++ b/docs/concepts/features/interfaces/http.mdx @@ -190,7 +190,7 @@ wget -nv -O- 'http://localhost:8123/?query=SELECT 1, 2, 3 FORMAT JSON' } ``` -You can use the `default_format` URL parameter to specify a default format other than `TabSeparated`. The `X-ClickHouse-Format` header selects the format of the response explicitly: it is an alias for the `output_format` setting, so it also overrides a `FORMAT` clause in the query. It never changes how the request body of an `INSERT` is parsed — use `input_format` or `format` for that. +You can use the `default_format` URL parameter to specify a default format other than `TabSeparated`. The `X-ClickHouse-Format` header selects the format of the response explicitly: it is an alias for the `output_format` setting, so it also overrides a `FORMAT` clause in the query. It never changes how the request body of an `INSERT` is parsed — use `input_format` or `format` for that. To restore the behavior of versions before 26.8, where the header was an alias for `default_format` and did not override a `FORMAT` clause, disable the `http_x_clickhouse_format_overrides_output_format` setting (in a user profile or as a URL parameter; like the format settings themselves, it can be changed in read-only mode). ```bash $ echo 'SELECT 1 FORMAT Pretty' | curl 'http://localhost:8123/?' --data-binary @- @@ -316,7 +316,7 @@ INSERT INTO t SELECT number FROM numbers(10) SETTINGS limit = 2; | Setting | Effect | |---------|--------| -| `output_format` | Overrides the output format. It takes precedence over the query's `FORMAT` clause, the path extension, `format`, and `default_format`. It can also be set using the `X-ClickHouse-Format` header. | +| `output_format` | Overrides the output format. It takes precedence over the query's `FORMAT` clause, the path extension, `format`, and `default_format`. It can also be set using the `X-ClickHouse-Format` header. With `http_x_clickhouse_format_overrides_output_format = 0`, the header sets `default_format` instead, as in versions before 26.8. | | `input_format` | Overrides the input format for `INSERT`. It takes precedence over the query's `FORMAT` clause and `format`. | | `format` | Overrides the format in both directions unless a direction-specific setting is provided. | | `default_format` | Sets the output format when the query has no `FORMAT` clause, no path extension, and no other format override. | @@ -644,6 +644,8 @@ curl -sS "http://localhost:8123?role=my_role&role=my_other_role" --data-binary " In this case, `?role=my_role&role=my_other_role` works similarly to executing `SET ROLE my_role, my_other_role` before the statement. +The `role` query parameter is not affected by the `access_control_improvements.readonly_restricts_set_role` server setting, which refuses only the `SET ROLE` statement. + ## HTTP response codes caveats {#http_response_codes_caveats} Because of limitations of the HTTP protocol, a HTTP 200 response code doesn't guarantee that a query was successful. diff --git a/docs/concepts/features/performance/allocation-profiling.mdx b/docs/concepts/features/performance/allocation-profiling.mdx index 7c51dfc01f933..383f8bca8cb8f 100644 --- a/docs/concepts/features/performance/allocation-profiling.mdx +++ b/docs/concepts/features/performance/allocation-profiling.mdx @@ -354,7 +354,7 @@ The same query can split the waste of each size class between its backtraces, in WITH 60e9 AS min_age_ns SELECT format('{} {}', if(s.est_old_objects > 0, s.stack, format('[unattributed];arena_{}_class_{}', toString(b.arena), toString(b.size))), - toString(toUInt64(if(s.est_old_objects > 0, b.waste * s.est_old_objects / sum(s.est_old_objects) OVER (PARTITION BY b.arena, b.index), b.waste)))) + toString(toUInt64(if(s.est_old_objects > 0, b.waste * s.est_old_objects / sum(s.est_old_objects) OVER (PARTITION BY b.arena, b.index), toFloat64(b.waste))))) FROM system.jemalloc_arena_bins AS b LEFT JOIN ( diff --git a/docs/concepts/features/security/tls/configuring-tls.mdx b/docs/concepts/features/security/tls/configuring-tls.mdx index be54610c4cdcd..ca80687a4e259 100644 --- a/docs/concepts/features/security/tls/configuring-tls.mdx +++ b/docs/concepts/features/security/tls/configuring-tls.mdx @@ -645,6 +645,19 @@ For `clickhouse-client`, you can also use the `--accept-invalid-certificate` CLI ``` +## Rotating certificates and CA certificates without a restart {#rotating-certificates-without-restart} + +ClickHouse server and ClickHouse Keeper watch the files referenced by `certificateFile`, `privateKeyFile` and `caConfig` +in the `openSSL.server` and `openSSL.client` sections (and in `protocols.*` for composable protocols), or the files in the +directory if `caConfig` is a directory. When one of these files changes, or when `SYSTEM RELOAD CONFIG` is executed, the +certificates are reloaded and used for all new TLS connections, including HTTPS, the secure native protocol, interserver +connections, connections to Keeper and the Raft connections between Keeper nodes. Established connections keep using the +certificates they were opened with. + +To rotate a CA without downtime, first replace the `caConfig` file with a bundle that contains both the old and the new +CA certificate, then switch the node and client certificates to ones issued by the new CA, and finally replace the bundle +with the new CA certificate only. + ## Summary {#summary} This article focused on getting a ClickHouse environment configured with TLS. The settings will differ for different requirements in production environments; for example, certificate verification levels, protocols, ciphers, etc. But you should now have a good understanding of the steps involved in configuring and implementing secure connections. diff --git a/docs/guides/clickhouse/performance-and-monitoring/analyzer.mdx b/docs/guides/clickhouse/performance-and-monitoring/analyzer.mdx index 730c453a6957d..2bf316f458c8e 100644 --- a/docs/guides/clickhouse/performance-and-monitoring/analyzer.mdx +++ b/docs/guides/clickhouse/performance-and-monitoring/analyzer.mdx @@ -277,22 +277,31 @@ SELECT category, sum(value) FROM t WHERE service = 'svc1' GROUP BY category; Error: `CTE with name ... already exists (MULTIPLE_EXPRESSIONS_FOR_ALIAS)`. Exception code: 179 -Cause: The old analyzer permitted defining multiple Common Table Expressions (WITH ...) with the same name shadowing the earlier one. The analyzer forbids this ambiguity. +Cause: The old analyzer permitted defining multiple Common Table Expressions (WITH ...) with the same name, a later definition shadowing the earlier one. The analyzer rejects this ambiguity by default. -Solution: Rename duplicate CTEs to be unique. +Solution: Rename duplicate CTEs to be unique. As a migration aid, enable `analyzer_compatibility_allow_cte_redefinition = 1` (available since ClickHouse `26.10`) to restore the legacy behavior: a reference binds to the latest definition of the name that is not being resolved at that moment, so a redefinition can read the previous definition and the query body reads the last one. + +Limitations: a CTE declared as `MATERIALIZED` and a CTE in a `WITH RECURSIVE` clause cannot be redefined even with the setting enabled. One shape differs from the old analyzer: a CTE declared between two definitions of a name also binds to the last definition, where the old analyzer bound it to the definition visible at its declaration point. ```sql /* ORIGINAL QUERY */ -WITH - data AS (SELECT 1 AS id), - data AS (SELECT 2 AS id) -- Redefined +WITH + data AS (SELECT 1 AS id), + data AS (SELECT id + 1 AS id FROM data) -- Redefined, reads the previous definition SELECT * FROM data; /* FIXED QUERY */ -WITH - raw_data AS (SELECT 1 AS id), - processed_data AS (SELECT 2 AS id) +WITH + raw_data AS (SELECT 1 AS id), + processed_data AS (SELECT id + 1 AS id FROM raw_data) SELECT * FROM processed_data; + +/* LEGACY BEHAVIOR AS A MIGRATION AID */ +WITH + data AS (SELECT 1 AS id), + data AS (SELECT id + 1 AS id FROM data) +SELECT * FROM data +SETTINGS analyzer_compatibility_allow_cte_redefinition = 1; ``` ### Ambiguous column identifiers {#ambiguous-column-identifiers} diff --git a/docs/products/cloud/guides/data-sources/accessing-s3-data-securely.mdx b/docs/products/cloud/guides/data-sources/accessing-s3-data-securely.mdx index d8002f31a5f2e..5c13a3cb8a13f 100644 --- a/docs/products/cloud/guides/data-sources/accessing-s3-data-securely.mdx +++ b/docs/products/cloud/guides/data-sources/accessing-s3-data-securely.mdx @@ -141,6 +141,7 @@ DESCRIBE TABLE s3('https://s3.amazonaws.com/BUCKETNAME/BUCKETOBJECT.csv','CSVWit Below is an example query that uses the `role_session_name` as a shared secret to query data from a bucket. If the `role_session_name` isn't correct, this operation will fail. +Because it can act as a shared secret, ClickHouse masks the value of `role_session_name` as `[HIDDEN]` in `system.query_log`, `SHOW CREATE TABLE`, and everywhere else the query text is shown, the same way as `external_id`. ```sql DESCRIBE TABLE s3('https://s3.amazonaws.com/BUCKETNAME/BUCKETOBJECT.csv','CSVWithNames',extra_credentials(role_arn = 'arn:aws:iam::111111111111:role/ClickHouseAccessRole-001', role_session_name = 'secret-role-name')) diff --git a/docs/reference/engines/table-engines/mergetree-family/annindexes.mdx b/docs/reference/engines/table-engines/mergetree-family/annindexes.mdx index 7ae7093ec9b81..e7b1d462d5dba 100644 --- a/docs/reference/engines/table-engines/mergetree-family/annindexes.mdx +++ b/docs/reference/engines/table-engines/mergetree-family/annindexes.mdx @@ -575,6 +575,14 @@ Example output: └──────────┴───────┴──────┴──────────────────────────┘ ``` +#### Deleted rows {#deleted-rows} + +A vector similarity index is built when a part is written, merged, or mutated, and it does not know about rows deleted later by a [lightweight `DELETE`](/reference/statements/delete) or by an `ALTER TABLE ... DELETE` mutation that is still pending. +Such rows would still be returned as candidates by the index and then removed after the read, so the query could return fewer than `LIMIT N` rows or miss the nearest remaining neighbors. +To keep the results correct, ClickHouse does not use the vector similarity index for parts with deleted rows and reads these parts in full (exact search) instead. +The index is used for the part again after a merge or a mutation rewrites it, for example after `OPTIMIZE TABLE ... FINAL`. +With setting `apply_deleted_mask = 0`, lightweight-deleted rows are returned by the query, so the index is still used unless the query filters by `_row_exists` explicitly. + #### Differences to regular skipping indexes {#differences-to-regular-skipping-indexes} As all regular [skipping indexes](/concepts/features/performance/skip-indexes/skipping-indexes), vector similarity indexes are constructed over granules and each indexed block consists of `GRANULARITY = [N]`-many granules (`[N]` = 1 by default for normal skipping indexes). diff --git a/docs/reference/engines/table-engines/mergetree-family/textindexes.mdx b/docs/reference/engines/table-engines/mergetree-family/textindexes.mdx index 2866642128ba2..29a4dbc5d465a 100644 --- a/docs/reference/engines/table-engines/mergetree-family/textindexes.mdx +++ b/docs/reference/engines/table-engines/mergetree-family/textindexes.mdx @@ -528,8 +528,8 @@ Search tokens that the postprocessor maps to an empty string are ignored, i.e. t | [hasAnyTokens(col, arr)](/reference/functions/regular-functions/string-search-functions#hasAnyTokens) | no (array elements are tokens as-is) | all | yes | | [hasAllTokens(col, arr)](/reference/functions/regular-functions/string-search-functions#hasAllTokens) | no (array elements are tokens as-is) | all | yes | | [hasPhrase](/reference/functions/regular-functions/string-search-functions#hasPhrase) | yes | `splitByNonAlpha`, `splitByString`, `splitByRegexp`³, `ngrams`, `asciiCJK`, `icu` | yes³ | -| [startsWith](/reference/functions/regular-functions/string-functions#startsWith) | yes | `splitByNonAlpha`, `ngrams`, `sparseGrams`, `asciiCJK` | yes | -| [endsWith](/reference/functions/regular-functions/string-functions#endsWith) | yes | `splitByNonAlpha`, `ngrams`, `sparseGrams`, `asciiCJK` | yes | +| [startsWith](/reference/functions/regular-functions/string-functions#startsWith) | yes⁴ | `splitByNonAlpha`, `ngrams`, `sparseGrams`, `asciiCJK`, `array`⁴ | yes⁴ | +| [endsWith](/reference/functions/regular-functions/string-functions#endsWith) | yes⁴ | `splitByNonAlpha`, `ngrams`, `sparseGrams`, `asciiCJK`, `array`⁴ | yes⁴ | | [like](/reference/functions/regular-functions/string-search-functions#like) | yes¹ | `splitByNonAlpha`, `ngrams`, `sparseGrams`, `asciiCJK`¹ | yes¹ | | [match](/reference/functions/regular-functions/string-search-functions#match) | yes¹ | `splitByNonAlpha`, `ngrams`, `sparseGrams`, `asciiCJK`¹ | yes¹ | | [ilike](/reference/functions/regular-functions/string-search-functions#like) | yes² (`lower`/`upper` only) | `splitByNonAlpha`, `array`² | no² | @@ -542,14 +542,19 @@ Search tokens that the postprocessor maps to an empty string are ignored, i.e. t | [hasAll](/reference/functions/regular-functions/array-functions#hasAll) | yes | `array` | yes | ¹ `LIKE` and `match` use direct read as a hint for the listed tokenizers, otherwise they fall back to brute-force scan. -`LIKE` additionally supports a *direct read (without hint)* (enabled via `use_text_index_like_evaluation_by_dictionary_scan`) for `splitByNonAlpha` and `array` tokenizers without preprocessor or postprocessor. +`LIKE` additionally supports evaluation by a dictionary scan (enabled via `use_text_index_like_evaluation_by_dictionary_scan`) for `splitByNonAlpha` and `array` tokenizers without preprocessor or postprocessor. +A `%value%` pattern is then a *direct read (without hint)*, while `value%` and `%value` patterns remain a hint, see [LIKE/ILIKE queries](#like-ilike-queries-perf). -² `ILIKE` is only supported via direct read (without hint) (`use_text_index_like_evaluation_by_dictionary_scan = 1`, `splitByNonAlpha` or `array` tokenizer). -There is no fallback to using the index as a hint: if the setting is disabled or the tokenizer is not in the supported set, the index is not used for `ILIKE`. +² `ILIKE` is only supported via evaluation by a dictionary scan (`use_text_index_like_evaluation_by_dictionary_scan = 1`, `splitByNonAlpha` or `array` tokenizer). +There is no fallback to using the index as a hint for patterns the dictionary scan does not support: if the setting is disabled or the tokenizer is not in the supported set, the index is not used for `ILIKE`. The preprocessor, if present, must be `lower` or `upper`; postprocessors are not supported. ³ `hasPhrase` on a `splitByRegexp` text index does **not** support a postprocessor: the combination is rejected with an exception, because the postprocessor row-level rewrite assumes whitespace-splitting `splitByNonAlpha`-style tokens. Without a postprocessor, `splitByRegexp` is fully supported by `hasPhrase`. +⁴ `startsWith` and `endsWith` search the complete tokens of the needle, and the token at the open end of the needle is incomplete because the value continues there: `startsWith(col, 'ClickHouse is')` searches the token `ClickHouse`, while `startsWith(col, 'ClickHouse')` has no complete token to search. +The latter is instead evaluated by a dictionary scan (`use_text_index_like_evaluation_by_dictionary_scan = 1`, `splitByNonAlpha` or `array` tokenizer, no preprocessor or postprocessor), which is also the path taken by `col LIKE 'ClickHouse%'` because the analyzer pass [optimize_rewrite_like_perfect_affix](/reference/settings/session-settings/optimize-rewrite#optimize_rewrite_like_perfect_affix) rewrites it into `startsWith`. +See [LIKE/ILIKE queries](#like-ilike-queries-perf). + **Experimental: Support phrase search argument (optional)**. Experimental parameter `support_phrase_search` (default: `0`) controls whether the index stores token positions. @@ -1393,10 +1398,20 @@ This ordering enables skipping even more data granules than the granules skipped ### LIKE/ILIKE queries {#like-ilike-queries-perf} -When a LIKE/ILIKE query pattern is `%%` and the text index tokenizer is `splitByNonAlpha` or `array`, ClickHouse leverages the inverted index to speed up LIKE/ILIKE queries significantly. To achieve that, ClickHouse scans the inverted index dictionary instead of a full-table scan to find the matching pattern. +When a LIKE/ILIKE query pattern is `%%`, `%` or `%` and the text index tokenizer is `splitByNonAlpha` or `array`, ClickHouse leverages the inverted index to speed up LIKE/ILIKE queries significantly. To achieve that, ClickHouse scans the inverted index dictionary instead of a full-table scan to find the matching pattern. + +How the result of the dictionary scan is used depends on where the needle is anchored: +- `%value%` matches a row if and only if it matches one of the row's tokens, so the index decides the query on its own: a [direct read (without hint)](#direct-read) that removes the original predicate. +- `value%` and `%value` anchor the needle at the whole value, whereas the dictionary scan can only anchor it at a token, so the scan returns a superset of the matching rows and is used as a [direct read as a hint](#direct-read). + +The same dictionary scan serves `startsWith(col, 'value')` and `endsWith(col, 'value')` when the needle has no complete token to search. +This is the path most `value%` and `%value` patterns actually take, because the analyzer pass [optimize_rewrite_like_perfect_affix](/reference/settings/session-settings/optimize-rewrite#optimize_rewrite_like_perfect_affix) (enabled by default) rewrites `col LIKE 'value%'` into `startsWith(col, 'value')` and `col LIKE '%value'` into `endsWith(col, 'value')`. +Needles that span several tokens keep using the complete tokens of the needle and do not need a dictionary scan. When the optimization is enabled, LIKE/ILIKE queries should be significantly faster than a full-table scan. However, when the pattern matches most dictionary tokens, the performance can be worse compared to a full-table scan. Luckily, there is a fallback mechanism to prevent that. +The speed-up of a `value%` or `%value` pattern comes from skipping granules, so it depends on how the matching rows are laid out. Needles that match rows in every granule prune nothing, and the query pays for the dictionary scan on top of the full scan it would have done anyway. Disable [use_text_index_like_evaluation_by_dictionary_scan](/reference/settings/session-settings/use-text#use_text_index_like_evaluation_by_dictionary_scan) for such workloads. + The optimization is controlled by a setting: - [use_text_index_like_evaluation_by_dictionary_scan](/reference/settings/session-settings/use-text#use_text_index_like_evaluation_by_dictionary_scan) @@ -1404,7 +1419,7 @@ The fallback mechanism is controlled by two settings: - [text_index_like_min_pattern_length](/reference/settings/session-settings/text-index#text_index_like_min_pattern_length) - [text_index_like_max_postings_to_read](/reference/settings/session-settings/text-index#text_index_like_max_postings_to_read) -This optimization supports only functions `like` and `ilike`. +This optimization supports only functions `like`, `ilike`, `startsWith`, and `endsWith`. ### Trivial count queries {#count-queries-perf} diff --git a/docs/reference/interfaces/specs/NativeFormat.mdx b/docs/reference/interfaces/specs/NativeFormat.mdx index 9a5cf1bfe6876..301aaa82b4895 100644 --- a/docs/reference/interfaces/specs/NativeFormat.mdx +++ b/docs/reference/interfaces/specs/NativeFormat.mdx @@ -1108,14 +1108,11 @@ Header blocks (rows = 0) and empty blocks therefore emit nothing, and a decoder | | | `1` | `STRING` | Native-format compatibility mode — Object transmitted as a single `String` column containing JSON text. | | | | `2` | `V2` | V1 layout minus the `max_dynamic_paths` parameter. | | | | `3` | `FLATTENED` | Native-format compatibility mode — flattened path representation. | -| | | `4` | `V3` | V2 plus a shared-data serialization version sub-field and a statistics flag. | -| **Object shared data** (sub-stream used in Object `V3`) | VarUInt | `0` | `MAP` | Shared data encoded as `Map(String, String)`. | -| | | `1` | `MAP_WITH_BUCKETS` | Same as `MAP` but split into N buckets for scan efficiency. | -| | | `2` | `ADVANCED` | Compact granule format with separate streams for paths / marks / metadata. | +| | | `4` | `V3` | Written into MergeTree parts only, never onto the wire; the layout is not specified here. | | **Dynamic** | UInt64 LE | `1` | `V1` | Original encoding. Includes `max_dynamic_types` and a list of runtime variant types. | | | | `2` | `V2` | V1 minus the `max_dynamic_types` parameter. | | | | `3` | `FLATTENED` | Native-format compatibility mode. | -| | | `4` | `V3` | V2 plus binary-encoded variant type names and empty-statistics support. | +| | | `4` | `V3` | Written into MergeTree parts only, never onto the wire; the layout is not specified here. | | **Variant** discriminators mode | UInt64 LE | `0` | `BASIC` | Every row's discriminator is written literally. | | | | `1` | `COMPACT` | If all rows in a granule share one discriminator, only a single value + granule marker is written. | | **Variant** granule format (when mode is `COMPACT`) | UInt8 | `0` | `PLAIN` | Granule has heterogeneous discriminators. | @@ -1127,7 +1124,7 @@ A few things worth noting about the table: - **The values are not contiguous.** `Dynamic` uses `1`, `2`, `3`, `4` with `V3` at `4` and `FLATTENED` at `3`. A higher number is not necessarily newer. - **Some values are native-format-only.** `Object::STRING`, `Object::FLATTENED`, and `Dynamic::FLATTENED` exist for native-protocol compatibility with clients that do not implement full Object/Dynamic. They do not appear in MergeTree on-disk storage. -- **`V3` is primarily on-disk.** Clients consuming the native TCP protocol typically see `FLATTENED` (value `3`) rather than `V3` (value `4`). +- **`V3` is on-disk only.** `Object::V3` and `Dynamic::V3` are listed because the values are assigned and must not be reused, not because a decoder can meet them: ClickHouse never sends them over `Native` and rejects them there with `INCORRECT_DATA`. Everything they carry — the shared-data sub-version of `Object::V3` and its bucket and granule layouts — describes a part rather than a block, and is therefore out of scope for this page. Clients consuming the native TCP protocol see `V1`, `V2`, `STRING` or `FLATTENED`. #### LowCardinality(T) {#lowcardinality} @@ -1313,11 +1310,17 @@ The state prefix (version + type list) is read at the start of every block with Do **not**, however, bound the **flattened** `num_types` by `ColumnDynamic::MAX_DYNAMIC_TYPES_LIMIT` (`254` in ClickHouse): the flattened type list carries *every* distinct runtime type, including those that had overflowed into the shared variant, so a valid flattened block can legitimately list far more than that. The `MAX_DYNAMIC_TYPES_LIMIT` bound applies only to the `num_dynamic_types` count in the non-flat `V1`/`V2`/`V3` prefixes, which counts regular-variant slots and is capped by the limit (ClickHouse validates it there before the `+ 1` for the shared variant). + +**Truncated streams** + +The discriminator run holds exactly `num_rows` entries, and a decoder must verify that it read that many rather than accepting whatever the stream contained. A shorter run does not contradict anything that follows it: the per-type value counts are derived from the discriminators actually read, so they stay consistent with the value runs, and a run made entirely of NULL markers demands no value bytes at all. ClickHouse rejects a short discriminator run with `INCORRECT_DATA` ("Mismatch in flattened `Dynamic` column: N rows are expected, but the indexes stream contains only M rows") before any per-type check, because the reconstructed column is then used for `num_rows` rows regardless of its actual size. + + Runtime types whose serialization is stateful (`LowCardinality`, `Variant`, `Dynamic`, `JSON`) carry nested state prefixes after the type-name list. -The runtime type list normally follows the `Variant` canonicalization — the regular variant slots are written in `DataTypeVariant` (type-name) order, so the wire order does not follow insertion order. It is **not always** globally sorted, however: types that overflowed into the shared variant (for example under `Dynamic(max_types=N)`) are appended after the regular slots in first-seen order, so the tail of the list can break type-name order. A decoder must therefore treat the transmitted type list as authoritative for discriminator assignment and must not re-sort it itself. For rows `[42::UInt64, "hi", NULL]` the two types are `String` and `UInt64`, and `"String"` sorts before `"UInt64"`, so the discriminators are `0` = String, `1` = UInt64, `2` = NULL: +The runtime type list normally follows the `Variant` canonicalization — the regular variant slots are written in `DataTypeVariant` (type-name) order, so the wire order does not follow insertion order. It is **not always** globally sorted, however: types that overflowed into the shared variant (for example under `Dynamic(max_types=N)`) are appended after the regular slots in first-seen order, so the tail of the list can break type-name order. A decoder must therefore treat the transmitted type list as authoritative for discriminator assignment and must not re-sort it itself. What the list may not contain is a repeated type or `Nothing`: two equal entries map onto one discriminator and `Nothing` is not a variant at all, so in both cases part of the list has nowhere to be reconstructed into. ClickHouse rejects either with `INCORRECT_DATA`. For rows `[42::UInt64, "hi", NULL]` the two types are `String` and `UInt64`, and `"String"` sorts before `"UInt64"`, so the discriminators are `0` = String, `1` = UInt64, `2` = NULL: ```text 03 00 00 00 00 00 00 00 state prefix: UInt64 version = 3 (FLATTENED) @@ -1355,14 +1358,18 @@ In FLATTENED mode there is **no shared-data column** (that overflow store belong [for each dynamic path: its Dynamic data] ← num_rows values (discriminators + runs) ``` -Note the two-phase shape: **all** path state prefixes come first, then **all** path data. A dynamic path's `Dynamic` prefix (in the prefix phase) is therefore separated from its data (in the data phase). The state prefix is read at the start of every block with rows > 0, and every path column (typed or dynamic) holds exactly `num_rows` values. Row `r`'s object is assembled by reading each path's value at index `r`; a dynamic path whose `Dynamic` discriminator is NULL for that row contributes no key. +Note the two-phase shape: **all** path state prefixes come first, then **all** path data. A dynamic path's `Dynamic` prefix (in the prefix phase) is therefore separated from its data (in the data phase). The state prefix is read at the start of every block with rows > 0, and every path column (typed or dynamic) holds exactly `num_rows` values. Row `r`'s object is assembled by reading each path's value at index `r`; a dynamic path whose `Dynamic` discriminator is NULL for that row contributes no key. The dynamic path names must be distinct and none of them may repeat a path that the type string already declares as typed — either would give the assembled object two values under one key, so ClickHouse rejects both with `INCORRECT_DATA`. The same applies to the dynamic path list of the non-flat encodings. **Malformed counts** -`num_dynamic_paths` in the FLATTENED layout documented here — and the dynamic-paths count in the non-flat `V1`/`V2`/`V3` encodings — is read from the stream before the path names. (There is no separate flattened-paths field in the non-flat prefixes: `V1`/`V2`/`V3` carry only the dynamic-paths count, plus a `max_dynamic_paths` value in `V1` that is read and discarded, and the `V3` shared-data metadata below.) As with [`Dynamic`](#dynamic), a decoder must treat these counts as untrusted and must not size an allocation on them directly — neither a `SIZE_MAX`-family count nor a large-but-representable one. ClickHouse reads the path names one entry at a time with only a capped pre-allocation hint, so a corrupted count is rejected either as `INCORRECT_DATA` ("JSON/Object column has too many paths", when it exceeds what the container can hold) or as an ordinary read error once the stream runs out of path names. +`num_dynamic_paths` in the FLATTENED layout documented here — and the dynamic-paths count in the non-flat `V1`/`V2` encodings — is read from the stream before the path names. (There is no separate flattened-paths field in the non-flat prefixes: they carry only the dynamic-paths count, plus a `max_dynamic_paths` value in `V1` that is read and discarded.) As with [`Dynamic`](#dynamic), a decoder must treat these counts as untrusted and must not size an allocation on them directly — neither a `SIZE_MAX`-family count nor a large-but-representable one. ClickHouse reads the path names one entry at a time with only a capped pre-allocation hint, so a corrupted count is rejected either as `INCORRECT_DATA` ("JSON/Object column has too many paths", when it exceeds what the container can hold) or as an ordinary read error once the stream runs out of path names. + + + +**Truncated streams** -The non-flat `V3` prefix additionally carries a `shared_data_buckets` count (present when the shared-data serialization version is `MAP_WITH_BUCKETS` or `ADVANCED`). It sizes the per-bucket reader state and column vectors directly (not via a grow-on-demand loop), so a decoder must reject an implausible bucket count up front. Unlike the path and type counts, this count has a tight writer-side invariant: the number of buckets is chosen from small MergeTree settings (`object_shared_data_buckets_for_compact_part` / `object_shared_data_buckets_for_wide_part`) that are non-zero and capped at `256`, so the only valid on-wire range is `1 … 256`. ClickHouse rejects any value outside that range — including a large-but-representable count such as `100000`, which is far below the container's `max_size()` — with `INCORRECT_DATA` ("JSON/Object column has an invalid number of shared data buckets"). +The `num_rows` values per path column are a hard requirement, not an upper bound: a decoder must check every path column, typed and dynamic, against the block's row count instead of trusting the stream to end where it should. The assembled object has no independent size to compare against — its row count is whatever the unflattening produced — so a path column that came up short is not detectable later, and reading row `r` of it goes past its end. ClickHouse rejects it with `INCORRECT_DATA` ("Unexpected size of typed path / flattened path *p* in flattened Object column"). For a dynamic path this is on top of the equivalent check inside its [`Dynamic`](#dynamic) column. `JSON` value `{"a": 42, "b": "hi"}` (one row, both paths dynamic). A JSON integer is inferred as `Int64`: @@ -1378,9 +1385,9 @@ The non-flat `V3` prefix additionally carries a `shared_data_buckets` count (pre 00 02 68 69 "b" data: discriminator 0, String "hi" ``` -#### JSON non-flat (V2/V3) {#json-non-flat} +#### JSON non-flat (V1/V2) {#json-non-flat} -The non-flattened `Object` encodings (`V1`/`V2`/`V3`) are used by MergeTree on-disk storage and are what the server emits over the wire when the flattened flag is off — `V1` over `clickhouse-client` / HTTP `FORMAT Native` (revision `0`), `V2` over the native TCP protocol. They carry a shared-data column and are **not** specified on this page. Note that they do **not** carry per-path statistics over the Native wire: `NativeWriter` leaves statistics disabled, so the `Object` structure prefix has no statistics section and the bytes after it are the typed/dynamic/shared-data prefixes and data directly. Statistics appear only on the MergeTree on-disk paths that enable them. To decode a `JSON` column with this page, a client must select one of the documented tiers: set `output_format_native_write_json_as_string = 1` for the [String fallback](#json-tier-1-string-fallback), or `output_format_native_use_flattened_dynamic_and_json_serialization = 1` (with `output_format_native_write_json_as_string = 0`) for the [FLATTENED Object](#json-tier-2-flattened-object) layout. +The non-flattened `Object` encodings are what the server emits over the wire when the flattened flag is off — `V1` over `clickhouse-client` / HTTP `FORMAT Native` (revision `0`), `V2` over the native TCP protocol. (`V3`, the third non-flat encoding, is written into MergeTree parts only and is rejected on the wire.) They carry a shared-data column and are **not** specified on this page. Note that they do **not** carry per-path statistics over the Native wire: `NativeWriter` leaves statistics disabled, so the `Object` structure prefix has no statistics section and the bytes after it are the typed/dynamic/shared-data prefixes and data directly. Statistics appear only on the MergeTree on-disk paths that enable them. To decode a `JSON` column with this page, a client must select one of the documented tiers: set `output_format_native_write_json_as_string = 1` for the [String fallback](#json-tier-1-string-fallback), or `output_format_native_use_flattened_dynamic_and_json_serialization = 1` (with `output_format_native_write_json_as_string = 0`) for the [FLATTENED Object](#json-tier-2-flattened-object) layout. ## Compression frame {#compression-frame} diff --git a/docs/reference/interfaces/specs/NativeProtocol.mdx b/docs/reference/interfaces/specs/NativeProtocol.mdx index 6fd620ad09c36..778b5a0f5c39c 100644 --- a/docs/reference/interfaces/specs/NativeProtocol.mdx +++ b/docs/reference/interfaces/specs/NativeProtocol.mdx @@ -84,6 +84,29 @@ SSH challenge-response authentication is available from protocol version 54466 o For distributed query execution, servers authenticate to one another by proving knowledge of a shared secret — without putting the secret on the wire. Each Query carries a 32-byte SHA-256 `auth_hash` in [`Query`](#query) field 4, computed over a salt, nonce, the configured secret, and the query, which the receiving server recomputes and compares. This is gated by the `INTERSERVER_SECRET` feature (v54441). External clients always send an empty string here. See [Inter-server authentication](#inter-server-authentication). +### Receiver-side size limits {#receiver-side-size-limits} + +Almost every string and collection on the wire is length-prefixed, and a receiver may allocate for the declared size before the data behind it arrives. A peer therefore has to bound what it accepts by what the data can plausibly hold — in inter-server mode the server reads `ClientInfo`, the settings and the externally granted roles of a query *before* it verifies the query's `auth_hash`, so these limits apply to unauthenticated input. + +The limits the ClickHouse server and `clickhouse-client` enforce, and which an implementation should expect to be rejected against, are: + +| Field | Limit | +|-------|-------| +| Each `String` of [`ClientHello`](#clienthello) — user, password, database, client name — and the query id of a [`Query`](#query) | 64 KiB | +| Each `String` of [`ClientInfo`](#clientinfo) — user, query id, host and client names, quota key, user agent, referer, URL, tracestate, JWT | 1 GiB — the generic bound, but read as the bytes arrive rather than allocated up front; `ClientInfo` is forwarded from the other interfaces, whose values can be longer, and persisted in async `Distributed` insert queue files | +| Number of current roles in [`ClientInfo`](#clientinfo), and the name of each | 65536 entries, 64 KiB each | +| Number of [`Setting`](#setting) entries and of [`Parameter`](#parameter) entries in a [`Query`](#query) | 65536 each | +| The name of each [`Setting`](#setting) and of each [`Parameter`](#parameter) | 64 KiB | +| The value of a [`Setting`](#setting) or a [`Parameter`](#parameter), and the query text | 1 GiB — the generic bound; a parameter is query-sized data, and the query text itself has to be read at this bound before the `auth_hash` covering it can be checked | +| The serialized externally granted roles of a [`Query`](#query) | 1 MiB in total, 65536 role names, 64 KiB each | +| Number of tables in a `TablesStatusRequest` / `TablesStatusResponse`, and the length of each database and table name | 100000 tables, 64 KiB per identifier | +| Number of UUIDs in a `PartUUIDs` packet (client-side) | 1000000 | +| Number of `out_of_order_buckets` in the block info of a [`Data`](#data) block | 256 — the number of two-level aggregation buckets | +| Number of `password_complexity_rules` in [`ServerHello`](#serverhello) | see the [note there](#serverhello) | +| Any other `String` or collection without a dedicated limit | 1 GiB | + +Exceeding a limit is answered with an [`Exception`](#exception) carrying `TOO_LARGE_ARRAY_SIZE` (128) or `TOO_LARGE_STRING_SIZE` (131). The values above are what the current server enforces; they are not negotiated, and a client should not rely on the server accepting anything close to them. + ## Versioning and feature gates {#versioning-and-feature-gates} ### Version negotiation {#version-negotiation} @@ -144,7 +167,7 @@ When a feature is active, its fields **must** be present on the wire. The protoc | SERVER_SETTINGS | 54474 | ServerHello | Server broadcasts its non-default settings as a list at the tail of ServerHello, after `nonce`. Format: `(key, flags, value)` triples terminated by an empty key — same as the Query packet's settings list. | | QUERY_AND_LINE_NUMBERS | 54475 | ClientInfo | Adds `script_query_number` (VarUInt) and `script_line_number` (VarUInt) at the tail of ClientInfo. Used by clickhouse-client for multi-statement script error attribution; external clients send `0, 0`. | | JWT_IN_INTERSERVER | 54476 | ClientInfo | Adds a JWT-presence UInt8 + optional `String jwt` at the tail of ClientInfo. External clients (no JWT) send byte `0x00`. (Spelled `DBMS_MIN_REVISON_WITH_JWT_IN_INTERSERVER` in C++ — note the typo in the constant name.) | -| QUERY_PLAN_SERIALIZATION | 54477 | ServerHello, QueryPlan packet | ServerHello appends `VarUInt query_plan_serialization_version` after server settings. Also introduces `ClientPacket::QueryPlan` (code `13`) for inter-server delivery of pre-built query plans — external clients never send. | +| QUERY_PLAN_SERIALIZATION | 54477 | ServerHello, QueryPlan packet | ServerHello appends `VarUInt query_plan_serialization_version` after server settings. Also introduces `ClientPacket::QueryPlan` (code `13`) for inter-server delivery of pre-built query plans — external clients never send. Query-plan serialization version `10` prefixes the serialized plan with `VarUInt max_threads` and `Bool concurrency_control`; when either is non-default, initiators use SQL shipping for peers below version `10` so the peer builds its plan from the transmitted query settings. | | PARALLEL_BLOCK_MARSHALLING | 54478 | Block (Column) | Server may wrap columns in `ColumnBLOB` (compressed inline) for parallel processing. Gated on the query having compression enabled AND `rows > 1`; otherwise the regular column wire format applies. Clients that never enable compression on outgoing Query packets see no wire change. | | VERSIONED_CLUSTER_FUNCTION_PROTOCOL | 54479 | ServerHello | Adds `VarUInt cluster_function_protocol_version` at the tail of ServerHello. Used for `*Cluster` table functions (`s3Cluster`, etc.). Current value: `8` (`DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION`); version `7` is reserved for a private-repository feature (Iceberg compaction), and `8` adds an optional `read_source_index` to the inter-server cluster read-task payload (the `ReadTaskResponse` body, which stays unspecified here — see below). External clients decode and ignore. | | OUT_OF_ORDER_BUCKETS_IN_AGGREGATION | 54480 | BlockInfo | Adds field 3 (`out_of_order_buckets: Vec`) to BlockInfo's field-tagged stream. Decoded as `[VarUInt count][Int32]*count`. External clients don't emit this themselves; the decoder reads any non-empty list the server sends. | @@ -908,7 +931,7 @@ Below version `8` the announcement is fire-and-forget regardless of mode, and th | 10 | MergeTreeReadTaskResponse | not specified | Parallel read task response | | 11 | SSHChallengeRequest | [SSH auth](#ssh-authentication) | SSH auth challenge request | | 12 | SSHChallengeResponse | [SSH auth](#ssh-authentication) | SSH auth challenge response | -| 13 | QueryPlan | not specified | Query plan | +| 13 | QueryPlan | [QueryPlan](#queryplan) | Query plan | | 14 | MergeTreeAllRangesAnnouncementResponse | [MergeTreeAllRangesAnnouncementResponse](#mergetreeallrangesannouncementresponse) | Initiator's reply to a follower's [`MergeTreeAllRangesAnnouncement`](#packet-type-reference) (gated on `parallel_replicas_protocol_version ≥ 8` — see [VERSIONED_PARALLEL_REPLICAS_PROTOCOL](#feature-table)). Inter-server only — external clients never send. | ### Server → Client {#server-to-client} @@ -935,6 +958,19 @@ Below version `8` the announcement is fire-and-forget regardless of mode, and th | 17 | TimezoneUpdate | [TimezoneUpdate](#timezoneupdate) | Server timezone update | | 18 | SSHChallenge | [SSH auth](#ssh-authentication) | SSH auth challenge | +### QueryPlan {#queryplan} + +`QueryPlan` is an inter-server-only client packet. It follows a [`Query`](#query) whose `stage` is `QueryPlan` (`7`) and contains one serialized plan stream. + +| # | Field | Type | Gate | Description | +|---|---|---|---|---| +| 1 | serialization_version | VarUInt | universal | Query-plan serialization version, no greater than the receiver's `query_plan_serialization_version` advertised in `ServerHello`. | +| 2 | max_threads | VarUInt | serialization version ≥ `10` | Plan-level thread limit. `0` means no plan-specific limit. | +| 3 | concurrency_control | Bool | serialization version ≥ `10` | Whether the plan participates in concurrency control. | +| 4 | plan | versioned query-plan stream | universal | Serialized plan tree, step settings, headers, and prepared sets. Its internal layout is owned by the query-plan serialization version. | + +Before version `10`, fields 2 and 3 are absent. An initiator with either non-default execution limit must not send that older layout: it ships the original SQL query instead, so the older receiver reconstructs the limits from the query settings. + ## Configuration {#configuration} This section covers the tunables that shape native protocol connections: diff --git a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx index 03357b3470b04..23af0e0fa6040 100644 --- a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx +++ b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx @@ -1159,16 +1159,18 @@ SSL client/server configuration. Support for SSL is provided by the `libpoco` library. The available configuration options are explained in [SSLManager.h](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h). Default values can be found in [SSLManager.cpp](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/src/SSLManager.cpp). +The files referenced by `certificateFile`, `privateKeyFile` and `caConfig` are reloaded without a restart when they change or on `SYSTEM RELOAD CONFIG`. New connections use the reloaded certificates, established connections are not affected. + Keys for server/client settings: | Option | Description | Default Value | |-------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------| | `cacheSessions` | Enables or disables caching sessions. Must be used in combination with `sessionIdContext`. Acceptable values: `true`, `false`. | `false` | -| `caConfig` | Path to the file or directory that contains trusted CA certificates. If this points to a file, it must be in PEM format and can contain several CA certificates. If this points to a directory, it must contain one .pem file per CA certificate. The filenames are looked up by the CA subject name hash value. Details can be found in the man page of [SSL_CTX_load_verify_locations](https://www.openssl.org/docs/man3.0/man3/SSL_CTX_load_verify_locations.html). | | +| `caConfig` | Path to the file or directory that contains trusted CA certificates. If this points to a file, it must be in PEM format and can contain several CA certificates. If this points to a directory, it must contain one .pem file per CA certificate. The filenames are looked up by the CA subject name hash value. Details can be found in the man page of [SSL_CTX_load_verify_locations](https://www.openssl.org/docs/man3.0/man3/SSL_CTX_load_verify_locations.html). Like `certificateFile` and `privateKeyFile`, the CA certificates are reloaded without a restart when the file changes or on `SYSTEM RELOAD CONFIG`. | | | `certificateFile` | Path to the client/server certificate file in PEM format. You can omit it if `privateKeyFile` contains the certificate. | | | `cipherList` | Supported OpenSSL encryptions. | `ALL:!ADH:!LOW:!EXP:!MD5:!3DES:@STRENGTH` | | `disableProtocols` | Protocols that are not allowed to be used. | | -| `extendedVerification` | If enabled, verify that the certificate CN or SAN matches the peer hostname. | `true` for `openSSL.client`, `false` for `openSSL.server` | +| `extendedVerification` | If enabled, verify that the certificate CN or SAN matches the peer hostname. | `false` | | `fips` | Activates OpenSSL FIPS mode. Supported if the library's OpenSSL version supports FIPS. | `false` | | `invalidCertificateHandler` | Class (a subclass of CertificateHandler) for verifying invalid certificates. For example: ` RejectCertificateHandler ` . | `RejectCertificateHandler` | | `loadDefaultCAFile` | Wether built-in CA certificates for OpenSSL will be used. ClickHouse assumes that builtin CA certificates are in the file `/etc/ssl/cert.pem` (resp. the directory `/etc/ssl/certs`) or in file (resp. directory) specified by the environment variable `SSL_CERT_FILE` (resp. `SSL_CERT_DIR`). | `true` | @@ -1688,11 +1690,22 @@ When adding a host with the `\` xml tag: - if the host is specified as an IP address, then it is checked as specified in the URL. For example: `[2a02:6b8:a::a]`. - if there are redirects and support for redirects is enabled, then every redirect (the location field) is checked. +When adding a regular expression with the `\` xml tag, the RE2 engine is used and the whole host is matched. For example: `^.*\.clickhouse\.com$` + +When adding a single S3 bucket with the `\` xml tag, as `host/bucket` or `host:port/bucket`: +- it is useful when the endpoint is shared by many parties, for example the path-style AWS endpoint `s3.us-west-2.amazonaws.com`, and allowing the whole host would allow every bucket on it. +- the bucket is taken from the parsed S3 URL, so the entry matches both `https://s3.us-west-2.amazonaws.com/my-bucket/...` and `https://my-bucket.s3.us-west-2.amazonaws.com/...`. +- it applies to S3 URLs only: the `s3` table functions and engines, the `S3` database engine, `BACKUP`/`RESTORE` to `S3`, and S3 disks created in SQL with the `disk` function. +- an S3 disk created in SQL is checked when the query creates it. A disk that already existed when the list was set, or that was created by a version without this check, still loads at server start, including a table that was dropped but not yet deleted, so `UNDROP TABLE` keeps working; the server reports it in the log and in `system.warnings`. +- a permanent (301) redirect returned by S3 is checked against the bucket entries as well, but a temporary (307) redirect is checked by host only, so it is rejected when the list contains only `\` entries. + For example: ```sql clickhouse.com + ^.*\.clickhouse\.com$ + s3.us-west-2.amazonaws.com/my-bucket ``` @@ -1807,6 +1820,7 @@ Settings for optional improvements in the access control system. | Setting | Description | Default | |-------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------| | `on_cluster_queries_require_cluster_grant` | Sets whether `ON CLUSTER` queries require the `CLUSTER` grant. | `true` | +| `readonly_restricts_set_role` | Sets whether the `SET ROLE` statement is refused when the session or the query is readonly. When `false`, `readonly` does not restrict `SET ROLE`. The `role` HTTP query parameter is not affected. | `false` | | `role_cache_expiration_time_seconds` | Sets the number of seconds since last access, that a role is stored in the Role Cache. | `600` | | `select_from_information_schema_requires_grant` | Sets whether `SELECT * FROM information_schema.` requires any grants and can be executed by any user. If set to true, then this query requires `GRANT SELECT ON information_schema.
`, just as for ordinary tables. | `true` | | `select_from_system_db_requires_grant` | Sets whether `SELECT * FROM system.
` requires any grants and can be executed by any user. If set to true then this query requires `GRANT SELECT ON system.
` just as for non-system tables. Exceptions: a few system tables (`tables`, `columns`, `databases`, and some constant tables like `one`, `contributors`) are still accessible for everyone; and if there is a `SHOW` privilege (e.g. `SHOW USERS`) granted then the corresponding system table (i.e. `system.users`) will be accessible. | `true` | @@ -1826,6 +1840,7 @@ Example: truetruefalse + false600 ``` diff --git a/docs/reference/statements/create/row-policy.mdx b/docs/reference/statements/create/row-policy.mdx index e036318fe9f50..47fcdef01d405 100644 --- a/docs/reference/statements/create/row-policy.mdx +++ b/docs/reference/statements/create/row-policy.mdx @@ -160,6 +160,10 @@ CREATE ROW POLICY filter ON mydb.local_table USING a < 1000 TO john; This works while the query is shipped as text, which is the default. With [`serialize_query_plan = 1`](/reference/settings/session-settings/serialize#serialize_query_plan) the initiator ships an already-built read plan instead, and a remote server executing such a plan does not apply its own row policies, so a read of a `Distributed` table over `local_table` returns unfiltered rows. Keep `serialize_query_plan = 0` for users whose row policies must be enforced. See [issue #112891](https://github.com/ClickHouse/ClickHouse/issues/112891). +## Join tables {#join-tables} + +A [Join](/reference/engines/table-engines/special/join) table is a prepared hash table that a `JOIN` or `joinGet` reads as is, so its rows cannot be filtered there. A policy on such a table, including a database-wide `ON db.*` policy, filters a plain `SELECT` from the table, but while it applies, `JOIN` and `joinGet` queries against the table fail with `ACCESS_DENIED`. + ## ON CLUSTER Clause {#on-cluster-clause} Allows creating row policies on a cluster, see [Distributed DDL](/reference/statements/distributed-ddl). This is also the convenient way to create the policy on the local tables of every server of the cluster. diff --git a/docs/reference/statements/select/join.mdx b/docs/reference/statements/select/join.mdx index a73f1de8fcde1..3ca7d583e1241 100644 --- a/docs/reference/statements/select/join.mdx +++ b/docs/reference/statements/select/join.mdx @@ -221,6 +221,8 @@ key2 a2 1 1 1 0 0 \N key4 f 2 3 4 0 0 \N ``` +With the `ie_join` algorithm, a condition that is evaluated during the join may not contain `arrayJoin`, because such a condition must preserve the number of rows; where the expansion depends on one side only, move it into an `ARRAY JOIN` in a subquery before the join. + ## NULL and NaN values in JOIN keys {#null-values-in-join-keys} `NULL` is not equal to any value, including itself. This means that if a `JOIN` key has a `NULL` value in one table, it won't match a `NULL` value in the other table. diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index fe530bd9d2f62..f86cb3a015219 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -626,14 +626,14 @@ try Coordination::EventPtr unused_event = std::make_shared(); - const std::string cert_path = config().getString("openSSL.server.certificateFile", ""); - const std::string key_path = config().getString("openSSL.server.privateKeyFile", ""); - + /// TLS certificates, keys and CA certificates are reloaded by CertificateReloader when these files change. std::vector extra_paths = {include_from_path}; - if (!cert_path.empty()) - extra_paths.emplace_back(cert_path); - if (!key_path.empty()) - extra_paths.emplace_back(key_path); + for (const auto * key : {"openSSL.server.certificateFile", "openSSL.server.privateKeyFile", "openSSL.server.caConfig", + "openSSL.client.certificateFile", "openSSL.client.privateKeyFile", "openSSL.client.caConfig"}) + { + if (auto file_path = config().getString(key, ""); !file_path.empty()) + extra_paths.emplace_back(std::move(file_path)); + } /// ConfigReloader have to strict parameters which are redundant in our case auto main_config_reloader = std::make_unique( diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index ad78a0d204a21..9884fc325c09d 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -487,6 +487,10 @@ namespace ServerSetting extern const ServerSettingsString logger_shutdown_level; extern const ServerSettingsString openssl_server_certificate_file; extern const ServerSettingsString openssl_server_private_key_file; + extern const ServerSettingsString openssl_server_ca_config; + extern const ServerSettingsString openssl_client_certificate_file; + extern const ServerSettingsString openssl_client_private_key_file; + extern const ServerSettingsString openssl_client_ca_config; extern const ServerSettingsString distributed_ddl_path; extern const ServerSettingsString distributed_ddl_replicas_path; extern const ServerSettingsInt32 distributed_ddl_pool_size; @@ -1132,7 +1136,7 @@ void loadStartupScripts(const Poco::Util::AbstractConfiguration & config, const auto condition_read_buffer = ReadBufferFromString(condition); auto condition_write_buffer = WriteBufferFromOwnString(); - LOG_DEBUG(log, "Checking startup query condition `{}`", condition); + LOG_DEBUG(log, "Checking startup query condition `{}`", formatQueryForLogging(condition, startup_context->getSettingsRef())); startup_context->setQueryKind(ClientInfo::QueryKind::INITIAL_QUERY); startup_context->setCurrentQueryId(""); @@ -1166,7 +1170,7 @@ void loadStartupScripts(const Poco::Util::AbstractConfiguration & config, const auto read_buffer = ReadBufferFromString(query); auto write_buffer = WriteBufferFromOwnString(); - LOG_DEBUG(log, "Executing query `{}`", query); + LOG_DEBUG(log, "Executing query `{}`", formatQueryForLogging(query, startup_context->getSettingsRef())); startup_context->setQueryKind(ClientInfo::QueryKind::INITIAL_QUERY); startup_context->setCurrentQueryId(""); @@ -2668,26 +2672,25 @@ try tryLogCurrentException(log, "Disabling cgroup memory observer because of an error during initialization"); } - std::string cert_path = server_settings[ServerSetting::openssl_server_certificate_file]; - std::string key_path = server_settings[ServerSetting::openssl_server_private_key_file]; - + /// TLS certificates, keys and CA certificates are reloaded by CertificateReloader when these files change. std::vector extra_paths = {include_from_path}; - if (!cert_path.empty()) - extra_paths.emplace_back(cert_path); - if (!key_path.empty()) - extra_paths.emplace_back(key_path); + auto watch_path = [&](const std::string & file_path) + { + if (!file_path.empty()) + extra_paths.emplace_back(file_path); + }; + watch_path(server_settings[ServerSetting::openssl_server_certificate_file]); + watch_path(server_settings[ServerSetting::openssl_server_private_key_file]); + watch_path(server_settings[ServerSetting::openssl_server_ca_config]); + watch_path(server_settings[ServerSetting::openssl_client_certificate_file]); + watch_path(server_settings[ServerSetting::openssl_client_private_key_file]); + watch_path(server_settings[ServerSetting::openssl_client_ca_config]); Poco::Util::AbstractConfiguration::Keys protocols; config().keys("protocols", protocols); for (const auto & protocol : protocols) - { - cert_path = config().getString("protocols." + protocol + ".certificateFile", ""); - key_path = config().getString("protocols." + protocol + ".privateKeyFile", ""); - if (!cert_path.empty()) - extra_paths.emplace_back(cert_path); - if (!key_path.empty()) - extra_paths.emplace_back(key_path); - } + for (const auto * key : {"certificateFile", "privateKeyFile", "caConfig"}) + watch_path(config().getString("protocols." + protocol + "." + key, "")); DNSResolver::instance().setFilterSettings(server_settings[ServerSetting::dns_allow_resolve_names_to_ipv4], server_settings[ServerSetting::dns_allow_resolve_names_to_ipv6]); /// DNSCacheUpdater uses BackgroundSchedulePool which lives in shared context diff --git a/programs/server/config.xml b/programs/server/config.xml index e99da390eb15d..e7cf36f663206 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -902,6 +902,10 @@ however you can change this behaviour by setting this to true --> false + + false + 600 @@ -1072,6 +1076,16 @@ Regexps are not aligned: don't forget to add ^ and $. Also don't forget to escape dot (.) metacharacter (forgetting to do so is a common source of error). --> + + - 9440 - - - - /etc/clickhouse-server/config.d/client-cert.pem - /etc/clickhouse-server/config.d/client-key.pem - /etc/clickhouse-server/config.d/ca-cert.pem - none - - - diff --git a/tests/integration/test_accept_invalid_certificate/test.py b/tests/integration/test_accept_invalid_certificate/test.py index b66f311b92355..afea8011c4b99 100644 --- a/tests/integration/test_accept_invalid_certificate/test.py +++ b/tests/integration/test_accept_invalid_certificate/test.py @@ -9,7 +9,6 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) MAX_RETRY = 5 -CA_CERT = f"{SCRIPT_DIR}/certs/ca-cert.pem" cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( @@ -36,18 +35,6 @@ ) -node2 = cluster.add_instance( - "node2", - main_configs=[ - "configs/ssl_config_ca_signed.xml", - "certs/client-key.pem", - "certs/client-cert.pem", - "certs/ca-cert.pem", - ], - with_zookeeper=False, -) - - @pytest.fixture(scope="module", autouse=True) def started_cluster(): try: @@ -74,36 +61,6 @@ def started_cluster(): """ -# node2 presents a certificate this CA signs, so the chain is valid and only the name can fail. -config_ca_signed = """ - - - {caConfig} - false - - -""" - -config_ca_signed_sni_override = """ - client - - - {caConfig} - false - - -""" - -config_ca_signed_no_extended_verification = """ - - - {caConfig} - false - false - - -""" - def execute_query_native(node, query, config): fd, config_path = tempfile.mkstemp( @@ -175,29 +132,3 @@ def test_strict_connection_reject(): config_connection_accept.format(ip_address=f"{instance.ip_address}"), ) assert "certificate verify failed" in str(err.value) - - -def test_hostname_mismatch_rejected_by_default(): - with pytest.raises(Exception) as err: - execute_query_native(node2, "SELECT 1", config_ca_signed.format(caConfig=CA_CERT)) - assert "Unacceptable certificate" in str(err.value) - - -def test_hostname_match_accepted(): - assert ( - execute_query_native( - node2, "SELECT 1", config_ca_signed_sni_override.format(caConfig=CA_CERT) - ) - == "1\n" - ) - - -def test_extended_verification_disabled(): - assert ( - execute_query_native( - node2, - "SELECT 1", - config_ca_signed_no_extended_verification.format(caConfig=CA_CERT), - ) - == "1\n" - ) diff --git a/tests/integration/test_acme_tls/configs/config.xml b/tests/integration/test_acme_tls/configs/config.xml index 0977ed12d7114..4594c72e885e3 100644 --- a/tests/integration/test_acme_tls/configs/config.xml +++ b/tests/integration/test_acme_tls/configs/config.xml @@ -14,9 +14,6 @@ - - false AcceptCertificateHandler diff --git a/tests/integration/test_acme_tls/configs/config_multi.xml b/tests/integration/test_acme_tls/configs/config_multi.xml index 9cda6eb0ba878..56c9d415886c4 100644 --- a/tests/integration/test_acme_tls/configs/config_multi.xml +++ b/tests/integration/test_acme_tls/configs/config_multi.xml @@ -14,9 +14,6 @@ - - false AcceptCertificateHandler diff --git a/tests/integration/test_ai_functions/test.py b/tests/integration/test_ai_functions/test.py index 5c20520e2fb76..1c7e063b7f5d1 100644 --- a/tests/integration/test_ai_functions/test.py +++ b/tests/integration/test_ai_functions/test.py @@ -2153,8 +2153,8 @@ def test_api_call_quota_ignores_subquery_settings(started_cluster): ) outer_wins = int(get_profile_events(qid)["api_calls"]) - # The quota is set only in the subquery; the outer query leaves it at the default (far - # above 64). The subquery cap is ignored, so all 64 rows run rather than stopping at 5 - + # The quota is set only in the subquery; the outer query leaves it at the default (0 - + # no limit). The subquery cap is ignored, so all 64 rows run rather than stopping at 5 - # a quota set only in a subquery has no effect. qid = unique_query_id("quota_levels_subquery_only") instance.query( diff --git a/tests/integration/test_arrowflight_interface/test_session_options_settings_profile.py b/tests/integration/test_arrowflight_interface/test_session_options_settings_profile.py new file mode 100644 index 0000000000000..d998b1690edf9 --- /dev/null +++ b/tests/integration/test_arrowflight_interface/test_session_options_settings_profile.py @@ -0,0 +1,76 @@ +# coding: utf-8 + +import pytest +import random +import string + +from .flight_sql_client import FlightSQLClient, SetSessionOptionsResult + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + main_configs=[ + "configs/flight_port.xml", + ], +) + +PROFILE_NAME = "profile_arrowflight_session_options_constraints" + + +def get_client(): + session_id = ''.join(random.choices(string.ascii_letters + string.digits, k=16)) + return FlightSQLClient( + host=node.ip_address, + port=8888, + insecure=True, + disable_server_verification=True, + metadata={'x-clickhouse-session-id': session_id}, + features={'metadata-reflection': 'true'}, + ) + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + node.wait_until_port_is_ready(8888, timeout=10) + yield cluster + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def settings_profile(): + node.query(f"DROP SETTINGS PROFILE IF EXISTS {PROFILE_NAME}") + node.query( + f"CREATE SETTINGS PROFILE {PROFILE_NAME} SETTINGS max_execution_time = 10 CONST" + ) + try: + yield PROFILE_NAME + finally: + node.query(f"DROP SETTINGS PROFILE IF EXISTS {PROFILE_NAME}") + + +def _query_scalar(client, query): + flight_info = client.execute(query) + table = client.do_get(flight_info.endpoints[0].ticket).read_all() + return table.column(0)[0].as_py() + + +def test_profile_constraints_apply_within_same_request(): + """A profile set in a SetSessionOptions request constrains the other options of the same request.""" + client = get_client() + + result = client.set_session_options( + {"profile": PROFILE_NAME, "max_execution_time": "999"} + ) + + assert "profile" not in result.errors + # A constraint violation is neither a parse nor an unknown-setting error, so it maps to UNSPECIFIED. + assert ( + result.errors["max_execution_time"].value + == SetSessionOptionsResult.UNSPECIFIED + ) + assert float(_query_scalar(client, "SELECT getSetting('max_execution_time')")) == 10 diff --git a/tests/integration/test_async_insert_pool_saturation/__init__.py b/tests/integration/test_async_insert_pool_saturation/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_async_insert_pool_saturation/configs/async_insert.xml b/tests/integration/test_async_insert_pool_saturation/configs/async_insert.xml new file mode 100644 index 0000000000000..287c263227223 --- /dev/null +++ b/tests/integration/test_async_insert_pool_saturation/configs/async_insert.xml @@ -0,0 +1,11 @@ + + 1 + true + + + tcp + true + 9010 + + + diff --git a/tests/integration/test_async_insert_pool_saturation/configs/no_flush_on_shutdown.xml b/tests/integration/test_async_insert_pool_saturation/configs/no_flush_on_shutdown.xml new file mode 100644 index 0000000000000..40503bcac555c --- /dev/null +++ b/tests/integration/test_async_insert_pool_saturation/configs/no_flush_on_shutdown.xml @@ -0,0 +1,3 @@ + + false + diff --git a/tests/integration/test_async_insert_pool_saturation/test.py b/tests/integration/test_async_insert_pool_saturation/test.py new file mode 100644 index 0000000000000..8d43a92d6612d --- /dev/null +++ b/tests/integration/test_async_insert_pool_saturation/test.py @@ -0,0 +1,260 @@ +from concurrent.futures import ThreadPoolExecutor, TimeoutError +from uuid import uuid4 + +import pytest + +from helpers.client import Client +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", main_configs=["configs/async_insert.xml"], stay_alive=True +) +node_no_flush = cluster.add_instance( + "node_no_flush", + main_configs=["configs/async_insert.xml", "configs/no_flush_on_shutdown.xml"], + stay_alive=True, +) + +FAILPOINT = "async_insert_flush_pause_in_executor" +ADMISSION_FAILPOINT = "async_insert_pause_before_schedule" +SETTINGS = { + "async_insert": 1, + "wait_for_async_insert": 0, + "async_insert_max_data_size": 2, + "async_insert_use_adaptive_busy_timeout": 0, + "async_insert_busy_timeout_min_ms": 3600000, + "async_insert_busy_timeout_max_ms": 3600000, +} + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + yield + finally: + cluster.shutdown() + + +def metric(name): + return int(node.query(f"SELECT value FROM system.metrics WHERE metric = '{name}'")) + + +@pytest.fixture +def tables(): + for table in ("saturated", "unrelated"): + node.query(f"CREATE TABLE {table} (x UInt64) ENGINE = MergeTree ORDER BY x") + try: + yield + finally: + node.query("SYSTEM FLUSH ASYNC INSERT QUEUE", timeout=30) + for table in ("saturated", "unrelated"): + node.query(f"DROP TABLE {table}") + + +@pytest.mark.parametrize("wait_for_async_insert", [0, 1]) +@pytest.mark.parametrize( + "flush_query", + ["SYSTEM FLUSH ASYNC INSERT QUEUE", "SYSTEM FLUSH ASYNC INSERT QUEUE unrelated"], +) +def test_saturated_pool_does_not_lock_queue_shard( + tables, wait_for_async_insert, flush_query +): + # One worker also means one queue shard: both tables necessarily share its mutex. + waiter_id = f"pool_waiter_{uuid4().hex}" + flush_id = f"explicit_flush_{uuid4().hex}" + with ThreadPoolExecutor(max_workers=2) as executor: + try: + node.query(f"SYSTEM ENABLE FAILPOINT {FAILPOINT}") + + # Exactly two bytes reach the size threshold without the oversized-input + # fallback. HTTP keeps parsing in the server, where the failpoint runs. + node.http_query( + "INSERT INTO saturated FORMAT TSV", + data="1\n", + params=SETTINGS, + timeout=30, + ) + node.query(f"SYSTEM WAIT FAILPOINT {FAILPOINT} PAUSE", timeout=30) + assert metric("AsynchronousInsertThreadsScheduled") == 1 + + waiter = executor.submit( + node.http_query, + "INSERT INTO saturated FORMAT TSV", + data="2\n", + params={ + **SETTINGS, + "wait_for_async_insert": wait_for_async_insert, + "query_id": waiter_id, + }, + timeout=120, + ) + # This is emitted with the shard lock held, just before the blocking + # submission. Waiting for it prevents the unrelated insert racing ahead. + node.wait_for_log_line( + rf"\{{{waiter_id}\}}.*Scheduling async insert processing job because enough bytes accumulated", + timeout=30, + ) + + # This insert only buffers. On the original code it times out waiting + # for the mutex held by the producer trying to submit the second flush. + assert ( + node.http_query( + "INSERT INTO unrelated FORMAT TSV", + data="3\n", + params={**SETTINGS, "async_insert_max_data_size": 1024}, + timeout=30, + ) + == "" + ) + assert not waiter.done() + assert metric("AsynchronousInsertThreadsScheduled") == 1 + assert metric("PendingAsyncInsert") == 3 + + flush = executor.submit( + node.query, flush_query, query_id=flush_id, timeout=120 + ) + node.wait_for_log_line( + rf"\{{{flush_id}\}}.*Requested to flush asynchronous insert queue", + timeout=30, + ) + assert not flush.done() + + node.query(f"SYSTEM DISABLE FAILPOINT {FAILPOINT}") + assert waiter.result(timeout=30) == "" + assert flush.result(timeout=30) == "" + node.query("SYSTEM FLUSH ASYNC INSERT QUEUE") + + assert node.query("SELECT x FROM saturated ORDER BY x") == "1\n2\n" + assert node.query("SELECT x FROM unrelated") == "3\n" + assert metric("PendingAsyncInsert") == 0 + assert metric("AsynchronousInsertThreadsScheduled") == 0 + finally: + # Release the worker even when the regression times out, before joining + # HTTP requests that need it to make progress. + node.query(f"SYSTEM DISABLE FAILPOINT {FAILPOINT}") + + +@pytest.mark.parametrize("trigger", ["size", "deadline"]) +@pytest.mark.parametrize("bad_data", [False, True]) +def test_forced_flush_waits_for_batch_before_pool_admission(tables, trigger, bad_data): + settings = {**SETTINGS, "async_insert_max_data_size": 4} + if trigger == "deadline": + settings["async_insert_busy_timeout_min_ms"] = 1000 + settings["async_insert_busy_timeout_max_ms"] = 1000 + + flush_id = f"drain_handoff_{uuid4().hex}" + with ThreadPoolExecutor(max_workers=2) as executor: + try: + node.query(f"SYSTEM ENABLE FAILPOINT {ADMISSION_FAILPOINT}") + + # The first request is acknowledged while still buffered. A subsequent + # producer or the deadline worker removes its batch from the shard queue. + node.http_query( + "INSERT INTO saturated FORMAT TSV", + data="x\n" if bad_data else "1\n", + params=settings, + timeout=30, + ) + producer = None + if trigger == "size": + producer = executor.submit( + node.http_query, + "INSERT INTO saturated FORMAT TSV", + data="x\n" if bad_data else "2\n", + params=settings, + timeout=120, + ) + node.query(f"SYSTEM WAIT FAILPOINT {ADMISSION_FAILPOINT} PAUSE", timeout=30) + assert metric("AsynchronousInsertThreadsScheduled") == 0 + + flush = executor.submit( + node.query, + "SYSTEM FLUSH ASYNC INSERT QUEUE", + query_id=flush_id, + timeout=120, + ) + node.wait_for_log_line( + rf"\{{{flush_id}\}}.*Will wait for finishing of 0 flushing jobs", + timeout=30, + ) + # An empty pool must not let the flush overlook the acknowledged row + # in the batch paused just before admission. + with pytest.raises(TimeoutError): + flush.result(timeout=1) + + # Waiting for that batch must also leave the shard mutex available. + node.http_query( + "INSERT INTO unrelated FORMAT TSV", + data="3\n", + params={**SETTINGS, "async_insert_max_data_size": 1024}, + timeout=30, + ) + + node.query(f"SYSTEM DISABLE FAILPOINT {ADMISSION_FAILPOINT}") + if producer is not None: + assert producer.result(timeout=30) == "" + assert flush.result(timeout=30) == "" + expected = "" if bad_data else "1\n2\n" if trigger == "size" else "1\n" + assert node.query("SELECT x FROM saturated ORDER BY x") == expected + node.query("SYSTEM FLUSH ASYNC INSERT QUEUE") + assert node.query("SELECT x FROM unrelated") == "3\n" + assert metric("PendingAsyncInsert") == 0 + finally: + node.query(f"SYSTEM DISABLE FAILPOINT {ADMISSION_FAILPOINT}") + + +@pytest.mark.parametrize("flush_on_shutdown", [False, True]) +def test_shutdown_handles_batch_before_pool_admission(flush_on_shutdown): + shutdown_node = node if flush_on_shutdown else node_no_flush + introspection = Client(shutdown_node.ip_address, 9010, command=cluster.client_bin_path) + table = f"shutdown_pending_{uuid4().hex}" + shutdown_node.query(f"CREATE TABLE {table} (x UInt64) ENGINE = MergeTree ORDER BY x") + + try: + with ThreadPoolExecutor(max_workers=1) as executor: + stop = None + try: + shutdown_node.query(f"SYSTEM ENABLE FAILPOINT {ADMISSION_FAILPOINT}") + # A deadline-triggered batch contains an acknowledged insert and has no + # producer connection left for server shutdown to wait on. + assert ( + shutdown_node.http_query( + f"INSERT INTO {table} FORMAT TSV", + data="1\n", + params={ + **SETTINGS, + "async_insert_max_data_size": 1024, + "async_insert_busy_timeout_min_ms": 1000, + "async_insert_busy_timeout_max_ms": 1000, + }, + timeout=30, + ) + == "" + ) + shutdown_node.query(f"SYSTEM WAIT FAILPOINT {ADMISSION_FAILPOINT} PAUSE", timeout=30) + assert shutdown_node.query("SELECT value FROM system.metrics WHERE metric = 'AsynchronousInsertThreadsScheduled'") == "0\n" + + stop = executor.submit(shutdown_node.stop_clickhouse, stop_wait_sec=60, kill=False) + shutdown_node.wait_for_log_line("Shutting down the asynchronous insertion queue", timeout=30) + assert not stop.done() + # Normal listeners have closed. The introspection listener remains + # available until after the async queue and database catalog shut down. + introspection.query(f"SYSTEM DISABLE FAILPOINT {ADMISSION_FAILPOINT}", timeout=30) + stop.result(timeout=90) + assert shutdown_node.contains_in_log("Asynchronous insertion queue finished", from_host=True) + assert shutdown_node.contains_in_log("Background threads finished", from_host=True) + finally: + if stop is None: + shutdown_node.query(f"SYSTEM DISABLE FAILPOINT {ADMISSION_FAILPOINT}") + elif not stop.done(): + introspection.query(f"SYSTEM DISABLE FAILPOINT {ADMISSION_FAILPOINT}", timeout=30) + stop.result(timeout=90) + if stop is not None: + shutdown_node.start_clickhouse() + + expected = "1\n" if flush_on_shutdown else "0\n" + assert shutdown_node.query(f"SELECT count() FROM {table}") == expected + finally: + shutdown_node.query(f"DROP TABLE {table}") diff --git a/tests/integration/test_auth_method_valid_until_stateful_protocols/__init__.py b/tests/integration/test_auth_method_valid_until_stateful_protocols/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_auth_method_valid_until_stateful_protocols/configs/protocols.xml b/tests/integration/test_auth_method_valid_until_stateful_protocols/configs/protocols.xml new file mode 100644 index 0000000000000..65eafc59c3f85 --- /dev/null +++ b/tests/integration/test_auth_method_valid_until_stateful_protocols/configs/protocols.xml @@ -0,0 +1,4 @@ + + 9001 + 5433 + diff --git a/tests/integration/test_auth_method_valid_until_stateful_protocols/test.py b/tests/integration/test_auth_method_valid_until_stateful_protocols/test.py new file mode 100644 index 0000000000000..1faf74afbbebd --- /dev/null +++ b/tests/integration/test_auth_method_valid_until_stateful_protocols/test.py @@ -0,0 +1,112 @@ +import time + +import psycopg2 +import pymysql.connections +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +# The authentication method's `VALID UNTIL` must be enforced per query, not only at login. +# Stateful protocols (MySQL, PostgreSQL) authenticate once at connection startup and then run +# every later command through `Session::makeQueryContext`, so without a per-query re-check a +# credential that expires after login would keep working for the lifetime of the connection. +# The check lives in `Session::makeQueryContextImpl`, so every protocol shares it. +node = cluster.add_instance("node", main_configs=["configs/protocols.xml"]) + +MYSQL_PORT = 9001 +POSTGRES_PORT = 5433 + +# Lifetime of the expiring credential, measured from user creation. The first query on each +# connection runs within a fraction of a second of creation (well inside the lifetime); the second +# query runs after sleeping past the expiry. +EXPIRING_LIFETIME_S = 6 + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def create_users(): + node.query("DROP USER IF EXISTS u_expiring, u_lasting") + expiry = node.query(f"SELECT toString(now() + INTERVAL {EXPIRING_LIFETIME_S} SECOND)").strip() + # Plaintext-stored passwords so both the MySQL and PostgreSQL frontends can verify them. + node.query(f"CREATE USER u_expiring IDENTIFIED WITH plaintext_password BY 'pw' VALID UNTIL '{expiry}'") + node.query("CREATE USER u_lasting IDENTIFIED WITH plaintext_password BY 'pw' VALID UNTIL '2999-01-01 00:00:00'") + node.query("GRANT SELECT ON system.one TO u_expiring, u_lasting") + return expiry + + +def sleep_past(expiry): + while int(node.query(f"SELECT now() > toDateTime('{expiry}')").strip()) != 1: + time.sleep(0.5) + # The check is `now > valid_until` with second precision, so cross the boundary decisively. + time.sleep(1.5) + + +def test_mysql_connection_stops_working_after_expiry(started_cluster): + expiry = create_users() + host = started_cluster.get_instance_ip("node") + + def connect(user): + return pymysql.connections.Connection(host=host, user=user, password="pw", database="default", port=MYSQL_PORT) + + expiring = connect("u_expiring") + lasting = connect("u_lasting") + + # Both credentials are valid at connection time and for the first query. + for client in (expiring, lasting): + cursor = client.cursor() + cursor.execute("SELECT 1") + assert cursor.fetchall() == ((1,),) + + sleep_past(expiry) + + # The same, still-open connection must stop working once the method has expired ... + with pytest.raises(pymysql.err.MySQLError, match="expired"): + expiring.cursor().execute("SELECT 1") + + # ... while a connection under a non-expired method keeps working. + cursor = lasting.cursor() + cursor.execute("SELECT 1") + assert cursor.fetchall() == ((1,),) + + lasting.close() + + +def test_postgresql_connection_stops_working_after_expiry(started_cluster): + expiry = create_users() + host = started_cluster.get_instance_ip("node") + + def connect(user): + conn = psycopg2.connect(host=host, user=user, password="pw", dbname="default", port=POSTGRES_PORT) + conn.autocommit = True + return conn + + expiring = connect("u_expiring") + lasting = connect("u_lasting") + + for client in (expiring, lasting): + cursor = client.cursor() + cursor.execute("SELECT 1") + assert cursor.fetchall() == [(1,)] + + sleep_past(expiry) + + # The PostgreSQL handler sends an ErrorResponse and then terminates the connection on an expired + # credential, and psycopg2 may surface either the message or the connection loss - both are + # fail-close; the point is that the query must not succeed. + with pytest.raises(psycopg2.Error, match="expired|closed the connection"): + expiring.cursor().execute("SELECT 1") + + cursor = lasting.cursor() + cursor.execute("SELECT 1") + assert cursor.fetchall() == [(1,)] + + lasting.close() diff --git a/tests/integration/test_backup_restore_s3/configs/remote_url_allow_hosts.xml b/tests/integration/test_backup_restore_s3/configs/remote_url_allow_hosts.xml new file mode 100644 index 0000000000000..294b4bc7dcb49 --- /dev/null +++ b/tests/integration/test_backup_restore_s3/configs/remote_url_allow_hosts.xml @@ -0,0 +1,5 @@ + + + minio1:9001 + + diff --git a/tests/integration/test_backup_restore_s3/configs/remote_url_allow_hosts_s3_bucket.xml b/tests/integration/test_backup_restore_s3/configs/remote_url_allow_hosts_s3_bucket.xml new file mode 100644 index 0000000000000..b7a6f3ca18157 --- /dev/null +++ b/tests/integration/test_backup_restore_s3/configs/remote_url_allow_hosts_s3_bucket.xml @@ -0,0 +1,5 @@ + + + minio1:9001/root + + diff --git a/tests/integration/test_backup_restore_s3/configs/remote_url_allow_hosts_s3_bucket_no_port.xml b/tests/integration/test_backup_restore_s3/configs/remote_url_allow_hosts_s3_bucket_no_port.xml new file mode 100644 index 0000000000000..ce808dc7c3ac9 --- /dev/null +++ b/tests/integration/test_backup_restore_s3/configs/remote_url_allow_hosts_s3_bucket_no_port.xml @@ -0,0 +1,5 @@ + + + minio1/root + + diff --git a/tests/integration/test_backup_restore_s3/test_remote_host_filter.py b/tests/integration/test_backup_restore_s3/test_remote_host_filter.py new file mode 100644 index 0000000000000..977d02c84e734 --- /dev/null +++ b/tests/integration/test_backup_restore_s3/test_remote_host_filter.py @@ -0,0 +1,56 @@ +import uuid + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.config_cluster import minio_secret_key + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + main_configs=["configs/remote_url_allow_hosts.xml"], + with_minio=True, +) + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + node.query( + "CREATE TABLE t (id UInt64, s String) ENGINE = MergeTree ORDER BY id" + ) + node.query("INSERT INTO t VALUES (1, 'a'), (2, 'b')") + yield cluster + finally: + cluster.shutdown() + + +def test_backup_to_allowed_host(): + name = uuid.uuid4().hex + destination = f"S3('http://minio1:9001/root/data/backups/{name}', 'minio', '{minio_secret_key}')" + node.query(f"BACKUP TABLE t TO {destination}") + node.query(f"RESTORE TABLE t AS t_{name} FROM {destination}") + assert node.query(f"SELECT count() FROM t_{name}") == "2\n" + node.query(f"DROP TABLE t_{name} SYNC") + + +@pytest.mark.parametrize( + "url", + [ + "http://minio1:9002/root/data/backups/x", + "http://resolver:8080/root/data/backups/x", + "http://127.0.0.1:9/root/data/backups/x", + ], +) +def test_backup_to_disallowed_host(url): + destination = f"S3('{url}', 'minio', '{minio_secret_key}')" + settings = "SETTINGS backup_restore_s3_retry_attempts = 0" + + error = node.query_and_get_error(f"BACKUP TABLE t TO {destination} {settings}") + assert "UNACCEPTABLE_URL" in error, error + + error = node.query_and_get_error( + f"RESTORE TABLE t AS t_restored FROM {destination} {settings}" + ) + assert "UNACCEPTABLE_URL" in error, error diff --git a/tests/integration/test_backup_restore_s3/test_remote_host_filter_s3_bucket.py b/tests/integration/test_backup_restore_s3/test_remote_host_filter_s3_bucket.py new file mode 100644 index 0000000000000..e946ad4402ac7 --- /dev/null +++ b/tests/integration/test_backup_restore_s3/test_remote_host_filter_s3_bucket.py @@ -0,0 +1,247 @@ +import uuid + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.config_cluster import minio_secret_key + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + main_configs=["configs/remote_url_allow_hosts_s3_bucket.xml"], + with_minio=True, +) +# The same bucket, allowed without a port: this is the form of a real entry for a shared cloud +# endpoint on the default port, for example `s3.eu-west-1.amazonaws.com/`. +node_no_port = cluster.add_instance( + "node_no_port", + main_configs=["configs/remote_url_allow_hosts_s3_bucket_no_port.xml"], + with_minio=True, +) + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + for instance in (node, node_no_port): + instance.query( + "CREATE TABLE t (id UInt64, s String) ENGINE = MergeTree ORDER BY id" + ) + instance.query("INSERT INTO t VALUES (1, 'a'), (2, 'b')") + yield cluster + finally: + cluster.shutdown() + + +def test_backup_to_allowed_bucket(): + name = uuid.uuid4().hex + destination = f"S3('http://minio1:9001/root/data/backups/{name}', 'minio', '{minio_secret_key}')" + node.query(f"BACKUP TABLE t TO {destination}") + node.query(f"RESTORE TABLE t AS t_{name} FROM {destination}") + assert node.query(f"SELECT count() FROM t_{name}") == "2\n" + node.query(f"DROP TABLE t_{name} SYNC") + + +@pytest.mark.parametrize( + "url", + [ + "http://minio1:9001/other/data/backups/x", + "http://minio1:9001/root-other/data/backups/x", + "http://minio1:9002/root/data/backups/x", + "http://resolver:8080/root/data/backups/x", + ], +) +def test_backup_to_disallowed_bucket(url): + destination = f"S3('{url}', 'minio', '{minio_secret_key}')" + settings = "SETTINGS backup_restore_s3_retry_attempts = 0" + + error = node.query_and_get_error(f"BACKUP TABLE t TO {destination} {settings}") + assert "UNACCEPTABLE_URL" in error, error + + error = node.query_and_get_error( + f"RESTORE TABLE t AS t_restored FROM {destination} {settings}" + ) + assert "UNACCEPTABLE_URL" in error, error + + +def test_s3_table_function_respects_bucket(): + name = uuid.uuid4().hex + node.query( + f"INSERT INTO FUNCTION s3('http://minio1:9001/root/data/{name}.csv', 'minio', '{minio_secret_key}', 'CSV') " + "SELECT * FROM t" + ) + assert ( + node.query( + f"SELECT count() FROM s3('http://minio1:9001/root/data/{name}.csv', 'minio', '{minio_secret_key}', 'CSV', 'id UInt64, s String')" + ) + == "2\n" + ) + + error = node.query_and_get_error( + f"SELECT * FROM s3('http://minio1:9001/other/data/{name}.csv', 'minio', '{minio_secret_key}', 'CSV', 'id UInt64, s String')" + ) + assert "UNACCEPTABLE_URL" in error, error + + error = node.query_and_get_error( + f"SELECT * FROM url('http://minio1:9001/root/data/{name}.csv', 'CSV', 'id UInt64, s String')" + ) + assert "UNACCEPTABLE_URL" in error, error + + +@pytest.mark.parametrize("disk_type", ["s3", "s3_plain_rewritable"]) +def test_custom_s3_disk_respects_bucket(disk_type): + name = uuid.uuid4().hex + + def create_query(table, endpoint): + return f""" + CREATE TABLE {table} (id UInt64, s String) ENGINE = MergeTree ORDER BY id + SETTINGS disk = disk( + type = '{disk_type}', + endpoint = '{endpoint}', + access_key_id = 'minio', + secret_access_key = '{minio_secret_key}') + """ + + node.query(create_query(f"t_disk_{name}", f"http://minio1:9001/root/data/disks/{name}/")) + node.query(f"INSERT INTO t_disk_{name} SELECT * FROM t") + assert node.query(f"SELECT count() FROM t_disk_{name}") == "2\n" + node.query(f"DROP TABLE t_disk_{name} SYNC") + + # Other buckets on the allowed host, another port and another host are rejected before the disk + # (and its S3 client) is created. + for endpoint in [ + f"http://minio1:9001/other/data/disks/{name}/", + f"http://minio1:9001/root-other/data/disks/{name}/", + f"http://minio1:9002/root/data/disks/{name}/", + f"http://resolver:8080/root/data/disks/{name}/", + ]: + error = node.query_and_get_error(create_query(f"t_disk_bad_{name}", endpoint)) + assert "UNACCEPTABLE_URL" in error, error + + +def test_database_s3_respects_bucket(): + name = uuid.uuid4().hex + node.query( + f"INSERT INTO FUNCTION s3('http://minio1:9001/root/data/{name}.csv', 'minio', '{minio_secret_key}', 'CSV') " + "SELECT * FROM t" + ) + node.query( + f"CREATE DATABASE db_{name} ENGINE = S3('http://minio1:9001/root', 'minio', '{minio_secret_key}')" + ) + assert node.query(f"EXISTS TABLE db_{name}.`data/{name}.csv`") == "1\n" + assert node.query(f"SELECT count() FROM db_{name}.`data/{name}.csv`") == "2\n" + + node.query( + f"CREATE DATABASE db_other_{name} ENGINE = S3('http://minio1:9001/other', 'minio', '{minio_secret_key}')" + ) + # `DatabaseS3` checks the URL with `throw_on_error = false` when it looks a table up, so a rejected + # bucket surfaces as a missing table and not as `UNACCEPTABLE_URL`. `CREATE DATABASE` itself does not + # check the URL. + assert node.query(f"EXISTS TABLE db_other_{name}.`data/{name}.csv`") == "0\n" + error = node.query_and_get_error( + f"SELECT * FROM db_other_{name}.`data/{name}.csv`" + ) + assert "UNKNOWN_TABLE" in error, error + + node.query(f"DROP DATABASE db_{name}") + node.query(f"DROP DATABASE db_other_{name}") + + +def test_bucket_entry_without_port(): + # `minio1/root` names no port, so it matches the bucket on every port, like a + # `` entry without a port. + name = uuid.uuid4().hex + destination = f"S3('http://minio1:9001/root/data/backups/{name}', 'minio', '{minio_secret_key}')" + node_no_port.query(f"BACKUP TABLE t TO {destination}") + node_no_port.query(f"RESTORE TABLE t AS t_{name} FROM {destination}") + assert node_no_port.query(f"SELECT count() FROM t_{name}") == "2\n" + node_no_port.query(f"DROP TABLE t_{name} SYNC") + + settings = "SETTINGS backup_restore_s3_retry_attempts = 0" + + # Another bucket on the same host is still rejected. + error = node_no_port.query_and_get_error( + f"BACKUP TABLE t TO S3('http://minio1:9001/other/data/backups/{name}', 'minio', '{minio_secret_key}') {settings}" + ) + assert "UNACCEPTABLE_URL" in error, error + + # The same bucket on another port passes the filter. Nothing listens on 9002, so the failure comes + # from the connection and not from `remote_url_allow_hosts`. + error = node_no_port.query_and_get_error( + f"BACKUP TABLE t TO S3('http://minio1:9002/root/data/backups/{name}', 'minio', '{minio_secret_key}') {settings}" + ) + assert "S3_ERROR" in error and "UNACCEPTABLE_URL" not in error, error + + +def test_base_backup_respects_bucket(): + # The `base_backup` of an incremental backup is a second S3 reference in the same statement and goes + # through the same check as the destination. + name = uuid.uuid4().hex + node.query( + f"CREATE TABLE t_{name} (id UInt64, s String) ENGINE = MergeTree ORDER BY id" + ) + node.query(f"INSERT INTO t_{name} VALUES (1, 'a'), (2, 'b')") + + base = f"S3('http://minio1:9001/root/data/backups/{name}_base', 'minio', '{minio_secret_key}')" + incremental = f"S3('http://minio1:9001/root/data/backups/{name}_inc', 'minio', '{minio_secret_key}')" + node.query(f"BACKUP TABLE t_{name} TO {base}") + node.query(f"INSERT INTO t_{name} VALUES (3, 'c')") + node.query(f"BACKUP TABLE t_{name} TO {incremental} SETTINGS base_backup = {base}") + node.query(f"RESTORE TABLE t_{name} AS t_{name}_restored FROM {incremental}") + assert node.query(f"SELECT count() FROM t_{name}_restored") == "3\n" + + rejected_base = f"S3('http://minio1:9001/other/data/backups/{name}_base', 'minio', '{minio_secret_key}')" + error = node.query_and_get_error( + f"BACKUP TABLE t_{name} TO S3('http://minio1:9001/root/data/backups/{name}_inc2', 'minio', '{minio_secret_key}') " + f"SETTINGS base_backup = {rejected_base}, backup_restore_s3_retry_attempts = 0" + ) + assert "UNACCEPTABLE_URL" in error, error + + node.query(f"DROP TABLE t_{name} SYNC") + node.query(f"DROP TABLE t_{name}_restored SYNC") + + +def test_reload_config_updates_buckets_and_keeps_previous_on_error(): + # `remote_url_allow_hosts` is applied on `SYSTEM RELOAD CONFIG` without a restart. A malformed + # `` makes the reload fail and keeps the list from the previous successful load. + # `root2` is the second bucket that the test MinIO creates. + config_path = "/etc/clickhouse-server/config.d/remote_url_allow_hosts_s3_bucket.xml" + name = uuid.uuid4().hex + settings = "SETTINGS backup_restore_s3_retry_attempts = 0" + + def backup_query(bucket, step): + return f"BACKUP TABLE t TO S3('http://minio1:9001/{bucket}/data/backups/{name}_{step}', 'minio', '{minio_secret_key}') {settings}" + + def backup_error(bucket, step): + return node.query_and_get_error(backup_query(bucket, step)) + + def set_config(buckets): + entries = "".join(f"{bucket}" for bucket in buckets) + node.replace_config( + config_path, + f"{entries}", + ) + + try: + assert "UNACCEPTABLE_URL" in backup_error("root2", 1) + + set_config(["minio1:9001/root", "minio1:9001/root2"]) + node.query("SYSTEM RELOAD CONFIG") + node.query(backup_query("root2", 2)) + assert "UNACCEPTABLE_URL" in backup_error("root-other", 3) + + # The malformed entry comes first: a parser that clears the lists before it reads the entries + # would leave nothing allowed here, so the checks below would reject `root` and `root2`. + set_config(["minio1:9001", "minio1:9001/root", "minio1:9001/root2"]) + error = node.query_and_get_error("SYSTEM RELOAD CONFIG") + assert "must have the form host/bucket or host:port/bucket" in error, error + node.query(backup_query("root", 4)) + node.query(backup_query("root2", 5)) + assert "UNACCEPTABLE_URL" in backup_error("root-other", 6) + finally: + set_config(["minio1:9001/root"]) + node.query("SYSTEM RELOAD CONFIG") + + assert "UNACCEPTABLE_URL" in backup_error("root2", 7) + node.query(backup_query("root", 8)) diff --git a/tests/integration/test_backup_restore_s3_role_arn/test.py b/tests/integration/test_backup_restore_s3_role_arn/test.py index a4ed1c43ca7dd..8c4126518e29a 100644 --- a/tests/integration/test_backup_restore_s3_role_arn/test.py +++ b/tests/integration/test_backup_restore_s3_role_arn/test.py @@ -9,9 +9,11 @@ from helpers.cluster import ClickHouseCluster from helpers.mock_servers import start_mock_servers -# `role_arn` and `role_session_name` are not secrets -- assuming the role still needs the server's -# own identity and a matching trust policy -- so they survive into the `` locator and -# let every hop of a chain reopen its base. The mock STS accepts the role and session name below. +# `role_arn` and `role_session_name` survive into the `` locator and let every hop of a +# chain reopen its base: assuming the role still needs the server's own identity and a matching trust +# policy. The session name is masked in logged query text, because a trust policy can pin it, but the +# stored locator keeps it on purpose (see `BackupInfo.cpp`). The mock STS accepts the role and +# session name below. # # Metadata written by a version that stripped the identifiers names no credentials and carries no # marker to reconstruct them from, so its base backup opens unauthenticated. Such a chain must still diff --git a/tests/integration/test_backup_source_grants/configs/backups.xml b/tests/integration/test_backup_source_grants/configs/backups.xml index 2d99cb1d1ef92..acf77f033a2f2 100644 --- a/tests/integration/test_backup_source_grants/configs/backups.xml +++ b/tests/integration/test_backup_source_grants/configs/backups.xml @@ -1,5 +1,14 @@ + + + + local + /var/lib/clickhouse/disks/backup_disk/ + + + /var/lib/clickhouse/backups + backup_disk diff --git a/tests/integration/test_backup_source_grants/test.py b/tests/integration/test_backup_source_grants/test.py index 42b44da713dcb..ff8efd1037bdd 100644 --- a/tests/integration/test_backup_source_grants/test.py +++ b/tests/integration/test_backup_source_grants/test.py @@ -256,9 +256,8 @@ def test_restore_on_cluster_authorizes_an_embedded_definition_over_an_existing_d # the check is skipped here and the restricted user reaches the embedded locator unchecked. # # The outer locator is File and IS granted, so a denial naming READ ON S3 can only come from the - # embedded S3 locator. `BACKUP DATABASE` serializes the locator as a string that - # `BackupInfo::fromAST` rejects before any check, so the function form is written into the - # manifest directly - the attacker-controlled-manifest shape this authorization exists for. + # embedded S3 locator, which is why the manifest is rewritten to hold one - the + # attacker-controlled-manifest shape this authorization exists for. # Every locator here is credential-free (File locally, a 1-argument S3 URL in the manifest): the # later definition-mismatch error logs both definitions, and a masked credential in that line # trips the tests-only `throw_on_match` masking rule and aborts the server. @@ -289,7 +288,7 @@ def test_restore_on_cluster_authorizes_an_embedded_definition_over_an_existing_d assert "READ ON S3" in error, error # With the grant, authorization passes: the restore proceeds past CHECKING_ACCESS_RIGHTS and - # fails later on the pre-existing string-vs-function definition mismatch instead. + # fails later on the mismatch between the pre-existing definition and the rewritten one instead. node.query(f"GRANT READ ON S3 TO {USER}") error = node.query_and_get_error( "RESTORE DATABASE dbembedded ON CLUSTER one_shard FROM File('outer10')", user=USER @@ -300,41 +299,103 @@ def test_restore_on_cluster_authorizes_an_embedded_definition_over_an_existing_d node.query("DROP DATABASE dbembedded SYNC") -def test_restore_on_cluster_of_a_real_backup_engine_manifest_is_unchanged(started_cluster): - # The manifest is NOT crafted here, which is the point: `BACKUP DATABASE` serializes the inner - # locator as a string, and authorizing it would parse it and reject it with BAD_ARGUMENTS before - # any access decision. Only this shape can catch that, so it is a separate case from the crafted - # one - which asserts the security property but cannot see this class. +def test_restore_on_cluster_refuses_a_quoted_embedded_locator_over_an_existing_database( + started_cluster, +): + # The pre-fix carrier: metadata an older server rewrote holds the locator as a string literal, and + # such a manifest can end up inside a backup. On this path the creation-time refusal never runs - + # the target database exists, so `CREATE DATABASE IF NOT EXISTS` returns before `DatabaseFactory`, + # and `allow_different_database_def = 1` waives the definition mismatch that would stop it next. + # The preflight is therefore the only place that can see the quoted locator, and it must refuse it + # rather than wave it through unauthorized: the function form of the very same manifest is denied + # for a missing `READ ON S3` in the case above. + # + # The refusal must not print the locator, which is why the assertion below looks for the host that + # a quoted locator could carry credentials next to. + node.query("DROP DATABASE IF EXISTS dbquoted SYNC") + node.query("BACKUP DATABASE d67785 TO File('inner13') FORMAT Null") + node.query("CREATE DATABASE dbquoted ENGINE = Backup('d67785', File('inner13'))") + node.query("BACKUP DATABASE dbquoted TO File('outer13') FORMAT Null") + manifest = ( + "CREATE DATABASE dbquoted ENGINE = Backup('d67785', " + "'S3(\\'http://minio1:9001/root/data/denied/b13\\')')" + ) + node.exec_in_container( + [ + "bash", + "-c", + "cat > /var/lib/clickhouse/backups/outer13/metadata/dbquoted.sql <<'MANIFEST'\n" + f"{manifest}\nMANIFEST", + ], + user="root", + ) + + assert node.query("SELECT count() FROM system.databases WHERE name = 'dbquoted'") == "1\n" + node.query(f"GRANT READ ON FILE TO {USER}") + + error = node.query_and_get_error( + "RESTORE DATABASE dbquoted ON CLUSTER one_shard FROM File('outer13') " + "SETTINGS allow_different_database_def = 1", + user=USER, + ) + assert "BAD_ARGUMENTS" in error, error + assert "Expected function as the backup destination" in error, error + assert "minio1" not in error, error + + node.query("DROP DATABASE dbquoted SYNC") + + +def test_restore_on_cluster_of_a_real_backup_engine_manifest_authorizes_the_inner_locator( + started_cluster, +): + # The manifest is NOT crafted here, which is the point: `BACKUP DATABASE` writes the inner locator + # as the function it is, so a real manifest carries a locator that authorization can decode, and + # the embedded one is checked exactly as a rewritten one is. Only this shape can see that, so it is + # a separate case from the rewritten one - which asserts the security property against a locator + # the definition never held. The outer locator is a Disk backup, which needs READ ON DISK, so a + # denial naming READ ON FILE can only come from the inner one. node.query("DROP DATABASE IF EXISTS dbreal SYNC") node.query("BACKUP DATABASE d67785 TO File('inner11') FORMAT Null") node.query("CREATE DATABASE dbreal ENGINE = Backup('d67785', File('inner11'))") - node.query("BACKUP DATABASE dbreal TO File('outer11') FORMAT Null") + node.query("BACKUP DATABASE dbreal TO Disk('backup_disk', 'outer11') FORMAT Null") manifest = node.exec_in_container( - ["bash", "-c", "cat /var/lib/clickhouse/backups/outer11/metadata/dbreal.sql"], + [ + "bash", + "-c", + "cat /var/lib/clickhouse/disks/backup_disk/outer11/metadata/dbreal.sql", + ], user="root", ) - # The locator really is the string form: an ASTLiteral, not an ASTFunction. - assert "Backup('d67785', 'File(\\'inner11\\')')" in manifest, manifest + # The locator really is the function form: an ASTFunction, not an ASTLiteral holding its text. + assert "Backup('d67785', File('inner11'))" in manifest, manifest - # Only the outer locator's own grant; the string-form inner one must not be authorized at all. - node.query(f"GRANT READ ON FILE TO {USER}") + # Only the outer locator's own grant, so the embedded File locator is the one that is missing. + node.query(f"GRANT READ ON DISK TO {USER}") + + error = node.query_and_get_error( + "RESTORE DATABASE dbreal ON CLUSTER one_shard FROM Disk('backup_disk', 'outer11')", + user=USER, + ) + assert "ACCESS_DENIED" in error, error + assert "READ ON FILE" in error, error - # Target exists, so nothing is created and the restore fully succeeds. Authorizing the string - # form turns this into `Code: 36` out of CHECKING_ACCESS_RIGHTS. + # Target exists and its definition matches the manifest, so nothing is created and the restore + # fully succeeds. + node.query(f"GRANT READ ON FILE TO {USER}") node.query( - "RESTORE DATABASE dbreal ON CLUSTER one_shard FROM File('outer11') FORMAT Null", user=USER + "RESTORE DATABASE dbreal ON CLUSTER one_shard FROM Disk('backup_disk', 'outer11') FORMAT Null", + user=USER, ) - # Target absent, so creation runs and rejects the string form - as it does without this feature. - # `While creating database` is what pins the failure to the creation stage rather than the - # access-check one, which is where the same code would report it. + # Target absent, so creation runs: the locator parses, and the created database reads the tables of + # the inner backup. node.query("DROP DATABASE dbreal SYNC") - error = node.query_and_get_error( - "RESTORE DATABASE dbreal ON CLUSTER one_shard FROM File('outer11')", user=USER + node.query( + "RESTORE DATABASE dbreal ON CLUSTER one_shard FROM Disk('backup_disk', 'outer11') FORMAT Null", + user=USER, ) - assert "ACCESS_DENIED" not in error, error - assert "BAD_ARGUMENTS" in error, error - assert "While creating database" in error, error + assert node.query("SELECT x FROM dbreal.secrets") == "42\n" + node.query("DROP DATABASE dbreal SYNC") def test_explicit_base_backup_locator_is_authorized_on_the_initiator(started_cluster): diff --git a/tests/integration/test_backward_compatibility/configs/drop_table_delay.xml b/tests/integration/test_backward_compatibility/configs/drop_table_delay.xml new file mode 100644 index 0000000000000..16393cfbdf0af --- /dev/null +++ b/tests/integration/test_backward_compatibility/configs/drop_table_delay.xml @@ -0,0 +1,4 @@ + + + 3600 + diff --git a/tests/integration/test_backward_compatibility/configs/remote_url_allow_hosts_without_minio.xml b/tests/integration/test_backward_compatibility/configs/remote_url_allow_hosts_without_minio.xml new file mode 100644 index 0000000000000..b62bb5c5623e8 --- /dev/null +++ b/tests/integration/test_backward_compatibility/configs/remote_url_allow_hosts_without_minio.xml @@ -0,0 +1,5 @@ + + + allowed.example.com + + diff --git a/tests/integration/test_backward_compatibility/test_custom_s3_disk_remote_url_allow_hosts.py b/tests/integration/test_backward_compatibility/test_custom_s3_disk_remote_url_allow_hosts.py new file mode 100644 index 0000000000000..89d8ade4bc71b --- /dev/null +++ b/tests/integration/test_backward_compatibility/test_custom_s3_disk_remote_url_allow_hosts.py @@ -0,0 +1,138 @@ +import uuid + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.config_cluster import minio_secret_key + +cluster = ClickHouseCluster(__file__) + +# 26.8.26.12 does not check SQL disks against `remote_url_allow_hosts`. The objects are created there with a list +# that does not name MinIO, then the server restarts on the current binary. +# Two instances: a failed check on the database metadata disk would stop the server. +node = cluster.add_instance( + "node", + main_configs=[ + "configs/remote_url_allow_hosts_without_minio.xml", + "configs/drop_table_delay.xml", + ], + image="clickhouse/clickhouse-server", + tag="26.8.26.12", + with_installed_binary=True, + with_minio=True, + stay_alive=True, +) +node_db = cluster.add_instance( + "node_db", + main_configs=[ + "configs/remote_url_allow_hosts_without_minio.xml", + "configs/drop_table_delay.xml", + ], + image="clickhouse/clickhouse-server", + tag="26.8.26.12", + with_installed_binary=True, + with_minio=True, + stay_alive=True, +) + +WARNING_IN_LOG = "is not allowed by `remote_url_allow_hosts`" + + +def s3_disk(path, disk_type="s3"): + return ( + f"disk(type = '{disk_type}', endpoint = 'http://minio1:9001/root/data/{path}/', " + f"access_key_id = 'minio', secret_access_key = '{minio_secret_key}')" + ) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_table_on_custom_s3_disk_loads_after_upgrade(started_cluster): + node.query( + f"CREATE TABLE t (id UInt64) ENGINE = MergeTree ORDER BY id SETTINGS disk = {s3_disk('upgrade_disk')}" + ) + node.query("INSERT INTO t VALUES (1), (2), (3)") + assert node.query("SELECT id FROM t ORDER BY id") == "1\n2\n3\n" + # Detached permanently before the upgrade, so its disk is first built by the short ATTACH on the new binary. + node.query( + f"CREATE TABLE t_detached (id UInt64) ENGINE = MergeTree ORDER BY id SETTINGS disk = {s3_disk('detached_disk')}" + ) + node.query("INSERT INTO t_detached VALUES (4), (5)") + node.query("DETACH TABLE t_detached PERMANENTLY") + # Dropped before the upgrade and still in the drop queue: the new binary rebuilds it at start from + # metadata_dropped under FORCE_RESTORE, which is the path UNDROP relies on. + node.query( + f"CREATE TABLE t_dropped (id UInt64) ENGINE = MergeTree ORDER BY id SETTINGS disk = {s3_disk('dropped_disk')}" + ) + node.query("INSERT INTO t_dropped VALUES (6), (7)") + node.query("DROP TABLE t_dropped") + + node.restart_with_latest_version() + + # Loaded from existing metadata: reported, not rejected. + assert node.query("SELECT id FROM t ORDER BY id") == "1\n2\n3\n" + assert node.contains_in_log(WARNING_IN_LOG) + assert ( + node.query( + "SELECT count() FROM system.warnings WHERE message LIKE '%remote_url_allow_hosts%' AND message LIKE '%minio1:9001/root%'" + ) + == "1\n" + ) + # Two disks were built at start and reported: the live table's and the dropped table's. + assert ( + node.query( + "SELECT countSubstrings(message, '->') FROM system.warnings WHERE message LIKE '%remote_url_allow_hosts%'" + ) + == "2\n" + ) + node.query("UNDROP TABLE t_dropped") + assert node.query("SELECT id FROM t_dropped ORDER BY id") == "6\n7\n" + + # Only loading existing metadata is exempt. A user ATTACH is checked like a CREATE, in the short form too: + # the first build of the parked table's disk is rejected, like every other disk check on a short ATTACH. + error = node.query_and_get_error("ATTACH TABLE t_detached") + assert "UNACCEPTABLE_URL" in error, error + error = node.query_and_get_error( + f"CREATE TABLE t_new (id UInt64) ENGINE = MergeTree ORDER BY id SETTINGS disk = {s3_disk('new_disk')}" + ) + assert "UNACCEPTABLE_URL" in error, error + error = node.query_and_get_error( + f"ATTACH TABLE t_attached UUID '{uuid.uuid4()}' (id UInt64) ENGINE = MergeTree ORDER BY id SETTINGS disk = {s3_disk('attached_disk')}" + ) + assert "UNACCEPTABLE_URL" in error, error + + # The operator adds the entry that the warning names; the parked table attaches without a restart. + node.replace_config( + "/etc/clickhouse-server/config.d/remote_url_allow_hosts_without_minio.xml", + "allowed.example.comminio1:9001/root", + ) + node.query("SYSTEM RELOAD CONFIG") + node.query("ATTACH TABLE t_detached") + assert node.query("SELECT id FROM t_detached ORDER BY id") == "4\n5\n" + + +def test_database_on_custom_s3_disk_loads_after_upgrade(started_cluster): + node_db.query( + f"CREATE DATABASE db ENGINE = Atomic SETTINGS disk = {s3_disk('upgrade_db_disk', 's3_plain_rewritable')}" + ) + node_db.query("CREATE TABLE db.t (id UInt64) ENGINE = MergeTree ORDER BY id") + node_db.query("INSERT INTO db.t VALUES (1), (2), (3)") + assert node_db.query("SELECT id FROM db.t ORDER BY id") == "1\n2\n3\n" + + node_db.restart_with_latest_version() + + # The server starts and the database loads; the disk is reported. + assert node_db.query("SELECT id FROM db.t ORDER BY id") == "1\n2\n3\n" + assert node_db.contains_in_log(WARNING_IN_LOG) + + # DETACH and ATTACH reuse the metadata disk built at server start, so no check runs. + node_db.query("DETACH DATABASE db") + node_db.query("ATTACH DATABASE db") + assert node_db.query("SELECT id FROM db.t ORDER BY id") == "1\n2\n3\n" diff --git a/tests/integration/test_backward_compatibility/test_native_type_names_secondary_query.py b/tests/integration/test_backward_compatibility/test_native_type_names_secondary_query.py new file mode 100644 index 0000000000000..9a3221a5725eb --- /dev/null +++ b/tests/integration/test_backward_compatibility/test_native_type_names_secondary_query.py @@ -0,0 +1,48 @@ +"""An older server queries a newer shard with the binary type encoding of the Native format enabled.""" + +import pytest + +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +new_node = cluster.add_instance("new_node") +old_node = cluster.add_instance( + "old_node", + image="clickhouse/clickhouse-server", + tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, + with_installed_binary=True, +) + +BINARY_TYPES = { + "output_format_native_encode_types_in_binary_format": 1, + "input_format_native_decode_types_in_binary_format": 1, +} + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_old_initiator_new_shard(start_cluster): + new_node.query("DROP TABLE IF EXISTS t_05317 SYNC") + new_node.query( + "CREATE TABLE t_05317 (x UInt64, s String) ENGINE = MergeTree ORDER BY x" + ) + + old_node.http_query( + "INSERT INTO FUNCTION remote('new_node', default, t_05317) VALUES (42, 'str')", + method="POST", + params=BINARY_TYPES, + ) + assert ( + old_node.http_query( + "SELECT x, s FROM remote('new_node', default, t_05317)", + params=BINARY_TYPES, + ) + == "42\tstr\n" + ) diff --git a/tests/integration/test_bind_host/configs/config.d/ssl_conf.xml b/tests/integration/test_bind_host/configs/config.d/ssl_conf.xml index e51749f354385..c11c5736d96f8 100644 --- a/tests/integration/test_bind_host/configs/config.d/ssl_conf.xml +++ b/tests/integration/test_bind_host/configs/config.d/ssl_conf.xml @@ -7,9 +7,6 @@ /etc/clickhouse-server/config.d/server.key - - false AcceptCertificateHandler diff --git a/tests/integration/test_database_backup/test.py b/tests/integration/test_database_backup/test.py index 8316d798e14f2..40aa406a050e1 100644 --- a/tests/integration/test_database_backup/test.py +++ b/tests/integration/test_database_backup/test.py @@ -12,6 +12,17 @@ with_minio=True, ) +# `test_database_backup_metadata_with_quoted_locator_loads_on_restart` rewrites a database metadata file +# in place, and such a file only exists when the metadata lives on the local disk, so that test runs on an +# instance which keeps the local database disk. +instance_local_metadata = cluster.add_instance( + "instance_local_metadata", + main_configs=["configs/backups.xml"], + stay_alive=True, + with_minio=True, + with_remote_database_disk=False, +) + @pytest.fixture(scope="module", autouse=True) def start_cluster(): @@ -271,3 +282,94 @@ def test_database_backup_unavailable_but_server_starts(backup_destination): instance.query("DROP DATABASE IF EXISTS test_database_backup SYNC") instance.query("DROP DATABASE IF EXISTS test_database SYNC") cleanup_backup_files(instance) + + +def test_database_backup_metadata_with_quoted_locator_loads_on_restart(): + # Regression test for https://github.com/ClickHouse/ClickHouse/issues/118349 + # An older server regenerated the definition of a `Backup` database with the locator quoted into a + # string literal, and `ALTER DATABASE ... MODIFY COMMENT` wrote that back into `metadata/.sql`. + # The next start replays the stored full `ATTACH DATABASE ... ENGINE = Backup(...)` statement, which + # is neither the short `ATTACH` nor a force-restore load, so it has to accept that form on its own. + # + # The metadata file is rewritten in place here, so this runs on the instance whose metadata is a file + # on the local disk rather than an object on a remote database disk. + instance = instance_local_metadata + + cleanup_backup_files(instance) + + instance.query( + """ + DROP DATABASE IF EXISTS test_database SYNC; + DROP DATABASE IF EXISTS test_database_backup SYNC; + + CREATE DATABASE test_database; + + CREATE TABLE test_database.test_table (id UInt64, value String) ENGINE=MergeTree ORDER BY id; + INSERT INTO test_database.test_table VALUES (0, 'test_database.test_table'); + + BACKUP DATABASE test_database TO File('test_database_backup_file'); + CREATE DATABASE test_database_backup ENGINE = Backup('test_database', File('test_database_backup_file')); + """ + ) + assert ( + instance.query("SELECT id, value FROM test_database_backup.test_table") + == "0\ttest_database.test_table\n" + ) + + # The metadata file exactly as a pre-fix server left it after a comment change. + instance.stop_clickhouse() + metadata = ( + "ATTACH DATABASE test_database_backup\n" + "ENGINE = Backup('test_database', 'File(\\'test_database_backup_file\\')')\n" + "COMMENT 'written by an older server'\n" + ) + instance.exec_in_container( + [ + "bash", + "-c", + "cat > /var/lib/clickhouse/metadata/test_database_backup.sql <<'SQL'\n" + + metadata + + "SQL\n", + ], + user="root", + ) + assert "'File(\\'test_database_backup_file\\')'" in instance.exec_in_container( + ["cat", "/var/lib/clickhouse/metadata/test_database_backup.sql"] + ) + instance.start_clickhouse() + + # The server started and the database loaded with its tables and comment. + assert ( + instance.query("SELECT id, value FROM test_database_backup.test_table") + == "0\ttest_database.test_table\n" + ) + assert ( + instance.query( + "SELECT comment FROM system.databases WHERE name = 'test_database_backup'" + ) + == "written by an older server\n" + ) + # The definition is regenerated with the locator as the function it is. + assert ( + "ENGINE = Backup('test_database', File('test_database_backup_file'))" + in instance.query("SHOW CREATE DATABASE test_database_backup FORMAT TSVRaw") + ) + + # A comment change on this server writes the function form, and that survives a restart too. + instance.query( + "ALTER DATABASE test_database_backup MODIFY COMMENT 'written by this server'" + ) + instance.restart_clickhouse() + assert ( + instance.query("SELECT id, value FROM test_database_backup.test_table") + == "0\ttest_database.test_table\n" + ) + assert ( + instance.query( + "SELECT comment FROM system.databases WHERE name = 'test_database_backup'" + ) + == "written by this server\n" + ) + + instance.query("DROP DATABASE test_database_backup SYNC") + instance.query("DROP DATABASE test_database SYNC") diff --git a/tests/integration/test_database_glue/test.py b/tests/integration/test_database_glue/test.py index 4ca41b7ac9880..145a84b0810a6 100644 --- a/tests/integration/test_database_glue/test.py +++ b/tests/integration/test_database_glue/test.py @@ -1371,11 +1371,104 @@ def test_sts_smoke(started_cluster): result = node.query(f"SELECT sum(value) FROM {db_name_success}.`{root_namespace}.{table_name}`") assert result.strip() == "60", f"Expected sum to be 60 but got: {result}" + # `aws_role_session_name` can act as a shared secret (the trust policy can pin it, which is what the + # STS mock does), so every display surface hides it while `aws_role_arn` stays visible. This database + # carries no other secret, so `[HIDDEN]` can only come from the session name. + show_create = node.query(f"SHOW CREATE DATABASE {db_name_success}") + assert "miniorole" not in show_create + assert "arn::role" in show_create + assert "[HIDDEN]" in show_create + + engine_full = node.query( + f"SELECT engine_full FROM system.databases WHERE name = '{db_name_success}'" + ) + assert "miniorole" not in engine_full + assert "[HIDDEN]" in engine_full + + node.query("SYSTEM FLUSH LOGS system.query_log") + logged_create = node.query( + f"SELECT arrayStringConcat(groupArray(query), '\\n') FROM system.query_log " + f"WHERE query_kind = 'Create' AND type = 'QueryFinish' AND query LIKE '%{db_name_success}%'" + ) + assert "[HIDDEN]" in logged_create + assert "miniorole" not in logged_create + # Cleanup node.query(f"DROP DATABASE IF EXISTS {db_name_fail} SYNC") node.query(f"DROP DATABASE IF EXISTS {db_name_success} SYNC") +def test_sts_backup_restore_keeps_role_session_name(started_cluster): + """A backup archives the database definition with the real `aws_role_session_name`, although every + display surface shows `[HIDDEN]`. The STS mock grants the role only for the session name `miniorole`, + so the restored catalog can read its table only if the archived value was the real one.""" + node = started_cluster.instances["node1"] + + test_ref = f"test_sts_backup_{uuid.uuid4()}" + table_name = f"{test_ref}_table" + root_namespace = f"{test_ref}_namespace" + + catalog = load_catalog_impl(started_cluster) + catalog.create_namespace(root_namespace) + + schema = Schema( + NestedField(field_id=1, name="id", field_type=StringType(), required=False), + NestedField(field_id=2, name="value", field_type=DoubleType(), required=False), + ) + table = create_table(catalog, root_namespace, table_name, schema, PartitionSpec(), DEFAULT_SORT_ORDER, dir=table_name) + table.append( + pa.Table.from_pylist( + [ + {"id": "row1", "value": 10.0}, + {"id": "row2", "value": 20.0}, + {"id": "row3", "value": 30.0}, + ] + ) + ) + + db_name = f"db_backup_{test_ref.replace('-', '_')}" + restored_db_name = f"{db_name}_restored" + create_clickhouse_glue_database( + started_cluster, + node, + db_name, + additional_settings={ + "aws_role_arn": "arn::role", + "aws_role_session_name": "miniorole", + }, + query_settings={"s3_allow_server_credentials_in_user_queries": 1}, + with_credentials=False, + ) + result = node.query(f"SELECT sum(value) FROM {db_name}.`{root_namespace}.{table_name}`") + assert result.strip() == "60", f"Expected sum to be 60 but got: {result}" + + # A DataLakeCatalog database owns no tables, so the backup holds just the database definition. The + # default server config allows File backups under its `backups` directory. + backup = f"File('{db_name}')" + node.query(f"BACKUP DATABASE {db_name} TO {backup}") + node.query(f"DROP DATABASE IF EXISTS {restored_db_name} SYNC") + node.query( + f"RESTORE DATABASE {db_name} AS {restored_db_name} FROM {backup}", + settings={ + "allow_database_glue_catalog": 1, + "s3_allow_server_credentials_in_user_queries": 1, + }, + ) + + # Readable only with the real session name: the STS mock rejects any other value. + result = node.query(f"SELECT sum(value) FROM {restored_db_name}.`{root_namespace}.{table_name}`") + assert result.strip() == "60", f"Expected sum to be 60 but got: {result}" + + # The archived value came back, and the display of the restored definition is masked like any other. + show_create = node.query(f"SHOW CREATE DATABASE {restored_db_name}") + assert "miniorole" not in show_create + assert "arn::role" in show_create + assert "[HIDDEN]" in show_create + + node.query(f"DROP DATABASE IF EXISTS {restored_db_name} SYNC") + node.query(f"DROP DATABASE IF EXISTS {db_name} SYNC") + + def test_sts_smoke_no_opt_in(started_cluster): """A Glue DataLakeCatalog with aws_role_arn set, no explicit aws_access_key_id/aws_secret_access_key, and no s3_allow_server_credentials_in_user_queries opt-in. GlueCatalog routes through the same diff --git a/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict_local_infile.xml b/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict_local_infile.xml new file mode 100644 index 0000000000000..9d5a3fe658f37 --- /dev/null +++ b/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict_local_infile.xml @@ -0,0 +1,30 @@ + + + dict_local_infile + + + test + mysql80 + 3306 + root + ClickHouse_MySQL_P@ssw0rd +
dict_local_infile_table
+ 1 + + + + + + + + id + + + value + String + + + + 0 + + diff --git a/tests/integration/test_dictionaries_mysql/test.py b/tests/integration/test_dictionaries_mysql/test.py index d2bd9de404019..b7be44a3ef57d 100644 --- a/tests/integration/test_dictionaries_mysql/test.py +++ b/tests/integration/test_dictionaries_mysql/test.py @@ -19,6 +19,7 @@ "configs/dictionaries/mysql_dict_compression.xml", "configs/dictionaries/mysql_dict_compression_wire.xml", "configs/dictionaries/mysql_dict_no_compression_wire.xml", + "configs/dictionaries/mysql_dict_local_infile.xml", ] CONFIG_FILES = [ "configs/remote_servers.xml", @@ -1032,3 +1033,74 @@ def reload(name): mysql_connection, "DROP TABLE IF EXISTS test.inherit_pool_test;" ) mysql_connection.close() + + +def test_enable_local_infile_xml_dict(started_cluster): + """`enable_local_infile` is rejected in a dictionary created with a DDL query, but a dictionary + defined in a server configuration file is written by an operator and keeps working.""" + mysql_connection = get_mysql_conn(started_cluster) + + try: + execute_mysql_query( + mysql_connection, "DROP TABLE IF EXISTS test.dict_local_infile_table;" + ) + execute_mysql_query( + mysql_connection, + "CREATE TABLE test.dict_local_infile_table (id INT NOT NULL, value TEXT, PRIMARY KEY(id));", + ) + execute_mysql_query( + mysql_connection, + "INSERT INTO test.dict_local_infile_table VALUES (1, 'local_infile');", + ) + + # The dictionary is lazily loaded, so nothing has touched the source yet; reload it now that + # its table exists. + for _ in range(10): + try: + instance.query("SYSTEM RELOAD DICTIONARY dict_local_infile") + break + except Exception: + time.sleep(0.5) + + # Loading the dictionary means the source was instantiated and connected with the option on, + # which is what the guard must not prevent for this route. + value = instance.query( + "SELECT dictGet('dict_local_infile', 'value', toUInt64(1))" + ).strip() + last_exception = instance.query( + "SELECT last_exception FROM system.dictionaries WHERE name = 'dict_local_infile'" + ).strip() + assert value == "local_infile", ( + " was rejected in the XML dict config: " + f"{last_exception!r}" + ) + + # The same option from a DDL query is rejected, at the source and at a replica alike. The + # endpoint here is the working MySQL server, so a rejection cannot be a connection failure + # in disguise. + credentials = f"USER 'root' PASSWORD '{mysql_pass}' DB 'test' TABLE 'dict_local_infile_table'" + for source in ( + f"HOST 'mysql80' PORT 3306 {credentials} ENABLE_LOCAL_INFILE 1", + f"{credentials} REPLICA(PRIORITY 1 HOST 'mysql80' PORT 3306 ENABLE_LOCAL_INFILE 1)", + ): + instance.query("DROP DICTIONARY IF EXISTS dict_local_infile_ddl") + instance.query( + f""" + CREATE DICTIONARY dict_local_infile_ddl (id UInt64, value String) + PRIMARY KEY id + SOURCE(MYSQL({source})) + LAYOUT(FLAT()) + LIFETIME(0) + """ + ) + with pytest.raises(Exception) as exc: + instance.query("SYSTEM RELOAD DICTIONARY dict_local_infile_ddl") + assert "cannot be enabled in a dictionary created with a DDL query" in str( + exc.value + ), f"Unexpected error for {source!r}: {exc.value}" + finally: + instance.query("DROP DICTIONARY IF EXISTS dict_local_infile_ddl") + execute_mysql_query( + mysql_connection, "DROP TABLE IF EXISTS test.dict_local_infile_table;" + ) + mysql_connection.close() diff --git a/tests/integration/test_grpc_protocol/test.py b/tests/integration/test_grpc_protocol/test.py index 915eaef3ad15f..9aa9080bf7379 100644 --- a/tests/integration/test_grpc_protocol/test.py +++ b/tests/integration/test_grpc_protocol/test.py @@ -611,6 +611,53 @@ def test_external_table(): ) +def test_settings_profile_constraints(): + profile = "grpc_profile_constraints" + query(f"DROP SETTINGS PROFILE IF EXISTS {profile}") + query( + f"CREATE SETTINGS PROFILE {profile} SETTINGS max_execution_time = 10 CONST, max_result_rows = 12345, format_csv_delimiter = '|'" + ) + try: + # `settings` is a protobuf map with no order: the profile goes first, so its constraints bind the + # other settings and they override the values it sets. + e = query_and_get_error( + "SELECT 1", settings={"profile": profile, "max_execution_time": "999"} + ) + assert "Setting max_execution_time should not be changed" in e.display_text + assert ( + query( + "SELECT getSetting('max_result_rows')", + settings={"profile": profile, "max_result_rows": "7"}, + ) + == "7\n" + ) + + columns = [ + clickhouse_grpc_pb2.NameAndType(name="UserID", type="UInt64"), + clickhouse_grpc_pb2.NameAndType(name="UserName", type="String"), + ] + + def ext(settings): + return clickhouse_grpc_pb2.ExternalTable( + name="ext1", columns=columns, data=b"1;Alex\n", format="CSV", settings=settings + ) + + e = query_and_get_error( + "SELECT * FROM ext1", + external_tables=[ext({"profile": profile, "max_execution_time": "999"})], + ) + assert "Setting max_execution_time should not be changed" in e.display_text + assert ( + query( + "SELECT * FROM ext1", + external_tables=[ext({"profile": profile, "format_csv_delimiter": ";"})], + ) + == "1\tAlex\n" + ) + finally: + query(f"DROP SETTINGS PROFILE {profile}") + + def test_external_table_streaming(): columns = [ clickhouse_grpc_pb2.NameAndType(name="UserID", type="UInt64"), diff --git a/tests/integration/test_keeper_session/test.py b/tests/integration/test_keeper_session/test.py index 07062f42f5e61..213e1f3b22bcb 100644 --- a/tests/integration/test_keeper_session/test.py +++ b/tests/integration/test_keeper_session/test.py @@ -36,6 +36,9 @@ reply_header_struct = struct.Struct("!iqi") stat_struct = struct.Struct("!qqqqiiiqiiq") +# Protocol version Keeper sends when it rejects a connection. +KEEPER_PROTOCOL_VERSION_CONNECTION_REJECT = 42 + @pytest.fixture(scope="module") def started_cluster(): @@ -86,7 +89,9 @@ def read_buffer(bytes, offset): return bytes[index : index + length], offset -def handshake(node_name=node1.name, session_timeout=1000, session_id=0): +def handshake( + node_name=node1.name, session_timeout=1000, session_id=0, full_reply=False +): client = None try: client = get_keeper_socket(node_name) @@ -130,6 +135,12 @@ def handshake(node_name=node1.name, session_timeout=1000, session_id=0): read_only = False print("negotiated_timeout - session_id", negotiated_timeout, session_id) + if full_reply: + (reply_length,) = int_struct.unpack_from(data, 0) + rest = data[int_struct.size + reply_length :] + while chunk := client.recv(1_000): + rest += chunk + return proto_version, negotiated_timeout, session_id, rest return negotiated_timeout, session_id finally: if client is not None: @@ -149,6 +160,35 @@ def test_session_timeout(started_cluster): assert negotiated_timeout == 10000 +def test_handshake_to_continue_session_is_expired(started_cluster): + wait_nodes() + negotiated_timeout, session_id = handshake( + node1.name, session_timeout=8000, session_id=0 + ) + assert negotiated_timeout == 8000 and session_id > 0 + # Keeper cannot restore a session, so a request to continue one must not be answered with a new session. + # The expired reply is the last thing the server sends before it closes the connection. + assert handshake( + node1.name, session_timeout=8000, session_id=session_id, full_reply=True + ) == (0, 0, 0, b"") + + try: + node2.stop_clickhouse() + node3.stop_clickhouse() + keeper_utils.wait_until_quorum_lost(cluster, node1) + # A rejected client gets no session id that it would send back as the session to continue. + assert handshake( + node1.name, session_timeout=8000, session_id=0, full_reply=True + ) == (KEEPER_PROTOCOL_VERSION_CONNECTION_REJECT, 8000, 0, b"") + assert handshake( + node1.name, session_timeout=8000, session_id=session_id, full_reply=True + ) == (0, 0, 0, b"") + finally: + node2.start_clickhouse() + node3.start_clickhouse() + wait_nodes() + + def test_session_close_shutdown(started_cluster): wait_nodes() diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py index d16d8f5623f74..24b95f0764938 100644 --- a/tests/integration/test_mask_sensitive_info/test.py +++ b/tests/integration/test_mask_sensitive_info/test.py @@ -396,8 +396,8 @@ def generate_create_table_numbered(tail): generate_create_table_numbered(f"(`x` int) ENGINE = AzureQueue('{azure_storage_account_url}', 'cont', '*', '{azure_account_name}', '[HIDDEN]', 'CSV', 'none') SETTINGS mode = 'unordered'"), generate_create_table_numbered(f"(`x` int) ENGINE = AzureBlobStorage('{masked_sas_conn_string}', 'exampledatasets', 'example.csv')"), generate_create_table_numbered("(`x` int) ENGINE = S3('https://my-s3-endpoint/bucket/data.csv', 'myaccess', '[HIDDEN]', 'CSV')"), - generate_create_table_numbered("(`x` int) ENGINE = Kafka SETTINGS kafka_broker_list = '127.0.0.1', kafka_topic_list = 'topic', kafka_group_name = 'group', kafka_format = 'JSONEachRow', kafka_security_protocol = 'sasl_ssl', kafka_sasl_mechanism = 'PLAIN', kafka_sasl_username = 'user', kafka_sasl_password = '[HIDDEN]', format_avro_schema_registry_url = 'http://schema_user:[HIDDEN]@'"), - generate_create_table_numbered("(`x` int) ENGINE = Kafka SETTINGS kafka_broker_list = '127.0.0.1', kafka_topic_list = 'topic', kafka_group_name = 'group', kafka_format = 'JSONEachRow', kafka_security_protocol = 'sasl_ssl', kafka_sasl_mechanism = 'PLAIN', kafka_sasl_username = 'user', kafka_sasl_password = '[HIDDEN]', format_avro_schema_registry_url = 'http://schema_user:[HIDDEN]@domain.com'"), + generate_create_table_numbered("(`x` int) ENGINE = Kafka SETTINGS kafka_broker_list = '127.0.0.1', kafka_topic_list = 'topic', kafka_group_name = 'group', kafka_format = 'JSONEachRow', kafka_security_protocol = 'sasl_ssl', kafka_sasl_mechanism = 'PLAIN', kafka_sasl_username = 'user', kafka_sasl_password = '[HIDDEN]', format_avro_schema_registry_url = 'http://[HIDDEN]@'"), + generate_create_table_numbered("(`x` int) ENGINE = Kafka SETTINGS kafka_broker_list = '127.0.0.1', kafka_topic_list = 'topic', kafka_group_name = 'group', kafka_format = 'JSONEachRow', kafka_security_protocol = 'sasl_ssl', kafka_sasl_mechanism = 'PLAIN', kafka_sasl_username = 'user', kafka_sasl_password = '[HIDDEN]', format_avro_schema_registry_url = 'http://[HIDDEN]@domain.com'"), generate_create_table_numbered("(`x` int) ENGINE = S3('http://minio1:9001/root/data/test5.csv.gz', 'CSV', access_key_id = 'minio', secret_access_key = '[HIDDEN]', compression_method = 'gzip')"), generate_create_table_numbered("(`x` int) ENGINE = ArrowFlight('arrowflight1:5006', 'dataset', 'arrowflight_user', '[HIDDEN]')"), generate_create_table_numbered("(`x` int) ENGINE = ArrowFlight(named_collection_1, host = 'arrowflight1', port = 5006, dataset = 'dataset', username = 'arrowflight_user', password = '[HIDDEN]')"), @@ -405,12 +405,12 @@ def generate_create_table_numbered(tail): generate_create_table_numbered("(`x` int) ENGINE = Redis('localhost', 0, '[HIDDEN]') PRIMARY KEY x"), generate_create_table_numbered("(`x` int) ENGINE = JDBC('[HIDDEN]', 'mydb', 'mytable')"), generate_create_table_numbered("(`x` int) ENGINE = ODBC('[HIDDEN]', 'mydb', 'mytable')"), - generate_create_table_numbered("(`x` int) ENGINE = JDBC('jdbc://user:[HIDDEN]@localhost:5432/mydb', 'mydb', 'mytable')"), - generate_create_table_numbered("(`x` int) ENGINE = ODBC('odbc://user:[HIDDEN]@localhost:5432/mydb', 'mydb', 'mytable')"), + generate_create_table_numbered("(`x` int) ENGINE = JDBC('[HIDDEN]', 'mydb', 'mytable')"), + generate_create_table_numbered("(`x` int) ENGINE = ODBC('[HIDDEN]', 'mydb', 'mytable')"), + generate_create_table_numbered("(`x` int) ENGINE = JDBC(named_collection_1, datasource = '[HIDDEN]', external_database = 'mydb', external_table = 'mytable')"), + generate_create_table_numbered("(`x` int) ENGINE = ODBC(named_collection_1, connection_settings = '[HIDDEN]', external_database = 'mydb', external_table = 'mytable')"), generate_create_table_numbered("(`x` int) ENGINE = JDBC(named_collection_1, datasource = '[HIDDEN]', external_database = 'mydb', external_table = 'mytable')"), generate_create_table_numbered("(`x` int) ENGINE = ODBC(named_collection_1, connection_settings = '[HIDDEN]', external_database = 'mydb', external_table = 'mytable')"), - generate_create_table_numbered("(`x` int) ENGINE = JDBC(named_collection_1, datasource = 'jdbc://user:[HIDDEN]@localhost:5432/mydb', external_database = 'mydb', external_table = 'mytable')"), - generate_create_table_numbered("(`x` int) ENGINE = ODBC(named_collection_1, connection_settings = 'odbc://user:[HIDDEN]@localhost:5432/mydb', external_database = 'mydb', external_table = 'mytable')"), generate_create_table_numbered("(`x` int) ENGINE = JDBC(named_collection_1, datasource = '[HIDDEN]', connection_settings = '[HIDDEN]', external_database = '[HIDDEN]', external_table = '[HIDDEN]')"), generate_create_table_numbered("(`x` int) ENGINE = JDBC(named_collection_1, connection_settings = '[HIDDEN]', external_database = '[HIDDEN]', datasource = '[HIDDEN]', external_table = '[HIDDEN]')"), generate_create_table_numbered("(`x` int) ENGINE = NATS SETTINGS nats_url = 'localhost:4222', nats_subjects = 'subject', nats_format = 'JSONEachRow', nats_token = '[HIDDEN]'"), @@ -656,12 +656,12 @@ def make_test_case(i): "CREATE TABLE tablefunc49 (`x` int) AS redis('localhost', 'key', 'key Int64', 0, '[HIDDEN]')", "CREATE TABLE tablefunc50 (`x` int) AS jdbc('[HIDDEN]', 'mydb', 'mytable')", "CREATE TABLE tablefunc51 (`x` int) AS odbc('[HIDDEN]', 'mydb', 'mytable')", - "CREATE TABLE tablefunc52 (`x` int) AS jdbc('jdbc://user:[HIDDEN]@localhost:5432/mydb', 'mydb', 'mytable')", - "CREATE TABLE tablefunc53 (`x` int) AS odbc('odbc://user:[HIDDEN]@localhost:5432/mydb', 'mydb', 'mytable')", + "CREATE TABLE tablefunc52 (`x` int) AS jdbc('[HIDDEN]', 'mydb', 'mytable')", + "CREATE TABLE tablefunc53 (`x` int) AS odbc('[HIDDEN]', 'mydb', 'mytable')", "CREATE TABLE tablefunc54 (`x` int) AS jdbc(named_collection_1, datasource = '[HIDDEN]')", "CREATE TABLE tablefunc55 (`x` int) AS odbc(named_collection_1, connection_settings = '[HIDDEN]')", - "CREATE TABLE tablefunc56 (`x` int) AS jdbc(named_collection_1, datasource = 'jdbc://user:[HIDDEN]@localhost:5432/mydb')", - "CREATE TABLE tablefunc57 (`x` int) AS odbc(named_collection_1, connection_settings = 'odbc://user:[HIDDEN]@localhost:5432/mydb')", + "CREATE TABLE tablefunc56 (`x` int) AS jdbc(named_collection_1, datasource = '[HIDDEN]')", + "CREATE TABLE tablefunc57 (`x` int) AS odbc(named_collection_1, connection_settings = '[HIDDEN]')", "CREATE TABLE tablefunc58 (`x` int) AS jdbc(named_collection_1, datasource = '[HIDDEN]', connection_settings = '[HIDDEN]')", "CREATE TABLE tablefunc59 (`x` int) AS jdbc(named_collection_1, connection_settings = '[HIDDEN]', external_database = '[HIDDEN]', datasource = '[HIDDEN]')", "CREATE TABLE tablefunc60 (`x` int) AS deltaLakeS3('http://minio1:9001/root/data/test11.csv.gz', 'minio', '[HIDDEN]')", @@ -1058,6 +1058,62 @@ def test_database_backup_engine_s3(): node.query("DROP DATABASE IF EXISTS backup_db_s3_2") +def test_database_backup_engine_azure_display_surfaces(): + """A live `Backup` database over an `AzureBlobStorage` locator: `account_key` must not reach + `SHOW CREATE DATABASE` or `system.databases.engine_full`. Only `S3` locators are reconstructed + argument by argument, so an Azure one keeps its engine name and arity and hides every argument.""" + azure_storage_account_url = cluster.env_variables["AZURITE_STORAGE_ACCOUNT_URL"] + azure_account_name = "devstoreaccount1" + azure_account_key = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + + # A backup destination is write-once and the container is emptied only when the cluster starts, + # so give every run its own blob path: a repeat inside one session hits BACKUP_ALREADY_EXISTS. + blob = "db_backup_azure_surfaces_" + "".join( + random.choice(string.ascii_lowercase) for _ in range(8) + ) + locator = ( + f"AzureBlobStorage('{azure_storage_account_url}', '{cluster.azurite_container}', " + f"'{blob}', '{azure_account_name}', '{azure_account_key}')" + ) + masked_locator = ( + "AzureBlobStorage('[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]')" + ) + surfaces = [ + "SHOW CREATE DATABASE backup_db_azure_view", + "SELECT engine_full FROM system.databases WHERE name = 'backup_db_azure_view'", + ] + + node.query("DROP DATABASE IF EXISTS backup_db_azure_src SYNC") + node.query("DROP DATABASE IF EXISTS backup_db_azure_view SYNC") + node.query("CREATE DATABASE backup_db_azure_src") + node.query( + "CREATE TABLE backup_db_azure_src.t (x int) ENGINE = MergeTree ORDER BY x" + ) + node.query("INSERT INTO backup_db_azure_src.t SELECT * FROM numbers(10)") + node.query(f"BACKUP DATABASE backup_db_azure_src TO {locator} FORMAT Null") + node.query( + f"CREATE DATABASE backup_db_azure_view ENGINE = Backup('backup_db_azure_src', {locator})" + ) + + # TSVRaw so the locator is compared as written rather than through TSV escaping. + for surface in surfaces: + shown = node.query(f"{surface} FORMAT TSVRaw") + assert masked_locator in shown, shown + assert azure_account_key not in shown, shown + + # The key is in the locator, so its absence above is masking rather than an argument that the + # regenerated definition never carried. + for surface in surfaces: + shown = node.query(f"{surface} {show_secrets}=1 FORMAT TSVRaw") + assert azure_account_key in shown, shown + + # The masked surfaces belong to a working database, not to one that failed to open. + assert node.query("SELECT count() FROM backup_db_azure_view.t") == "10\n" + + node.query("DROP DATABASE IF EXISTS backup_db_azure_view SYNC") + node.query("DROP DATABASE IF EXISTS backup_db_azure_src SYNC") + + def test_backup_table_azure_named_collection(): """Test that secrets in Azure named collection backups are masked in system.backups and logs.""" azure_storage_account_url = cluster.env_variables["AZURITE_STORAGE_ACCOUNT_URL"] diff --git a/tests/integration/test_mq_remote_host_filter/__init__.py b/tests/integration/test_mq_remote_host_filter/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_mq_remote_host_filter/configs/allowed_hosts.xml b/tests/integration/test_mq_remote_host_filter/configs/allowed_hosts.xml new file mode 100644 index 0000000000000..054acd185f73a --- /dev/null +++ b/tests/integration/test_mq_remote_host_filter/configs/allowed_hosts.xml @@ -0,0 +1,8 @@ + + + localhost:19092 + localhost:14222 + allowed-any-port + .*\.allowed\.example\.com + + diff --git a/tests/integration/test_mq_remote_host_filter/configs/kafka_bootstrap_override.xml b/tests/integration/test_mq_remote_host_filter/configs/kafka_bootstrap_override.xml new file mode 100644 index 0000000000000..e6b7a5f8f93fd --- /dev/null +++ b/tests/integration/test_mq_remote_host_filter/configs/kafka_bootstrap_override.xml @@ -0,0 +1,5 @@ + + + localhost:9999 + + diff --git a/tests/integration/test_mq_remote_host_filter/test.py b/tests/integration/test_mq_remote_host_filter/test.py new file mode 100644 index 0000000000000..7d3e4b1afebea --- /dev/null +++ b/tests/integration/test_mq_remote_host_filter/test.py @@ -0,0 +1,193 @@ +""" +Tests that the `Kafka` and `NATS` table engines respect `remote_url_allow_hosts`. + +Both engines used to hand the user-supplied broker addresses (`kafka_broker_list`, +`nats_url`, `nats_server_list`) to the client library without consulting +`RemoteHostFilter`, so `CREATE TABLE` opened outbound TCP connections to hosts the +operator had explicitly forbidden. `RabbitMQ`, the sibling engine, rejects the same +addresses at `CREATE` with `UNACCEPTABLE_URL`. + +The addresses are parsed, validated element-wise, and rebuilt before they reach the +client library, so an entry whose re-parse by the library could disagree with the +validated form (a path, an empty host, a NUL, ...) is rejected with `BAD_ARGUMENTS` +instead of being repaired. + +No broker runs in this test: a forbidden or malformed address must be rejected before +any connection attempt, and an allowed one must get past the filter (and then fail to +connect, for the engine which connects at DDL time). + +Each case is a triplet: the engine input, the expected error code (`None` means the +table must be created), and an optional string the error message must contain. +""" + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + main_configs=["configs/allowed_hosts.xml"], +) +# The section of the server configuration (and named collections, loaded by the same code) +# can override the broker list librdkafka receives, behind the back of the validated +# `kafka_broker_list` setting. This instance carries such an override to a forbidden host. +node_kafka_override = cluster.add_instance( + "node_kafka_override", + main_configs=["configs/allowed_hosts.xml", "configs/kafka_bootstrap_override.xml"], +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +# (kafka_broker_list value, expected error code or None for success, required error message part) +KAFKA_CASES = [ + pytest.param("localhost:9093", "UNACCEPTABLE_URL", "localhost:9093", id="forbidden_broker"), + # One allowed broker must not smuggle a forbidden one past the filter. + pytest.param("localhost:19092,localhost:9093", "UNACCEPTABLE_URL", "localhost:9093", id="element_wise"), + # librdkafka accepts `scheme://host:port` entries; the filter must see host:port. + pytest.param("PLAINTEXT://localhost:9093", "UNACCEPTABLE_URL", "localhost:9093", id="scheme_prefix"), + # A bare host must be checked with librdkafka's default port 9092. + pytest.param("localhost", "UNACCEPTABLE_URL", "localhost:9092", id="default_port"), + # A bare `` allowlist entry allows the host on any port, as for other engines. + pytest.param("allowed-any-port:9095", None, None, id="host_allowed_for_any_port"), + pytest.param("localhost:19092", None, None, id="allowed_broker"), + # librdkafka cuts an entry at the first `/` after `scheme://`. Without the strict parse, the + # filter would see the host `evil.com/.allowed.example.com`, the `host_regexp` allowlist entry + # would fully match it, and librdkafka would dial `evil.com`. + pytest.param( + "PLAINTEXT://evil.com/.allowed.example.com", "BAD_ARGUMENTS", "Unexpected character '/'", + id="path_does_not_bypass_host_regexp", + ), + # librdkafka splits the broker list on spaces too. Without doing the same, the filter would see + # the single host `evil.com .allowed.example.com`, the `host_regexp` allowlist entry would fully + # match it, and librdkafka would dial `evil.com`. + pytest.param("evil.com .allowed.example.com", "UNACCEPTABLE_URL", "evil.com:9092", id="space_separator"), + # librdkafka substitutes `localhost` for an empty host. + pytest.param(":9092", "BAD_ARGUMENTS", "Empty host", id="empty_host"), + # The broker list reaches librdkafka as a C string, so a NUL must not truncate it after the check. + pytest.param("evil.com\\0.allowed.example.com", "BAD_ARGUMENTS", "NUL", id="nul"), + # The validated entries are rebuilt and rejoined; the original text never reaches librdkafka. + pytest.param("localhost:19092 , localhost:19092", None, None, id="canonicalized"), + # An allowed host by the `host_regexp` allowlist entry. + pytest.param("sub.allowed.example.com", None, None, id="host_regexp_allowed"), +] + +# (NATS settings, expected error code or None for success, required error message part) +NATS_CASES = [ + pytest.param("nats_url = 'localhost:9999'", "UNACCEPTABLE_URL", "localhost:9999", id="forbidden_url"), + # libnats accepts `nats://user:password@host:port`; the filter must see host:port. + pytest.param( + "nats_url = 'nats://user:password@localhost:9999'", "UNACCEPTABLE_URL", "localhost:9999", + id="scheme_and_credentials", + ), + # One allowed server must not smuggle a forbidden one past the filter. + pytest.param( + "nats_server_list = 'localhost:14222,localhost:9999'", "UNACCEPTABLE_URL", "localhost:9999", + id="server_list_element_wise", + ), + # A bare host must be checked with libnats' default port 4222. + pytest.param("nats_url = 'localhost'", "UNACCEPTABLE_URL", "localhost:4222", id="default_port"), + # An allowed address must get past the filter and reach the connection attempt. `NATS` connects + # at `CREATE` time and nothing listens on the allowed port, so the statement fails - but with a + # connection error, not `UNACCEPTABLE_URL`. + pytest.param( + "nats_url = 'localhost:14222', nats_startup_connect_tries = 1", "CANNOT_CONNECT_NATS", None, + id="allowed_url_passes_filter", + ), + # libnats allows a `/path` after the port, so a path must not reach the filter as part of the host. + pytest.param( + "nats_url = 'nats://evil.com:4222/x.allowed.example.com'", "BAD_ARGUMENTS", "Unexpected character '/'", + id="path_does_not_bypass_host_regexp", + ), + # libnats substitutes `localhost` for an empty host. + pytest.param("nats_url = 'nats://:4222'", "BAD_ARGUMENTS", "Empty host", id="empty_host"), + # The URL reaches libnats as a C string, so a NUL must not truncate it after the check. + pytest.param("nats_url = 'evil.com\\0.allowed.example.com'", "BAD_ARGUMENTS", "NUL", id="nul"), + # The credentials must survive the rebuild of the validated address. + pytest.param( + "nats_url = 'nats://user:password@localhost:14222', nats_startup_connect_tries = 1", + "CANNOT_CONNECT_NATS", None, + id="credentials_preserved", + ), +] + + +def check(query, expected_error, message_part): + if expected_error is None: + node.query(query) + return + + error = node.query_and_get_error(query) + assert expected_error in error, error + if message_part is not None: + assert message_part in error, error + + +@pytest.mark.parametrize("broker_list, expected_error, message_part", KAFKA_CASES) +def test_kafka(started_cluster, broker_list, expected_error, message_part): + node.query("DROP TABLE IF EXISTS kafka_filtered SYNC") + check( + f""" + CREATE TABLE kafka_filtered (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = '{broker_list}', + kafka_topic_list = 'topic', + kafka_group_name = 'group', + kafka_format = 'JSONEachRow' + """, + expected_error, + message_part, + ) + + +def test_kafka_config_override_is_validated(started_cluster): + """A broker list supplied by the server configuration must be validated too. + + `getConsumerConfiguration` seeds `metadata.broker.list` from the validated + `kafka_broker_list`, but the `` section of the server configuration (or a named + collection) is merged afterwards and can override it - here through the + `bootstrap.servers` alias. The merged value is validated again when the consumer is + created, so the `CREATE` (which sees only the allowed setting) succeeds and the read + fails on the forbidden override. + """ + node_kafka_override.query( + """ + CREATE TABLE kafka_override (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'localhost:19092', + kafka_topic_list = 'topic', + kafka_group_name = 'group', + kafka_format = 'JSONEachRow' + """ + ) + error = node_kafka_override.query_and_get_error( + "SELECT * FROM kafka_override LIMIT 1" + " SETTINGS stream_like_engine_allow_direct_select = 1" + ) + assert "UNACCEPTABLE_URL" in error, error + assert "localhost:9999" in error, error + node_kafka_override.query("DROP TABLE kafka_override SYNC") + + +@pytest.mark.parametrize("settings, expected_error, message_part", NATS_CASES) +def test_nats(started_cluster, settings, expected_error, message_part): + node.query("DROP TABLE IF EXISTS nats_filtered SYNC") + check( + f""" + CREATE TABLE nats_filtered (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS {settings}, + nats_subjects = 'subject', + nats_format = 'JSONEachRow' + """, + expected_error, + message_part, + ) diff --git a/tests/integration/test_paimon_incremental_read/test.py b/tests/integration/test_paimon_incremental_read/test.py index 4e256a4a1de61..2ceb6c2987300 100644 --- a/tests/integration/test_paimon_incremental_read/test.py +++ b/tests/integration/test_paimon_incremental_read/test.py @@ -21,6 +21,7 @@ CH_MV_PAIMON_TABLE = "paimon_mv_source" CH_MV_MERGETREE_TABLE = "paimon_mv_dest" CH_MV_NAME = "paimon_refresh_mv" +CH_TABLE_NAME_ACTIVATE_RECLAIM = "paimon_inc_read_activate_reclaim" cluster = ClickHouseCluster(__file__) node = cluster.add_instance( @@ -425,3 +426,105 @@ def test_paimon_to_mergetree_via_refresh_mv(started_cluster): node.query(f"DROP VIEW IF EXISTS {CH_MV_NAME} SYNC;") node.query(f"DROP TABLE IF EXISTS {CH_MV_MERGETREE_TABLE} SYNC;") node.query(f"DROP TABLE IF EXISTS {CH_MV_PAIMON_TABLE} SYNC;") + + +def test_paimon_incremental_read_activate_tolerates_reaped_is_active(started_cluster): + """A read after a Keeper session loss reclaims the `is_active` marker its own + previous session left behind, and must survive Keeper reaping that ephemeral + concurrently: the marker is visible to the reclaim path's `tryGet` and already + gone by its versioned `tryRemove`, which then reports ZNONODE. The marker here + is fabricated because it reproduces exactly that observable state, whereas + waiting for the real reap to land inside a two-round-trip window would make the + test nondeterministic.""" + writer_container_id = cluster.get_instance_docker_id("paimon-incremental-writer") + + warehouse_name = "warehouse_activate_reclaim" + warehouse_uri = f"file://{USER_FILES_PATH}/{warehouse_name}/" + warehouse_dir = f"{USER_FILES_PATH}/{warehouse_name}" + table_path = f"{USER_FILES_PATH}/{warehouse_name}/test.db/test_table" + # Unique per run: committed_snapshot persists in Keeper, so a rerun against + # the same cluster must not inherit an earlier run's watermark. + keeper_path = f"/clickhouse/paimon_activate_reclaim_{uuid.uuid4().hex}" + is_active_path = f"{keeper_path}/replicas/r1/is_active" + failpoint = "paimon_incremental_read_pause_before_is_active_remove" + + _clean_warehouse(writer_container_id, warehouse_dir) + + # Warm-up commit (snapshot 1), consumed to establish the baseline. + _run_writer(writer_container_id, warehouse_uri=warehouse_uri, start_id=0, rows_per_commit=1, commit_times=1) + _create_clickhouse_table_for_paimon_incremental_read( + CH_TABLE_NAME_ACTIVATE_RECLAIM, table_path, keeper_path=keeper_path + ) + count_query = f"SELECT count() FROM {CH_TABLE_NAME_ACTIVATE_RECLAIM}" + _wait_until_query_result(count_query, "1\n", database="default") + _wait_until_query_result(count_query, "0\n", database="default") + + zk = cluster.get_kazoo_client("zoo1") + reader = None + reader_result = {} + try: + # Read the server's own marker instead of hardcoding its payload format. + identifier = zk.get(is_active_path)[0] + + # Snapshot 2: the batch the post-reconnect read must deliver. + _run_writer(writer_container_id, warehouse_uri=warehouse_uri, start_id=1, rows_per_commit=10, commit_times=1) + + session_query = ( + "SELECT client_id FROM system.zookeeper_connection WHERE name = 'default'" + ) + old_session = node.query(session_query) + # Finalizes the shared session, so the Keeper handle latched in + # PaimonStreamState stays expired and the next read takes the + # needsNewKeeper() branch that calls activate(). + node.query("SYSTEM RECONNECT ZOOKEEPER") + assert node.query(session_query) != old_session, ( + f"the Keeper session was not replaced (still {old_session!r})" + ) + + deadline = time.monotonic() + 60 + while zk.exists(is_active_path) is not None: + assert time.monotonic() < deadline, "the old session's is_active was never reaped" + time.sleep(0.5) + + # Persistent, not ephemeral: activate() never inspects stat.ephemeralOwner, + # so persistence is invisible to the code under test while keeping a second + # Keeper session out of the test's failure modes. + zk.create(is_active_path, identifier) + + node.query(f"SYSTEM ENABLE FAILPOINT {failpoint}") + reader = threading.Thread( + target=lambda: reader_result.update( + zip(("out", "err"), node.query_and_get_answer_with_error(count_query)) + ) + ) + reader.start() + + # Returns only once a thread has parked at the failpoint. Since the + # failpoint sits inside the identifier-matched branch, that proves the read + # entered the reclaim path with a marker it recognises as its own. + node.query(f"SYSTEM WAIT FAILPOINT {failpoint} PAUSE", timeout=60) + assert reader.is_alive(), ( + f"the reader returned before parking at the failpoint: {reader_result!r}" + ) + + # The reap lands inside the read's tryGet/tryRemove window. + zk.delete(is_active_path) + node.query(f"SYSTEM NOTIFY FAILPOINT {failpoint}") + + reader.join(timeout=120) + assert not reader.is_alive(), "the reader thread never finished" + assert not reader_result.get("err"), f"the read failed: {reader_result!r}" + # Not just "did not throw": the pending snapshot must still be delivered. + assert reader_result.get("out") == "10\n", ( + f"the pending snapshot was not delivered: {reader_result!r}" + ) + assert zk.get(f"{keeper_path}/committed_snapshot")[0] == b"2" + _wait_until_query_result(count_query, "0\n", database="default") + finally: + # The failpoint is process-global: an early failure must leave neither it + # armed nor the reader thread parked on it. + node.query(f"SYSTEM DISABLE FAILPOINT {failpoint}") + zk.stop() + if reader is not None: + reader.join(timeout=60) + node.query(f"DROP TABLE IF EXISTS {CH_TABLE_NAME_ACTIVATE_RECLAIM} SYNC;") diff --git a/tests/integration/test_parallel_replicas_dist_over_dist/__init__.py b/tests/integration/test_parallel_replicas_dist_over_dist/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_parallel_replicas_dist_over_dist/configs/hub.xml b/tests/integration/test_parallel_replicas_dist_over_dist/configs/hub.xml new file mode 100644 index 0000000000000..ed214c14ffe84 --- /dev/null +++ b/tests/integration/test_parallel_replicas_dist_over_dist/configs/hub.xml @@ -0,0 +1,34 @@ + + + + + + true + n19000 + n29000 + n39000 + + + + + + false + n19000 + + + false + n49000 + + + false + n69000 + + + + diff --git a/tests/integration/test_parallel_replicas_dist_over_dist/configs/single.xml b/tests/integration/test_parallel_replicas_dist_over_dist/configs/single.xml new file mode 100644 index 0000000000000..598ee98360974 --- /dev/null +++ b/tests/integration/test_parallel_replicas_dist_over_dist/configs/single.xml @@ -0,0 +1,12 @@ + + + + + + true + n69000 + + + + diff --git a/tests/integration/test_parallel_replicas_dist_over_dist/configs/spoke.xml b/tests/integration/test_parallel_replicas_dist_over_dist/configs/spoke.xml new file mode 100644 index 0000000000000..340c7488a6930 --- /dev/null +++ b/tests/integration/test_parallel_replicas_dist_over_dist/configs/spoke.xml @@ -0,0 +1,12 @@ + + + + + + true + n49000 + n59000 + + + + diff --git a/tests/integration/test_parallel_replicas_dist_over_dist/test.py b/tests/integration/test_parallel_replicas_dist_over_dist/test.py new file mode 100644 index 0000000000000..15a744a350a05 --- /dev/null +++ b/tests/integration/test_parallel_replicas_dist_over_dist/test.py @@ -0,0 +1,171 @@ +import uuid + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +# Three instances in a hub-and-spoke arrangement. Each carries its own `remote_servers`, where the +# cluster named `default` - the one a Cloud instance always has - is that instance's own replicas: +# three for the hub, two for the spoke and one for the single-node instance, which therefore cannot +# use parallel replicas at all. Only the hub knows `outer_cluster`, the entry point that fans a query +# out to all three. +# One `ClickHouseCluster` is used for all of them: separate `ClickHouseCluster` objects get separate docker +# compose projects and therefore separate networks, so their nodes could not reach each other. +hub_nodes = [ + cluster.add_instance(f"n{i}", main_configs=["configs/hub.xml"], with_zookeeper=True) + for i in (1, 2, 3) +] +spoke_nodes = [ + cluster.add_instance( + f"n{i}", main_configs=["configs/spoke.xml"], with_zookeeper=True + ) + for i in (4, 5) +] +single_nodes = [ + cluster.add_instance( + "n6", main_configs=["configs/single.xml"], with_zookeeper=True + ) +] +INSTANCES = [("hub", hub_nodes), ("spoke", spoke_nodes), ("single", single_nodes)] +nodes = hub_nodes + spoke_nodes + single_nodes + +TABLE = "logs" +INNER_DIST = "logs_dist" +OUTER_DIST = "logs_dist_dist" + +SERVICE = "id-api-auth-service" +WINDOW_START = "2026-09-28 15:40:00" +WINDOW_END = "2026-09-28 15:45:00" + +QUERY = f""" + SELECT count(), avg(length(Body)) + FROM {OUTER_DIST} + WHERE (ServiceName = '{SERVICE}') + AND (Timestamp >= '{WINDOW_START}') AND (Timestamp < '{WINDOW_END}') + """ + +# The hub holds 1000 matching rows with a body of 10 bytes, the spoke 3000 with a body of 20 and the +# single-node instance 1000 with a body of 10, so an instance read twice or not at all moves both the +# count and the average: (1000 * 10 + 3000 * 20 + 1000 * 10) / 5000. +EXPECTED = "5000\t16\n" + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + create_tables() + yield cluster + finally: + cluster.shutdown() + + +def create_tables(): + for instance, instance_nodes in INSTANCES: + for i, node in enumerate(instance_nodes): + node.query(f""" + CREATE TABLE {TABLE} + ( + Timestamp DateTime, + ServiceName LowCardinality(String), + Body String + ) + Engine=ReplicatedMergeTree('/test_pr_dist_over_dist/{instance}/{TABLE}', 'r{i}') + ORDER BY (ServiceName, Timestamp) + -- Small granules so there are mark segments to spread over the replicas of a shard. + SETTINGS index_granularity = 8 + """) + + # Every server reads its own instance through the same cluster name. + node.query(f""" + CREATE TABLE {INNER_DIST} AS {TABLE} + Engine=Distributed(default, currentDatabase(), {TABLE}, rand()) + """) + + hub_nodes[0].query(f""" + CREATE TABLE {OUTER_DIST} AS {INNER_DIST} + Engine=Distributed(outer_cluster, currentDatabase(), {INNER_DIST}, rand()) + """) + + insert_data(hub_nodes[0], matching_rows=1000, body_length=10) + insert_data(spoke_nodes[0], matching_rows=3000, body_length=20) + insert_data(single_nodes[0], matching_rows=1000, body_length=10) + + for node in nodes: + node.query(f"SYSTEM SYNC REPLICA {TABLE}") + + +def insert_data(node, matching_rows, body_length): + node.query(f""" + INSERT INTO {TABLE} + SELECT toDateTime('{WINDOW_START}') + (number % 300), '{SERVICE}', repeat('a', {body_length}) + FROM numbers({matching_rows}) + """) + # Rows the WHERE has to exclude. Their bodies are far longer than any matching row, so letting one + # through shows up in the average and not only in the count. + node.query(f""" + INSERT INTO {TABLE} VALUES + ('{WINDOW_START}', 'some-other-service', '{'x' * 100}'), + (toDateTime('{WINDOW_START}') - 1, '{SERVICE}', '{'x' * 100}'), + ('{WINDOW_END}', '{SERVICE}', '{'x' * 100}') + """) + + +def parallel_replicas_coordinators(query_id): + """Names of the nodes that hosted a parallel-replicas reading coordinator """ + + coordinators = set() + for node in nodes: + # SYSTEM FLUSH LOGS is not cluster-aware, it has to be issued on each node separately. + node.query("SYSTEM FLUSH LOGS query_log") + if int( + node.query( + f""" + SELECT countIf(ProfileEvents['ParallelReplicasNumRequests'] > 0) + FROM system.query_log + WHERE initial_query_id = '{query_id}' AND type = 'QueryFinish' + SETTINGS enable_parallel_replicas = 0 + """ + ) + ): + coordinators.add(node.name) + return coordinators + + +@pytest.mark.parametrize("prefer_localhost_replica", [0, 1]) +def test_parallel_replicas_over_distributed_over_distributed( + start_cluster, prefer_localhost_replica +): + # Without parallel replicas + assert ( + hub_nodes[0].query(QUERY, settings={"enable_parallel_replicas": 0}) == EXPECTED + ) + + query_id = str(uuid.uuid4()) + assert ( + hub_nodes[0].query( + QUERY, + query_id=query_id, + settings={ + "enable_parallel_replicas": 2, + "max_parallel_replicas": 3, + "prefer_localhost_replica": prefer_localhost_replica, + # The automatic decision must not turn parallel replicas off on this small data, and + # one mark segment per granule lets the reading spread over both replicas. + "automatic_parallel_replicas_mode": 0, + "parallel_replicas_mark_segment_size": 1, + }, + ) + == EXPECTED + ) + + # Every instance with more than one replica read with parallel replicas over its own `default` + # cluster, and the single-node one did not use them at all. + coordinators = parallel_replicas_coordinators(query_id) + for instance, instance_nodes in INSTANCES: + used = any(node.name in coordinators for node in instance_nodes) + assert used == (len(instance_nodes) > 1), ( + f"{instance}: parallel replicas used={used}, coordinators: {coordinators}" + ) diff --git a/tests/integration/test_query_runner/test.py b/tests/integration/test_query_runner/test.py index 9b2a055705ffb..49036e56c360b 100644 --- a/tests/integration/test_query_runner/test.py +++ b/tests/integration/test_query_runner/test.py @@ -134,6 +134,22 @@ def test_cluster_query_log(): ) +def test_cluster_query_log_hides_secrets(): + node_query_runner.query(runner_ddl("query String, settings Map(String, String)", "synchronous", "cluster")) + node_query_runner.query( + "INSERT INTO runner VALUES (" + "'SELECT * FROM url(''http://cluster_user:cluster_secret@node_cluster:8123/?query=SELECT+1'', ''LineAsString'', ''s String'')', " + "{'log_comment': 'qr_cluster_log_secret'})" + ) + node_query_runner.query("SYSTEM FLUSH LOGS query_log") + assert_eq_with_retry( + node_query_runner, + "SELECT type, query LIKE '%[HIDDEN]%', query LIKE '%cluster_secret%' FROM system.query_log " + "WHERE log_comment = 'qr_cluster_log_secret' AND is_internal ORDER BY type", + "QueryStart\t1\t0\nQueryFinish\t1\t0", + ) + + def test_cluster_insert_requires_remote(): node_query_runner.query(runner_ddl("query String", "synchronous", "cluster")) node_query_runner.query("CREATE USER no_remote_user") diff --git a/tests/integration/test_readonly_restricts_set_role/__init__.py b/tests/integration/test_readonly_restricts_set_role/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role.xml b/tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role.xml new file mode 100644 index 0000000000000..80d7145131cdf --- /dev/null +++ b/tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role.xml @@ -0,0 +1,5 @@ + + + true + + diff --git a/tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role_absent.xml b/tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role_absent.xml new file mode 100644 index 0000000000000..76b2225c37b3e --- /dev/null +++ b/tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role_absent.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/tests/integration/test_readonly_restricts_set_role/test.py b/tests/integration/test_readonly_restricts_set_role/test.py new file mode 100644 index 0000000000000..c36af86fa7203 --- /dev/null +++ b/tests/integration/test_readonly_restricts_set_role/test.py @@ -0,0 +1,116 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +# Every instance gets a copy of the production `config.xml`, which sets the element explicitly, so +# `node_off` removes it to run at the compatibility default an upgraded deployment has. +node_on = cluster.add_instance( + "node_on", main_configs=["configs/readonly_restricts_set_role.xml"] +) +node_off = cluster.add_instance( + "node_off", main_configs=["configs/readonly_restricts_set_role_absent.xml"] +) + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + + for node in (node_on, node_off): + node.query( + "CREATE ROLE ro_role, other_role;" + "CREATE USER probe IDENTIFIED WITH plaintext_password BY 'x';" + "GRANT ro_role, other_role TO probe;" + "ALTER USER probe DEFAULT ROLE ro_role;" + ) + + yield cluster + + finally: + cluster.shutdown() + + +def as_probe(node, sql, readonly): + return node.query_and_get_answer_with_error( + sql, user="probe", password="x", settings={"readonly": readonly} + ) + + +@pytest.mark.parametrize( + "readonly, statement", + [ + (1, "SET ROLE other_role"), + # The other `ASTSetRoleQuery::Kind` reaching the check. + (1, "SET ROLE DEFAULT"), + # `readonly = 2` is what the HTTP execution path sets for safe methods. + (2, "SET ROLE other_role"), + ], +) +def test_set_role_refused_when_enabled(readonly, statement): + _, error = as_probe(node_on, statement, readonly) + assert "READONLY" in error + + +def test_set_role_allowed_without_readonly(): + answer, error = as_probe(node_on, "SET ROLE other_role; SELECT currentRoles()", 0) + assert error == "" + assert answer.strip() == "['other_role']" + + +def test_set_role_allowed_at_default_settings(): + answer, error = as_probe(node_off, "SET ROLE other_role; SELECT currentRoles()", 1) + assert error == "" + assert answer.strip() == "['other_role']" + + +def in_session(node, session_id): + # Every request is a POST so the HTTP method contributes no `readonly` of its own + # (`setReadOnlyIfHTTPMethodIdempotent` exempts POST), leaving the session's own value as the + # only source. Reading the roles back needs the same session, which is why this is HTTP. + def post(sql): + return node.http_query_and_get_answer_with_error( + sql, method="POST", params={"session_id": session_id}, user="probe", password="x" + ) + + assert post("SET readonly = 1")[1] is None + _, error = post("SET ROLE other_role") + roles, roles_error = post("SELECT currentRoles()") + assert roles_error is None + return error, roles.strip() + + +def test_set_role_refused_mid_session(): + error, roles = in_session(node_on, "refused_mid_session") + assert error and "READONLY" in error + assert roles == "['ro_role']" + + +def test_set_role_allowed_mid_session_at_default_settings(): + error, roles = in_session(node_off, "allowed_mid_session") + assert error is None + assert roles == "['other_role']" + + +def test_http_role_parameter_not_restricted_when_enabled(): + # The `role` URL parameter reaches `Context::setCurrentRoles` without passing through + # `InterpreterSetRoleQuery`, so the restriction does not cover it. + def post(sql, **extra): + return node_on.http_query_and_get_answer_with_error( + sql, + method="POST", + params={"session_id": "http_role_parameter", **extra}, + user="probe", + password="x", + ) + + assert post("SET readonly = 1")[1] is None + answer, error = post("SELECT currentRoles()", role="other_role") + assert error is None + assert answer.strip() == "['other_role']" + # The parameter applies to one request, so the session keeps its default role. + answer, error = post("SELECT currentRoles()") + assert error is None + assert answer.strip() == "['ro_role']" diff --git a/tests/integration/test_reload_ca_certificate/__init__.py b/tests/integration/test_reload_ca_certificate/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_reload_ca_certificate/certs/ca.crt b/tests/integration/test_reload_ca_certificate/certs/ca.crt new file mode 100644 index 0000000000000..b836c066c43e6 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/ca.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDRzCCAi+gAwIBAgIURxZvEwx6gcq/IbNHsPts6W5c7P0wDQYJKoZIhvcNAQEL +BQAwMzEYMBYGA1UECgwPQ2xpY2tIb3VzZSBUZXN0MRcwFQYDVQQDDA5UZXN0IFJv +b3QgQ0EgMTAeFw0yNjA4MzExNzI4MTRaFw0zNjA4MjgxNzI4MTRaMDMxGDAWBgNV +BAoMD0NsaWNrSG91c2UgVGVzdDEXMBUGA1UEAwwOVGVzdCBSb290IENBIDEwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCQMESmjuEmo6A25FY5Bt11iOrs +ySvgaWC7DkFaRRy/EZOvQg/HiGXG/oRtt1BsKLYdxRrvtxgJ9H5Uei/SsY0kLTiz +PZFsEomesNwxV1Y8dyeKf9EDSKCjCnrlJwaTmpJ2FpjJRQHf5YJtwVf9E13eKGw7 +qGoCxGGUx14kU7ftdfpEtCC8HRjktWlkXt16nCAHInXx8WSi6SAgLoLUeVC/HpNQ +dVWyPHUhYyFppkYnYONIAJa2UaXB4OlaE+d9/edJeKWGc46oIFK4bJV53nOJ+1Jw +IEf13ax0IcKc1I0q1hJrsnB68Wowi6NR33THHbPNCvU4h/+1qcTYdzicaA9JAgMB +AAGjUzBRMB0GA1UdDgQWBBSJsd7zDPOJkBExjUxJ8bKVSg2pHTAfBgNVHSMEGDAW +gBSJsd7zDPOJkBExjUxJ8bKVSg2pHTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA4IBAQBiopVu+5LqgH4FwKh6YLvWEWdtw5gWs1Eda27cN2b1nTnJ+Xfn +UlwAdaS1lzGBkgUDYfbL0OJo9wx/yg5MuakE2kNO1QALb4r5mpGqTBMYT1XVzcGx +6LCOweHBI4a/ff4s1rj1xRR+PIPIHnCd5mKEgscKMvCmR5kUzkkmjRPFnBJFLHH9 +Z88L4J5NBwF8wJsFCv9td8aeeeTc9rg19zKOn6vkR5AczeXVAeBDBxAtcmX2YRMt +ytIWGmKfW3OE8kxsZdJJn4b+9jE5pLbTEL/So0YuNELRgcpK+9MGApOKjcc3DJg0 +LhsYev318QV4SjluwR1T/jgRvAlFbwk8+10L +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_ca_certificate/certs/ca1.crt b/tests/integration/test_reload_ca_certificate/certs/ca1.crt new file mode 100644 index 0000000000000..b836c066c43e6 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/ca1.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDRzCCAi+gAwIBAgIURxZvEwx6gcq/IbNHsPts6W5c7P0wDQYJKoZIhvcNAQEL +BQAwMzEYMBYGA1UECgwPQ2xpY2tIb3VzZSBUZXN0MRcwFQYDVQQDDA5UZXN0IFJv +b3QgQ0EgMTAeFw0yNjA4MzExNzI4MTRaFw0zNjA4MjgxNzI4MTRaMDMxGDAWBgNV +BAoMD0NsaWNrSG91c2UgVGVzdDEXMBUGA1UEAwwOVGVzdCBSb290IENBIDEwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCQMESmjuEmo6A25FY5Bt11iOrs +ySvgaWC7DkFaRRy/EZOvQg/HiGXG/oRtt1BsKLYdxRrvtxgJ9H5Uei/SsY0kLTiz +PZFsEomesNwxV1Y8dyeKf9EDSKCjCnrlJwaTmpJ2FpjJRQHf5YJtwVf9E13eKGw7 +qGoCxGGUx14kU7ftdfpEtCC8HRjktWlkXt16nCAHInXx8WSi6SAgLoLUeVC/HpNQ +dVWyPHUhYyFppkYnYONIAJa2UaXB4OlaE+d9/edJeKWGc46oIFK4bJV53nOJ+1Jw +IEf13ax0IcKc1I0q1hJrsnB68Wowi6NR33THHbPNCvU4h/+1qcTYdzicaA9JAgMB +AAGjUzBRMB0GA1UdDgQWBBSJsd7zDPOJkBExjUxJ8bKVSg2pHTAfBgNVHSMEGDAW +gBSJsd7zDPOJkBExjUxJ8bKVSg2pHTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA4IBAQBiopVu+5LqgH4FwKh6YLvWEWdtw5gWs1Eda27cN2b1nTnJ+Xfn +UlwAdaS1lzGBkgUDYfbL0OJo9wx/yg5MuakE2kNO1QALb4r5mpGqTBMYT1XVzcGx +6LCOweHBI4a/ff4s1rj1xRR+PIPIHnCd5mKEgscKMvCmR5kUzkkmjRPFnBJFLHH9 +Z88L4J5NBwF8wJsFCv9td8aeeeTc9rg19zKOn6vkR5AczeXVAeBDBxAtcmX2YRMt +ytIWGmKfW3OE8kxsZdJJn4b+9jE5pLbTEL/So0YuNELRgcpK+9MGApOKjcc3DJg0 +LhsYev318QV4SjluwR1T/jgRvAlFbwk8+10L +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_ca_certificate/certs/ca1.key b/tests/integration/test_reload_ca_certificate/certs/ca1.key new file mode 100644 index 0000000000000..562b4debf1ac9 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/ca1.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCQMESmjuEmo6A2 +5FY5Bt11iOrsySvgaWC7DkFaRRy/EZOvQg/HiGXG/oRtt1BsKLYdxRrvtxgJ9H5U +ei/SsY0kLTizPZFsEomesNwxV1Y8dyeKf9EDSKCjCnrlJwaTmpJ2FpjJRQHf5YJt +wVf9E13eKGw7qGoCxGGUx14kU7ftdfpEtCC8HRjktWlkXt16nCAHInXx8WSi6SAg +LoLUeVC/HpNQdVWyPHUhYyFppkYnYONIAJa2UaXB4OlaE+d9/edJeKWGc46oIFK4 +bJV53nOJ+1JwIEf13ax0IcKc1I0q1hJrsnB68Wowi6NR33THHbPNCvU4h/+1qcTY +dzicaA9JAgMBAAECggEABbduL1gkknXkBcJw2IRec8OaBHQs2cdfwwbGM0b1xEL/ +2Dp9+j7ngcJTO9or1XVElCKoMNxmgaAzK9RSJfqtV3j2AaWKyOO6wk3YKwTKXy2+ +WvS6k7RgTwUnjm3TykwTa1xgZ2wQQ8EmC3CCCtScMPu+CnGgw1Y9GVXmqaQf5jQ1 +xAoxP8/POBLjco8PAFWtai+wK9X1OVxQ6gr5Tw3TwAIHrYGsTY19g5c5IM4Y2k34 +QoaSGHkmSFOrBZs2DaWCaJ08Iz3zWUBP0VV2/H0szdIIrkv/Tqw2K6peRbJB8mYN +TID/BpbAadbEllzfn31O3Si0urHNZMrJI5IS2IChEQKBgQDHXrc6AUQQArbEKCxB +dq6B9+tLukaiMctxLHNXhY2NYlJxtASxkmYJewmU/d16lMyo3S+IPkEueo0Q1u3Z +k/MotSiis5kkL2EJWJjHq4H/ktw6k2a0vRd4DtcoysJ2qkCcgubSGYqSE+YcI1KC +MyOoKfb2IffCucyYAzntKmh+2QKBgQC5JQH9+CdyPlcWTJsXMztgDwyhzBixSqIl +Ovfrzzi7QSIX9csccZoXWkOfDEUJp2CZImErASO06+RM2ETQ1n2art4eWxX3UubD +pLXfr1m8NL/LJb56k9sWT+YwLXm4JeHeaErK6xTQCDNxH6szXUXQdXUpnbrrjg0S +6xVG0tCt8QKBgBI+U4vmQ8EnTmwitPIElzFja0+Rqxb6cYBYrfFLUkmmvp6S9378 +Q4QIkzbkCBlIdnXZT5krATHsmu34jOlFBZIrCZ3hy1ipUTrWtZxH0Gx/ltFxXYua +ZgRhb0TXUPYk3Ca2P8Ln/WsikQLwJIOvhErGFEgvkYlrERKz8OAH6mn5AoGAXQxd +YO9bm8365KkhdNp5p8BIf/RcIJY6wW1OdkPR5kJIyTPtnWD2qW/i9kcrVzu4j524 +qe1Lrby0I265vx9dRuVFmon6ky8l7QOVqFKvTahRD97rSR2QCTmknWfteYAIcUeG +906ISjkk6WCaIRlqYeb2ODEeZQ4iQfTF369J03ECgYA3NPk8j5ma59w/TYUPgZWN +aFlEKWLO8xOal9kWhx5dpXi1YCv3YoQztrfYus5b2nj7iDshK5cK6TzZaxPahqxA +rW9ExDjN2q89AvqP9vKPDJZ8Rk7+vTlLSNkFSe9LvvtfnU6IHdNcj+t/CI2823GC +Myj/NocqikSkeRCcq4SGCA== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_ca_certificate/certs/ca2.crt b/tests/integration/test_reload_ca_certificate/certs/ca2.crt new file mode 100644 index 0000000000000..eff5f7fd97bc9 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/ca2.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDRzCCAi+gAwIBAgIUdDLEtjezpK/A2pIn/0YjbO3JhyAwDQYJKoZIhvcNAQEL +BQAwMzEYMBYGA1UECgwPQ2xpY2tIb3VzZSBUZXN0MRcwFQYDVQQDDA5UZXN0IFJv +b3QgQ0EgMjAeFw0yNjA4MzExNzI4MTVaFw0zNjA4MjgxNzI4MTVaMDMxGDAWBgNV +BAoMD0NsaWNrSG91c2UgVGVzdDEXMBUGA1UEAwwOVGVzdCBSb290IENBIDIwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC1bXrQUqqds9kXelFc7J+IrXsE +UdFZtxktFglh9eFrffqr0/+vthrcnqZ/OZrLmZV5PSW+gSNuaAVArbgGwYWW1lqL +0eRfQOktX/NcjoYlWUYlK7lWReFQj2CTCUrP3ojrdo/Oe0/0LsV3kctfUagcNxMX +oV5NAwHd9dDf9aL1ulljc+RO4ym8BlRqSApShkyH3xklkDLBneHOUJncwrWcC12p +1kV3fvt9iBURGK/MYAhhBJGnBeHM+OHLcyVZaUhqXcZ4+fFA2dEA5yGvPs2Gaf/1 +aAE/FsU/zLZkHmI8gslCoBrLtc10dgiUne07Hphu14wplVrSCWJImFMNN31fAgMB +AAGjUzBRMB0GA1UdDgQWBBRQyTcy9OASBuFcqS92wJ9LmtdwejAfBgNVHSMEGDAW +gBRQyTcy9OASBuFcqS92wJ9LmtdwejAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA4IBAQCnd9ONRHb484+BSpRjfWpuDwGgjdqW9aglEFHOVsUestWe8rSl +9lZBveng/4zIqpIKZpOK04shPwP28lKwPim7EFp9iMSd7lnvSy0vHUhYoVzDtK/V ++QoYcrS/LgijOd+ohaszn2hzWWenNdWkgzSfjzp0WgiGg6j2R5pM9fmX/Wh0KtaC +XF23cM8eLaA5CyVCGRmH637qircr9wLbulld14GNBa9hTqcsp12TE+UX5ULwrGdO +PfiFeC8dmtiNW1p1dhdas52kB92e8EWU1wjFMZACyWpY6x7fxmdLexys5TqKkSn+ +JODwownsXCO4RGHTgQUuZNKFcyfbQGUbF3Ig +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_ca_certificate/certs/ca2.key b/tests/integration/test_reload_ca_certificate/certs/ca2.key new file mode 100644 index 0000000000000..14928a74fcad6 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/ca2.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC1bXrQUqqds9kX +elFc7J+IrXsEUdFZtxktFglh9eFrffqr0/+vthrcnqZ/OZrLmZV5PSW+gSNuaAVA +rbgGwYWW1lqL0eRfQOktX/NcjoYlWUYlK7lWReFQj2CTCUrP3ojrdo/Oe0/0LsV3 +kctfUagcNxMXoV5NAwHd9dDf9aL1ulljc+RO4ym8BlRqSApShkyH3xklkDLBneHO +UJncwrWcC12p1kV3fvt9iBURGK/MYAhhBJGnBeHM+OHLcyVZaUhqXcZ4+fFA2dEA +5yGvPs2Gaf/1aAE/FsU/zLZkHmI8gslCoBrLtc10dgiUne07Hphu14wplVrSCWJI +mFMNN31fAgMBAAECggEADGEHnovqwk88LSLrxvMKQPF9UaJ3g1nqk4NLvphf+iKi +u53/yzBvYsWccf/Z37HUsMIK/5WgzO63X7NlZBNI7ILRs3VDJ3tZWFQ8rwBhMiPb +lREHrgBJ9beNYlr29M9So6ZM/Pdi/DPuG1DNAe2jZ9Fw3GLmo/XkPGMtDlWcpIwm +DpJDpFDCPfpi0g0z5gIyqVQQW3m9dE8WFvBPUYoDALmKfj8xvPuw1ezEsSkUj06N +YD+/PTOvpQyDhJGVVS9fZyQvdfCszTPMYsNx7e3FVsDdrXRwCFr4gbzxkjQhcwoq +VeZc+1k9R6V17uY3lPL5hfUpLBz8CxfitBzHJjH7QQKBgQDhiAXlNGn0+ggKg3r0 +H2wJ9E6ecyhUVOXoS9f65b+V6sqPSEHWk0jKPJqMXZV6WY4lQHq90WU3hxx2sNiX +jIP6AZdTUVfqJ7gAvBIn14kkLrirmPYx3MURFYoPm+B4LIHcvMlnMtY7BzO3eiv9 +1GDZYDl0mNcx/m5ipYxYLIH34QKBgQDN8CKzju8R1Ogp7m7bbjxSv3PCEzV0jHrs +jQBTSjb60d+Lz/STI6N0H7BMkwvNewnMwQU1xZ8DgGsDXQa4/6x2UiDGHvsCDFTF +HHsXzQCHP8n10vaU4xeCFqyj9/KYElqS8WOn/faH+c1N246+ATTMRDiU3lhI64Bo +iXKmuQcdPwKBgHKYPwabf0su0G8nJ45reOYF8Pyp3tAa40cJYpDltFdkmc/8ExgI +dm/sI0s3MgCdCJD9FmDkyN1SFbBpY2R9zYF21YFMT7N2wxP8e+0qo1BzPPpUGqRz +XN61ZxVPSttFIica9est9ZTAsBKGTVwIUb2iGw+XqaCJe2U8YPdchh2BAoGBAJ+L +fAbqJHMHJDpgG4hqlddxtafUo+RAdXdQIcFlTMTy1aKGoK9hu99aMYaRoWI3ATed +DoFDMldPJRj8+BlZEu6z3+o91C8ZCI+Q6hhdXRxrIfcN0rU0XmENWgDKNir0hTE0 +TAW5LkbYE+NOxv6TBql97OwAehs8QEY8vhNGY6mXAoGASwKe9byhQKXk0X6Q31g+ +5qaDc+fAqirKnPkqexw+1pUv/j6pK0QFBmPAT46Iwt/znAXqXCfSCNwIpR2UW78n +2J9UQR076SUat71MvcbPhesoUhLw4ZyFQYZ1Lo3piokhzjI0zwlmMusDpUdOjdB6 +JJcZmm30hrRMYNncfobFvXc= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_ca_certificate/certs/cert1.crt b/tests/integration/test_reload_ca_certificate/certs/cert1.crt new file mode 100644 index 0000000000000..3c2faa29525a0 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/cert1.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbjCCAlagAwIBAgIUB2rETXTp2XN0iWipA9fQo9alsGQwDQYJKoZIhvcNAQEL +BQAwMzEYMBYGA1UECgwPQ2xpY2tIb3VzZSBUZXN0MRcwFQYDVQQDDA5UZXN0IFJv +b3QgQ0EgMTAeFw0yNjA4MzExNzI4MTRaFw0zNjA4MjgxNzI4MTRaMBoxGDAWBgNV +BAMMD2NsaWNraG91c2UtdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAN7xjZT2K9ocTmXgCKTgRU70rTvDrW9lpupOAASUm+pwWjtwtYs6AyI1eOlM +5ecZlMHC5zhgngxHlQ1MKqMmvFeMJmPZlpG4xPiwXQXxoFvbQ9T6nRvcnxhB2R/p +wSxBvVwwX2Tf7h+f03XwVndC3CkPwdmwR3VsnVld7WPsLaTemUDKefZ6yTR0OVYZ +fN86bw5Ao9ZzjyH079fAvL2BAQ27I64WJz2480dJX9xgp9wzo91oYyy/ox6cA+ls +lPxUIvV/Se9mV0NhtkkdEgC0DidXyKZM79PsDTxnQUYiqF8p6YGv+bDbvBgyaoTJ +uJQ5MyEDKEl6PcTWq06mHWGF7iMCAwEAAaOBkjCBjzAJBgNVHRMEAjAAMAsGA1Ud +DwQEAwIFoDA1BgNVHREELjAsgglsb2NhbGhvc3SCBG5vZGWCBW5vZGUxggVub2Rl +MoIFbm9kZTOHBH8AAAEwHQYDVR0OBBYEFCLJtr15jan+ipoJ4MHQ8hHuj5SmMB8G +A1UdIwQYMBaAFImx3vMM84mQETGNTEnxspVKDakdMA0GCSqGSIb3DQEBCwUAA4IB +AQAhoPNnfpwuqjZClvtMNGZz4He2g09XO7RgE6uFBPG4DKZyJoRwNkrUrdqpkG13 +2D1lV8IUUzgRBqzP4JmVjmm6Vb7106V6epaZXQPar9PaM9pS/whf6G+5lXTVbn0G +skjMAuDeqZLIPh+547hvdI3WguJ+Lt+iqtlpVpqjPqySgG21H7CFMEZ7pKz7bKgc +Q7qzlvVNJc/XDE8nkuxfdBDZODQD3HEQXOVVGx/1x82HXy/EsTS6nBZnO9gOyj5H +gd/+uN2S5oIuxs1a1b1b54MuWZaFz9OGaqGr3CAjSZO1imIfgT4Ftfoa0FAFrSdO +pXpfpZLT5iOYwG+WxcYu1fW5 +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_ca_certificate/certs/cert1.key b/tests/integration/test_reload_ca_certificate/certs/cert1.key new file mode 100644 index 0000000000000..61143a040b98e --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/cert1.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDe8Y2U9ivaHE5l +4Aik4EVO9K07w61vZabqTgAElJvqcFo7cLWLOgMiNXjpTOXnGZTBwuc4YJ4MR5UN +TCqjJrxXjCZj2ZaRuMT4sF0F8aBb20PU+p0b3J8YQdkf6cEsQb1cMF9k3+4fn9N1 +8FZ3QtwpD8HZsEd1bJ1ZXe1j7C2k3plAynn2esk0dDlWGXzfOm8OQKPWc48h9O/X +wLy9gQENuyOuFic9uPNHSV/cYKfcM6PdaGMsv6MenAPpbJT8VCL1f0nvZldDYbZJ +HRIAtA4nV8imTO/T7A08Z0FGIqhfKemBr/mw27wYMmqEybiUOTMhAyhJej3E1qtO +ph1hhe4jAgMBAAECggEAOTrp3OTqseVVTLqbjXOS5ydRNwfOxEtkcz5Nq99YPPDV +gO+4csKUHlp6rO0UEWSUNr8pKuRGfiF8BjtYsKQXciPkkPpAuCylx69CWe3CfAIH +4irpXMcgQhJZQeN4Nruzd/Bk9Ji1YIHfPyXQlHHh4VqNqSui1GZq6A+ACogM2Ybe +Qu2tEKGYC7LIYATfcnWMF8QpdF+FDCQmjL8yZTz6WhCDTk+h58y/lDfFwzhCKEwx +Y3g69S16lgIRlzxqoXU9UR31ieGsG+op6hmgrdO57chTUF/+g3qS7fGoFN6g6gsm +GeanWfM2nHpM4A73+TF7aI8xf4IHIgkXYa/KEXXh4QKBgQDz+Kj7OEQvv2NIs+lY +RkYTl0ljof8IzFPp8qZEo9z0PpD/Nl9sphfoV/ZYj79Ijnb8bkPcK2aqn2f9Ortx +I28XtzZog/p4XVSLw0o6hRRENsZEpsxYebaGp6kighZM9JHPm67T4neQ5PcR0r4g +Kz6DpoN/bM+uiWiULIMBKeHlwwKBgQDp73x77kvFXGb/Xc67qjfT7H8gPdBcAajd +SGeQHP+AbvsxZaEp0hMSXHYoJRq4eSsA5h8BOHDjM39LZ4uvlcphQDpuCHC/cbUI +bt5PmuddY/YTDkvEi/WIZi9+HSuAZ+RnVgbZowuTV0/t776Fj8R/MefDuWSuiDvF +Bt2z1XZwIQKBgBGZxtcY4BJxxD/ietsbdsLDD1BYx4Vi+ErQbp5VFAOq39sJmSjF +csQYVHVfKXWakYr0iYDAwM9eYKosKomm/MTBOvOfUdqNISRUGm7OWv/w06zwO53G +ahycy97pc6JpontPx/URSX7yhcCLa5v2grQMtz/iIbl9wEWwUGMtGlbxAoGANQD3 +IplWf6w1Bg06JxklNxYxo5t91yrlGOYr2OJJHc+HiKSvRGt9uL5MY0Is8Lk7fiOl +yMACC+iCIhKe+rSkuy4zTvUInsfjrbp5Em5Vl7pradvmXO0dP79vaVKwpZJklOlP ++gXQPJ0e1hlpAJgXfH5RNe6OmmDxse2hU/q8sCECgYBmK1MbPoX0SbuzPmAT6iu8 +4spHZp/AsiWOAnXU42VgEJCRfH+086CvZdf8N7Q6cJDfxofO95syDIMkC/7bkTGT +WT+R+1s0ouZFBtcX+6uPf9eR0/5W3SCzV1YkQtw6iXhoFociRdXDGO8JON80+x8e +BWtbo3oEMlwc6/rNAELLbQ== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_ca_certificate/certs/cert2.crt b/tests/integration/test_reload_ca_certificate/certs/cert2.crt new file mode 100644 index 0000000000000..4fb702bf143b2 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/cert2.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbjCCAlagAwIBAgIUIcogGF2yo/W8Z+f1vXaSVWnpDO4wDQYJKoZIhvcNAQEL +BQAwMzEYMBYGA1UECgwPQ2xpY2tIb3VzZSBUZXN0MRcwFQYDVQQDDA5UZXN0IFJv +b3QgQ0EgMjAeFw0yNjA4MzExNzI4MTVaFw0zNjA4MjgxNzI4MTVaMBoxGDAWBgNV +BAMMD2NsaWNraG91c2UtdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAJKHwEUY5CUuRs7G9Zv+slTng5op18vSYWpj9m1/VagGhkRLlp6FGqf0+pnb +m1A0T0sXKVeuKFgJWqkTTJ5BuP2IOnS+HsfJ2oc0g2vRpO8v0mtZBdT8KA/W6fur +bxQeUOAoEIGca1r0ha/nRAdwvptFRIiIbDgIe9NXR7DQVBWe7Q+DxpzDhjR3/EvZ +W1/jjvTyg/WRhnM3WhrxfeBrHm6o1d8SsDR1t2FTe+7tiBh4ItI1ZWQUSpPhs9qf +LGe5jDx0+9fy4bj8isyMfGvnT+Ll5QFQn34DZrhQ+CAnU6AukZw2Bc3tIfLqtx28 +dh1VUDl0j/BRdbXerEzla1AVCb8CAwEAAaOBkjCBjzAJBgNVHRMEAjAAMAsGA1Ud +DwQEAwIFoDA1BgNVHREELjAsgglsb2NhbGhvc3SCBG5vZGWCBW5vZGUxggVub2Rl +MoIFbm9kZTOHBH8AAAEwHQYDVR0OBBYEFCjHe0KJ1rhLUhZfsMAtv5KSBYC0MB8G +A1UdIwQYMBaAFFDJNzL04BIG4VypL3bAn0ua13B6MA0GCSqGSIb3DQEBCwUAA4IB +AQAbU63n0EM1JWgCDaHYTo9FvnCmnmc1Ge2Nnz5lCwwom5kzB7GPjHljsx3L93OG +ETPbFD1On4OZI9ynURh+9mGlTAE3RiCHT+UjnHnQlKpn0mEjNihXGGM9iMIe4a/q +iVM2To3ayfjuMviLqgpRk48941rEbSSlnaZu+9bXG87QfRdRww2brRF1YOh3IvdT +lawdGc4Rlw/Hh9XjEBJ4u9dtRneoScKI//dzkTKuY0Jj3LeWRq8quX4+DnJZaBlx +vhU9YHstiNIooT1WGlHfNErOB7rsjWbAexWAdIo4gJVTM8caQtpaYPbOBxvCMaMX +/GD8THgD62RZxT80PWXdr4RW +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_ca_certificate/certs/cert2.key b/tests/integration/test_reload_ca_certificate/certs/cert2.key new file mode 100644 index 0000000000000..beeebe86de1fa --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/cert2.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCSh8BFGOQlLkbO +xvWb/rJU54OaKdfL0mFqY/Ztf1WoBoZES5aehRqn9PqZ25tQNE9LFylXrihYCVqp +E0yeQbj9iDp0vh7HydqHNINr0aTvL9JrWQXU/CgP1un7q28UHlDgKBCBnGta9IWv +50QHcL6bRUSIiGw4CHvTV0ew0FQVnu0Pg8acw4Y0d/xL2Vtf44708oP1kYZzN1oa +8X3gax5uqNXfErA0dbdhU3vu7YgYeCLSNWVkFEqT4bPanyxnuYw8dPvX8uG4/IrM +jHxr50/i5eUBUJ9+A2a4UPggJ1OgLpGcNgXN7SHy6rcdvHYdVVA5dI/wUXW13qxM +5WtQFQm/AgMBAAECggEAF9DYR21vbhBH8kV0gl69Zb8H7O9zw2KAcoDTJY072FFQ +fbAwHQ1dku071UmpbuSZpgbv3Brn3sCNIOGMo0TjstLIoX83KdodL7rcHhLsdPMi +XE/opUCN/bO8wwAO+pz0yyy9MLvCLSiExAguzpYcvMg02+DEf2pNlJY2hj6jXqyq +PKQXqTdtclrTfsjLBDJfqysZA5TrEV8tglU4uYr7fD2q3iFJ90VsBOKZaNs5jEsC +TJfXYWOf5OngwUD5nVCWHfsltJyysbhHUPQJcsk4G2zzQdX9ASS6wMc67rBuz25B +EMzR0ZP2eGAYqhbmYa6oVr0kE1GnukxLkQqtHOeAuQKBgQDEKEu4CRri/fw+RXOB +S33jdEPFP5hzPhx/41bKQzvZ/VkVRH7SLxMGIQjgkd1j7Klx7o42iUJPUYtlN8Gc +0+HxmnUjAh81F2fAq+C3rqRo2D0KQT3izHPGKxfmyPhPXVKrL8FVXGWMv1wvSHpJ +HQ3M4KaD9GMKmnswwyPArM0OBwKBgQC/O5/bqJf+ltCVeE4q77VxsHNcrJfxwgn6 +1I6ZcoAUPPyywOAG4WLaiH41I3CPydFYLfDk6AnbO9JQybvnu69OPa+At++Dggm6 +MSAYvYigQ4RsZuc+kzaV9//zhWIVa5kcfljyo1L49cLf2uBz3JWziEqYsl4TmCtp +bs8oWME4iQKBgAJU5EmEujAWisgGtU/FIPLyL9gJYHuGMnqGrkJrOCvoKgXpsYQ4 +EQbSn7NjqHkGmCEFj+UwDny44GpMll2R2y6vAlNvNAXCiHYu1NX6GnQwldEoY17t +xTaGzprsqp7u4gus3qRwG7jnkWXye5mg4cgcp34MCp1Wpr42o5cntqxDAoGAKP89 +XDgercPjX8f06huNyJvNf5a41GmG/jFHiPoVH0Gb4y6aWJ9FNBiDBh1c6laX/NGM +jWZ5hniitBMrp5iDEsECuRO103mzYClb+jHX8pPG9f5xoOaqkyghxTFZP8JbhtJH +e20sQpddeeRQrkYiCeU0KNxEcuryk53f54RvmBECgYAO8Yo/NA6Cz7FIidmb4tL5 +n3frL1fdA7bY74ltjMJp4dCmvlezbKG2xdSKqDlEvfoWgRrZEBtjH/i5esZhzH/H +6z0rK/TmoLpW5HLmBJ5Fqmj5lWOvmtNVh63QcW/hTFOGRSzuESzTD1qafMjtAAvZ +HBvkHzbEbZ6tVlKpNnNZzg== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_ca_certificate/certs/generate_certs.sh b/tests/integration/test_reload_ca_certificate/certs/generate_certs.sh new file mode 100755 index 0000000000000..c487b3941426a --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/generate_certs.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Generates the certificates for test_reload_ca_certificate: +# ca1.crt / ca1.key - first root CA +# ca2.crt / ca2.key - second, unrelated root CA +# cert1.crt / cert1.key - leaf certificate issued by ca1 +# cert2.crt / cert2.key - leaf certificate issued by ca2 +# Both leaf certificates are valid for all host names used in the test and carry no extended key usage, +# so they can be used as server, client and Keeper (Raft) certificates. +# ca.crt, node.crt, node.key - copies of ca1.crt, cert1.crt, cert1.key: the initial content of the files +# that the configs point to and that the test overwrites to rotate certificates. +set -e +cd "$(dirname "${BASH_SOURCE[0]}")" + +DAYS=3650 + +cat > leaf.cnf << 'EOC' +[req] +distinguished_name = dn +prompt = no +[dn] +CN = clickhouse-test +[v3_leaf] +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment +subjectAltName = DNS:localhost, DNS:node, DNS:node1, DNS:node2, DNS:node3, IP:127.0.0.1 +EOC + +for i in 1 2; do + openssl req -x509 -newkey rsa:2048 -nodes -batch -sha256 -days $DAYS \ + -subj "/O=ClickHouse Test/CN=Test Root CA $i" -keyout ca$i.key -out ca$i.crt + + openssl req -newkey rsa:2048 -nodes -batch -config leaf.cnf -keyout cert$i.key -out cert$i.csr + openssl x509 -req -in cert$i.csr -CA ca$i.crt -CAkey ca$i.key -CAcreateserial -sha256 -days $DAYS \ + -extfile leaf.cnf -extensions v3_leaf -out cert$i.crt + rm -f cert$i.csr ca$i.srl +done +rm -f leaf.cnf + +cp ca1.crt ca.crt +cp cert1.crt node.crt +cp cert1.key node.key diff --git a/tests/integration/test_reload_ca_certificate/certs/node.crt b/tests/integration/test_reload_ca_certificate/certs/node.crt new file mode 100644 index 0000000000000..3c2faa29525a0 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/node.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbjCCAlagAwIBAgIUB2rETXTp2XN0iWipA9fQo9alsGQwDQYJKoZIhvcNAQEL +BQAwMzEYMBYGA1UECgwPQ2xpY2tIb3VzZSBUZXN0MRcwFQYDVQQDDA5UZXN0IFJv +b3QgQ0EgMTAeFw0yNjA4MzExNzI4MTRaFw0zNjA4MjgxNzI4MTRaMBoxGDAWBgNV +BAMMD2NsaWNraG91c2UtdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAN7xjZT2K9ocTmXgCKTgRU70rTvDrW9lpupOAASUm+pwWjtwtYs6AyI1eOlM +5ecZlMHC5zhgngxHlQ1MKqMmvFeMJmPZlpG4xPiwXQXxoFvbQ9T6nRvcnxhB2R/p +wSxBvVwwX2Tf7h+f03XwVndC3CkPwdmwR3VsnVld7WPsLaTemUDKefZ6yTR0OVYZ +fN86bw5Ao9ZzjyH079fAvL2BAQ27I64WJz2480dJX9xgp9wzo91oYyy/ox6cA+ls +lPxUIvV/Se9mV0NhtkkdEgC0DidXyKZM79PsDTxnQUYiqF8p6YGv+bDbvBgyaoTJ +uJQ5MyEDKEl6PcTWq06mHWGF7iMCAwEAAaOBkjCBjzAJBgNVHRMEAjAAMAsGA1Ud +DwQEAwIFoDA1BgNVHREELjAsgglsb2NhbGhvc3SCBG5vZGWCBW5vZGUxggVub2Rl +MoIFbm9kZTOHBH8AAAEwHQYDVR0OBBYEFCLJtr15jan+ipoJ4MHQ8hHuj5SmMB8G +A1UdIwQYMBaAFImx3vMM84mQETGNTEnxspVKDakdMA0GCSqGSIb3DQEBCwUAA4IB +AQAhoPNnfpwuqjZClvtMNGZz4He2g09XO7RgE6uFBPG4DKZyJoRwNkrUrdqpkG13 +2D1lV8IUUzgRBqzP4JmVjmm6Vb7106V6epaZXQPar9PaM9pS/whf6G+5lXTVbn0G +skjMAuDeqZLIPh+547hvdI3WguJ+Lt+iqtlpVpqjPqySgG21H7CFMEZ7pKz7bKgc +Q7qzlvVNJc/XDE8nkuxfdBDZODQD3HEQXOVVGx/1x82HXy/EsTS6nBZnO9gOyj5H +gd/+uN2S5oIuxs1a1b1b54MuWZaFz9OGaqGr3CAjSZO1imIfgT4Ftfoa0FAFrSdO +pXpfpZLT5iOYwG+WxcYu1fW5 +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_ca_certificate/certs/node.key b/tests/integration/test_reload_ca_certificate/certs/node.key new file mode 100644 index 0000000000000..61143a040b98e --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/node.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDe8Y2U9ivaHE5l +4Aik4EVO9K07w61vZabqTgAElJvqcFo7cLWLOgMiNXjpTOXnGZTBwuc4YJ4MR5UN +TCqjJrxXjCZj2ZaRuMT4sF0F8aBb20PU+p0b3J8YQdkf6cEsQb1cMF9k3+4fn9N1 +8FZ3QtwpD8HZsEd1bJ1ZXe1j7C2k3plAynn2esk0dDlWGXzfOm8OQKPWc48h9O/X +wLy9gQENuyOuFic9uPNHSV/cYKfcM6PdaGMsv6MenAPpbJT8VCL1f0nvZldDYbZJ +HRIAtA4nV8imTO/T7A08Z0FGIqhfKemBr/mw27wYMmqEybiUOTMhAyhJej3E1qtO +ph1hhe4jAgMBAAECggEAOTrp3OTqseVVTLqbjXOS5ydRNwfOxEtkcz5Nq99YPPDV +gO+4csKUHlp6rO0UEWSUNr8pKuRGfiF8BjtYsKQXciPkkPpAuCylx69CWe3CfAIH +4irpXMcgQhJZQeN4Nruzd/Bk9Ji1YIHfPyXQlHHh4VqNqSui1GZq6A+ACogM2Ybe +Qu2tEKGYC7LIYATfcnWMF8QpdF+FDCQmjL8yZTz6WhCDTk+h58y/lDfFwzhCKEwx +Y3g69S16lgIRlzxqoXU9UR31ieGsG+op6hmgrdO57chTUF/+g3qS7fGoFN6g6gsm +GeanWfM2nHpM4A73+TF7aI8xf4IHIgkXYa/KEXXh4QKBgQDz+Kj7OEQvv2NIs+lY +RkYTl0ljof8IzFPp8qZEo9z0PpD/Nl9sphfoV/ZYj79Ijnb8bkPcK2aqn2f9Ortx +I28XtzZog/p4XVSLw0o6hRRENsZEpsxYebaGp6kighZM9JHPm67T4neQ5PcR0r4g +Kz6DpoN/bM+uiWiULIMBKeHlwwKBgQDp73x77kvFXGb/Xc67qjfT7H8gPdBcAajd +SGeQHP+AbvsxZaEp0hMSXHYoJRq4eSsA5h8BOHDjM39LZ4uvlcphQDpuCHC/cbUI +bt5PmuddY/YTDkvEi/WIZi9+HSuAZ+RnVgbZowuTV0/t776Fj8R/MefDuWSuiDvF +Bt2z1XZwIQKBgBGZxtcY4BJxxD/ietsbdsLDD1BYx4Vi+ErQbp5VFAOq39sJmSjF +csQYVHVfKXWakYr0iYDAwM9eYKosKomm/MTBOvOfUdqNISRUGm7OWv/w06zwO53G +ahycy97pc6JpontPx/URSX7yhcCLa5v2grQMtz/iIbl9wEWwUGMtGlbxAoGANQD3 +IplWf6w1Bg06JxklNxYxo5t91yrlGOYr2OJJHc+HiKSvRGt9uL5MY0Is8Lk7fiOl +yMACC+iCIhKe+rSkuy4zTvUInsfjrbp5Em5Vl7pradvmXO0dP79vaVKwpZJklOlP ++gXQPJ0e1hlpAJgXfH5RNe6OmmDxse2hU/q8sCECgYBmK1MbPoX0SbuzPmAT6iu8 +4spHZp/AsiWOAnXU42VgEJCRfH+086CvZdf8N7Q6cJDfxofO95syDIMkC/7bkTGT +WT+R+1s0ouZFBtcX+6uPf9eR0/5W3SCzV1YkQtw6iXhoFociRdXDGO8JON80+x8e +BWtbo3oEMlwc6/rNAELLbQ== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_ca_certificate/configs/keeper1.xml b/tests/integration/test_reload_ca_certificate/configs/keeper1.xml new file mode 100644 index 0000000000000..427a4b8f6872c --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/configs/keeper1.xml @@ -0,0 +1,36 @@ + + 0.0.0.0 + 0.0.0.0 + + 0 + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + trace + + + + true + + 1 + node1 + 9234 + + + 2 + node2 + 9234 + + + 3 + node3 + 9234 + + + + diff --git a/tests/integration/test_reload_ca_certificate/configs/keeper2.xml b/tests/integration/test_reload_ca_certificate/configs/keeper2.xml new file mode 100644 index 0000000000000..5dac68a6af56d --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/configs/keeper2.xml @@ -0,0 +1,36 @@ + + 0.0.0.0 + 0.0.0.0 + + 0 + 9181 + 2 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + trace + + + + true + + 1 + node1 + 9234 + + + 2 + node2 + 9234 + + + 3 + node3 + 9234 + + + + diff --git a/tests/integration/test_reload_ca_certificate/configs/keeper3.xml b/tests/integration/test_reload_ca_certificate/configs/keeper3.xml new file mode 100644 index 0000000000000..28a155e926314 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/configs/keeper3.xml @@ -0,0 +1,36 @@ + + 0.0.0.0 + 0.0.0.0 + + 0 + 9181 + 3 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + trace + + + + true + + 1 + node1 + 9234 + + + 2 + node2 + 9234 + + + 3 + node3 + 9234 + + + + diff --git a/tests/integration/test_reload_ca_certificate/configs/ssl.xml b/tests/integration/test_reload_ca_certificate/configs/ssl.xml new file mode 100644 index 0000000000000..3b93b0a7552ab --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/configs/ssl.xml @@ -0,0 +1,31 @@ + + 8443 + + + + + /etc/clickhouse-server/config.d/node.crt + /etc/clickhouse-server/config.d/node.key + /etc/clickhouse-server/config.d/ca.crt + false + + relaxed + false + sslv2,sslv3 + true + + + /etc/clickhouse-server/config.d/node.crt + /etc/clickhouse-server/config.d/node.key + /etc/clickhouse-server/config.d/ca.crt + false + relaxed + false + sslv2,sslv3 + true + + RejectCertificateHandler + + + + diff --git a/tests/integration/test_reload_ca_certificate/configs/ssl_with_default_cas.xml b/tests/integration/test_reload_ca_certificate/configs/ssl_with_default_cas.xml new file mode 100644 index 0000000000000..6a1077ce40b97 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/configs/ssl_with_default_cas.xml @@ -0,0 +1,29 @@ + + 8443 + + + + + /etc/clickhouse-server/config.d/node.crt + /etc/clickhouse-server/config.d/node.key + /etc/clickhouse-server/config.d/ca.crt + + relaxed + false + sslv2,sslv3 + true + + + /etc/clickhouse-server/config.d/node.crt + /etc/clickhouse-server/config.d/node.key + /etc/clickhouse-server/config.d/ca.crt + relaxed + false + sslv2,sslv3 + true + + RejectCertificateHandler + + + + diff --git a/tests/integration/test_reload_ca_certificate/test.py b/tests/integration/test_reload_ca_certificate/test.py new file mode 100644 index 0000000000000..473a2e850e2cb --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/test.py @@ -0,0 +1,275 @@ +""" +Hot reload of the trusted CA certificates (`openSSL.*.caConfig`). + +The certificates in certs/ are produced by certs/generate_certs.sh: +two unrelated root CAs (ca1, ca2) and one leaf certificate issued by each of them (cert1, cert2). +Every instance starts with ca.crt = ca1 as the trusted CA and node.crt/node.key = cert1 as its own certificate, +and the tests overwrite these files in the container to rotate them without restarting anything. +""" + +import time +import uuid + +import pytest + +import helpers.keeper_utils as ku +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +CONFIG_DIR = "/etc/clickhouse-server/config.d" +CERT_FILES = [ + "certs/ca.crt", + "certs/ca1.crt", + "certs/ca2.crt", + "certs/node.crt", + "certs/node.key", + "certs/cert1.crt", + "certs/cert1.key", + "certs/cert2.crt", + "certs/cert2.key", +] + +# Serves HTTPS and acts as a TLS client towards itself. +node = cluster.add_instance("node", main_configs=["configs/ssl.xml"] + CERT_FILES) + +# Does not set `loadDefaultCAFile`, and OpenSSL's default CA file is ca2 for this instance. +node_with_default_cas = cluster.add_instance( + "node_with_default_cas", + main_configs=["configs/ssl_with_default_cas.xml"] + CERT_FILES, + env_variables={"SSL_CERT_FILE": f"{CONFIG_DIR}/ca2.crt"}, +) + +# Three nodes with embedded Keeper talking Raft over TLS. `loadDefaultCAFile` is not set for them: Keeper assumes `false` for +# the Raft connections then, unlike everything else, and their CA certificates have to be reloaded all the same. +keeper_nodes = [ + cluster.add_instance(f"node{i}", main_configs=[f"configs/keeper{i}.xml", "configs/ssl_with_default_cas.xml"] + CERT_FILES) for i in (1, 2, 3) +] + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def set_trusted_cas(instance, *cas): + """Overwrite ca.crt (the configured `caConfig`) with the given CA certificates.""" + sources = " ".join(f"{CONFIG_DIR}/{ca}.crt" for ca in cas) + instance.exec_in_container(["bash", "-c", f"cat {sources} > {CONFIG_DIR}/ca.crt.tmp && mv {CONFIG_DIR}/ca.crt.tmp {CONFIG_DIR}/ca.crt"]) + + +def set_own_certificate(instance, cert): + """Overwrite node.crt/node.key (the configured `certificateFile`/`privateKeyFile`) with the given leaf certificate.""" + instance.exec_in_container( + [ + "bash", + "-c", + f"cp {CONFIG_DIR}/{cert}.crt {CONFIG_DIR}/node.crt.tmp && mv {CONFIG_DIR}/node.crt.tmp {CONFIG_DIR}/node.crt && " + f"cp {CONFIG_DIR}/{cert}.key {CONFIG_DIR}/node.key.tmp && mv {CONFIG_DIR}/node.key.tmp {CONFIG_DIR}/node.key", + ] + ) + + +@pytest.fixture(autouse=True) +def restore_certificates(started_cluster): + yield + for instance in [node, node_with_default_cas] + keeper_nodes: + set_trusted_cas(instance, "ca1") + set_own_certificate(instance, "cert1") + instance.query("SYSTEM RELOAD CONFIG") + for instance in keeper_nodes: + kill_raft_connections(instance) + ku.wait_nodes(cluster, keeper_nodes) + + +def https_request_with_client_certificate(cert, instance=node): + """Query the HTTPS port of `instance` presenting the given client certificate. Returns the response, or None if the TLS handshake failed.""" + result = instance.exec_in_container( + [ + "bash", + "-c", + f"curl --silent --show-error --cacert {CONFIG_DIR}/ca1.crt --cert {CONFIG_DIR}/{cert}.crt --key {CONFIG_DIR}/{cert}.key " + f"'https://localhost:8443/?query=SELECT%201' 2>&1 || echo CURL_FAILED", + ] + ) + return None if "CURL_FAILED" in result else result + + +def assert_eventually(predicate, description, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.5) + assert predicate(), description + + +def test_server_reloads_ca(started_cluster): + """The CAs used to verify client certificates follow the content of `caConfig` without a restart or an explicit reload.""" + assert https_request_with_client_certificate("cert1") == "1\n" + assert https_request_with_client_certificate("cert2") is None + + # Trust both CAs: the file change alone triggers the reload. + set_trusted_cas(node, "ca1", "ca2") + assert_eventually(lambda: https_request_with_client_certificate("cert2") == "1\n", "cert2 is accepted after ca2 was added") + assert https_request_with_client_certificate("cert1") == "1\n" + + # Drop the old CA: certificates issued by it are not accepted anymore. + set_trusted_cas(node, "ca2") + assert_eventually(lambda: https_request_with_client_certificate("cert1") is None, "cert1 is rejected after ca1 was removed") + assert https_request_with_client_certificate("cert2") == "1\n" + + assert node.contains_in_log("Reloaded CA certificates") + + +def test_client_reloads_ca(started_cluster): + """The CAs used to verify server certificates of outgoing connections follow the content of `caConfig`.""" + # `node` connects to its own HTTPS port, which serves cert1. `Connection: close` rules out reusing a pooled connection. + query = "SELECT * FROM url('https://localhost:8443/?query=SELECT%201', 'TSV', 'x UInt8', headers('Connection'='close'))" + + set_trusted_cas(node, "ca2") + node.query("SYSTEM RELOAD CONFIG") + error = node.query_and_get_error(query) + assert "certificate verify failed" in error, error + + set_trusted_cas(node, "ca1") + node.query("SYSTEM RELOAD CONFIG") + assert node.query(query) == "1\n" + + +def test_default_cas_are_kept(started_cluster): + """With `loadDefaultCAFile` left at its default, both the CAs from `caConfig` and the default ones are trusted, before and after a reload.""" + # cert1 is trusted through `caConfig` (ca.crt = ca1), cert2 through the default CA file (SSL_CERT_FILE = ca2). + assert https_request_with_client_certificate("cert1", node_with_default_cas) == "1\n" + assert https_request_with_client_certificate("cert2", node_with_default_cas) == "1\n" + + reloads = int(node_with_default_cas.count_in_log("Reloaded CA certificates").strip()) + set_trusted_cas(node_with_default_cas, "ca1") # same content, new modification time + node_with_default_cas.query("SYSTEM RELOAD CONFIG") + assert int(node_with_default_cas.count_in_log("Reloaded CA certificates").strip()) > reloads + + assert https_request_with_client_certificate("cert1", node_with_default_cas) == "1\n" + assert https_request_with_client_certificate("cert2", node_with_default_cas) == "1\n" + + +def test_system_certificates_follows_reload(started_cluster): + """`system.certificates` shows the CA certificates that are currently used, also after `caConfig` is changed to another file.""" + query = "SELECT path, subject LIKE '%Test Root CA {}%' FROM system.certificates WHERE NOT default" + assert node.query(query.format(1)) == f"{CONFIG_DIR}/ca.crt\t1\n" + + node.replace_in_config(f"{CONFIG_DIR}/ssl.xml", f"{CONFIG_DIR}/ca.crt", f"{CONFIG_DIR}/ca2.crt") + try: + node.query("SYSTEM RELOAD CONFIG") + assert node.query(query.format(2)) == f"{CONFIG_DIR}/ca2.crt\t1\n" + finally: + node.replace_in_config(f"{CONFIG_DIR}/ssl.xml", f"{CONFIG_DIR}/ca2.crt", f"{CONFIG_DIR}/ca.crt") + node.query("SYSTEM RELOAD CONFIG") + + +def test_ca_directory(started_cluster): + """`caConfig` can be a directory with certificates named by their subject hash. Replacing a certificate in it is noticed too.""" + ca_dir = f"{CONFIG_DIR}/ca_dir" + node.exec_in_container(["bash", "-c", f"mkdir -p {ca_dir} && cp {CONFIG_DIR}/ca1.crt {ca_dir}/$(openssl x509 -noout -subject_hash -in {CONFIG_DIR}/ca1.crt).0"]) + node.replace_in_config(f"{CONFIG_DIR}/ssl.xml", f"{CONFIG_DIR}/ca.crt", ca_dir) + try: + node.query("SYSTEM RELOAD CONFIG") + assert https_request_with_client_certificate("cert1") == "1\n" + assert https_request_with_client_certificate("cert2") is None + + # Overwrite the only file in place: the set of file names in the directory does not change. + node.exec_in_container(["bash", "-c", f"cat {CONFIG_DIR}/ca2.crt > {ca_dir}/$(openssl x509 -noout -subject_hash -in {CONFIG_DIR}/ca1.crt).0"]) + node.query("SYSTEM RELOAD CONFIG") + assert https_request_with_client_certificate("cert1") is None + finally: + node.replace_in_config(f"{CONFIG_DIR}/ssl.xml", ca_dir, f"{CONFIG_DIR}/ca.crt") + node.exec_in_container(["rm", "-rf", ca_dir]) + node.query("SYSTEM RELOAD CONFIG") + + +def kill_raft_connections(instance): + instance.exec_in_container( + ["bash", "-c", "ss --kill -tn state established '( dport = :9234 or sport = :9234 )' > /dev/null"], nothrow=True + ) + + +def raft_port_accepts_client_certificate(instance, target, cert): + """Whether the Raft port of `target` completes a TLS handshake with a client that presents `cert`.""" + # With TLS 1.2 the server verifies the client certificate before the handshake completes on the client side, + # so a rejected certificate reliably shows up as a failed handshake in s_client. + result = instance.exec_in_container( + [ + "bash", + "-c", + f"openssl s_client -brief -tls1_2 -connect {target.name}:9234 -cert {CONFIG_DIR}/{cert}.crt -key {CONFIG_DIR}/{cert}.key " + f"&1 || true", + ] + ) + return "CONNECTION ESTABLISHED" in result + + +def raft_port_certificate_issuer(instance, target): + return instance.exec_in_container( + [ + "bash", + "-c", + f"openssl s_client -connect {target.name}:9234 /dev/null | openssl x509 -noout -issuer 2>/dev/null || true", + ] + ).strip() + + +def check_keeper_cluster_works(path): + connections = [] + try: + for instance in keeper_nodes: + connections.append(ku.get_fake_zk(cluster, instance.name)) + connections[0].create(path, b"data") + for connection in connections: + connection.sync(path) + assert connection.get(path)[0] == b"data" + finally: + for connection in connections: + connection.stop() + connection.close() + + +def reload_and_reconnect_raft(): + for instance in keeper_nodes: + instance.query("SYSTEM RELOAD CONFIG") + for instance in keeper_nodes: + kill_raft_connections(instance) + ku.wait_nodes(cluster, keeper_nodes) + + +def test_keeper_raft_reloads_ca(started_cluster): + """Rotate the CA of the Raft connections between Keeper nodes without restarting them.""" + run = uuid.uuid4().hex + ku.wait_nodes(cluster, keeper_nodes) + check_keeper_cluster_works(f"/before_{run}") + assert "Test Root CA 1" in raft_port_certificate_issuer(keeper_nodes[0], keeper_nodes[1]) + + # 1. Trust the new CA in addition to the old one. + for instance in keeper_nodes: + set_trusted_cas(instance, "ca1", "ca2") + reload_and_reconnect_raft() + check_keeper_cluster_works(f"/both_cas_{run}") + + # 2. Switch the nodes to certificates issued by the new CA. + for instance in keeper_nodes: + set_own_certificate(instance, "cert2") + reload_and_reconnect_raft() + check_keeper_cluster_works(f"/new_certs_{run}") + for target in keeper_nodes[1:]: + assert "Test Root CA 2" in raft_port_certificate_issuer(keeper_nodes[0], target) + + # 3. Stop trusting the old CA. + for instance in keeper_nodes: + set_trusted_cas(instance, "ca2") + reload_and_reconnect_raft() + check_keeper_cluster_works(f"/new_ca_{run}") + + assert raft_port_accepts_client_certificate(keeper_nodes[0], keeper_nodes[1], "cert2") + assert not raft_port_accepts_client_certificate(keeper_nodes[0], keeper_nodes[1], "cert1") diff --git a/tests/integration/test_s3_cluster/test.py b/tests/integration/test_s3_cluster/test.py index 30d5a6190217c..76b8f0df2881b 100644 --- a/tests/integration/test_s3_cluster/test.py +++ b/tests/integration/test_s3_cluster/test.py @@ -2,23 +2,21 @@ import logging import os import shutil -import uuid +import time +from email.errors import HeaderParseError import pytest from helpers.cluster import ClickHouseCluster -from helpers.config_cluster import minio_access_key, minio_secret_key +from helpers.config_cluster import minio_secret_key from helpers.mock_servers import start_mock_servers from helpers.test_tools import TSV -from helpers.utility import random_string logging.getLogger().setLevel(logging.INFO) logging.getLogger().addHandler(logging.StreamHandler()) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) - -# Base CSV files committed to git. Immutable on disk, safe to read concurrently. -S3_BASE_FILES = [ +S3_DATA = [ "data/clickhouse/part1.csv", "data/clickhouse/part123.csv", "data/database/part2.csv", @@ -26,48 +24,14 @@ ] -def _generated_host_dir(): - """Per-xdist-worker host directory for the dynamically generated CSV files. - - Concurrent xdist workers running this module (especially under - `--dist=each`, used by the targeted/flaky integration jobs) all execute - this fixture in parallel. Their MinIO containers are isolated by - `ClickHouseCluster` (it appends `PYTEST_XDIST_WORKER` to `project_name`), - but the host filesystem path is not. With a shared host directory, one - worker's teardown (`shutil.rmtree(...)`) deletes files while another - worker is still uploading them, producing setup-phase - `FileNotFoundError` on `data/generated/file_*.csv` and failing every - test in the module via fixture error. - - The MinIO bucket key for tests stays `data/generated/file_N.csv` (each - worker uploads to its own MinIO instance); only the host write/read path - is scoped per worker. - """ - worker_id = os.environ.get("PYTEST_XDIST_WORKER", "") - suffix = f"_{worker_id}" if worker_id else "" - return os.path.join(SCRIPT_DIR, f"data/generated{suffix}") - - def create_buckets_s3(cluster): minio = cluster.minio_client - host_dir = _generated_host_dir() - os.makedirs(host_dir, exist_ok=True) - - files_to_upload = [] # list of (bucket_object_key, host_file_path) - - # Base files: committed in git, read from the shared SCRIPT_DIR location. - for relative_path in S3_BASE_FILES: - files_to_upload.append( - (relative_path, os.path.join(SCRIPT_DIR, relative_path)) - ) - - # Generated files: written into a per-worker host directory but uploaded - # to MinIO under the canonical `data/generated/` prefix that tests query. for file_number in range(100): - bucket_object_key = f"data/generated/file_{file_number}.csv" - host_path = os.path.join(host_dir, f"file_{file_number}.csv") - with open(host_path, "w+", encoding="utf-8") as f: + file_name = f"data/generated/file_{file_number}.csv" + os.makedirs(os.path.join(SCRIPT_DIR, "data/generated/"), exist_ok=True) + S3_DATA.append(file_name) + with open(os.path.join(SCRIPT_DIR, file_name), "w+", encoding="utf-8") as f: # a String, b UInt64 data = [] @@ -79,13 +43,12 @@ def create_buckets_s3(cluster): writer = csv.writer(f) writer.writerows(data) - files_to_upload.append((bucket_object_key, host_path)) - for bucket_object_key, host_path in files_to_upload: + for file in S3_DATA: minio.fput_object( bucket_name=cluster.minio_bucket, - object_name=bucket_object_key, - file_path=host_path, + object_name=file, + file_path=os.path.join(SCRIPT_DIR, file), ) for obj in minio.list_objects(cluster.minio_bucket, recursive=True): print(obj.object_name) @@ -104,7 +67,6 @@ def run_s3_mocks(started_cluster): @pytest.fixture(scope="module") def started_cluster(): - cluster = None try: cluster = ClickHouseCluster(__file__) cluster.add_instance( @@ -140,10 +102,8 @@ def started_cluster(): yield cluster finally: - # Only this worker's directory; never touches other workers' data. - shutil.rmtree(_generated_host_dir(), ignore_errors=True) - if cluster is not None: - cluster.shutdown() + shutil.rmtree(os.path.join(SCRIPT_DIR, "data/generated/")) + cluster.shutdown() def test_select_all(started_cluster): @@ -549,218 +509,3 @@ def test_cluster_default_expression(started_cluster): ) assert result == expected_result - - -@pytest.mark.parametrize("allow_experimental_analyzer", [0, 1]) -@pytest.mark.parametrize("use_partition_strategy", [False, True]) -def test_hive_partitioning(started_cluster, allow_experimental_analyzer, use_partition_strategy): - node = started_cluster.instances["s0_0_0"] - - data_path = f"root/data/hive_{allow_experimental_analyzer}/{random_string(6)}" - - if use_partition_strategy: - node.query( - f""" - INSERT - INTO FUNCTION s3( - 'http://minio1:9001/{data_path}', - 'minio', - '{minio_secret_key}', - format='Parquet', - partition_strategy='hive', - structure='key Int32, value Int32') - PARTITION BY key - SELECT number+1 as key, number+1 as value - FROM system.numbers - LIMIT 4 - """ - ) - else: - for i in range(1, 5): - node.query( - f""" - INSERT - INTO FUNCTION s3('http://minio1:9001/{data_path}/key={i}/data.parquet', 'minio', '{minio_secret_key}', 'Parquet', 'key Int32, value Int32') - SELECT {i}, {i} - """ - ) - - # With use_partition_strategy=1 and use_hive_partitioning=0 we have no 'key' column - if not use_partition_strategy: - query_id_full = str(uuid.uuid4()) - result = node.query( - f""" - SELECT count() - FROM s3('http://minio1:9001/{data_path}/key=**.parquet', 'minio', '{minio_secret_key}', 'Parquet', 'key Int32, value Int32') - WHERE key <= 2 - FORMAT TSV - SETTINGS enable_filesystem_cache = 0, use_query_cache = 0, use_cache_for_count_from_files = 0, use_hive_partitioning = 0, allow_experimental_analyzer={allow_experimental_analyzer} - """, - query_id=query_id_full, - ) - result = int(result) - assert result == 2 - - query_id_optimized = str(uuid.uuid4()) - result = node.query( - f""" - SELECT count() - FROM s3('http://minio1:9001/{data_path}/key=**.parquet', 'minio', '{minio_secret_key}', 'Parquet', 'key Int32, value Int32') - WHERE key <= 2 - FORMAT TSV - SETTINGS enable_filesystem_cache = 0, use_query_cache = 0, use_cache_for_count_from_files = 0, use_hive_partitioning = 1, allow_experimental_analyzer={allow_experimental_analyzer} - """, - query_id=query_id_optimized, - ) - result = int(result) - assert result == 2 - - if not use_partition_strategy: - query_id_cluster_full = str(uuid.uuid4()) - result = node.query( - f""" - SELECT count() - FROM s3Cluster(cluster_simple, 'http://minio1:9001/{data_path}/key=**.parquet', 'minio', '{minio_secret_key}', 'Parquet', 'key Int32, value Int32') - WHERE key <= 2 - FORMAT TSV - SETTINGS enable_filesystem_cache = 0, use_query_cache = 0, use_cache_for_count_from_files = 0, use_hive_partitioning = 0, allow_experimental_analyzer={allow_experimental_analyzer} - """, - query_id=query_id_cluster_full, - ) - result = int(result) - assert result == 2 - - query_id_cluster_optimized = str(uuid.uuid4()) - result = node.query( - f""" - SELECT count() - FROM s3Cluster(cluster_simple, 'http://minio1:9001/{data_path}/key=**.parquet', 'minio', '{minio_secret_key}', 'Parquet', 'key Int32, value Int32') - WHERE key <= 2 - FORMAT TSV - SETTINGS enable_filesystem_cache = 0, use_query_cache = 0, use_cache_for_count_from_files = 0, use_hive_partitioning = 1, allow_experimental_analyzer={allow_experimental_analyzer} - """, - query_id=query_id_cluster_optimized, - ) - result = int(result) - assert result == 2 - - node.query("SYSTEM FLUSH LOGS ON CLUSTER 'cluster_simple'") - - event = "EngineFileLikeReadFiles" - - if not use_partition_strategy: - full_traffic = node.query( - f""" - SELECT sum(ProfileEvents['{event}']) - FROM clusterAllReplicas(cluster_simple, system.query_log) - WHERE type='QueryFinish' AND initial_query_id='{query_id_full}' - FORMAT TSV - """ - ) - full_traffic = int(full_traffic) - assert full_traffic == 4 - - optimized_traffic = node.query( - f""" - SELECT sum(ProfileEvents['{event}']) - FROM clusterAllReplicas(cluster_simple, system.query_log) - WHERE type='QueryFinish' AND initial_query_id='{query_id_optimized}' - FORMAT TSV - """ - ) - optimized_traffic = int(optimized_traffic) - assert optimized_traffic == 2 - - if not use_partition_strategy: - cluster_full_traffic = node.query( - f""" - SELECT sum(ProfileEvents['{event}']) - FROM clusterAllReplicas(cluster_simple, system.query_log) - WHERE type='QueryFinish' AND initial_query_id='{query_id_cluster_full}' - FORMAT TSV - """ - ) - cluster_full_traffic = int(cluster_full_traffic) - assert cluster_full_traffic == full_traffic - - cluster_optimized_traffic = node.query( - f""" - SELECT sum(ProfileEvents['{event}']) - FROM clusterAllReplicas(cluster_simple, system.query_log) - WHERE type='QueryFinish' AND initial_query_id='{query_id_cluster_optimized}' - FORMAT TSV - """ - ) - cluster_optimized_traffic = int(cluster_optimized_traffic) - assert cluster_optimized_traffic == optimized_traffic - -def test_iceberg_s3_cluster_read_task_failpoint(started_cluster): - """INSERT INTO SELECT FROM icebergS3Cluster fails and does not hang. - Reproduces the scenario from https://github.com/ClickHouse/ClickHouse/issues/98165 - """ - node = started_cluster.instances["s0_0_0"] - run_id = uuid.uuid4().hex[:8] - iceberg_table = f"iceberg_src_{run_id}" - dst_table = f"local_dst_{run_id}" - iceberg_url = f"http://minio1:9001/root/{iceberg_table}/" - - # Create and populate the source Iceberg table before enabling the failpoint. - node.query( - f""" - CREATE TABLE {iceberg_table} (id UInt64, data String) - ENGINE = IcebergS3('{iceberg_url}', '{minio_access_key}', '{minio_secret_key}') - """ - ) - node.query( - f"INSERT INTO {iceberg_table} SELECT number AS id, randomString(10) AS data FROM numbers(50)", - settings={"allow_insert_into_iceberg": 1}, - ) - - # Local destination table — we only care that the query fails, not the data. - node.query( - f""" - CREATE TABLE {dst_table} (id UInt64, data String) - ENGINE = MergeTree() ORDER BY id - """ - ) - - all_node_names = ["s0_0_0", "s0_0_1", "s0_1_0"] - try: - for name in all_node_names: - started_cluster.instances[name].query( - "SYSTEM ENABLE FAILPOINT storage_cluster_read_sleep" - ) - - # The failpoint sleeps 10s uninterruptibly before max_execution_time can - # fire, so the server's legitimate abort lands well after 5s. The client - # timeout must outlast that abort (plus sanitizer/runner slowdown), not - # the 5s execution limit, otherwise a slow runner raises a false - # "Client timed out!". - _, error = node.query_and_get_answer_with_error( - f""" - INSERT INTO {dst_table} - SELECT * FROM icebergS3Cluster( - 'cluster_simple', - '{iceberg_url}', - '{minio_access_key}', '{minio_secret_key}') - SETTINGS max_execution_time = 5 - """, - timeout=120, - ) - - assert error, ( - "Expected a timeout error but the query succeeded. " - "The `storage_cluster_read_sleep` failpoint should have caused the query " - "to hang inside ReadTaskIterator::next() on every worker, " - "and `max_execution_time` should have aborted it." - ) - assert "Timeout" in error or "timeout" in error.lower(), ( - f"Expected a timeout-related error, got: {error}" - ) - finally: - for name in all_node_names: - started_cluster.instances[name].query( - "SYSTEM DISABLE FAILPOINT storage_cluster_read_sleep" - ) - node.query(f"DROP TABLE IF EXISTS {dst_table}") - node.query(f"DROP TABLE IF EXISTS {iceberg_table}") diff --git a/tests/integration/test_s3_style_link/configs/config.d/parallel_replicas.xml b/tests/integration/test_s3_style_link/configs/config.d/parallel_replicas.xml new file mode 100644 index 0000000000000..f8e015f0b761f --- /dev/null +++ b/tests/integration/test_s3_style_link/configs/config.d/parallel_replicas.xml @@ -0,0 +1,23 @@ + + + + + + + http://minio1:9001/root/ + minio + ClickHouse_Minio_P@ssw0rd + + + + + + + node9000 + node9000 + node9000 + + + + diff --git a/tests/integration/test_s3_style_link/test.py b/tests/integration/test_s3_style_link/test.py index da7e64676d173..353c3a0764bf0 100644 --- a/tests/integration/test_s3_style_link/test.py +++ b/tests/integration/test_s3_style_link/test.py @@ -10,6 +10,7 @@ "node", main_configs=[ "configs/config.d/minio.xml", + "configs/config.d/parallel_replicas.xml", ], user_configs=[ "configs/users.d/users.xml", @@ -139,3 +140,55 @@ def test_s3_question_mark_wildcards(started_cluster): assert result_s3_scheme == result_http_scheme assert result_s3_scheme.startswith('20\t') assert "['a1','a2']" in result_s3_scheme or "['a2','a1']" in result_s3_scheme + + +def test_url_s3_scheme_with_parallel_replicas(started_cluster): + """ + `url('s3://...')` is delegated to the `s3` backend, but the query text still names `url`. + The cluster fan-out of `parallel_replicas_for_cluster_engines` rewrites the forwarded query + from that surface name, so it used to send `urlCluster('s3://...')` - a shape `urlCluster` + rejects - both for a plain `SELECT` and for the distributed `INSERT ... SELECT`. + """ + node.query( + f""" + INSERT INTO FUNCTION s3 + ( + 'minio://data/parallel_replicas_url.csv', 'minio', '{minio_secret_key}', + 'CSV', 'a UInt32' + ) SETTINGS s3_truncate_on_insert=1 + SELECT number FROM numbers(10); + """ + ) + + parallel_replicas_settings = """ + SETTINGS cluster_for_parallel_replicas = 'parallel_replicas', + enable_parallel_replicas = 1, + max_parallel_replicas = 3, + parallel_replicas_for_cluster_engines = 1 + """ + + assert ( + node.query( + f""" + SELECT count() FROM url('s3://data/parallel_replicas_url.csv', 'CSV', 'a UInt32') + {parallel_replicas_settings} + """ + ) + == "10\n" + ) + + node.query("DROP TABLE IF EXISTS url_s3_parallel_replicas SYNC") + node.query( + "CREATE TABLE url_s3_parallel_replicas (a UInt32) ENGINE = MergeTree ORDER BY a" + ) + node.query( + f""" + INSERT INTO url_s3_parallel_replicas + SELECT * FROM url('s3://data/parallel_replicas_url.csv', 'CSV', 'a UInt32') + {parallel_replicas_settings}, parallel_distributed_insert_select = 2 + """ + ) + + # The rows must be inserted exactly once, not once per replica. + assert node.query("SELECT count() FROM url_s3_parallel_replicas") == "10\n" + node.query("DROP TABLE url_s3_parallel_replicas SYNC") diff --git a/tests/integration/test_ssh/test.py b/tests/integration/test_ssh/test.py index 5fa570bfe6c08..350aa0023645f 100644 --- a/tests/integration/test_ssh/test.py +++ b/tests/integration/test_ssh/test.py @@ -1,6 +1,7 @@ import os import re import socket +import struct import subprocess import time @@ -495,3 +496,117 @@ def test_interactive_tab_completion_respects_session_user(started_cluster): instance.query("DROP USER IF EXISTS completer") instance.query("DROP TABLE IF EXISTS default.visible_completion_target") instance.query("DROP TABLE IF EXISTS default.hidden_completion_target") + + +def test_interactive_session_torn_down_with_a_dead_pty(started_cluster): + """Losing the pty while the embedded client is shutting down must not kill the server. + + `ReplxxLineReader::~ReplxxLineReader` writes an escape sequence to the + terminal to reset cursor blinking when overwrite mode was ever enabled. + `Replxx::print` throws `std::runtime_error("write failed")` when that write + does not go through, and a destructor is implicitly `noexcept`, so the + exception used to `std::terminate` the whole server process. + + Reproduce it the way a real disconnect does: turn overwrite mode on with + the `Insert` key, then drop the TCP connection with a RST so that the + server's side of the pty is gone by the time the line reader is destroyed. + """ + # The daemon watchdog restarts the server after `std::terminate`, so "the + # server answers queries again" is not evidence of anything, and neither is + # a growing `uptime()`: on a fresh cluster the uptime before the disconnect + # is only a few seconds, so a restarted server reaches a larger value well + # within the sampling window below. Pin the process identity instead: the + # watchdog restarts the server by forking a new child, so the set of + # `clickhouse-server` pids in the container changes and cannot recover. + # The pattern is anchored at argv0 so that the shell running `pgrep` (whose + # own command line contains the pattern) does not match itself. + def server_pids(): + return instance.exec_in_container( + ["bash", "-c", "pgrep -f '^[^ ]*clickhouse(-| )server' | sort -n"], + nothrow=True, + ).split() + + pids_before = server_pids() + assert pids_before, "no `clickhouse-server` process in the container" + uptime_before = float(instance.query("SELECT uptime()").strip()) + + pkey = paramiko.Ed25519Key.from_private_key_file(f"{SCRIPT_DIR}/keys/lucy_ed25519") + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.connect( + hostname=instance.ip_address, + port=9022, + username="lucy", + pkey=pkey, + timeout=30, + ) + try: + channel = client.invoke_shell(term="xterm", width=80, height=24) + channel.settimeout(20) + output = _read_channel_until(channel, timeout=20, marker=":) ") + assert ":) " in output, f"no prompt from the embedded client: {output!r}" + + # `Insert` toggles overwrite mode, which is what makes the destructor + # print the "reset cursor blinking" sequence in the first place. Wait + # for the raw `\033[5 q` ("blinking cursor") escape the key handler + # prints: it is the only observable proof that the server really + # consumed the key and that `overwrite_mode` became true. Without it + # the destructor writes nothing and the test would pass even unfixed. + channel.sendall("\x1b[2~") + raw = b"" + deadline = time.time() + 10 + while time.time() < deadline and b"\x1b[5 q" not in raw: + if channel.recv_ready(): + raw += channel.recv(65536) + else: + time.sleep(0.05) + assert ( + b"\x1b[5 q" in raw + ), f"overwrite mode was not enabled, the destructor would write nothing: {raw!r}" + + # Abort the connection with a RST instead of a graceful shutdown, so + # writes on the server side fail rather than being silently discarded. + sock = client.get_transport().sock + sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) + sock.close() + finally: + client.close() + + # The session teardown is asynchronous, so keep sampling for a while. A + # sample that fails only means the teardown is still in flight, but the + # *last* sample of the window must succeed and must still show the very + # same process: the server that survived the disconnect, not a fresh one. + last_failure = None + last_uptime = None + deadline = time.time() + 30 + while time.time() < deadline: + try: + uptime = float(instance.query("SELECT uptime()").strip()) + except Exception as e: # down or restarting — the next sample decides + last_failure = e + last_uptime = None + time.sleep(0.5) + continue + last_uptime = uptime + pids = server_pids() + assert pids == pids_before, ( + "the server process was replaced after the SSH disconnect, i.e. it " + "died while tearing the session down and was started again " + f"(pids {pids} != {pids_before})" + ) + assert uptime >= uptime_before, ( + "the server restarted after the SSH disconnect, i.e. it died while " + f"tearing the session down (uptime {uptime} < {uptime_before})" + ) + time.sleep(0.5) + + assert last_uptime is not None, ( + "the server did not answer a query at the end of the 30 s window after " + f"the SSH disconnect, i.e. it died while tearing the session down: {last_failure}" + ) + + # `from_host=True` also greps the rotated logs, so a restart cannot hide the + # fatal line by rotating it out of the active `clickhouse-server.log`. + assert not instance.contains_in_log( + "std::terminate", from_host=True + ), "the server called `std::terminate` while tearing down the SSH session" diff --git a/tests/integration/test_ssl_cert_authentication/certs/client13-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client13-cert.pem new file mode 100644 index 0000000000000..82687debfcdec --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client13-cert.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFPzCCAyegAwIBAgIUb8W/doRsWnXgJSqDgDz8Yb+cLVwwDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjAwMTAx +MDAwMDAwWhcNMjkxMjI5MDAwMDAwWjBhMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRow +GAYDVQQDDBFjbGllbnQxAC5ldmlsLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQADggIP +ADCCAgoCggIBAOMBJ2yqK72eew557aeGID0RXFzVs6lLTN4/71C4/309gAcIoqsw +Phulwzj3IfH7v2PPntsROxGtRCVS5LWarXUTEviEaiWUp1MwoFlrhn2Dm78hIgYC +7NrdtI48tb9t9y6eTio99WdIiXGY2KNlctfnTyCxKeanZi7JZUmb9VW95gh/5dl3 +sajZPjUDrFxqUQbzsSzbgElFkjvOzN5xEoUM1qbjHcyML7aKweXJKhSswdi4K3oA +v/FbFleJQgoRPB43JDqc5YjpPARKJlt5VZqpkgiKFAm0R+o4TYeCBd0kZFtL+7Y3 +TO27ZdQiOFKR/DINPCdA3/nxtQ7GJ17g/nwXLQRKUDxXain7jo+87OHghMA8WfwQ +lMSkQ+sqJdxA/VunQEHNCljw64BHeuU6CqQDU9eEnFNq9+3Qb0INiMRn6EGj20LK +5+q/fJqGm/N8BcuoYHUymbtojAjnJmCIWwvTzo7OYaG84nRwOPkZZvcSOrzn2CbT +zhmXHiSnP7ZAL/r24ZW5oikjw6Ie40iqQ2MyknTv3Ko1LWIKkjAbMcE6fo3SlzMA +fVsm9wAyVa/UyjO1TcnPNNta1TdqSgdRsEPDvR/IMPQYJ7Be0H0SZLj2/v8ZmHmZ +KVfGKdWnFBQKiTSHVEBHBdXADcXi2grYjnOBKnQ7H2UQK9jfGMu4FPYlAgMBAAEw +DQYJKoZIhvcNAQELBQADggIBAF4piZQxvyxwfhQxmNbyc8EtUo5qBIWE026e5Hsl +4QX3glKhAo+xLEQbKAImsMdXXYNdGaBJDTEZRbGSvyEgVa8e4YF0FP+CkFDcM0oF +k7YUgQHkZvlR/lsd5idMM7JJ1WXnux6jay9hpF9GEi2QMB2QkwuRwBHkgtjEun8C +qdyvApSecY6BXQllAJyoalSVR4ZzaMGVgSGL99RKyn320icevO6WDOVocEiohJTK +KOwf8q9G6riKWsRbNNPhkt9QwZzC0/RuKV6r25QR7H+GSjmllcBnbyqW3+DGGuJL +iMtKrCmx4xHedvDXRxi8ZxsfIrWVI7f5hgcWCXFWFMeqkU9nD3TGJrAuamBBQMZn +aC3q9AyVudroJs/1ojrSD92+H3qSzhqoLZ4bd531ShPasQ/EHIiOeN794j3Rxo0d +ituD2LSboe2Zr4TNhi7ou/dSfDXPEwJSb5V1Xgp9OyhQCKnFa+RkbQVtME5dVBMR +oO4HMTqaCgYSzCKhrxIO/vid2p6NQp7ZI8V2hAhUR4aBwnWFFspmuWaXpZqXxE6+ +a46rTA4HLpQ7TKwbi4fS2zWSh64nCHmv72GOHQ86dlXDNvfqLo8vyz79chKPo+31 +fsTZJks9wBzeSsq0tqAWflN/hQQCZJT1IpIc5VwU45FLSraKBGYRpCP4FnciXUuJ +RiyF +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client13-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client13-key.pem new file mode 100644 index 0000000000000..452e632233838 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client13-key.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEA4wEnbKorvZ57Dnntp4YgPRFcXNWzqUtM3j/vULj/fT2ABwii +qzA+G6XDOPch8fu/Y8+e2xE7Ea1EJVLktZqtdRMS+IRqJZSnUzCgWWuGfYObvyEi +BgLs2t20jjy1v233Lp5OKj31Z0iJcZjYo2Vy1+dPILEp5qdmLsllSZv1Vb3mCH/l +2XexqNk+NQOsXGpRBvOxLNuASUWSO87M3nEShQzWpuMdzIwvtorB5ckqFKzB2Lgr +egC/8VsWV4lCChE8HjckOpzliOk8BEomW3lVmqmSCIoUCbRH6jhNh4IF3SRkW0v7 +tjdM7btl1CI4UpH8Mg08J0Df+fG1DsYnXuD+fBctBEpQPFdqKfuOj7zs4eCEwDxZ +/BCUxKRD6yol3ED9W6dAQc0KWPDrgEd65ToKpANT14ScU2r37dBvQg2IxGfoQaPb +Qsrn6r98moab83wFy6hgdTKZu2iMCOcmYIhbC9POjs5hobzidHA4+Rlm9xI6vOfY +JtPOGZceJKc/tkAv+vbhlbmiKSPDoh7jSKpDYzKSdO/cqjUtYgqSMBsxwTp+jdKX +MwB9Wyb3ADJVr9TKM7VNyc8021rVN2pKB1GwQ8O9H8gw9BgnsF7QfRJkuPb+/xmY +eZkpV8Yp1acUFAqJNIdUQEcF1cANxeLaCtiOc4EqdDsfZRAr2N8Yy7gU9iUCAwEA +AQKCAgAhsUz9M2q866tUaY4iXxAk5ZFBvtQ6NztzAfv5YrlFCPOaLP1xOX6rvYLd +eczCLP0/Xr8IwSTKdF7uHUC7Svhzgn++fsTUdse5Bg7JXDdregphUr/JMvZIa+n9 +xzlV7t07wEerZw59Txuwo8uFFlchKtX97QhiPDa4kLAlve4AKmiHRW3DmnY2MKcA +2Qk5nuwPg0Go/fFWBBX20N8WevjmjOfFKxDZXctCeujZkW3bVg2K0ZcJC4McWx/U +oUI52qIDfWgKrQ6NL3T20VyibL64bx0GLccDQ7dS0RrXbISF9Kin5lNiEbuAzjTd +yLB3DO+TgY8L58n2lUSsKzFdGI7yFjJTr6yzmKIayS11c00TCBQ0w/l+oCWOhrvM +zn/k3mrORcRA1aqX5mcqAdzs6hYFIyFbKXNH8hlE7JnUUbcvR76WbzMl6IxPxMT+ +ikbTIQnG+T2PMgmaxVbe9YhyHT2YbMHF7igocz5mI0PHsdNaB1IBSV1G2Knq8QJi +rbYUdCShTJksrNIXghyjbCPx5VIo05fx8NleCuNND18WPs32tI2rceLlF3KSk8GF +moCnPJFTtEgk5IvzWavOGO+abkU1A9xGOhNHCMZVxOca/ZGlhHSRBCuhagpLPx9e +QprxfFVIFG8jNrN0/Mhgv3ivKmbYZuv03wdcWDtKCdMrxkah9wKCAQEA8Xp+V4zk +Ta4nqkjL4TTv+uczszGt/Kb5OYZyb1EzUseabRyOFkb2iMUf+Ol/92UMBlh/FzTa +RFHE1lRed45l8Jhgsbaq9r8gmJZ1WZM2IUByUI++z+USlEtG8O+8wlniyTn9Fc+n +y+wkZwSeJSLWZAWIzeygJs/ceKG+4TpnLZnC2rQqOoKGc+0Dimc1E5HV3mOkhwe1 +CtiYOUvgNf0rEV7E5D7smjpAeiJ8xPhYRg0XXLztN+9rQA+6AvwpbI3VnyAKbPUQ +/J+ZVg0cVMH7pTDXMj0jB2wevrIa7xj2Pn4w/jR8i/tNjFauMMvnofr6RW6KqYk/ +opXhfSE1Pk9w6wKCAQEA8KfWOQEmKlC0KJrN1/D1ZchNZf57pNFtjyWdjLUah05k +a0QLUe1yme7lTw59tsFIfrd/ozeiWKSvhum9/mUNvwBgPjnqp73pmrEEu1bgQ8Ib +cCm55GAmBWlNMO52iD+EPdsYvj+KgAWxu6W8B6XsyjoVLBYqHwNzitZlW9D1PAbY +KWBppmntVAhj7oNvk40uWEFIuTU0o0RIrvcYmX795sr2PY69ozOdiY6UziR4oRAs +DhJ+0a8Sjct0/RMq+sOpBONrpOBHqJzoCX+/alOjbJ8FYIMuc6OU2cNtcxjL3tcZ +cJKInsMl7bMYhi0SIEnttuAgh6mzD4Iox1WX3hbxLwKCAQEAySWPHmCnQCnEsqzG +38aX7DkwsKC+XEm+KnPa2O0bwmWwNhlmJhpgfBcwBciDJtHODW8kFnGZKvWt8BcB +RbehJKPZT89oY/dbMJ+MCtx8Z4BmeML0X/ph2pNF+abJZl83cQVT0xpRnKUMwZ9w +GGEHkvOlPFtSIGJfNUEOXlCm565ASKtwzaIyW1hf7acA6Fc/fmsj/rKl1O4NBxU0 +I+TKPkLh4Xqk3eeOE+6sXeq8pUV8Y2ygcUqG3Sr8eEYSP3F6M32eEZql1rkfIjXz +loqrkrO3XgrdJe6jQZfcpbP2UqINL4MLCBOCSldd7Bm9zgjg9nsZGBXSox0UYoYJ +8uh7lwKCAQA+I3gi+/L41iHOojooWeVjRJcHkPAoHJNndNT7cf/JlCpFsCokG2WN +7at0AE/hkoK/hW4FnOXkcZGJCm2udDVabiRrrNS0P0tEUBTisonxtPsUuRFwsIrg +ttHhopEkmRHyTtJSvWFrsQy1YRPt/Z/oj5rL6WUy4NdCsB032fqYZ0QFWwmsmIlZ +O1liSrnSpY+j5id6+wv+ZDFITDEj4TB6GUn/lw3MSBWTYSd3Gt+y5tQZRhlM1yG5 +TyGD/yEH4uGPi5FN30NhfDJF0aCBOdtmvqDKzNR/s2tJ1zY5k9uATJYbBRsVs1nl +yGq6qoSVpcEliTWdEepURM12utkd1VqtAoIBAFLriLD2AF0+ayheiDFgsbRo78V3 +dygUPBZi7b6nPWv3gugnwvWrTFaCAndACxNZPWNwEcING2Utjp8PTvPTzad9yibl +3xqNSKR9gaqmDYPEY1DwYGCpHRc969PIYcyK6z3dICj1ySEo0RwTa+Ab0OHeRWzS +ROnmmy9a9Vpx7KyMZcmOGhSXL3q7nr2imI+qZ4tGiEJts85d2fXAVhGeBOagzy5n +uD00wYzBc8rZ9u+d7ZOyQetgoJBnhyQNzW7IBCN+Z3EWW5lPlHQJpTXjHCdXgiZU +28pX/4J/mIcaURA0xhexuuLqTEjLlFoAnjNjWCCXv0u5uP8Ba35g4BSAz0o= +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client14-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client14-cert.pem new file mode 100644 index 0000000000000..25c2687c0edf6 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client14-cert.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFazCCA1OgAwIBAgIUTgWFBpS4cJ/+Fgg5jAfOubkwokowDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjAwMTAx +MDAwMDAwWhcNMjkxMjI5MDAwMDAwWjBmMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMR8w +HQYDVQQDDBZldmlsAC5jb3JwLmV4YW1wbGUuY29tMIICIjANBgkqhkiG9w0BAQEF +AAOCAg8AMIICCgKCAgEAs6uGg+vk4GlmN+ZlJb9gUI6AS6TJVQvLCSw7RPHt0k8x +qMXeoHid3mmVIC7tWK6BDEbAWJB814cWzOviXy5GbuUZB5Yq1MR0bQV968PBa/jp +/X6eDAW3KDfvgsk2eBo+aZqbFh1EJqXzN04d255X2mlAGpwW97T+XhXa/ahxj1Ey +G5smL6jaybcrKj/Xce4qVgSQk0kVlun/35Rfd6kcFTdq+Laap3nO0JKBYwfcoeMU +vVfRZV0iaP7sbOpiZlv0+zFDDGX2Xi18VH2xeclICN4ar8K0h+VNE1uqf27r+3NA +iZbZphA2huN60KYP+IV09KirWhFM3Xcd5LlqbluRUwWxp4xXZNWFIa6AeSbuC+43 +hAa0vBfN9+CE03ifX6stqfUy/eiJsghuPEZOUXqJpBpz4VHpxXVqiLX11hrGw60S +rEQJ30SVahULl4v1/zhFaRgFnZE2yRdQG0h1AejA5OUykKTH5vAkfZRxPmL3hnBB +fg+iti93EFvdC2bXNMB04X+VkvF8CcqRUr2Ka7ZpfleJuO9ps2e0LBtjSJOLxYRA +N13RUMp9m/45F5iGE4QXdUFzOrasrclg8KfP0Zl+6/FQcygV4bLQywqm/zgCX1zC +SAb88JLEElIUmyMFT/cvst1uscy1fvoLseuKcz+9woAF1LA8XrZzMPRA22xSABMC +AwEAAaMlMCMwIQYDVR0RBBowGIIWZXZpbAAuY29ycC5leGFtcGxlLmNvbTANBgkq +hkiG9w0BAQsFAAOCAgEAFuyPgncpYKGYffQPoOWR2yTn6L56XxvQcFNrchyKkwAn +zsBQFLOX4T4z1FuIQTwwKiSRwz+hwuIfcK2++I1yyxgYzw8Cxw7uqpvRi6AgN9pk +LEsXbSOvqaQyFEukVFenbkIejBVexF74gk2LKJYnT7MRVgs/k8C5Lqy5lKpB/2Gk +7xnzVe6mLhdZrMRbtCmt+Pr2m5l+XMGucnoaNCK87zSIGEt8fCYNB//vQ61GeANv +hRYiWgdWcOCnl2ip+EWbq0+ZK2JvJ8Ih2d0S+Qtbhek8XQsuCoa5P3YjLyAfzhpk +/CZb+VbOH41qZ2uWi05xVE3IRbJBlRDnX/bXC4Nx2ojzOnWAqzmrJffsKal/04oP +BcA6dOzm+CvT7AlzImlRI/6ZEC6DlrXSkvm5yRNwdUhTZ5ZjQnSyZBF4lNY3u9Sa +G1ipxyR9wRcA7uac8cFv/MBAGP5godwi3WPvqC/7noGDbZqDxcsZOeVFdP+NFtFg +dilITLxlgaQimN2nxhPvAAz6EcJ2owFgJJk6J3f6ICq8gdbJzFq2uVJrKBBL35St +/aOpflDRhQkMdJwrCuI/+4NBcXG5fQywjH8H+9dvbUONIlttbwE2zZJgXtY0gb+A +kY8fmy5pWBYnYAB8lFdAZxW0Sx6ehkmLA/ExtH/7jz/GwnwS12/of6wSvQHj8so= +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client14-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client14-key.pem new file mode 100644 index 0000000000000..10d1b89a27011 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client14-key.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEAs6uGg+vk4GlmN+ZlJb9gUI6AS6TJVQvLCSw7RPHt0k8xqMXe +oHid3mmVIC7tWK6BDEbAWJB814cWzOviXy5GbuUZB5Yq1MR0bQV968PBa/jp/X6e +DAW3KDfvgsk2eBo+aZqbFh1EJqXzN04d255X2mlAGpwW97T+XhXa/ahxj1EyG5sm +L6jaybcrKj/Xce4qVgSQk0kVlun/35Rfd6kcFTdq+Laap3nO0JKBYwfcoeMUvVfR +ZV0iaP7sbOpiZlv0+zFDDGX2Xi18VH2xeclICN4ar8K0h+VNE1uqf27r+3NAiZbZ +phA2huN60KYP+IV09KirWhFM3Xcd5LlqbluRUwWxp4xXZNWFIa6AeSbuC+43hAa0 +vBfN9+CE03ifX6stqfUy/eiJsghuPEZOUXqJpBpz4VHpxXVqiLX11hrGw60SrEQJ +30SVahULl4v1/zhFaRgFnZE2yRdQG0h1AejA5OUykKTH5vAkfZRxPmL3hnBBfg+i +ti93EFvdC2bXNMB04X+VkvF8CcqRUr2Ka7ZpfleJuO9ps2e0LBtjSJOLxYRAN13R +UMp9m/45F5iGE4QXdUFzOrasrclg8KfP0Zl+6/FQcygV4bLQywqm/zgCX1zCSAb8 +8JLEElIUmyMFT/cvst1uscy1fvoLseuKcz+9woAF1LA8XrZzMPRA22xSABMCAwEA +AQKCAgAEaIj8Y6VR/EQNyxFgQ7nRQC3VrU1jUM7CgttRbb4wEtFdGr3DojH9awnF +qGEac+2mp3XAtorZnu7oSEFdpH0F64kZro2OeuOAaUoVps/wHkNffOPT17AOxJCT +3OwBNmOho7F6cW1ipV+6U6hX4yK0sTBpdrr5iO9Uz6R35NIkehGIq93b/YCgwmXE +u5xFp1pSkfoaIwjskwE8Mx/Eh9mwi5OMVq6kvVBdvbp++4pmTnQL0UPKAOb/PIIA +ih+v80GniCXk//tzhBow2ISqQE4MKabt+REE5JNnjjA4wDf6C3Hh7lmYwX0VAi/Z +PrnVlzCvcBQEObhxFqMdIY+C9awzGrf0iJ1UGZms5IrcKwvEL7VsrQqO2COEv42K +KtbVNpRMYG5n+VwtylChmh2g6oX4wMLbYIykDfXZVv/FIjP3vgPxs19t/CRVUEer +x6GztwhAmKvp4WzerVH1xJ7QKcaIuy4cjhNLVUQCcjPbFbtHsHzvuScXHfppCM67 +6rpNQ5DrQ0Wj0a1kCal/57aKQQdD37k4aTTEoCbDJkbD10SoaljJYElHleB26xCV +AwbyXz6gwJ80CxbXS1uco3c86keDekrGV5f73XX53MZouRR90Ii/TJvvdGutXFkX +mpAIaOZlfeDSFMJoFaSpOj3UaY4RBwcXXDB69QpFMDe2ND0gwQKCAQEA6jw1G0dP +HwDYfQSMzTQkhle7hHqxPoyuooBe4Vl6MJ64S22tgvRLE+FuLfubR9tMBYYr/Ub4 +CmSBE/XMbNGpH31yct5I/8rc9OnRg0bcCw2dz/Q7b84LKas6fkBFaf1mMlPvH2zi +nwtS3Bw3Q6DAky1ra6rqjiZDtDCN+vQOh0zXVdOyVjApDBbngX/x8hVhFLs3SuwS +jm3P8YloEQ6aQ0wCjvsemJs+IpDlH7cHwYlgxBREqQ1wOKdo8IZ/MTTvlPOXW04F +pP995LCvtVacjhfOLaWdHTs442yuFvFBoPWFEvx2CGYTQe+dk8zmzLFdYVNpEvbV +dh/m1hMrA3lUdwKCAQEAxF1d+QuDanP39u+16u4+u61p3gzYv2f9m+bjViiYMv6N +kSDzdXrKidD1I+iEgIy1QjRfdHlQMCIpP8D4zuWCrNY57QKdV4bRKs0RSgpOXPv+ +FGeVf9taPIH/lUH8FUJ1gqfoLX+6W2RbZxcNPOSOf13o6jTjBbB6oVetljNjsnDS +us7CJe3+MDk20pO2+E49Q0TuyeALDPRSciSvJrVCVmoPs6rpu22z5zMCxd6+WztL +YBWfFM38QpIMUg1miz7hU7m7lQHlYE9/p9RSI5ePD9mgepPQN+bQg0fGQaD0tzad +MJr/BI0OvHLdwFQNLya66nWK7s85aD/UjDRO/A6kRQKCAQB1zGes21ToM6WsYeBp +xsJjqbWNb6K54UhmQwb0b+pqjzgB/xuW00L6sZGWoIW8QoZd9Ncknk9Z8qeToTb4 +twxF4PHw4Od3dM9ggEK0sasyB9wI3DwUA1xLzWgyXCJMpnqB7wJAHKNv9uLp/Wqx +oSOYIOx4DlG9wXKlKRIOVjUESFm3OSrj+355LP+qeez0oVnccjbhgA3pAULlpwPm +KCDenVhgDdyaROCfw5znMUY+R9eZZNQO7Mo2Q8Mby5gl6AhhMYw6B+gAzdjDbTRA +j1lWgJRZEoQMUl9OyLZYpWYrC66sGLlHigY/T8FAtniQEtbyfl9GgUpjCLIvkR49 +tgQLAoIBAF/WefD9D4y7QQDCifU5hmCvCIaZmogAxyR6EeaRNYdd+dYlUO27mnKd +C6gU6eabxjOjwBrmwp5bbepx0n2YQqj8fZURu51mbVwIbjHGyexUCPQIgky+0FHL +2OQOKmxt3VCBhq3+MwQ7/OhZtdpMasf7G5yDZ3H1akSouE4gkr4alp8aHmPIvlDm ++7zW32xdM0VLtYfN01blP//5q4qm2NO4PCWieyVBK5bhrK7KQfng/K7Onq/WwRH0 +mhLJ+4xmii8E7WqSXFMfOdy9ocFBTU+dFdf9oJhIDOil9Ts+xXFONHXukBy2g8Sy +A0zFORIUQxH/gGmBtjENRj2PoiUfOEkCggEBAMiBLM8L+k7ovGd7KVoeSgk7FjsV +5gq0GDwXGov9HEyeY/mmmuS5mfxZUYvRgzbpQutqy88MvbMsCS7VDtT9m0IFs8zR +qXIUQmOrYJuI37BrA16K5EDTj6wY/7DKRSM4guNqQjYnJ2bYBrXCZLWFUF/OVl3w +VF5reN7BCMOTXGc4aAeJzcmvhGU7M3WnGwL7yGkiX8BCDXo+ddLuf1SyDUttfrmo +Kbb4h4LAe2ATtRO0F9SfEetdmHUwO6iRJGQNQQWO7pWOjUStSry3Z8AM0uVDBRDG +Ol3DmKQlxbFAhLD62KkcWt30eNnNMlYaq8huKY4Y2juyi5ZQ31sDij4worA= +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh b/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh index 19354a5a3e364..b8bbd34567a21 100755 --- a/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh +++ b/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh @@ -60,3 +60,76 @@ openssl x509 -req -days 36525 -in client_far_future-req.pem -CA ca-cert.pem -CAk # 6. Generate one more self-signed certificate and private key for using as wrong certificate (because it's not signed by CA) openssl req -newkey rsa:4096 -x509 -days 3650 -nodes -batch -keyout wrong-key.pem -out wrong-cert.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client" + +# 7. Generate a CA-signed certificate whose CN carries an embedded NUL byte ("client1\0.evil.com"), +# to test that server-side CN extraction does not truncate at the NUL. User 'john' is configured with +# client1, so a server that truncated the CN at the NUL would extract +# "client1" and wrongly authenticate this certificate as 'john'; the full CN must be preserved so the +# match fails. openssl's CLI cannot place a NUL inside a -subj field, so we build this certificate +# with the 'cryptography' library instead. +python3 - <<'PY' +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +import datetime + +with open("ca-key.pem", "rb") as f: + ca_key = serialization.load_pem_private_key(f.read(), password=None) +with open("ca-cert.pem", "rb") as f: + ca_cert = x509.load_pem_x509_certificate(f.read()) + +key = rsa.generate_private_key(public_exponent=65537, key_size=4096) +subject = x509.Name([ + x509.NameAttribute(NameOID.COUNTRY_NAME, "RU"), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "Some-State"), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Internet Widgits Pty Ltd"), + x509.NameAttribute(NameOID.COMMON_NAME, "client1\x00.evil.com"), +]) +now = datetime.datetime(2020, 1, 1) +cert = (x509.CertificateBuilder() + .subject_name(subject).issuer_name(ca_cert.subject) + .public_key(key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now).not_valid_after(now + datetime.timedelta(days=3650)) + .sign(ca_key, hashes.SHA256())) +with open("client13-key.pem", "wb") as f: + f.write(key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.TraditionalOpenSSL, serialization.NoEncryption())) +with open("client13-cert.pem", "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) +PY + +# 8. Generate a CA-signed certificate whose CN and DNS SAN both carry an embedded NUL byte +# ("evil\0.corp.example.com"), to test that wildcard matching rejects it. Users 'wildcard_cn' and +# 'wildcard_dns' are configured with '*.corp.example.com' and 'DNS:*.corp.example.com': without an +# explicit check, '*' would match the single "label" "evil\0" and authenticate this certificate. +python3 - <<'PY' +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +import datetime + +with open("ca-key.pem", "rb") as f: + ca_key = serialization.load_pem_private_key(f.read(), password=None) +with open("ca-cert.pem", "rb") as f: + ca_cert = x509.load_pem_x509_certificate(f.read()) + +key = rsa.generate_private_key(public_exponent=65537, key_size=4096) +subject = x509.Name([ + x509.NameAttribute(NameOID.COUNTRY_NAME, "RU"), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "Some-State"), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Internet Widgits Pty Ltd"), + x509.NameAttribute(NameOID.COMMON_NAME, "evil\x00.corp.example.com"), +]) +now = datetime.datetime(2020, 1, 1) +cert = (x509.CertificateBuilder() + .subject_name(subject).issuer_name(ca_cert.subject) + .public_key(key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now).not_valid_after(now + datetime.timedelta(days=3650)) + .add_extension(x509.SubjectAlternativeName([x509.DNSName("evil\x00.corp.example.com")]), critical=False) + .sign(ca_key, hashes.SHA256())) +with open("client14-key.pem", "wb") as f: + f.write(key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.TraditionalOpenSSL, serialization.NoEncryption())) +with open("client14-cert.pem", "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) +PY diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 977bc82ea2766..ca4e6147f928b 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -44,7 +44,6 @@ def started_cluster(): config = """ - {sslHost} strict @@ -63,7 +62,6 @@ def execute_query_native(node, query, user, cert_name, password=None): certificateFile=f"{SCRIPT_DIR}/certs/{cert_name}-cert.pem", privateKeyFile=f"{SCRIPT_DIR}/certs/{cert_name}-key.pem", caConfig=f"{SCRIPT_DIR}/certs/ca-cert.pem", - sslHost=SSL_HOST, ) file = open(config_path, "w") @@ -147,6 +145,18 @@ def test_native_fallback_to_password(): assert "AUTHENTICATION_FAILED" in str(err.value) +def test_native_cn_nul_byte_no_bypass(): + # Authentication bypass: client13's CN is "client1\0.evil.com" and user 'john' is configured with + # client1. If server-side CN extraction truncated at the embedded NUL + # byte, the CN would collapse to "client1" and the certificate would authenticate as user 'john'. + # The full CN must be preserved, so the match must fail. + with pytest.raises(Exception) as err: + execute_query_native( + instance, "SELECT currentUser()", user="john", cert_name="client13" + ) + assert "AUTHENTICATION_FAILED" in str(err.value) + + def get_ssl_context(cert_name): context = WrapSSLContextWithSNI(SSL_HOST, ssl.PROTOCOL_TLS_CLIENT) context.load_verify_locations(cafile=f"{SCRIPT_DIR}/certs/ca-cert.pem") @@ -221,6 +231,14 @@ def test_https_wrong_cert(): ) +def test_https_cn_nul_byte_no_bypass(): + # Same bypass as test_native_cn_nul_byte_no_bypass, over the HTTPS interface: client13's CN + # "client1\0.evil.com" must not be truncated to "client1" and authenticate as user 'john'. + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user="john", cert_name="client13") + assert "403" in str(err.value) + + def test_https_non_ssl_auth(): # Users with non-SSL authentication are allowed, in this case we can skip sending a client certificate at all (because "verificationMode" is set to "relaxed"). # assert execute_query_https("SELECT currentUser()", user="peter", enable_ssl_auth=False) == "peter\n" @@ -546,6 +564,21 @@ def test_x509_cn_wildcard_single_label(): assert "403" in str(err.value) +def test_x509_wildcard_nul_byte_no_bypass(): + # Authentication bypass: client14's CN and DNS SAN are both "evil\0.corp.example.com". Users + # 'wildcard_cn' and 'wildcard_dns' are configured with '*.corp.example.com' and + # 'DNS:*.corp.example.com'. The '*' must not match the span "evil\0", so both must fail. + for user in ["wildcard_cn", "wildcard_dns"]: + with pytest.raises(Exception) as err: + execute_query_native( + instance, "SELECT currentUser()", user=user, cert_name="client14" + ) + assert "AUTHENTICATION_FAILED" in str(err.value) + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user=user, cert_name="client14") + assert "403" in str(err.value) + + def test_x509_uri_san_wildcard_dot_in_segment(): # Non-regression: '.' separates labels for DNS/CN but is NOT a separator for URI SANs, # whose separator is '/'. A wildcard URI path segment may legitimately contain dots, so diff --git a/tests/integration/test_statistics_mutation_session_gate/__init__.py b/tests/integration/test_statistics_mutation_session_gate/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_statistics_mutation_session_gate/config/disable_statistics.xml b/tests/integration/test_statistics_mutation_session_gate/config/disable_statistics.xml new file mode 100644 index 0000000000000..1d8d744300526 --- /dev/null +++ b/tests/integration/test_statistics_mutation_session_gate/config/disable_statistics.xml @@ -0,0 +1,12 @@ + + + + + 0 + + + diff --git a/tests/integration/test_statistics_mutation_session_gate/test.py b/tests/integration/test_statistics_mutation_session_gate/test.py new file mode 100644 index 0000000000000..ae270af5cea80 --- /dev/null +++ b/tests/integration/test_statistics_mutation_session_gate/test.py @@ -0,0 +1,99 @@ +# https://github.com/ClickHouse/ClickHouse/issues/115448 +# +# `allow_statistics` gates the statistics DDL. It used to be checked twice: once when the statement +# was validated, with the submitting session's value, and once more while the mutation ran - where +# the background context reads the server-default profile rather than that session. With +# `allow_statistics = 0` in the profile and `allow_statistics = 1` in the statement, the mutation was +# therefore accepted and then failed on every retry, sitting in `system.mutations` with +# `is_done = 0` forever and blocking every later mutation on the table. +# +# The server profile is what a stateless test cannot set, which is why this half of the reproducer +# lives here; `05175_materialize_statistics_session_gate` pins the submission-time rejection. + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance( + "node", + user_configs=["config/disable_statistics.xml"], +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def unfinished_mutations(table): + return node.query( + f"SELECT countIf(is_done = 0) FROM system.mutations WHERE database = 'default' AND table = '{table}'" + ).strip() + + +def test_statistics_mutations_with_session_opt_in(started_cluster): + node.query("DROP TABLE IF EXISTS t_session_gate SYNC") + node.query( + """ + CREATE TABLE t_session_gate (k UInt64, a Int64) ENGINE = MergeTree ORDER BY k + """ + ) + node.query("INSERT INTO t_session_gate SELECT number, number FROM numbers(100)") + + # `ADD STATISTICS` has always honoured the session opt-in. + node.query( + "ALTER TABLE t_session_gate ADD STATISTICS a TYPE tdigest", + settings={"allow_statistics": 1, "mutations_sync": 2}, + ) + + # `MATERIALIZE STATISTICS` used to be accepted here and then fail forever in the background. + node.query( + "ALTER TABLE t_session_gate MATERIALIZE STATISTICS a", + settings={"allow_statistics": 1, "mutations_sync": 2}, + ) + assert unfinished_mutations("t_session_gate") == "0" + + node.query( + "ALTER TABLE t_session_gate DROP STATISTICS a", + settings={"allow_statistics": 1, "mutations_sync": 2}, + ) + assert unfinished_mutations("t_session_gate") == "0" + + # The table is not wedged: a later, unrelated mutation still completes. + node.query( + "ALTER TABLE t_session_gate UPDATE a = a + 1 WHERE k < 10", + settings={"mutations_sync": 2}, + ) + assert unfinished_mutations("t_session_gate") == "0" + assert node.query("SELECT count() FROM t_session_gate").strip() == "100" + + node.query("DROP TABLE t_session_gate SYNC") + + +def test_statistics_mutations_without_session_opt_in(started_cluster): + node.query("DROP TABLE IF EXISTS t_session_gate_off SYNC") + node.query( + """ + CREATE TABLE t_session_gate_off (k UInt64, a Int64) ENGINE = MergeTree ORDER BY k + """ + ) + + # Without the opt-in the profile's value still refuses the statements up front, and nothing + # reaches the mutation queue. + for statement in ( + "ALTER TABLE t_session_gate_off ADD STATISTICS a TYPE tdigest", + "ALTER TABLE t_session_gate_off MATERIALIZE STATISTICS a", + "ALTER TABLE t_session_gate_off DROP STATISTICS a", + ): + assert "Alter table with statistics is disabled" in node.query_and_get_error( + statement + ) + + assert unfinished_mutations("t_session_gate_off") == "0" + node.query("DROP TABLE t_session_gate_off SYNC") diff --git a/tests/integration/test_storage_azure_blob_storage/test_cluster.py b/tests/integration/test_storage_azure_blob_storage/test_cluster.py index 602af40f7c666..e7341ad6d62c8 100644 --- a/tests/integration/test_storage_azure_blob_storage/test_cluster.py +++ b/tests/integration/test_storage_azure_blob_storage/test_cluster.py @@ -82,6 +82,47 @@ def test_select_all(cluster): assert TSV(pure_azure) == TSV(distributed_azure) +def test_select_all_split_by_buckets(cluster): + """ + With `cluster_table_function_split_granularity = 'bucket'` the initiator opens every Parquet file + itself to split it into row-group buckets, using the object info of the cluster iterator, which + skips the metadata probe. That placeholder must not be taken as a known size of zero: the Azure + read is bounded by the size recorded in the object info, so the file would be read as empty. + """ + node = cluster.instances["node_0"] + storage_account_url = cluster.env_variables["AZURITE_STORAGE_ACCOUNT_URL"] + azure_query( + node, + f"INSERT INTO TABLE FUNCTION azureBlobStorage('{storage_account_url}', 'cont', 'test_cluster_split_by_buckets.parquet', 'devstoreaccount1'," + f"'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'Parquet', 'auto', 'key UInt64, data String') " + f"SELECT number, toString(number) FROM numbers(100000)", + settings={ + "azure_truncate_on_insert": 1, + "output_format_parquet_row_group_size": 1000, + }, + ) + + pure_azure = azure_query( + node, + f"SELECT count(), sum(key), sum(length(data)) from azureBlobStorage('{storage_account_url}', 'cont', 'test_cluster_split_by_buckets.parquet', 'devstoreaccount1'," + f"'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'Parquet')", + ) + assert TSV(pure_azure) == TSV("100000\t4999950000\t488890") + + for validate_etag_on_read in (0, 1): + distributed_azure = azure_query( + node, + f"SELECT count(), sum(key), sum(length(data)) from azureBlobStorageCluster('simple_cluster', '{storage_account_url}', 'cont', 'test_cluster_split_by_buckets.parquet', 'devstoreaccount1'," + f"'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'Parquet')", + settings={ + "cluster_table_function_split_granularity": "bucket", + "cluster_table_function_buckets_batch_size": 10, + "azure_validate_etag_on_read": validate_etag_on_read, + }, + ) + assert TSV(pure_azure) == TSV(distributed_azure) + + def test_count(cluster): node = cluster.instances["node_0"] port = cluster.env_variables["AZURITE_PORT"] diff --git a/tests/integration/test_storage_delta/test.py b/tests/integration/test_storage_delta/test.py index cc8f7034012f3..e04c74cbeedea 100644 --- a/tests/integration/test_storage_delta/test.py +++ b/tests/integration/test_storage_delta/test.py @@ -3280,14 +3280,14 @@ def test_delta_kernel_internal_pruning(started_cluster): ) ) - assert result == 1 + assert result == 3 instance.query("SYSTEM FLUSH LOGS") - assert 1 == int( + assert 3 == int( instance.query( f"SELECT count() FROM system.text_log WHERE query_id = '{query_id}' and message ILIKE '%Scanned file%'" ) ) - assert 1 == int( + assert 3 == int( instance.query( f"SELECT count() FROM system.text_log WHERE query_id = '{query_id}' and message ILIKE '%Scanned file: {TABLE_NAME}/b=test2%'" ) @@ -5358,24 +5358,16 @@ def test_early_return_limit(started_cluster, use_delta_kernel): assert first_check_hits > 0 or queue_check_hits > 0 - assert 1 == int(instance.query( - f"SELECT count() FROM system.text_log WHERE query_id = '{query_id}' AND message LIKE '%List batch size is 1/1, shutdown: true%'" - )) - # Early return should scan significantly fewer files # With s3_list_object_keys_size=1, queue pauses frequently forcing shutdown checks # Should stop very early after consuming just a few files assert scanned_files < full_scan_files, \ f"Early return should scan fewer files: {scanned_files} >= {full_scan_files}" - # 3 because: - # we have async reader creation with 2 existing readers at a moment of time, - # each calls next() and consumes 2 files from the scan. - # It takes 1 file for the query to stop because of LIMIT 1. - # But because scan is also asynchronous and continues once batch limit is not reached, - # we get +1 scanned file. - assert scanned_files == 3, \ - f"Early return should scan 3 files with LIMIT 1, but scanned {scanned_files}" + # At most 3: two async readers each consume one file, plus one the scan produces before + # it observes shutdown. Fewer is legal when shutdown lands earlier. + assert scanned_files <= 3, \ + f"Early return should scan at most 3 files with LIMIT 1, but scanned {scanned_files}" def test_struct_dotted_field_names(started_cluster): diff --git a/tests/integration/test_storage_iceberg_no_spark/test_writes_statistics_by_minmax_pruning.py b/tests/integration/test_storage_iceberg_no_spark/test_writes_statistics_by_minmax_pruning.py index 2cc7d839778a9..63ac6063258f6 100644 --- a/tests/integration/test_storage_iceberg_no_spark/test_writes_statistics_by_minmax_pruning.py +++ b/tests/integration/test_storage_iceberg_no_spark/test_writes_statistics_by_minmax_pruning.py @@ -1,11 +1,22 @@ +import glob +import os + +import avro.datafile +import avro.io +import pyarrow.parquet as pq import pytest from helpers.iceberg_utils import ( check_validity_and_get_prunned_files_general, create_iceberg_table, - get_uuid_str + default_download_directory, + get_last_snapshot, + get_uuid_str, + unescape_path, ) +TABLE_ROOT = "/var/lib/clickhouse/user_files/iceberg_data/default" + @pytest.mark.parametrize("format_version", [1, 2]) @pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) @@ -133,4 +144,102 @@ def check_validity_and_get_prunned_files(select_expression): f"SELECT * FROM {TABLE_NAME} WHERE number <= 5 ORDER BY ALL" ) == 3 - ) \ No newline at end of file + ) + + +def _read_avro(path): + with open(path, "rb") as f: + return list(avro.datafile.DataFileReader(f, avro.io.DatumReader())) + + +def _data_files_of_last_snapshot(table_path): + """The `data_file` record of every manifest entry of the newest snapshot.""" + snapshot_id = get_last_snapshot(table_path) + manifest_lists = glob.glob(f"{table_path}/metadata/snap-{snapshot_id}-*.avro") + assert len(manifest_lists) == 1, manifest_lists + + data_files = [] + for list_entry in _read_avro(manifest_lists[0]): + manifest = os.path.join( + table_path, "metadata", os.path.basename(unescape_path(list_entry["manifest_path"])) + ) + data_files.extend(entry["data_file"] for entry in _read_avro(manifest)) + return data_files + + +def _column_sizes_from_parquet(path): + """Per-column compressed size as the Parquet footer of one data file reports it, keyed by the + Iceberg field id the column carries.""" + parquet_file = pq.ParquetFile(path) + field_ids = [ + int(field.metadata[b"PARQUET:field_id"]) for field in parquet_file.schema_arrow + ] + + sizes = dict.fromkeys(field_ids, 0) + metadata = parquet_file.metadata + for row_group_index in range(metadata.num_row_groups): + row_group = metadata.row_group(row_group_index) + assert row_group.num_columns == len(field_ids) + for column_index, field_id in enumerate(field_ids): + sizes[field_id] += row_group.column(column_index).total_compressed_size + return sizes + + +@pytest.mark.parametrize("format_version", [1, 2]) +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_writes_column_sizes_are_on_disk_sizes( + started_cluster_iceberg_no_spark, format_version, storage_type +): + """`data_file.column_sizes` is the size the column occupies inside the data file, so every + entry must repeat what the Parquet footer of the file it names says. An in-memory size passes + neither check below: `s` holds 1000000 bytes of one repeated character, which compresses to a + fraction of the file, and the sum over a file cannot exceed the file itself.""" + instance = started_cluster_iceberg_no_spark.instances["node1"] + TABLE_NAME = ( + "test_writes_column_sizes_are_on_disk_sizes_" + + storage_type + + "_" + + get_uuid_str() + ) + + create_iceberg_table( + storage_type, + instance, + TABLE_NAME, + started_cluster_iceberg_no_spark, + "(id Int32, s String)", + format_version, + order_by="id", + ) + # Blocks of 1000 rows that the insert pipeline does not squash, so the writer rolls over at + # exactly 4000 rows and the commit holds three data files of known sizes. + instance.query( + f"INSERT INTO {TABLE_NAME} SELECT number, repeat('x', 100) FROM numbers(10000)", + settings={ + "iceberg_insert_max_rows_in_data_file": 4000, + "max_block_size": 1000, + "max_insert_block_size": 1000, + "min_insert_block_size_rows": 0, + "min_insert_block_size_bytes": 0, + "max_insert_threads": 1, + }, + ) + + table_path = f"{TABLE_ROOT}/{TABLE_NAME}/" + default_download_directory( + started_cluster_iceberg_no_spark, storage_type, table_path, table_path + ) + + data_files = _data_files_of_last_snapshot(table_path) + assert len(data_files) == 3 + + for data_file in data_files: + data_path = os.path.join( + table_path, "data", os.path.basename(unescape_path(data_file["file_path"])) + ) + written = {pair["key"]: pair["value"] for pair in data_file["column_sizes"]} + + assert written == _column_sizes_from_parquet(data_path) + assert sum(written.values()) <= data_file["file_size_in_bytes"] + + assert instance.query(f"SELECT count(), uniqExact(s) FROM {TABLE_NAME}") == "10000\t1\n" diff --git a/tests/integration/test_storage_iceberg_with_spark/test_partition_pruning_with_functions.py b/tests/integration/test_storage_iceberg_with_spark/test_partition_pruning_with_functions.py new file mode 100644 index 0000000000000..51bb446b5f316 --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_partition_pruning_with_functions.py @@ -0,0 +1,268 @@ +import pytest + +from helpers.iceberg_utils import ( + check_validity_and_get_prunned_files_general, + execute_spark_query_general, + get_creation_expression, + get_uuid_str, +) + + +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_partition_pruning_with_functions(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_partition_pruning_with_functions_" + storage_type + "_" + get_uuid_str() + + def execute_spark_query(query: str): + return execute_spark_query_general( + spark, started_cluster_iceberg_with_spark, storage_type, TABLE_NAME, query + ) + + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} ( + ts_day TIMESTAMP, + ts_hour TIMESTAMP, + ts_month TIMESTAMP, + d_year DATE, + tag INT + ) + USING iceberg + PARTITIONED BY (days(ts_day), hours(ts_hour), months(ts_month), years(d_year)) + OPTIONS('format-version'='2') + """ + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '2024-01-20 10:00:00', TIMESTAMP '2024-01-20 10:00:00', TIMESTAMP '2024-01-20 10:00:00', DATE '2024-01-20', 1), + (TIMESTAMP '2024-01-21 11:00:00', TIMESTAMP '2024-01-20 11:00:00', TIMESTAMP '2024-02-20 10:00:00', DATE '2025-01-20', 2), + (TIMESTAMP '2024-02-20 10:00:00', TIMESTAMP '2024-01-21 10:00:00', TIMESTAMP '2024-03-20 10:00:00', DATE '2026-01-20', 3), + (TIMESTAMP '2025-02-20 10:00:00', TIMESTAMP '2024-01-21 11:00:00', TIMESTAMP '2025-01-20 10:00:00', DATE '2027-01-20', 4); + """ + ) + + creation_expression = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_with_spark, table_function=True + ) + + def check_validity_and_get_prunned_files(select_expression): + settings1 = {"use_iceberg_partition_pruning": 0, "session_timezone": "UTC"} + settings2 = {"use_iceberg_partition_pruning": 1, "session_timezone": "UTC"} + return check_validity_and_get_prunned_files_general( + instance, TABLE_NAME, settings1, settings2, "IcebergPartitionPrunedFiles", select_expression + ) + + def select(where): + return f"SELECT * FROM {creation_expression} WHERE {where} ORDER BY ALL" + + # A filter that wraps the partition source column in a monotonic function must still be able to + # use the partition value, which is what https://github.com/ClickHouse/ClickHouse/issues/103433 + # reported for `toDate`. + assert check_validity_and_get_prunned_files(select("toDate(ts_day) = toDate('2024-01-20')")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfDay(ts_day) = toDateTime64('2024-01-20 00:00:00', 6)")) == 3 + assert check_validity_and_get_prunned_files(select("toYYYYMMDD(ts_day) = 20240120")) == 3 + assert check_validity_and_get_prunned_files(select("toDate(ts_day) IN (toDate('2024-01-20'), toDate('2024-02-20'))")) == 2 + assert check_validity_and_get_prunned_files(select("toDate(ts_day) > toDate('2024-02-01')")) == 2 + assert check_validity_and_get_prunned_files(select("toYear(ts_day) = 2025")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfHour(ts_hour) = toDateTime64('2024-01-20 10:00:00', 6)")) == 3 + assert check_validity_and_get_prunned_files(select("toDate(ts_hour) = toDate('2024-01-21')")) == 2 + assert check_validity_and_get_prunned_files(select("toStartOfMonth(ts_month) = toDate('2024-02-01')")) == 3 + assert check_validity_and_get_prunned_files(select("toYYYYMM(ts_month) = 202503")) == 4 + assert check_validity_and_get_prunned_files(select("toYear(d_year) = 2026")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfYear(d_year) = toDate('2028-01-01')")) == 4 + + # A constant that is not aligned to the transform's granularity: 10:30 is inside the 10:00 hour, + # so the file of that hour must survive, and the same for a day and a month boundary. + assert check_validity_and_get_prunned_files(select("toStartOfHour(ts_hour) < toDateTime64('2024-01-20 10:30:00', 6)")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfHour(ts_hour) >= toDateTime64('2024-01-20 10:30:00', 6)")) == 1 + assert check_validity_and_get_prunned_files(select("toDate(ts_day) < toDate('2024-01-20') + INTERVAL 12 HOUR")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfMonth(ts_month) > toDate('2024-02-10')")) == 2 + + # A single day is one weekday, so the partition value answers this too: only 2024-01-20 is a Saturday. + assert check_validity_and_get_prunned_files(select("toDayOfWeek(ts_day) = 6")) == 3 + + # The partition value of a `day` transform says nothing about the hour of the day, so a filter on + # it must not prune anything. + assert check_validity_and_get_prunned_files(select("toHour(ts_day) = 10")) == 0 + + +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_partition_pruning_with_functions_before_epoch(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_partition_pruning_before_epoch_" + storage_type + "_" + get_uuid_str() + + def execute_spark_query(query: str): + return execute_spark_query_general( + spark, started_cluster_iceberg_with_spark, storage_type, TABLE_NAME, query + ) + + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} (ts TIMESTAMP, d DATE, tag INT) + USING iceberg + PARTITIONED BY (days(ts), years(d)) + OPTIONS('format-version'='2') + """ + ) + + # The transforms count from 1970, so these partition values are negative. + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '1969-11-15 07:00:00', DATE '1969-11-15', 1), + (TIMESTAMP '1969-12-31 23:00:00', DATE '1970-01-01', 2), + (TIMESTAMP '1970-01-01 00:30:00', DATE '1971-06-01', 3), + (TIMESTAMP '2024-01-20 10:00:00', DATE '2024-01-20', 4); + """ + ) + + creation_expression = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_with_spark, table_function=True + ) + + def check_validity_and_get_prunned_files(select_expression): + settings1 = {"use_iceberg_partition_pruning": 0, "session_timezone": "UTC"} + settings2 = {"use_iceberg_partition_pruning": 1, "session_timezone": "UTC"} + return check_validity_and_get_prunned_files_general( + instance, TABLE_NAME, settings1, settings2, "IcebergPartitionPrunedFiles", select_expression + ) + + def select(where): + return f"SELECT * FROM {creation_expression} WHERE {where} ORDER BY ALL" + + assert check_validity_and_get_prunned_files(select("toDate32(ts) = toDate32('1969-11-15')")) == 3 + assert check_validity_and_get_prunned_files(select("toDate32(ts) >= toDate32('1970-01-01')")) == 2 + assert check_validity_and_get_prunned_files(select("toDate32(ts) < toDate32('1970-01-01')")) == 2 + assert check_validity_and_get_prunned_files(select("toYear(d) = 1969")) == 3 + assert check_validity_and_get_prunned_files(select("toYear(d) >= 1971")) == 2 + + +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_partition_pruning_with_functions_before_epoch_separate_commits(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_partition_pruning_before_epoch_separate_commits_" + storage_type + "_" + get_uuid_str() + + def execute_spark_query(query: str): + return execute_spark_query_general( + spark, started_cluster_iceberg_with_spark, storage_type, TABLE_NAME, query + ) + + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} (ts TIMESTAMP, d DATE, tag INT) + USING iceberg + PARTITIONED BY (days(ts), years(d)) + OPTIONS('format-version'='2') + """ + ) + + # One manifest file per commit, so the partition summaries of the first manifest are negative on + # both of their sides, while the ones of the second manifest are positive on both. + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '1969-11-15 07:00:00', DATE '1969-11-15', 1), + (TIMESTAMP '1969-12-31 23:00:00', DATE '1969-06-01', 2); + """ + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '2024-01-20 10:00:00', DATE '2024-01-20', 3); + """ + ) + + creation_expression = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_with_spark, table_function=True + ) + + def check_validity_and_get_prunned_files(select_expression): + settings1 = {"use_iceberg_partition_pruning": 0, "session_timezone": "UTC"} + settings2 = {"use_iceberg_partition_pruning": 1, "session_timezone": "UTC"} + return check_validity_and_get_prunned_files_general( + instance, TABLE_NAME, settings1, settings2, "IcebergPartitionPrunedFiles", select_expression + ) + + def select(where): + return f"SELECT * FROM {creation_expression} WHERE {where} ORDER BY ALL" + + assert check_validity_and_get_prunned_files(select("toDate32(ts) = toDate32('1969-11-15')")) == 2 + assert check_validity_and_get_prunned_files(select("toDate32(ts) >= toDate32('1970-01-01')")) == 2 + assert check_validity_and_get_prunned_files(select("toYear(d) = 1969")) == 1 + + +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_partition_pruning_without_functions(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_partition_pruning_without_functions_" + storage_type + "_" + get_uuid_str() + + def execute_spark_query(query: str): + return execute_spark_query_general( + spark, started_cluster_iceberg_with_spark, storage_type, TABLE_NAME, query + ) + + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} ( + ts_day TIMESTAMP, + ts_hour TIMESTAMP, + ts_month TIMESTAMP, + d_year DATE, + tag INT + ) + USING iceberg + PARTITIONED BY (days(ts_day), hours(ts_hour), months(ts_month), years(d_year)) + OPTIONS('format-version'='2') + """ + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '2024-01-20 10:00:00', TIMESTAMP '2024-01-20 10:00:00', TIMESTAMP '2024-01-20 10:00:00', DATE '2024-01-20', 1), + (TIMESTAMP '2024-01-21 11:00:00', TIMESTAMP '2024-01-20 11:00:00', TIMESTAMP '2024-02-20 10:00:00', DATE '2025-01-20', 2), + (TIMESTAMP '2024-02-20 10:00:00', TIMESTAMP '2024-01-21 10:00:00', TIMESTAMP '2024-03-20 10:00:00', DATE '2026-01-20', 3), + (TIMESTAMP '2025-02-20 10:00:00', TIMESTAMP '2024-01-21 11:00:00', TIMESTAMP '2025-01-20 10:00:00', DATE '2027-01-20', 4); + """ + ) + + creation_expression = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_with_spark, table_function=True + ) + + def check_validity_and_get_prunned_files(select_expression): + settings1 = {"use_iceberg_partition_pruning": 0, "session_timezone": "UTC"} + settings2 = {"use_iceberg_partition_pruning": 1, "session_timezone": "UTC"} + return check_validity_and_get_prunned_files_general( + instance, TABLE_NAME, settings1, settings2, "IcebergPartitionPrunedFiles", select_expression + ) + + def select(where): + return f"SELECT * FROM {creation_expression} WHERE {where} ORDER BY ALL" + + assert check_validity_and_get_prunned_files(select("ts_day = '2024-01-20 10:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("ts_day < '2024-01-21 00:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("ts_day >= '2024-02-01 00:00:00'")) == 2 + assert check_validity_and_get_prunned_files(select("ts_day BETWEEN '2024-01-21 00:00:00' AND '2024-02-21 00:00:00'")) == 2 + assert check_validity_and_get_prunned_files(select("ts_day IN ('2024-01-20 10:00:00', '2024-02-20 10:00:00')")) == 2 + + assert check_validity_and_get_prunned_files(select("ts_hour = '2024-01-20 11:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("ts_hour > '2024-01-21 09:00:00'")) == 2 + + assert check_validity_and_get_prunned_files(select("ts_month = '2024-02-20 10:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("ts_month < '2024-03-01 00:00:00'")) == 2 + + assert check_validity_and_get_prunned_files(select("d_year = '2026-01-20'")) == 3 + assert check_validity_and_get_prunned_files(select("d_year < '2025-01-01'")) == 3 + assert check_validity_and_get_prunned_files(select("d_year >= '2026-01-01'")) == 2 + assert check_validity_and_get_prunned_files(select("d_year IN ('2024-01-20', '2026-01-20')")) == 2 + + assert check_validity_and_get_prunned_files(select("ts_day = '2024-01-20 12:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("d_year = '2026-06-01'")) == 3 diff --git a/tests/integration/test_storage_kafka/test_batch_fast.py b/tests/integration/test_storage_kafka/test_batch_fast.py index c36d42a7b0dfe..f55249b5a7d63 100644 --- a/tests/integration/test_storage_kafka/test_batch_fast.py +++ b/tests/integration/test_storage_kafka/test_batch_fast.py @@ -1931,6 +1931,57 @@ def test_kafka_producer_consumer_separate_settings( assert property_in_log in kafka_producer_applied_properties +@pytest.mark.parametrize( + "create_query_generator", + [ + k.generate_old_create_table_query, + k.generate_new_create_table_query, + ], +) +def test_kafka_password_not_logged(kafka_cluster, create_query_generator): + suffix = k.random_string(6) + kafka_table = f"kafka_{suffix}" + username = f"kafka_user_{suffix}" + password = f"secret_kafka_password_{suffix}" + + instance.rotate_logs() + instance.query( + create_query_generator( + kafka_table, + "key UInt64", + topic_list="password_not_logged", + consumer_group="test", + settings={ + "kafka_sasl_username": username, + "kafka_sasl_password": password, + }, + ) + ) + + # Create an mv to initialize the librdkafka consumers + instance.query(f"CREATE MATERIALIZED VIEW test.{kafka_table}_view ENGINE=MergeTree ORDER BY tuple() AS SELECT * FROM test.{kafka_table}") + instance.wait_for_log_line(f"{kafka_table}.*Created #0 consumer") + instance.query(f"DROP TABLE test.{kafka_table}_view") + instance.query(f"INSERT INTO test.{kafka_table} VALUES (1)") + + assert instance.contains_in_log(f"{kafka_table}.*Kafka producer created") + + # The property-logging loops ran for both the consumer and the producer, + # but they hid the values of the sensitive properties. `sasl.username` is + # hidden because librdkafka marks it with the _RK_SENSITIVE flag, not + # because of the name, so it validates the generated blacklist. + for client_type in ["Consumer", "Producer"]: + for property_name in ["sasl.username", "sasl.password"]: + assert instance.contains_in_log( + f"{kafka_table}.*{client_type} set property {property_name}:\\[HIDDEN\\]" + ) + # The username still appears in the logged CREATE TABLE text (only + # kafka_sasl_password is masked there), so check only the password value. + assert not instance.contains_in_log(password) + + instance.query(f"DROP TABLE test.{kafka_table}") + + @pytest.mark.parametrize( "create_query_generator, log_line", [ diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 252fc892b9e48..6b4f639cc6b4c 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -2796,7 +2796,7 @@ def onReceived(channel, method, properties, body): check_expected_result_polling(num_rows, f"SELECT count() FROM {db}.view") - result = instance.query(f"SELECT * FROM {db}.view") + result = instance.query(f"SELECT * FROM {db}.view ORDER BY key") assert result == "0\t0\n10\t100\n20\t200\n30\t300\n40\t400\n" @@ -2884,7 +2884,7 @@ def onReceived(channel, method, properties, body): for i in range(num_rows): expected += str(i * 10) + "\t" + str(i * 100) + "\n" - result = instance.query(f"SELECT * FROM {db}.view") + result = instance.query(f"SELECT * FROM {db}.view ORDER BY key") assert result == expected @@ -3602,7 +3602,7 @@ def test_hiding_credentials(rabbitmq_cluster, db, unique): instance.query("SYSTEM FLUSH LOGS") message = instance.query(f"SELECT message FROM system.text_log WHERE message ILIKE '%CREATE TABLE {db}.{table_name}%'") assert "rabbitmq_password = \\'[HIDDEN]\\'" in message - assert "rabbitmq_address = \\'amqp://root:[HIDDEN]@rabbitmq1:5672/\\'" in message + assert "rabbitmq_address = \\'[HIDDEN]\\'" in message def test_rabbitmq_default_mode_nack_on_parse_error(rabbitmq_cluster, db, unique): diff --git a/tests/integration/test_storage_redis/fake_redis.py b/tests/integration/test_storage_redis/fake_redis.py new file mode 100644 index 0000000000000..2407443329828 --- /dev/null +++ b/tests/integration/test_storage_redis/fake_redis.py @@ -0,0 +1,128 @@ +""" +A minimal RESP server that answers MGET with a reply of the test's choosing. + +`StorageRedis` pairs the elements of an MGET reply with the requested keys by position, so both +the element count and the position of a nil decide what the engine does. Usage: + + fake_redis.py [ ] + +Every MGET is answered with `max(0, len(keys) + delta)` elements: the requested keys echoed back +as their own values, followed by RESP nils. `` and `` are comma-separated +key names; when they are given, SCAN answers with those keys and MGET answers nil for the names +in ``, so a full scan meets a fixed valid/nil sequence rather than Redis's own key +order. Anything else is answered `+OK`. +""" + +import socket +import sys +import threading + + +def read_exactly(stream, size): + data = stream.read(size) + if len(data) != size: + raise EOFError + return data + + +def read_line(stream): + line = stream.readline() + if not line: + raise EOFError + return line.rstrip(b"\r\n") + + +def read_command(stream): + """Read one RESP array. + + Argument payloads are read by declared length, never up to a newline: the keys on the wire + are ClickHouse's serializeBinary output and may contain \\r, \\n or non-UTF-8 bytes. + """ + header = read_line(stream) + if not header.startswith(b"*"): + return None + args = [] + for _ in range(int(header[1:])): + arg_header = read_line(stream) + if not arg_header.startswith(b"$"): + raise EOFError + args.append(read_exactly(stream, int(arg_header[1:]))) + read_exactly(stream, 2) # trailing CRLF + return args + + +def serialize_string(name): + """A String shorter than 128 bytes as ClickHouse serializes it: one length byte, then bytes. + + The engine deserializes the keys SCAN reports and the values MGET returns, so both have to + arrive in that form. See serialize_binary_for_string in test.py. + """ + return bytes([len(name)]) + name.encode() + + +def mget_reply(keys, delta, nil_keys): + count = max(0, len(keys) + delta) + out = [b"*%d\r\n" % count] + for i in range(count): + if i < len(keys) and keys[i] not in nil_keys: + out.append(b"$%d\r\n%s\r\n" % (len(keys[i]), keys[i])) + else: + out.append(b"$-1\r\n") + return b"".join(out) + + +def scan_reply(scan_keys): + """Cursor 0 with every key, so the engine reads the whole keyspace in one batch.""" + out = [b"*2\r\n$1\r\n0\r\n", b"*%d\r\n" % len(scan_keys)] + for key in scan_keys: + out.append(b"$%d\r\n%s\r\n" % (len(key), key)) + return b"".join(out) + + +def handle(conn, delta, scan_keys, nil_keys): + try: + with conn.makefile("rb") as stream: + while True: + args = read_command(stream) + if not args: + break + command = args[0].upper() + if command == b"MGET": + conn.sendall(mget_reply(args[1:], delta, nil_keys)) + elif command == b"SCAN" and scan_keys: + conn.sendall(scan_reply(scan_keys)) + else: + conn.sendall(b"+OK\r\n") + except (EOFError, OSError, ValueError): + pass + finally: + conn.close() + + +def parse_names(argument): + return [serialize_string(name) for name in argument.split(",") if name] + + +def main(): + port = int(sys.argv[1]) + delta = int(sys.argv[2]) + scan_keys = parse_names(sys.argv[3]) if len(sys.argv) > 3 else [] + nil_keys = parse_names(sys.argv[4]) if len(sys.argv) > 4 else [] + + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("0.0.0.0", port)) + # The engine holds a connection pool and a direct join reads on several pipeline threads, so + # connections have to be served concurrently. + server.listen(64) + print(f"listening on {port}, MGET reply length = requested + {delta}", flush=True) + + while True: + conn, _ = server.accept() + threading.Thread( + target=handle, args=(conn, delta, scan_keys, nil_keys), daemon=True + ).start() + + +if __name__ == "__main__": + main() diff --git a/tests/integration/test_storage_redis/test.py b/tests/integration/test_storage_redis/test.py index af2892125fce3..8bf1f96526a71 100644 --- a/tests/integration/test_storage_redis/test.py +++ b/tests/integration/test_storage_redis/test.py @@ -1,5 +1,6 @@ ## sudo -H pip install redis import json +import os import struct import sys @@ -8,7 +9,9 @@ from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV +from helpers.test_tools import TSV, wait_condition + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) @@ -529,3 +532,161 @@ def check_query(query, read_type, keys_count, rows_read): plan = node.query("EXPLAIN actions=1, optimize=0 SELECT * FROM test_get_keys") assert 'ReadType: FullScan' in plan + + +# Ports of the three fake Redis endpoints, inside the ClickHouse container. +FAKE_REDIS_PORT_OK = 16379 +FAKE_REDIS_PORT_LONG = 16380 +FAKE_REDIS_PORT_SHORT = 16381 +FAKE_REDIS_PORT_SCAN = 16382 + + +def start_fake_redis(port, delta, keys=""): + node.exec_in_container( + [ + "bash", + "-c", + f"python3 /fake_redis.py {port} {delta} {keys}" + f" > /var/log/clickhouse-server/fake_redis_{port}.log 2>&1", + ], + detach=True, + user="root", + ) + wait_condition( + lambda: node.exec_in_container( + ["bash", "-c", f"exec 3<>/dev/tcp/127.0.0.1/{port} && echo OK"], + nothrow=True, + ), + lambda r: "OK" in r, + max_attempts=40, + delay=0.5, + ) + + +def test_malformed_mget_reply(started_cluster): + """An MGET reply whose element count differs from the request must be rejected. + + The result loop of StorageRedis was bounded by the reply length while the null map it + indexes is sized from the request, so an over-long reply read and wrote past the end of the + null map. A short reply produced fewer rows than keys, which breaks the row-per-key contract + that IKeyValueEntity::getByKeys promises to a direct join. + """ + tables = ("redis_fake_ok", "redis_fake_long", "redis_fake_short", "t_fake_left") + for table in tables: + drop_table(table) + + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "fake_redis.py"), "/fake_redis.py" + ) + start_fake_redis(FAKE_REDIS_PORT_OK, 0) + start_fake_redis(FAKE_REDIS_PORT_LONG, 1) + start_fake_redis(FAKE_REDIS_PORT_SHORT, -1) + + node.query( + f""" + CREATE TABLE redis_fake_ok (key String, value String) + Engine=Redis('127.0.0.1:{FAKE_REDIS_PORT_OK}') PRIMARY KEY (key); + + CREATE TABLE redis_fake_long (key String, value String) + Engine=Redis('127.0.0.1:{FAKE_REDIS_PORT_LONG}') PRIMARY KEY (key); + + CREATE TABLE redis_fake_short (key String, value String) + Engine=Redis('127.0.0.1:{FAKE_REDIS_PORT_SHORT}') PRIMARY KEY (key); + + CREATE TABLE t_fake_left (k String) ENGINE = TinyLog; + INSERT INTO t_fake_left VALUES ('a'), ('b'); + """ + ) + + def direct_join(table): + return node.query( + f"SELECT key, value FROM (SELECT k AS key FROM t_fake_left) AS t " + f"INNER JOIN {table} USING (key) ORDER BY key " + f"SETTINGS join_algorithm = 'direct' FORMAT TSV" + ) + + # An endpoint that answers with one element per key still works, so a mock that never + # listens or frames RESP wrongly reddens here instead of green-washing the arms below. + assert TSV.toMat(direct_join("redis_fake_ok")) == [["a", "a"], ["b", "b"]] + + with pytest.raises(QueryRuntimeException) as long_join: + direct_join("redis_fake_long") + assert "INTERNAL_REDIS_ERROR" in str(long_join.value) + assert "for MGET of" in str(long_join.value) + + with pytest.raises(QueryRuntimeException) as short_join: + direct_join("redis_fake_short") + assert "INTERNAL_REDIS_ERROR" in str(short_join.value) + + # The other caller of the reply reads it with no null map at all. + with pytest.raises(QueryRuntimeException) as long_in: + node.query("SELECT * FROM redis_fake_long WHERE key IN ('a', 'b')") + assert "INTERNAL_REDIS_ERROR" in str(long_in.value) + + # A zero-element reply is a null array in Poco, not an empty one, so it is the isNull() + # term of the guard that rejects it. One key against the delta = -1 endpoint produces it. + with pytest.raises(QueryRuntimeException) as zero_in: + node.query("SELECT * FROM redis_fake_short WHERE key IN ('a')") + assert "INTERNAL_REDIS_ERROR" in str(zero_in.value) + assert "returned 0 values" in str(zero_in.value) + + for table in tables: + drop_table(table) + + +def test_full_scan_skips_missing_values(started_cluster): + """A full scan must skip the keys MGET answers with nil, not stop at the first one. + + MGET answers by position, and a key SCAN listed can hold a non-string type or expire + before the MGET runs, so a nil marks one absent value and not the end of the batch. + """ + address = get_address_for_ch() + table = "test_full_scan_missing" + fake_table = "redis_fake_scan" + + client = get_redis_connection(db_id=4) + client.flushdb() + drop_table(table) + drop_table(fake_table) + + # Redis alone decides in what order SCAN reports keys, so the mock pins the one thing this + # test is about: a nil arriving before a key that still has a value. + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "fake_redis.py"), "/fake_redis.py" + ) + start_fake_redis(FAKE_REDIS_PORT_SCAN, 0, "a,b,c b") + + node.query( + f""" + CREATE TABLE {fake_table} (key String, value String) + Engine=Redis('127.0.0.1:{FAKE_REDIS_PORT_SCAN}') PRIMARY KEY (key); + """ + ) + rows = node.query(f"SELECT key, value FROM {fake_table} ORDER BY key FORMAT TSV") + assert TSV.toMat(rows) == [["a", "a"], ["c", "c"]] + + # The same thing on a real Redis, which answers nil for any key that holds another type. + node.query( + f""" + CREATE TABLE {table}(k String, v String) + Engine=Redis('{address}', 4, 'clickhouse') PRIMARY KEY (k); + + INSERT INTO {table} SELECT toString(number), toString(number) FROM numbers(16); + """ + ) + + # Control: every row is readable before the keys below exist, so a fixture that writes + # nothing reddens here instead of leaving the assertion after it vacuous. + assert int(node.query(f"SELECT uniqExact(k) FROM {table}")) == 16 + + for i in range(16): + client.rpush(f"list_{i}", "x") + + # SCAN can report a key twice while Redis rehashes and no read path dedupes the rows, so the + # oracle here is the key set: a lost key is the defect, a repeated one is not. + keys = node.query(f"SELECT DISTINCT k FROM {table} ORDER BY toUInt32(k) FORMAT TSV") + assert TSV.toMat(keys) == [[str(i)] for i in range(16)] + + client.flushdb() + drop_table(table) + drop_table(fake_table) diff --git a/tests/integration/test_storage_s3_queue/test_sts_smoke.py b/tests/integration/test_storage_s3_queue/test_sts_smoke.py index 761606c336af9..5b02ffebace71 100644 --- a/tests/integration/test_storage_s3_queue/test_sts_smoke.py +++ b/tests/integration/test_storage_s3_queue/test_sts_smoke.py @@ -111,14 +111,14 @@ def get_count(node, table_name): assert get_count(node, dst_table_name) == 10 assert ( - "extra_credentials(\\'role_arn\\' = \\'arn::role\\', \\'role_session_name\\' = \\'miniorole\\')" + "extra_credentials(\\'role_arn\\' = \\'arn::role\\', \\'role_session_name\\' = \\'[HIDDEN]\\')" in node.query(f"SHOW CREATE TABLE {table_name}") ) node.restart_clickhouse() assert ( - "extra_credentials(\\'role_arn\\' = \\'arn::role\\', \\'role_session_name\\' = \\'miniorole\\')" + "extra_credentials(\\'role_arn\\' = \\'arn::role\\', \\'role_session_name\\' = \\'[HIDDEN]\\')" in node.query(f"SHOW CREATE TABLE {table_name}") ) @@ -181,7 +181,7 @@ def test_s3_queue_extra_credentials_backup(started_cluster): # are equally invalid for MinIO, so a restored table that silently dropped # the clause would fail with the same error. Pin the round trip explicitly. assert ( - "extra_credentials(\\'role_arn\\' = \\'arn::role\\', \\'role_session_name\\' = \\'miniorole\\')" + "extra_credentials(\\'role_arn\\' = \\'arn::role\\', \\'role_session_name\\' = \\'[HIDDEN]\\')" in node.query(f"SHOW CREATE TABLE {table_name}") ) diff --git a/tests/integration/test_webterminal/test.py b/tests/integration/test_webterminal/test.py index bd6588c28ebce..6f5e76944392a 100644 --- a/tests/integration/test_webterminal/test.py +++ b/tests/integration/test_webterminal/test.py @@ -568,3 +568,90 @@ def test_tab_completion_respects_session_user(): instance.query("DROP USER IF EXISTS completer") instance.query("DROP TABLE IF EXISTS default.visible_completion_target") instance.query("DROP TABLE IF EXISTS default.hidden_completion_target") + + +def test_ctrl_z_does_not_stop_server(): + """Ctrl-Z in the embedded client must not suspend the server process. + + The embedded client runs inside the server, so the "suspend" action of the + line editor (`raise(SIGSTOP)`) would stop the whole server. It is disabled by + the key binding, but the incremental history search (Ctrl-R) used to handle + Ctrl-Z by itself, bypassing the binding. + """ + sock = _open_ws(instance.ip_address, 8123) + try: + _ws_send_text(sock, json.dumps({"type": "auth", "user": "default", "password": ""})) + output = _read_pty_until(sock, timeout=20, marker=":) ") + assert ":) " in output, f"no prompt from the embedded client: {output!r}" + + # Ctrl-Z in the main loop, then Ctrl-R (history search) followed by Ctrl-Z. + _ws_send_binary(sock, b"\x1a") + _ws_send_binary(sock, b"\x12\x1a") + _read_pty_until(sock, timeout=1) + + response = instance.http_request("ping", method="GET", timeout=10) + assert response.status_code == 200 + + # The session is still usable: Ctrl-U clears the line, then run a query. + _ws_send_binary(sock, b"\x15SELECT 1234 + 1\r") + output = _read_pty_until(sock, timeout=10, marker="1235") + assert "1235" in output, f"the embedded client does not respond: {output!r}" + finally: + sock.close() + + +def test_session_keeps_user_settings(): + """The embedded client must keep the settings of the authenticated user. + + It used to replace the session settings with default values, dropping + everything that comes from the user's profiles, roles and + `CREATE USER ... SETTINGS`, so a `readonly` user got write access and + resource limits were removed. Settings sent along with a query must also + be checked against the constraints. + """ + instance.query( + "CREATE USER restricted_terminal_user IDENTIFIED WITH plaintext_password BY 'restricted_password' " + "SETTINGS readonly = 1, max_memory_usage = 5000000 CONST" + ) + + try: + sock = _open_ws(instance.ip_address, 8123) + try: + _ws_send_text( + sock, + json.dumps( + { + "type": "auth", + "user": "restricted_terminal_user", + "password": "restricted_password", + } + ), + ) + + output = _read_pty_until(sock, timeout=20, marker=":) ") + assert ":) " in output, f"no prompt from the embedded client: {output!r}" + + _ws_send_binary( + sock, + b"SELECT concat('settings_', getSetting('readonly')::String, '_', getSetting('max_memory_usage')::String) FORMAT TSV\r", + ) + output = _read_pty_until(sock, timeout=10, marker="settings_1_5000000\r\n") + assert "settings_1_5000000\r\n" in output, ( + f"the session settings of the user were not preserved: {output!r}" + ) + + # Wait for the next prompt: while a query runs, the client reads the + # keyboard (to toggle the progress table) and would swallow the next query. + if ":) " not in output.split("settings_1_5000000\r\n", 1)[1]: + output = _read_pty_until(sock, timeout=10, marker=":) ") + assert ":) " in output, f"no prompt after the first query: {output!r}" + + _ws_send_binary(sock, b"SELECT 'bypassed' SETTINGS readonly = 0 FORMAT TSV\r") + output = _read_pty_until(sock, timeout=10, marker="READONLY") + assert "READONLY" in output and "bypassed\r\n" not in output, ( + f"changing `readonly` in the query was not rejected: {output!r}" + ) + finally: + sock.close() + finally: + instance.query("DROP USER IF EXISTS restricted_terminal_user") diff --git a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/counter.clj b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/counter.clj index 160599290aae4..cc8e3f678a10f 100644 --- a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/counter.clj +++ b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/counter.clj @@ -34,9 +34,12 @@ :type :ok :value (long (count (zk-list conn root-path)))) (catch Exception _ (assoc op :type :info, :error :connect-error))) - :final-read (chu/exec-with-retries 30 (fn [] (assoc op - :type :ok - :value (long (count (zk-list conn root-path)))))) + :final-read (chu/exec-with-retries 30 (fn [] + (with-fresh-conn nodename (:with-auth test) + (fn [conn] + (assoc op + :type :ok + :value (long (count (zk-list conn root-path)))))))) :add (try (do (zk-multi-create-many-seq-nodes conn (concat-path root-path "seq-") (:value op) :with-acl (:with-auth test)) diff --git a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/queue.clj b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/queue.clj index d26849ee127e1..be4b3f1481a63 100644 --- a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/queue.clj +++ b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/queue.clj @@ -45,8 +45,10 @@ :drain ; drain via delete is to long, just list all nodes (chu/exec-with-retries 30 (fn [] - (zk-sync conn) - (assoc op :type :ok :value (into #{} (map #(str %1) (zk-list conn root-path)))))))) + (with-fresh-conn nodename (:with-auth test) + (fn [conn] + (zk-sync conn) + (assoc op :type :ok :value (into #{} (map #(str %1) (zk-list conn root-path)))))))))) (teardown! [_ test]) diff --git a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/set.clj b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/set.clj index 87b4f91b5863c..afe25f28a8097 100644 --- a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/set.clj +++ b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/set.clj @@ -25,10 +25,12 @@ (invoke! [this test op] (case (:f op) :read (chu/exec-with-retries 30 (fn [] - (zk-sync conn) - (assoc op - :type :ok - :value (read-string (:data (zk-get-str conn k)))))) + (with-fresh-conn nodename (:with-auth test) + (fn [conn] + (zk-sync conn) + (assoc op + :type :ok + :value (read-string (:data (zk-get-str conn k)))))))) :add (try (do (zk-add-to-set conn k (:value op)) diff --git a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/utils.clj b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/utils.clj index 9947973e51aea..cfdf7d1a829aa 100644 --- a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/utils.clj +++ b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/utils.clj @@ -32,6 +32,16 @@ conn) conn))) +(defn with-fresh-conn + "Calls (f conn) on a new connection to node and closes it. Keeper cannot continue a session, + so a worker's connection that dropped after its last operation is expired." + [node with-auth f] + (let [conn (zk-connect node 9181 30000 with-auth)] + (try + (f conn) + (finally + (zk/close conn))))) + (defn zk-create-range [conn n & {:keys [with-acl] :or {with-acl false}}] (dorun (map (fn [v] (zk/create-all conn v diff --git a/tests/performance/aggregation_single_level_result_fanout.xml b/tests/performance/aggregation_single_level_result_fanout.xml new file mode 100644 index 0000000000000..e7a9f16cc9f3e --- /dev/null +++ b/tests/performance/aggregation_single_level_result_fanout.xml @@ -0,0 +1,111 @@ + + + + 8 + 0 + 0 + + + CREATE TABLE single_level_fanout_t1 (oid UInt64, cid Int32, cr DateTime, h String, s1 Int32) ENGINE = MergeTree ORDER BY oid + CREATE TABLE single_level_fanout_t2 (oid UInt64) ENGINE = MergeTree ORDER BY oid + CREATE TABLE single_level_fanout_t3 (cid Int32, h String, ref DateTime, s2 Float64) ENGINE = MergeTree ORDER BY (cid, h, ref) + + + INSERT INTO single_level_fanout_t1 + SELECT + intDiv(number, 5), + toInt32(intDiv(number, 5) % 4), + toDateTime('2026-09-08 00:00:00') + toIntervalMinute(intDiv(number, 5) % 1440) + toIntervalSecond(number % 5), + toString(intDiv(number, 5) % 200), + toInt32(number % 300) + FROM numbers(300000) + + INSERT INTO single_level_fanout_t2 SELECT number FROM numbers(60000) + + INSERT INTO single_level_fanout_t3 + SELECT + toInt32(number % 4), + toString(intDiv(number, 4) % 200), + toDateTime('2026-09-08 00:00:00') + toIntervalMinute(intDiv(number, 800)), + toFloat64(number % 5) / 2 + FROM numbers(1152000) + + + + + join_algorithm + + hash + parallel_hash + + + + + = e.t0 - toIntervalMinute(3) AND p.ref <= e.t0 + toIntervalMinute(1) + GROUP BY e.oid, e.cid, e.h, e.s + ) + GROUP BY cid + FORMAT Null + SETTINGS join_algorithm = '{join_algorithm}' + ]]> + + + = e.t0 - toIntervalMinute(3) AND p.ref <= e.t0 + toIntervalMinute(1) + GROUP BY e.oid, e.cid, e.h, e.s + ) + GROUP BY cid + FORMAT Null + SETTINGS enable_parallel_single_level_merge = 1, enable_adaptive_aggregator = 1 + ]]> + + DROP TABLE IF EXISTS single_level_fanout_t1 + DROP TABLE IF EXISTS single_level_fanout_t2 + DROP TABLE IF EXISTS single_level_fanout_t3 + diff --git a/tests/performance/array_join.xml b/tests/performance/array_join.xml index b3fc24d316652..af586967907b0 100644 --- a/tests/performance/array_join.xml +++ b/tests/performance/array_join.xml @@ -1,4 +1,6 @@ + CREATE TABLE array_join_wide_chain (A Array(String), B Array(String), C Array(String)) ENGINE = MergeTree ORDER BY tuple() + INSERT INTO array_join_wide_chain SELECT arrayMap(i -> concat('a', toString(number), '_', toString(i)), range(100)), arrayMap(i -> concat('b', toString(number), '_', toString(i)), range(100)), arrayMap(i -> concat('c', toString(number), '_', toString(i)), range(100)) FROM numbers(100000) @@ -10,4 +12,10 @@ SELECT count() FROM (SELECT [number] a, [number * 2, number] b FROM numbers(10000000)) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 with 'clickhouse' as str select arrayJoin(range(number % 10)), materialize(str) from numbers(10000000) + + + SELECT a, b, c FROM array_join_wide_chain ARRAY JOIN A AS a ARRAY JOIN B AS b ARRAY JOIN C AS c LIMIT 31 SETTINGS max_threads = 1 FORMAT Null + SELECT a, b, c FROM array_join_wide_chain LEFT ARRAY JOIN A AS a LEFT ARRAY JOIN B AS b LEFT ARRAY JOIN C AS c LIMIT 31 SETTINGS max_threads = 1 FORMAT Null + + DROP TABLE IF EXISTS array_join_wide_chain diff --git a/tests/performance/full_sorting_merge_join.xml b/tests/performance/full_sorting_merge_join.xml index 874b4be213944..39aef42d749e6 100644 --- a/tests/performance/full_sorting_merge_join.xml +++ b/tests/performance/full_sorting_merge_join.xml @@ -67,6 +67,11 @@ OPTIMIZE TABLE fsm_nullable_left FINAL OPTIMIZE TABLE fsm_nullable_right FINAL + + CREATE TABLE fsm_u64_two_keys (k1 UInt64, k2 UInt64) ENGINE = MergeTree ORDER BY (k1, k2) + INSERT INTO fsm_u64_two_keys SELECT number % 10000, sipHash64(number) FROM numbers_mt(5000000) + OPTIMIZE TABLE fsm_u64_two_keys FINAL + SELECT count() FROM test_u64_left_{card} AS a JOIN test_u64_right_{card} AS b ON a.k = b.k FORMAT Null SETTINGS join_algorithm = 'full_sorting_merge' SELECT count() FROM test_str_left AS a JOIN test_str_right AS b ON a.k = b.k FORMAT Null SETTINGS join_algorithm = 'full_sorting_merge' SELECT count() FROM test_lc_left AS a JOIN test_lc_right AS b ON a.k = b.k FORMAT Null SETTINGS join_algorithm = 'full_sorting_merge' @@ -78,6 +83,7 @@ SELECT count() FROM fsm_u64_mult2 AS a JOIN fsm_u64_mult3 AS b ON a.k = b.k FORMAT Null SETTINGS join_algorithm = 'full_sorting_merge' SELECT count() FROM fsm_str_sparse_left AS a JOIN fsm_str_sparse_right AS b ON a.k = b.k FORMAT Null SETTINGS join_algorithm = 'full_sorting_merge' SELECT count() FROM fsm_nullable_left AS a JOIN fsm_nullable_right AS b ON a.k = b.k FORMAT Null SETTINGS join_algorithm = 'full_sorting_merge', join_use_nulls = 1 + SELECT count() FROM fsm_u64_two_keys AS a JOIN fsm_u64_two_keys AS b ON a.k1 = b.k1 AND a.k2 = b.k2 FORMAT Null SETTINGS join_algorithm = 'full_sorting_merge' DROP TABLE IF EXISTS test_u64_left_{card} DROP TABLE IF EXISTS test_u64_right_{card} @@ -93,4 +99,5 @@ DROP TABLE IF EXISTS fsm_str_sparse_right DROP TABLE IF EXISTS fsm_nullable_left DROP TABLE IF EXISTS fsm_nullable_right + DROP TABLE IF EXISTS fsm_u64_two_keys diff --git a/tests/performance/hash_join_low_cardinality_many_matches.xml b/tests/performance/hash_join_low_cardinality_many_matches.xml new file mode 100644 index 0000000000000..8c6d625176024 --- /dev/null +++ b/tests/performance/hash_join_low_cardinality_many_matches.xml @@ -0,0 +1,11 @@ + + + + 8 + 0 + + + SELECT count() FROM (SELECT number % 100 AS k FROM numbers(4000)) AS a INNER JOIN (SELECT intDiv(number, 4000) AS k, toLowCardinality(toString(number % 7)) AS lc FROM numbers(400000)) AS b ON a.k = b.k WHERE NOT ignore(b.lc) SETTINGS join_algorithm = 'hash' + SELECT count() FROM (SELECT number % 100 AS k FROM numbers(4000)) AS a INNER JOIN (SELECT intDiv(number, 4000) AS k, toLowCardinality(toString(number % 7)) AS lc FROM numbers(400000)) AS b ON a.k = b.k WHERE NOT ignore(b.lc) SETTINGS join_algorithm = 'parallel_hash' + diff --git a/tests/performance/lazy_replicated_array_functions.xml b/tests/performance/lazy_replicated_array_functions.xml new file mode 100644 index 0000000000000..189daf39ec41d --- /dev/null +++ b/tests/performance/lazy_replicated_array_functions.xml @@ -0,0 +1,15 @@ + + + + lazy_replication + + 0 + 1 + + + + + + SELECT sum(length(arraySlice(big, x, 2))) FROM (SELECT materialize(range(10000)) AS big, range(100) AS r FROM numbers(100)) ARRAY JOIN r AS x SETTINGS enable_lazy_columns_replication = {lazy_replication} FORMAT Null + SELECT sum(big[x * 50]) FROM (SELECT materialize(range(10000)) AS big, range(100) AS r FROM numbers(100)) ARRAY JOIN r AS x SETTINGS enable_lazy_columns_replication = {lazy_replication} FORMAT Null + diff --git a/tests/performance/limit_by_in_pk_order.xml b/tests/performance/limit_by_in_pk_order.xml index b7eed01dcb338..75e8f67ccd33d 100644 --- a/tests/performance/limit_by_in_pk_order.xml +++ b/tests/performance/limit_by_in_pk_order.xml @@ -201,6 +201,28 @@ SETTINGS allow_limit_by_partitions_independently = 1 + + + CREATE TABLE limit_by_in_pk_order_str + (a String, b String, z UInt64) + ENGINE = MergeTree + ORDER BY (a, b) + + + + INSERT INTO limit_by_in_pk_order_str + SELECT concat('k', toString(sipHash64(number % 10000))), hex(sipHash64(number)), number + FROM numbers(5000000) + + OPTIMIZE TABLE limit_by_in_pk_order_str FINAL + + + SELECT a, b, z FROM limit_by_in_pk_order_str + ORDER BY a, b + LIMIT 1 BY a, b + FORMAT NULL + + DROP TABLE IF EXISTS limit_by_in_pk_order_int DROP TABLE IF EXISTS limit_by_in_pk_order_dt DROP TABLE IF EXISTS limit_by_in_pk_order_low @@ -209,4 +231,5 @@ DROP TABLE IF EXISTS limit_by_in_pk_order_merge_1 DROP TABLE IF EXISTS limit_by_in_pk_order_merge_2 DROP TABLE IF EXISTS limit_by_in_pk_order_part_in_order + DROP TABLE IF EXISTS limit_by_in_pk_order_str diff --git a/tests/performance/map_lowcardinality_key_lookup.xml b/tests/performance/map_lowcardinality_key_lookup.xml new file mode 100644 index 0000000000000..ee61624b8bf31 --- /dev/null +++ b/tests/performance/map_lowcardinality_key_lookup.xml @@ -0,0 +1,43 @@ + + + + + CREATE TABLE map_lc_keys (id UInt64, m Map(LowCardinality(String), String)) + ENGINE = MergeTree ORDER BY id + + + + INSERT INTO map_lc_keys + SELECT number, mapFromArrays( + arrayMap(x -> concat('k', leftPad(toString(x), 2, '0')), range(20)), + arrayMap(x -> concat('v', toString(cityHash64(number, x) % 1000)), range(20))) + FROM numbers(1500000) + + + SELECT sum(ignore(m['k00'])) FROM map_lc_keys SETTINGS max_threads = 1 + SELECT sum(ignore(m['k09'])) FROM map_lc_keys SETTINGS max_threads = 1 + SELECT sum(ignore(m['k19'])) FROM map_lc_keys SETTINGS max_threads = 1 + SELECT sum(ignore(m['absent'])) FROM map_lc_keys SETTINGS max_threads = 1 + + DROP TABLE map_lc_keys + + + + + CREATE TABLE map_uuid_keys (id UInt64, m Map(UUID, DateTime64(3))) + ENGINE = MergeTree ORDER BY id + + + + INSERT INTO map_uuid_keys + SELECT number, mapFromArrays( + arrayMap(x -> reinterpretAsUUID(toFixedString(leftPad(toString(x), 16, '0'), 16)), range(20)), + arrayMap(x -> toDateTime64(number + x, 3), range(20))) + FROM numbers(1000000) + + + SELECT sum(ignore(m[reinterpretAsUUID(toFixedString('0000000000000019', 16))])) FROM map_uuid_keys SETTINGS max_threads = 1 + + DROP TABLE map_uuid_keys + diff --git a/tests/performance/negative_limit_by_in_order.xml b/tests/performance/negative_limit_by_in_order.xml index 70445dde56953..b8dfeb025a41d 100644 --- a/tests/performance/negative_limit_by_in_order.xml +++ b/tests/performance/negative_limit_by_in_order.xml @@ -31,15 +31,22 @@ INSERT INTO test_datetime SELECT toDateTime(1700000000 + number % 100) FROM numbers_mt(10000000) OPTIMIZE TABLE test_datetime FINAL + + CREATE TABLE test_str_two_keys (a String, b String) ENGINE = MergeTree ORDER BY (a, b) + INSERT INTO test_str_two_keys SELECT concat('k', toString(sipHash64(number % 10000))), hex(sipHash64(number)) FROM numbers_mt(5000000) + OPTIMIZE TABLE test_str_two_keys FINAL + SELECT k FROM test_u64_{card} ORDER BY k LIMIT -1 BY k FORMAT Null SELECT s FROM test_str ORDER BY s LIMIT -1 BY s FORMAT Null SELECT lc FROM test_lc ORDER BY lc LIMIT -1 BY lc FORMAT Null SELECT nk FROM test_nullable ORDER BY nk LIMIT -1 BY nk FORMAT Null SELECT dt FROM test_datetime ORDER BY dt LIMIT -1 BY dt FORMAT Null + SELECT a, b FROM test_str_two_keys ORDER BY a, b LIMIT -1 BY a, b FORMAT Null DROP TABLE IF EXISTS test_u64_{card} DROP TABLE IF EXISTS test_str DROP TABLE IF EXISTS test_lc DROP TABLE IF EXISTS test_nullable DROP TABLE IF EXISTS test_datetime + DROP TABLE IF EXISTS test_str_two_keys diff --git a/tests/performance/skip_index_planning_with_patch_parts.xml b/tests/performance/skip_index_planning_with_patch_parts.xml new file mode 100644 index 0000000000000..e7b4261bd1f48 --- /dev/null +++ b/tests/performance/skip_index_planning_with_patch_parts.xml @@ -0,0 +1,90 @@ + + + + 1 + 0 + + + + + CREATE TABLE test_patch_parts_planning + ( + project_id UInt64, + id UInt64, + value UInt64, + s String, + INDEX idx_id id TYPE bloom_filter GRANULARITY 1, + INDEX idx_s s TYPE text(tokenizer = splitByNonAlpha) + ) + ENGINE = MergeTree ORDER BY project_id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1, apply_patches_on_merge = 0, + parts_to_delay_insert = 100000, parts_to_throw_insert = 100000, + refresh_statistics_interval = 1 + + + SYSTEM STOP MERGES test_patch_parts_planning + + + + INSERT INTO test_patch_parts_planning SELECT intDiv(number, 50), number, 0, concat('tok', toString(number)) + FROM numbers(100000) + SETTINGS max_block_size = 50, min_insert_block_size_rows = 50, min_insert_block_size_bytes = 0, max_insert_threads = 1 + + + + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 0 AND id = 7 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 40 AND id = 2007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 80 AND id = 4007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 120 AND id = 6007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 160 AND id = 8007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 200 AND id = 10007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 240 AND id = 12007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 280 AND id = 14007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 320 AND id = 16007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 360 AND id = 18007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 400 AND id = 20007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 440 AND id = 22007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 480 AND id = 24007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 520 AND id = 26007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 560 AND id = 28007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 600 AND id = 30007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 640 AND id = 32007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 680 AND id = 34007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 720 AND id = 36007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 760 AND id = 38007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 800 AND id = 40007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 840 AND id = 42007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 880 AND id = 44007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 920 AND id = 46007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 960 AND id = 48007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1000 AND id = 50007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1040 AND id = 52007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1080 AND id = 54007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1120 AND id = 56007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1160 AND id = 58007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1200 AND id = 60007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1240 AND id = 62007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1280 AND id = 64007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1320 AND id = 66007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1360 AND id = 68007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1400 AND id = 70007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1440 AND id = 72007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1480 AND id = 74007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1520 AND id = 76007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1560 AND id = 78007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1600 AND id = 80007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1640 AND id = 82007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1680 AND id = 84007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1720 AND id = 86007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1760 AND id = 88007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1800 AND id = 90007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1840 AND id = 92007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1880 AND id = 94007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1920 AND id = 96007 + UPDATE test_patch_parts_planning SET value = 1 WHERE project_id = 1960 AND id = 98007 + + SELECT count() FROM test_patch_parts_planning WHERE project_id = 777 AND id = 38857 + SELECT count() FROM test_patch_parts_planning WHERE project_id = 777 AND hasToken(s, 'tok38857') + + DROP TABLE IF EXISTS test_patch_parts_planning + diff --git a/tests/performance/sorted_key_prefix_runs.xml b/tests/performance/sorted_key_prefix_runs.xml new file mode 100644 index 0000000000000..289a942b6f471 --- /dev/null +++ b/tests/performance/sorted_key_prefix_runs.xml @@ -0,0 +1,11 @@ + + + CREATE TABLE sorted_key_prefix_runs (a String, b String, x UInt64) ENGINE = MergeTree ORDER BY (a, b) + INSERT INTO sorted_key_prefix_runs SELECT concat('k', toString(sipHash64(intDiv(number, 2) % 1000))), hex(sipHash64(intDiv(number, 2))), number FROM numbers_mt(5000000) + OPTIMIZE TABLE sorted_key_prefix_runs FINAL + + SELECT DISTINCT a, b FROM sorted_key_prefix_runs FORMAT Null SETTINGS max_threads = 1 + SELECT count() FROM sorted_key_prefix_runs AS l INNER JOIN sorted_key_prefix_runs AS r USING (a, b) FORMAT Null SETTINGS join_algorithm = 'partial_merge' + + DROP TABLE IF EXISTS sorted_key_prefix_runs + diff --git a/tests/performance/text_index_like.xml b/tests/performance/text_index_like.xml index 5f38d81015702..935b025a51654 100644 --- a/tests/performance/text_index_like.xml +++ b/tests/performance/text_index_like.xml @@ -1,4 +1,9 @@ + + 0 + 4 + + CREATE TABLE tab ( @@ -14,14 +19,23 @@ INSERT INTO tab SELECT number, - if(number % 100 = 0, + trimRight(if(number < 100000, repeat('clickhouse is a fast column oriented database system ', 4), - repeat('the quick brown fox jumps over the lazy dog near the river ', 4)) + repeat('the quick brown fox jumps over the lazy dog near the river ', 4))) FROM numbers(10000000) SELECT count() FROM tab WHERE message LIKE '%clickhouse%' SELECT count() FROM tab WHERE message ILIKE '%CLICKHOUSE%' + SELECT count() FROM tab WHERE message LIKE 'clickhouse%' + SELECT count() FROM tab WHERE message ILIKE 'CLICKHOUSE%' + + SELECT count() FROM tab WHERE message LIKE '%system' + SELECT count() FROM tab WHERE message ILIKE '%SYSTEM' + + SELECT count() FROM tab WHERE startsWith(message, 'clickhouse') + SELECT count() FROM tab WHERE endsWith(message, 'system') + DROP TABLE IF EXISTS tab diff --git a/tests/queries/0_stateless/01780_column_sparse.reference b/tests/queries/0_stateless/01780_column_sparse.reference index 3393361a19fb0..7452be01648e6 100644 --- a/tests/queries/0_stateless/01780_column_sparse.reference +++ b/tests/queries/0_stateless/01780_column_sparse.reference @@ -132,7 +132,7 @@ SELECT id % 7, sum(u) FROM t_sparse GROUP BY id % 7 ORDER BY id % 7; 4 190 5 330 6 270 -SELECT arrayFilter(x -> x % 2 = 1, arr2) FROM t_sparse WHERE arr2 != [] LIMIT 5; +SELECT arrayFilter(x -> x % 2 = 1, arr2) FROM t_sparse WHERE arr2 != [] ORDER BY id LIMIT 5; [1] [1,3] [1,3,5] diff --git a/tests/queries/0_stateless/01780_column_sparse.sql b/tests/queries/0_stateless/01780_column_sparse.sql index 8e3c4372d05aa..c5b98b1fd422f 100644 --- a/tests/queries/0_stateless/01780_column_sparse.sql +++ b/tests/queries/0_stateless/01780_column_sparse.sql @@ -27,7 +27,7 @@ SELECT * FROM t_sparse WHERE arr2 != [] ORDER BY id; SELECT sum(u) FROM t_sparse; SELECT id % 7, sum(u) FROM t_sparse GROUP BY id % 7 ORDER BY id % 7; -SELECT arrayFilter(x -> x % 2 = 1, arr2) FROM t_sparse WHERE arr2 != [] LIMIT 5; +SELECT arrayFilter(x -> x % 2 = 1, arr2) FROM t_sparse WHERE arr2 != [] ORDER BY id LIMIT 5; CREATE TABLE t_sparse_1 (id UInt64, v Int64) ENGINE = MergeTree ORDER BY tuple() diff --git a/tests/queries/0_stateless/01890_stem.reference b/tests/queries/0_stateless/01890_stem.reference index 7b5f7af344898..6b0de99cce390 100644 --- a/tests/queries/0_stateless/01890_stem.reference +++ b/tests/queries/0_stateless/01890_stem.reference @@ -103,6 +103,17 @@ run bless bless disguis +- Lengthening stemmers. +-- Turkish stems a 3-byte word to a 5-byte word, so the output is longer than the input. +uagı +-- Over a multi-block scan the output must stay correct even when it overflows the input-sized estimate. +0 +-- Turkish also lengthens via a multi-byte substitution (3 bytes to 4 bytes). +aaç +-- Estonian lengthens by appending ASCII letters (4 bytes to 5 bytes), a different mechanism. +keesi +-- Estonian over a multi-block scan must also stay correct past the input-sized estimate. +0 - Negative tests. -- Whitespace in a String input raises BAD_ARGUMENTS. -- Whitespace in an Array element raises BAD_ARGUMENTS. diff --git a/tests/queries/0_stateless/01890_stem.sql b/tests/queries/0_stateless/01890_stem.sql index cc5dd09fac708..c678bc88fb960 100644 --- a/tests/queries/0_stateless/01890_stem.sql +++ b/tests/queries/0_stateless/01890_stem.sql @@ -144,6 +144,27 @@ INSERT INTO stem_test_lc VALUES ('blessing'), ('disguise'), ('blessing'); SELECT stem(word, 'en') FROM stem_test_lc ORDER BY word; DROP TABLE stem_test_lc; +SELECT '- Lengthening stemmers.'; + +SELECT '-- Turkish stems a 3-byte word to a 5-byte word, so the output is longer than the input.'; +SELECT stem('uag', 'tr'); + +SELECT '-- Over a multi-block scan the output must stay correct even when it overflows the input-sized estimate.'; +SELECT countIf(s != 'uagı') +FROM (SELECT stem(materialize('uag'), 'tr') AS s FROM numbers(6800)) +SETTINGS max_block_size = 1700; + +SELECT '-- Turkish also lengthens via a multi-byte substitution (3 bytes to 4 bytes).'; +SELECT stem('aac', 'tr'); + +SELECT '-- Estonian lengthens by appending ASCII letters (4 bytes to 5 bytes), a different mechanism.'; +SELECT stem('keeb', 'et'); + +SELECT '-- Estonian over a multi-block scan must also stay correct past the input-sized estimate.'; +SELECT countIf(s != 'keesi') +FROM (SELECT stem(materialize('keeb'), 'et') AS s FROM numbers(6800)) +SETTINGS max_block_size = 1700; + SELECT '- Negative tests.'; SELECT '-- Whitespace in a String input raises BAD_ARGUMENTS.'; diff --git a/tests/queries/0_stateless/02346_text_index_function_like_affix.reference b/tests/queries/0_stateless/02346_text_index_function_like_affix.reference new file mode 100644 index 0000000000000..89bb276d2e317 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_function_like_affix.reference @@ -0,0 +1,123 @@ +Test results are same with/without the optimization +-- without optimization +[1,3] +[2,5] +[1,2,3,4,5,6] +[2,4,5,6,7] +[1,3,4,6,7] +[] +[] +[1] +[1,2,3,5] +[1] +[1,3] +[2,5] +[1] +[6] +[1,3] +[2,5] +[1,3] +[2,5] +-- with optimization +[1,3] +[2,5] +[1,2,3,4,5,6] +[2,4,5,6,7] +[1,3,4,6,7] +[] +[] +[1] +[1,2,3,5] +[1] +[1,3] +[2,5] +[1] +[6] +[1,3] +[2,5] +[1,3] +[2,5] +-- with optimization but without hints +[1,3] +[2,5] +[1,2,3,4,5,6] +[2,4,5,6,7] +[1,3,4,6,7] +[1,3] +[2,5] +Prefix and suffix patterns keep the original condition, infix patterns do not +prefix 1 1 +suffix 1 1 +prefix, no rewrite 1 1 +suffix, no rewrite 1 1 +infix 1 0 +ilike prefix 1 1 +Needles shorter than text_index_like_min_pattern_length are not evaluated by a dictionary scan +prefix 0 1 +[1,3] +[1] +Text index analysis +-- Prefix pattern should choose 1 part and 1024 granules out of 4 parts and 4096 granules +Description: text GRANULARITY 100000000 +Parts: 1/4 +Granules: 1024/4096 +-- Suffix pattern should choose 3 parts and 3072 granules out of 4 parts and 4096 granules +Description: text GRANULARITY 100000000 +Parts: 3/4 +Granules: 3072/4096 +-- Prefix pattern with a non-existent token should choose none +Description: text GRANULARITY 100000000 +Parts: 0/4 +Granules: 0/4096 +1024 +1024 +2048 +2048 +Test results are same with/without the optimization with array tokenizer +-- without optimization +[1,3] +[3] +[2,4] +[1,3] +[3] +[1,2,3,4] +[3] +-- with optimization +[1,3] +[3] +[2,4] +[1,3] +[3] +[1,2,3,4] +[3] +With the array tokenizer a token is the whole value, so an affix is exact +prefix 1 0 +suffix 1 0 +ilike prefix 1 0 +A nullable value is left to the original condition +[1,3] +[1,3] +[2] +[2] +[1,2] +[1,2] +[] +[] +[2] +[2] +[] +[] +An affix hint that prunes nothing is discarded, granule pruning is kept +1000 +0 +affix_hint_nonselective 0 1 1 +affix_hint_selective 1 0 1 +A nullable value reached through mapValues is left to the original condition +499 +499 +499 +499 +499 +499 +1 +1 diff --git a/tests/queries/0_stateless/02346_text_index_function_like_affix.sql b/tests/queries/0_stateless/02346_text_index_function_like_affix.sql new file mode 100644 index 0000000000000..bf57731ff6808 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_function_like_affix.sql @@ -0,0 +1,338 @@ +-- Tags: no-parallel-replicas +-- Tests that affix LIKE/ILIKE patterns, i.e. prefix ('value%') and suffix ('%value'), use the text index as a hint. +-- By default the analyzer rewrites such patterns into startsWith/endsWith (optimize_rewrite_like_perfect_affix), +-- so both spellings are covered here. +SET explain_query_plan_default = 'legacy'; + +SET enable_analyzer = 1; +SET use_skip_indexes_on_data_read = 1; +SET query_plan_direct_read_from_text_index = 1; +SET query_plan_text_index_add_hint = 1; +-- Pinned because the queries below assert which of the two spellings the plan ends up with. +SET optimize_rewrite_like_perfect_affix = 1; + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab +( + id UInt32, + message String, + INDEX idx(message) TYPE text(tokenizer = splitByNonAlpha) +) +ENGINE = MergeTree +ORDER BY (id); + +-- The dictionary of this table contains tokens that match the prefix/suffix patterns below in rows +-- where the whole value does not match them, so a hint that is not verified would return extra rows. +INSERT INTO tab(id, message) VALUES + (1, 'foobar baz'), + (2, 'baz foobar'), + (3, 'foobarqux end'), + (4, 'end foobarqux'), + (5, 'quuxfoobar'), + (6, 'quuxfoobar tail'), + (7, 'nothing here'); + +SELECT 'Test results are same with/without the optimization'; + +SELECT '-- without optimization'; + +SET use_text_index_like_evaluation_by_dictionary_scan = 0; + +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar%'; +SELECT groupArray(id) FROM tab WHERE message NOT LIKE 'foobar%'; +SELECT groupArray(id) FROM tab WHERE message NOT LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'nonexistent%'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%nonexistent'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' AND message LIKE '%baz'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' OR message LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' AND hasToken(message, 'baz'); +SELECT groupArray(id) FROM tab WHERE startsWith(message, 'foobar'); +SELECT groupArray(id) FROM tab WHERE endsWith(message, 'foobar'); +SELECT groupArray(id) FROM tab WHERE startsWith(message, 'foobar baz'); +SELECT groupArray(id) FROM tab WHERE endsWith(message, 'quuxfoobar tail'); +SELECT groupArray(id) FROM tab WHERE message ILIKE 'FOOBAR%'; +SELECT groupArray(id) FROM tab WHERE message ILIKE '%FOOBAR'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' SETTINGS optimize_rewrite_like_perfect_affix = 0; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar' SETTINGS optimize_rewrite_like_perfect_affix = 0; + +SELECT '-- with optimization'; + +SET use_text_index_like_evaluation_by_dictionary_scan = 1; + +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar%'; +SELECT groupArray(id) FROM tab WHERE message NOT LIKE 'foobar%'; +SELECT groupArray(id) FROM tab WHERE message NOT LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'nonexistent%'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%nonexistent'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' AND message LIKE '%baz'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' OR message LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' AND hasToken(message, 'baz'); +SELECT groupArray(id) FROM tab WHERE startsWith(message, 'foobar'); +SELECT groupArray(id) FROM tab WHERE endsWith(message, 'foobar'); +SELECT groupArray(id) FROM tab WHERE startsWith(message, 'foobar baz'); +SELECT groupArray(id) FROM tab WHERE endsWith(message, 'quuxfoobar tail'); +SELECT groupArray(id) FROM tab WHERE message ILIKE 'FOOBAR%'; +SELECT groupArray(id) FROM tab WHERE message ILIKE '%FOOBAR'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' SETTINGS optimize_rewrite_like_perfect_affix = 0; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar' SETTINGS optimize_rewrite_like_perfect_affix = 0; + +SELECT '-- with optimization but without hints'; + +SET query_plan_text_index_add_hint = 0; + +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar%'; +SELECT groupArray(id) FROM tab WHERE message NOT LIKE 'foobar%'; +SELECT groupArray(id) FROM tab WHERE message NOT LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message ILIKE 'FOOBAR%'; +SELECT groupArray(id) FROM tab WHERE message ILIKE '%FOOBAR'; + +SET query_plan_text_index_add_hint = 1; + +SELECT 'Prefix and suffix patterns keep the original condition, infix patterns do not'; + +-- The columns are: whether the query plan contains a text index virtual column, and whether it still +-- evaluates the original search function. The latter is absent only for an exact direct read. +SELECT 'prefix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION startsWith(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message LIKE 'foobar%'); + +SELECT 'suffix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION endsWith(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message LIKE '%foobar'); + +SELECT 'prefix, no rewrite', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION like(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message LIKE 'foobar%' SETTINGS optimize_rewrite_like_perfect_affix = 0); + +SELECT 'suffix, no rewrite', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION like(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message LIKE '%foobar' SETTINGS optimize_rewrite_like_perfect_affix = 0); + +SELECT 'infix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION like(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message LIKE '%foobar%'); + +SELECT 'ilike prefix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION ilike(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message ILIKE 'FOOBAR%'); + +SELECT 'Needles shorter than text_index_like_min_pattern_length are not evaluated by a dictionary scan'; + +SELECT 'prefix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION startsWith(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message LIKE 'foo%'); + +SELECT groupArray(id) FROM tab WHERE message LIKE 'foo%'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%baz'; + +DROP TABLE tab; + +SELECT 'Text index analysis'; + +CREATE TABLE tab +( + id UInt32, + message String, + INDEX idx(message) TYPE text(tokenizer = splitByNonAlpha) GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY (id) +SETTINGS index_granularity = 1; + +INSERT INTO tab SELECT number, 'Hello ClickHouse' FROM numbers(1024); +INSERT INTO tab SELECT number, 'Hello World, ClickHouse is fast!' FROM numbers(1024); +INSERT INTO tab SELECT number, 'Hallo xClickHouse' FROM numbers(1024); +INSERT INTO tab SELECT number, 'ClickHousez rocks' FROM numbers(1024); + +SELECT '-- Prefix pattern should choose 1 part and 1024 granules out of 4 parts and 4096 granules'; +-- Only 'ClickHousez rocks' has a token starting with 'ClickHousez'. +SELECT trimLeft(explain) AS explain FROM ( + EXPLAIN indexes=1 + SELECT count() FROM tab WHERE message LIKE 'ClickHousez%' +) WHERE explain LIKE '%Description:%' OR explain LIKE '%Parts:%' OR explain LIKE '%Granules:%' +LIMIT 2, 3; + +SELECT '-- Suffix pattern should choose 3 parts and 3072 granules out of 4 parts and 4096 granules'; +-- 'ClickHouse' and 'xClickHouse' are tokens ending with 'ClickHouse', only 'ClickHousez' is not. +SELECT trimLeft(explain) AS explain FROM ( + EXPLAIN indexes=1 + SELECT count() FROM tab WHERE message LIKE '%ClickHouse' +) WHERE explain LIKE '%Description:%' OR explain LIKE '%Parts:%' OR explain LIKE '%Granules:%' +LIMIT 2, 3; + +SELECT '-- Prefix pattern with a non-existent token should choose none'; +SELECT trimLeft(explain) AS explain FROM ( + EXPLAIN indexes=1 + SELECT count() FROM tab WHERE message LIKE 'random%' +) WHERE explain LIKE '%Description:%' OR explain LIKE '%Parts:%' OR explain LIKE '%Granules:%' +LIMIT 2, 3; + +-- Three parts have a token starting with 'ClickHouse' but only one of them has rows starting with it, +-- so the hint must be verified by the original condition. +SELECT count() FROM tab WHERE message LIKE 'ClickHouse%'; +SELECT count() FROM tab WHERE message LIKE 'ClickHouse%' SETTINGS use_skip_indexes = 0; + +SELECT count() FROM tab WHERE message LIKE '%ClickHouse'; +SELECT count() FROM tab WHERE message LIKE '%ClickHouse' SETTINGS use_skip_indexes = 0; + +DROP TABLE tab; + +SELECT 'Test results are same with/without the optimization with array tokenizer'; + +CREATE TABLE tab +( + id UInt32, + tag String, + INDEX idx(tag) TYPE text(tokenizer = array) +) +ENGINE = MergeTree +ORDER BY (id); + +INSERT INTO tab(id, tag) VALUES + (1, 'ClickHouseServer'), + (2, 'clickhouseClient'), + (3, 'ClickHouseCloud'), + (4, 'CLICKHOUSE_SQL'); + +SELECT '-- without optimization'; + +SET use_text_index_like_evaluation_by_dictionary_scan = 0; + +SELECT groupArray(id) FROM tab WHERE tag LIKE 'ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE tag LIKE '%Cloud'; +SELECT groupArray(id) FROM tab WHERE tag NOT LIKE 'ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE startsWith(tag, 'ClickHouse'); +SELECT groupArray(id) FROM tab WHERE endsWith(tag, 'Cloud'); +SELECT groupArray(id) FROM tab WHERE tag ILIKE 'clickhouse%'; +SELECT groupArray(id) FROM tab WHERE tag ILIKE '%cloud'; + +SELECT '-- with optimization'; + +SET use_text_index_like_evaluation_by_dictionary_scan = 1; + +SELECT groupArray(id) FROM tab WHERE tag LIKE 'ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE tag LIKE '%Cloud'; +SELECT groupArray(id) FROM tab WHERE tag NOT LIKE 'ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE startsWith(tag, 'ClickHouse'); +SELECT groupArray(id) FROM tab WHERE endsWith(tag, 'Cloud'); +SELECT groupArray(id) FROM tab WHERE tag ILIKE 'clickhouse%'; +SELECT groupArray(id) FROM tab WHERE tag ILIKE '%cloud'; + +DROP TABLE tab; + +SELECT 'With the array tokenizer a token is the whole value, so an affix is exact'; + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab +( + id UInt32, + tag String, + INDEX idx(tag) TYPE text(tokenizer = array) +) +ENGINE = MergeTree +ORDER BY (id); + +INSERT INTO tab(id, tag) VALUES + (1, 'ClickHouseServer'), + (2, 'clickhouseClient'), + (3, 'ClickHouseCloud'), + (4, 'CLICKHOUSE_SQL'); + +SELECT 'prefix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION startsWith(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE tag LIKE 'ClickHouse%'); + +SELECT 'suffix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION endsWith(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE tag LIKE '%Cloud'); + +SELECT 'ilike prefix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION ilike(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE tag ILIKE 'clickhouse%'); + +DROP TABLE tab; + +SELECT 'A nullable value is left to the original condition'; + +CREATE TABLE tab +( + id UInt32, + tag Nullable(String), + INDEX idx(tag) TYPE text(tokenizer = array) +) +ENGINE = MergeTree +ORDER BY (id); + +INSERT INTO tab(id, tag) VALUES + (1, 'ClickHouseServer'), + (2, 'clickhouseClient'), + (3, 'ClickHouseCloud'), + (4, NULL); + +SELECT groupArray(id) FROM tab WHERE tag LIKE 'ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE tag LIKE 'ClickHouse%' SETTINGS use_skip_indexes = 0; +SELECT groupArray(id) FROM tab WHERE tag NOT LIKE 'ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE tag NOT LIKE 'ClickHouse%' SETTINGS use_skip_indexes = 0; +SELECT groupArray(id) FROM tab WHERE NOT endsWith(tag, 'Cloud'); +SELECT groupArray(id) FROM tab WHERE NOT endsWith(tag, 'Cloud') SETTINGS use_skip_indexes = 0; +SELECT groupArray(id) FROM tab WHERE tag NOT ILIKE 'clickhouse%'; +SELECT groupArray(id) FROM tab WHERE tag NOT ILIKE 'clickhouse%' SETTINGS use_skip_indexes = 0; +SELECT groupArray(id) FROM tab WHERE tag NOT LIKE '%ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE tag NOT LIKE '%ClickHouse%' SETTINGS use_skip_indexes = 0; +SELECT groupArray(id) FROM tab WHERE tag NOT ILIKE '%clickhouse%'; +SELECT groupArray(id) FROM tab WHERE tag NOT ILIKE '%clickhouse%' SETTINGS use_skip_indexes = 0; + +DROP TABLE tab; + +SELECT 'An affix hint that prunes nothing is discarded, granule pruning is kept'; + +CREATE TABLE tab +( + id UInt64, + message String, + INDEX idx(message) TYPE text(tokenizer = splitByNonAlpha) GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id; + +INSERT INTO tab SELECT number, multiIf(number < 1000, 'clickhouse is fast', number < 30000, 'bank of the river', 'alpha beta gamma') FROM numbers(100000); + +SELECT count() FROM tab WHERE message LIKE 'clickhouse%' SETTINGS log_comment = 'affix_hint_selective'; +SELECT count() FROM tab WHERE message LIKE 'river%' SETTINGS log_comment = 'affix_hint_nonselective'; + +SYSTEM FLUSH LOGS query_log; + +SELECT log_comment, + max(ProfileEvents['TextIndexUseHint'] > 0) AS hint_used, + max(ProfileEvents['TextIndexDiscardHint'] > 0) AS hint_discarded, + max(read_rows < 100000) AS granules_pruned +FROM system.query_log +WHERE event_date >= yesterday() AND event_time >= now() - 600 AND current_database = currentDatabase() + AND type = 'QueryFinish' AND log_comment IN ('affix_hint_selective', 'affix_hint_nonselective') +GROUP BY log_comment +ORDER BY log_comment; + +DROP TABLE tab; + +SELECT 'A nullable value reached through mapValues is left to the original condition'; + +CREATE TABLE tab +( + id UInt64, + m Map(String, Nullable(String)), + INDEX idx mapValues(m) TYPE text(tokenizer = splitByNonAlpha) +) +ENGINE = MergeTree +ORDER BY id; + +INSERT INTO tab SELECT number, map('k', if(number = 0, 'foobar value', 'zulu yankee')) FROM numbers(500); +INSERT INTO tab VALUES (1000, map('k', NULL)); + +SELECT count() FROM tab WHERE NOT startsWith(m['k'], 'foobar'); +SELECT count() FROM tab WHERE NOT startsWith(m['k'], 'foobar') SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab WHERE NOT m['k'] LIKE 'foobar%'; +SELECT count() FROM tab WHERE NOT m['k'] LIKE 'foobar%' SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab WHERE NOT endsWith(m['k'], 'value'); +SELECT count() FROM tab WHERE NOT endsWith(m['k'], 'value') SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab WHERE startsWith(m['k'], 'foobar'); +SELECT count() FROM tab WHERE startsWith(m['k'], 'foobar') SETTINGS use_skip_indexes = 0; + +DROP TABLE tab; diff --git a/tests/queries/0_stateless/02346_text_token_index_transform_null_in.reference b/tests/queries/0_stateless/02346_text_token_index_transform_null_in.reference new file mode 100644 index 0000000000000..7f82e6a6845d9 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_token_index_transform_null_in.reference @@ -0,0 +1,35 @@ +Granules: 1/2 +1 +1 +1 +Granules: 1/2 +1 +7 +1 +7 +1 +Granules: 1/2 +1 +7 +1 +7 +1 +1 +1 +Granules: 1/2 +1 +7 +1 +7 +1 +Granules: 1/2 +2 +3 +1 +1 +2 +2 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/02346_text_token_index_transform_null_in.sql b/tests/queries/0_stateless/02346_text_token_index_transform_null_in.sql new file mode 100644 index 0000000000000..6127b05004cdb --- /dev/null +++ b/tests/queries/0_stateless/02346_text_token_index_transform_null_in.sql @@ -0,0 +1,136 @@ +-- Tests that the `text`, `tokenbf_v1`, `ngrambf_v1` and `sparse_grams` skip indexes prune an `IN` +-- when `transform_null_in = 1`, where the predicate arrives as `nullIn`/`globalNullIn`, and that a +-- set holding a NULL element is refused because `nullIn` also matches the column's NULL rows. +-- +-- `index_granularity = 4` over 8 rows, so every granule is mixed: granule 0 holds word0..word3. + +SET enable_full_text_index = 1; +SET transform_null_in = 1; + +DROP TABLE IF EXISTS tab; + +-- The same block runs for every index type. `tokenbf_v1`, `ngrambf_v1` and `sparse_grams` reject a +-- `Nullable` column at DDL, so the common block uses `String` and NULL enters through the set only. + +-- text + +CREATE TABLE tab (s String, INDEX idx s TYPE text(tokenizer = splitByNonAlpha)) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4; +INSERT INTO tab SELECT 'word' || toString(number) FROM numbers(8); + +SELECT extract(explain, 'Granules: \\d+/\\d+') FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE s IN ('word1')) WHERE explain LIKE '%Granules: %/%'; +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- The text index does not model NOT IN at all, unlike the token bloom filter family below. +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +-- Without transform_null_in the same predicates keep the globalIn/globalNotIn spellings, which must prune too. +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +-- A Nullable element type is what transform_null_in = 1 adds, so a null-free set of it must prune. +SELECT count() FROM tab WHERE s IN (SELECT CAST('word1', 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s IN (SELECT CAST(NULL, 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } + +-- tokenbf_v1 + +DROP TABLE tab; +CREATE TABLE tab (s String, INDEX idx s TYPE tokenbf_v1(256, 2, 0)) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4; +INSERT INTO tab SELECT 'word' || toString(number) FROM numbers(8); + +SELECT extract(explain, 'Granules: \\d+/\\d+') FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE s IN ('word1')) WHERE explain LIKE '%Granules: %/%'; +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- NOT IN never prunes on these indexes, but the index must still be used. +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- Without transform_null_in the same predicates keep the globalIn/globalNotIn spellings, which must prune too. +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +-- A Nullable element type is what transform_null_in = 1 adds, so a null-free set of it must prune. +SELECT count() FROM tab WHERE s IN (SELECT CAST('word1', 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s IN (SELECT CAST(NULL, 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } + +-- ngrambf_v1 + +DROP TABLE tab; +CREATE TABLE tab (s String, INDEX idx s TYPE ngrambf_v1(3, 256, 2, 0)) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4; +INSERT INTO tab SELECT 'word' || toString(number) FROM numbers(8); + +SELECT extract(explain, 'Granules: \\d+/\\d+') FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE s IN ('word1')) WHERE explain LIKE '%Granules: %/%'; +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- NOT IN never prunes on these indexes, but the index must still be used. +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- Without transform_null_in the same predicates keep the globalIn/globalNotIn spellings, which must prune too. +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +-- A Nullable element type is what transform_null_in = 1 adds, so a null-free set of it must prune. +SELECT count() FROM tab WHERE s IN (SELECT CAST('word1', 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s IN (SELECT CAST(NULL, 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } + +-- A FixedString element has its NUL padding stripped before tokenization, so it keeps pruning. +SELECT count() FROM tab WHERE s IN (SELECT toFixedString('word1', 12)) SETTINGS force_data_skipping_indices = 'idx'; +SELECT (SELECT count() FROM tab WHERE s IN (SELECT toFixedString('word1', 12))) = (SELECT count() FROM tab WHERE s IN (SELECT toFixedString('word1', 12)) SETTINGS use_skip_indexes = 0); + +-- sparse_grams + +DROP TABLE tab; +CREATE TABLE tab (s String, INDEX idx s TYPE sparse_grams(3, 100, 512, 2, 0)) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4; +INSERT INTO tab SELECT 'word' || toString(number) FROM numbers(8); + +SELECT extract(explain, 'Granules: \\d+/\\d+') FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE s IN ('word1')) WHERE explain LIKE '%Granules: %/%'; +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- NOT IN never prunes on these indexes, but the index must still be used. +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- Without transform_null_in the same predicates keep the globalIn/globalNotIn spellings, which must prune too. +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +-- A Nullable element type is what transform_null_in = 1 adds, so a null-free set of it must prune. +SELECT count() FROM tab WHERE s IN (SELECT CAST('word1', 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s IN (SELECT CAST(NULL, 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } + +-- A Nullable column: only the text index accepts one, and only there can a NULL element of the set +-- meet a NULL row of the column. The set is a disjunction, so one NULL element refuses the index +-- whatever else the set holds. + +DROP TABLE tab; +CREATE TABLE tab (s Nullable(String), INDEX idx s TYPE text(tokenizer = splitByNonAlpha)) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4; +INSERT INTO tab SELECT if(number = 7, NULL, 'word' || toString(number)) FROM numbers(8); + +SELECT extract(explain, 'Granules: \\d+/\\d+') FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE s IN ('word1')) WHERE explain LIKE '%Granules: %/%'; +SELECT count() FROM tab WHERE s IN ('word1', NULL) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +SELECT count() FROM tab WHERE s IN ('word1', NULL); +SELECT count() FROM tab WHERE s IN ('word1', NULL, 'word2') SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +SELECT count() FROM tab WHERE s IN ('word1', NULL, 'word2'); + +-- A tuple set is one Tuple column unpacked by position, so the indexed component is addressed by it. + +DROP TABLE tab; +CREATE TABLE tab (id UInt64, s String, INDEX idx s TYPE text(tokenizer = splitByNonAlpha)) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 4; +INSERT INTO tab SELECT number, 'word' || toString(number) FROM numbers(8); + +SELECT count() FROM tab WHERE (id, s) IN ((1, 'word1')) SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE (id, s) IN (SELECT tuple(number, 'word1') FROM numbers(8)) SETTINGS force_data_skipping_indices = 'idx'; + +-- An absent map key reads the value type's default, which mapValues never stores. The FixedString default +-- is all NUL and the array tokenizer keeps the padding, so it must be recognised as the default. + +DROP TABLE tab; +CREATE TABLE tab (id UInt64, m Map(String, FixedString(4)), INDEX idx mapValues(m) TYPE text(tokenizer = array)) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 1; +INSERT INTO tab VALUES (0, {'k':'val0'}), (1, {'other':'xxxx'}), (2, {'k':'val2'}), (3, {}); + +SELECT count() FROM tab WHERE m['k'] IN (toFixedString('', 4)) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +SELECT count() FROM tab WHERE m['k'] IN (toFixedString('', 4)); +SELECT count() FROM tab WHERE m['k'] = toFixedString('', 4) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +SELECT count() FROM tab WHERE m['k'] = toFixedString('', 4); +SELECT count() FROM tab WHERE m['k'] IN ('val0') SETTINGS force_data_skipping_indices = 'idx'; + +-- A String map value defaults to '' only, so an all-NUL value is a real value and must prune. + +DROP TABLE tab; +CREATE TABLE tab (id UInt64, m Map(String, String), INDEX idx mapValues(m) TYPE text(tokenizer = array)) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 1; +INSERT INTO tab SELECT 0, map('k', unhex('00')); +INSERT INTO tab SELECT number, map('k', 'val' || toString(number)) FROM numbers(1, 2); +INSERT INTO tab SELECT 3, map('other', 'x'); + +SELECT count() FROM tab WHERE m['k'] = unhex('00') SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE m['k'] IN (unhex('00')) SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE m['k'] = '' SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +SELECT count() FROM tab WHERE m['k'] IN ('') SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +SELECT count() FROM tab WHERE m['k'] = ''; + +DROP TABLE tab; diff --git a/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql b/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql index e0e49b0609047..25efb3237844b 100644 --- a/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql +++ b/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql @@ -62,6 +62,8 @@ INSERT INTO tab_bf16 VALUES (0, [1.0, 0.0]), (9, [0.0, 2.4]); -- The nearest neighbours to [0.0, 0.2] are 5,6,7,8 +-- Round distances: ARM sanitizer builds disable SimSIMD and scalar USearch +-- differs in the 3rd decimal (0.203125 vs 0.1875). Neighbor order is unchanged. SELECT 'Column: Array(Float32)'; diff --git a/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql b/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql index 6c42607cc565a..79d53d6a2e77a 100644 --- a/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql +++ b/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql @@ -16,6 +16,6 @@ SELECT '--'; SELECT * FROM t1 AS t2, (SELECT 1) AS t1; -SELECT * FROM (SELECT 1) AS t1, t1.nested AS t2; -- { serverError UNKNOWN_IDENTIFIER } +SELECT * FROM (SELECT 1) AS t1, t1.nested AS t2; -- { serverError UNKNOWN_DATABASE } -SELECT * FROM t1.nested AS t2, (SELECT 1) AS t1; -- { serverError UNKNOWN_IDENTIFIER } +SELECT * FROM t1.nested AS t2, (SELECT 1) AS t1; -- { serverError UNKNOWN_DATABASE } diff --git a/tests/queries/0_stateless/03279_database_backup_database_disk_engine.reference b/tests/queries/0_stateless/03279_database_backup_database_disk_engine.reference index 4f8c3ae7dc13c..80083db7617a9 100644 --- a/tests/queries/0_stateless/03279_database_backup_database_disk_engine.reference +++ b/tests/queries/0_stateless/03279_database_backup_database_disk_engine.reference @@ -20,7 +20,7 @@ 8 1500 9 1500 -- -CREATE DATABASE default_inner_backup_database\nENGINE = Backup(\'default_inner\', \'Disk(\\\'backups\\\', \\\'default_inner\\\')\') +CREATE DATABASE default_inner_backup_database\nENGINE = Backup(\'default_inner\', Disk(\'backups\', \'default_inner\')) test_table_1 15000 test_table_2 15000 -- diff --git a/tests/queries/0_stateless/03300_ai_functions.reference b/tests/queries/0_stateless/03300_ai_functions.reference index 2e1f69e2d613c..7fd7da0270370 100644 --- a/tests/queries/0_stateless/03300_ai_functions.reference +++ b/tests/queries/0_stateless/03300_ai_functions.reference @@ -44,9 +44,9 @@ String -- Setting defaults ai_function_embedding_default_credentials ai_function_embedding_max_batch_size 100 -ai_function_max_api_calls_per_query 1000 -ai_function_max_input_tokens_per_query 1000000 -ai_function_max_output_tokens_per_query 500000 +ai_function_max_api_calls_per_query 0 +ai_function_max_input_tokens_per_query 0 +ai_function_max_output_tokens_per_query 0 ai_function_max_retries 1 ai_function_request_timeout_sec 60 ai_function_retry_initial_delay_ms 1000 diff --git a/tests/queries/0_stateless/04032_and_comparison_filter_optimization_1.reference b/tests/queries/0_stateless/04032_and_comparison_filter_optimization_1.reference index a0f46fa1a31f6..148993579eca6 100644 --- a/tests/queries/0_stateless/04032_and_comparison_filter_optimization_1.reference +++ b/tests/queries/0_stateless/04032_and_comparison_filter_optimization_1.reference @@ -4,10 +4,10 @@ eq_eq_same SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, 3), equals(__table1.i, 3))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.i, 3)\nSETTINGS optimize_redundant_comparisons = 1 eq_eq_diff -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, 3), equals(__table1.i, 5))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 eq_eq_flip -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(3, __table1.i), equals(__table1.i, 5))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 eq_eq_cross_same 3 30 3 c y 2024-06-15 12:00:00 @@ -15,7 +15,7 @@ eq_eq_cross_same SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, 3), equals(__table1.i, toUInt8(3)))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.i, 3)\nSETTINGS optimize_redundant_comparisons = 1 eq_eq_cross_diff -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, 3), equals(__table1.i, toUInt8(5)))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 eq_eq_float_same 3 30 3 c y 2024-06-15 12:00:00 @@ -194,5 +194,5 @@ multi_expr SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(greater(__table1.i, 1), less(__table1.i, 5), greater(__table1.f, 2.), less(__table1.f, 6.))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(greater(__table1.i, 1), less(__table1.i, 5), greater(__table1.f, 2.), less(__table1.f, 6.))\nSETTINGS optimize_redundant_comparisons = 1 multi_expr_conflict -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, 3), equals(__table1.i, 5), greater(__table1.f, 1.))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 diff --git a/tests/queries/0_stateless/04032_and_comparison_filter_optimization_2.reference b/tests/queries/0_stateless/04032_and_comparison_filter_optimization_2.reference index 08f40bf45839f..4e7ebdf27a6d3 100644 --- a/tests/queries/0_stateless/04032_and_comparison_filter_optimization_2.reference +++ b/tests/queries/0_stateless/04032_and_comparison_filter_optimization_2.reference @@ -15,7 +15,7 @@ lc_eq_eq SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.lc, \'y\'), equals(__table1.lc, \'y\'))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE _CAST(equals(__table1.lc, \'y\'), \'UInt8\')\nSETTINGS optimize_redundant_comparisons = 1 lc_eq_conflict -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.lc, \'y\'), equals(__table1.lc, \'z\'))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 str_eq_eq 3 30 3 c y 2024-06-15 12:00:00 @@ -23,10 +23,10 @@ str_eq_eq SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.s, \'c\'), equals(__table1.s, \'c\'))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.s, \'c\')\nSETTINGS optimize_redundant_comparisons = 1 str_eq_conflict -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.s, \'c\'), equals(__table1.s, \'a\'))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 transitive_conflict -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, 3), equals(__table1.i, __table1.u), equals(__table1.u, 5), equals(__table1.i, 5), equals(__table1.u, 3))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 transitive_no_redundant SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(less(__table1.i, 3), greater(__table1.u, 3), less(__table1.u, 10))\nSETTINGS optimize_redundant_comparisons = 0 diff --git a/tests/queries/0_stateless/04032_and_comparison_filter_type_conversion.reference b/tests/queries/0_stateless/04032_and_comparison_filter_type_conversion.reference index bceb5ddeb7ed3..76b4c36132c46 100644 --- a/tests/queries/0_stateless/04032_and_comparison_filter_type_conversion.reference +++ b/tests/queries/0_stateless/04032_and_comparison_filter_type_conversion.reference @@ -4,7 +4,7 @@ int_str_eq_same SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, \'3\'), equals(__table1.i, 3))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.i, \'3\')\nSETTINGS optimize_redundant_comparisons = 1 int_str_eq_diff -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, \'3\'), equals(__table1.i, \'5\'))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 int_str_range 3 30 3 c y 2024-06-15 12:00:00 @@ -17,7 +17,7 @@ float_int_eq_same SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.f, 3), equals(__table1.f, 3.))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.f, 3)\nSETTINGS optimize_redundant_comparisons = 1 float_int_eq_diff -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.f, 3), equals(__table1.f, 4))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 float_int_range 1 10 1.5 a x 2024-01-01 00:00:00 @@ -42,7 +42,7 @@ float_str_and_int SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.f, \'3.0\'), less(__table1.f, 5))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.f, \'3.0\')\nSETTINGS optimize_redundant_comparisons = 1 float_str_eq_diff -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.f, \'3.0\'), equals(__table1.f, \'5.0\'))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 float_str_range 1 10 1.5 a x 2024-01-01 00:00:00 @@ -57,7 +57,7 @@ dt_str_eq_same SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.dt, \'2024-06-15 12:00:00\'), equals(__table1.dt, \'2024-06-15 12:00:00\'))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.dt, \'2024-06-15 12:00:00\')\nSETTINGS optimize_redundant_comparisons = 1 dt_str_eq_diff -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.dt, \'2024-06-15 12:00:00\'), equals(__table1.dt, \'2025-01-01 00:00:00\'))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 dt_str_range 3 30 3 c y 2024-06-15 12:00:00 @@ -78,7 +78,7 @@ dt_int_eq_prune SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.dt, 1718452800), greater(__table1.dt, 1704067200))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.dt, 1718452800)\nSETTINGS optimize_redundant_comparisons = 1 dt_int_eq_conflict -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.dt, 1718452800), equals(__table1.dt, 1735689600))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 dt_int_range 3 30 3 c y 2024-06-15 12:00:00 diff --git a/tests/queries/0_stateless/04050_text_index_starts_ends_with.reference b/tests/queries/0_stateless/04050_text_index_starts_ends_with.reference index 51f83ff27583f..6090a2c07c4d5 100644 --- a/tests/queries/0_stateless/04050_text_index_starts_ends_with.reference +++ b/tests/queries/0_stateless/04050_text_index_starts_ends_with.reference @@ -6,7 +6,13 @@ Expression ((Project names + Projection)) Condition: true Parts: 3/3 Granules: 3/3 - Ranges: 3 + Skip + Name: idx_name_fts + Description: text GRANULARITY 100000000 + Condition: (mode: All; tokens: []) + Parts: 1/3 + Granules: 1/3 + Ranges: 1 2 Full-text search is now generally available Expression ((Project names + Projection)) ReadFromMergeTree (default.test_fts) @@ -15,7 +21,13 @@ Expression ((Project names + Projection)) Condition: true Parts: 3/3 Granules: 3/3 - Ranges: 3 + Skip + Name: idx_name_fts + Description: text GRANULARITY 100000000 + Condition: (mode: All; tokens: []) + Parts: 1/3 + Granules: 1/3 + Ranges: 1 2 Full-text search is now generally available Expression ((Project names + Projection)) ReadFromMergeTree (default.test_fts) @@ -112,9 +124,9 @@ Expression ((Project names + Projection)) Name: idx_name_fts Description: text GRANULARITY 100000000 Condition: (mode: All; tokens: []) - Parts: 3/3 - Granules: 3/3 - Ranges: 3 + Parts: 1/3 + Granules: 1/3 + Ranges: 1 2 Full-text search is now generally available Expression ((Project names + Projection)) ReadFromMergeTree (default.test_fts) @@ -127,9 +139,9 @@ Expression ((Project names + Projection)) Name: idx_name_fts Description: text GRANULARITY 100000000 Condition: (mode: All; tokens: []) - Parts: 3/3 - Granules: 3/3 - Ranges: 3 + Parts: 1/3 + Granules: 1/3 + Ranges: 1 2 Full-text search is now generally available Expression ((Project names + Projection)) ReadFromMergeTree (default.test_fts) diff --git a/tests/queries/0_stateless/04056_ast_json_limits.sql b/tests/queries/0_stateless/04056_ast_json_limits.sql index e448e8ff8bb7c..6aac3b4200510 100644 --- a/tests/queries/0_stateless/04056_ast_json_limits.sql +++ b/tests/queries/0_stateless/04056_ast_json_limits.sql @@ -91,3 +91,17 @@ SELECT formatQueryFromJSON('{"type":"RefreshStrategy","schedule_kind":"EVERY"}') -- `clickhouse_json` client/server entry points (so a huge shallow document is rejected before Poco -- materializes it). Here `parseQueryToJSON('SELECT 1')` is well over 10 bytes. SELECT formatQueryFromJSON(parseQueryToJSON('SELECT 1')) SETTINGS max_query_size = 10; -- { serverError SYNTAX_ERROR } + +-- Reading an AST back from JSON descends over the document, so raising `max_ast_depth` must not turn +-- a deep payload into a stack overflow: each shape below must fail with a controlled error, never +-- crash. The depths sit far past every bound so that holds in any build; which of the descents +-- reports it first follows the build's stack budget, so no arm asserts a particular one. +-- A chain of nested AST nodes. +SELECT formatQueryFromJSON(concat(repeat('{"type":"ExpressionList","children":[', 20000), '{"type":"Literal","value":{"field_type":"UInt64","value":1}}', repeat(']}', 20000))) SETTINGS max_ast_depth = 100000, max_ast_elements = 0, max_query_size = 100000000; -- { serverError TOO_DEEP_RECURSION } + +-- A nested `field_type` value inside a single literal, which adds no AST nodes of its own. +SELECT formatQueryFromJSON(concat('{"type":"Literal","value":', repeat('{"field_type":"Array","value":[', 25000), '{"field_type":"UInt64","value":1}', repeat(']}', 25000), '}')) SETTINGS max_ast_depth = 200000, max_ast_elements = 0, max_query_size = 100000000; -- { serverError TOO_DEEP_RECURSION } + +-- The JSON text is parsed before any AST node exists, so a deep member that the reader never descends +-- into still has to be bounded while the text itself is being read. +SELECT formatQueryFromJSON(concat('{"type":"Literal","value":{"field_type":"UInt64","value":1},"junk":', repeat('[', 200000), repeat(']', 200000), '}')) SETTINGS max_ast_depth = 100000, max_ast_elements = 0, max_query_size = 100000000; -- { serverError TOO_DEEP_RECURSION } diff --git a/tests/queries/0_stateless/04061_text_index_json_all_values.reference b/tests/queries/0_stateless/04061_text_index_json_all_values.reference index 494d66d4afc24..1bb69101acef6 100644 --- a/tests/queries/0_stateless/04061_text_index_json_all_values.reference +++ b/tests/queries/0_stateless/04061_text_index_json_all_values.reference @@ -22,8 +22,8 @@ SELECT id FROM tab WHERE startsWith(data.key1, 'lazy') ORDER BY id 1 Description: text GRANULARITY 100000000 Condition: (mode: All; tokens: []) -Parts: 4/4 -Granules: 4/4 +Parts: 1/4 +Granules: 1/4 SELECT id FROM tab WHERE endsWith(data.key1, 'fox') ORDER BY id 0 Description: text GRANULARITY 100000000 @@ -61,8 +61,8 @@ SELECT id FROM tab WHERE startsWith(data.key1::String, 'lazy') ORDER BY id 1 Description: text GRANULARITY 100000000 Condition: (mode: All; tokens: []) -Parts: 4/4 -Granules: 4/4 +Parts: 1/4 +Granules: 1/4 SELECT id FROM tab WHERE hasToken(data.key1::String, 'quick') ORDER BY id 0 2 diff --git a/tests/queries/0_stateless/04070_url_base_setting.reference b/tests/queries/0_stateless/04070_url_base_setting.reference index 8c23601338af0..958e5fdfc6c78 100644 --- a/tests/queries/0_stateless/04070_url_base_setting.reference +++ b/tests/queries/0_stateless/04070_url_base_setting.reference @@ -28,3 +28,5 @@ url = 'http://base.invalid/dir/persist.csv' 0 0 must contain a scheme +must contain a scheme +0 diff --git a/tests/queries/0_stateless/04070_url_base_setting.sh b/tests/queries/0_stateless/04070_url_base_setting.sh index 46289ca8b2c14..9127cb87d0660 100755 --- a/tests/queries/0_stateless/04070_url_base_setting.sh +++ b/tests/queries/0_stateless/04070_url_base_setting.sh @@ -172,3 +172,10 @@ DROP TABLE IF EXISTS ${CLICKHOUSE_TEST_UNIQUE_NAME}_url_pos_creds; # Invalid url_base (no scheme) should produce an error $CLICKHOUSE_CLIENT --query "SELECT * FROM url('data.csv', CSV, 'c String') SETTINGS url_base = 'example.invalid/def/', $FAST" 2>&1 | grep -oF 'must contain a scheme' | head -1 + +# The rejected value is not echoed back: it can carry a credential, and the message reaches the client, +# the exception column of the query log and the server log, none of which the display-secrets setting +# gates. (last line counts the leaks; clickhouse-local is used because the client also prints back the +# query it was given, which is the caller's own input rather than something the message disclosed) +$CLICKHOUSE_LOCAL --query "SELECT * FROM url('data.csv', CSV, 'c String') SETTINGS url_base = 'user:SEKRIT_PW@example.invalid/def/', $FAST" 2>&1 | grep -oF 'must contain a scheme' | head -1 +$CLICKHOUSE_LOCAL --query "SELECT * FROM url('data.csv', CSV, 'c String') SETTINGS url_base = 'user:SEKRIT_PW@example.invalid/def/', $FAST" 2>&1 | grep -c SEKRIT_PW ||: diff --git a/tests/queries/0_stateless/04072_jit_decimal_expressions.sql b/tests/queries/0_stateless/04072_jit_decimal_expressions.sql index e735f68165d7b..eda70438d6ae3 100644 --- a/tests/queries/0_stateless/04072_jit_decimal_expressions.sql +++ b/tests/queries/0_stateless/04072_jit_decimal_expressions.sql @@ -34,6 +34,7 @@ SELECT toFloat32(d32), toFloat64(d64) FROM test_jit_dec_expr ORDER BY d32; SELECT 'Test integer to Decimal conversions'; SELECT toDecimal32(i64, 2), toDecimal64(i64, 4) FROM test_jit_dec_expr ORDER BY i64; +-- The `Float ->` direction is evaluated by the interpreter: see #117442. SELECT 'Test float to Decimal conversions'; SELECT toDecimal64(f64, 4) FROM test_jit_dec_expr ORDER BY f64; diff --git a/tests/queries/0_stateless/04143_if_decimal_int_literal_jit.sql b/tests/queries/0_stateless/04143_if_decimal_int_literal_jit.sql index 0900b1e0533e2..d54a37585d5a1 100644 --- a/tests/queries/0_stateless/04143_if_decimal_int_literal_jit.sql +++ b/tests/queries/0_stateless/04143_if_decimal_int_literal_jit.sql @@ -45,7 +45,7 @@ WITH materialize(3::Decimal(38, 30)) AS r, materialize(2) AS k SELECT sum(multiI WITH materialize(1::Decimal(76, 60)) AS r, materialize(1) AS k SELECT sum(if(k != 1, r, 1)); WITH materialize(1::Decimal(76, 73)) AS r, materialize(1) AS k SELECT sum(if(k != 1, r, 1)); --- `Float` -> `Decimal` branch lift (the analyzer promotes the result to `Decimal`). +-- A float branch is never lifted: the result is `Variant(Decimal(18, 4), Float64)`, which is not compiled. WITH materialize(toDecimal64(1.5, 4)) AS r, materialize(1) AS k SELECT if(k != 1, r, 2.5::Float64); -- Sanity check: the non-JIT path produces the same answers. diff --git a/tests/queries/0_stateless/04205_jit_decimal_cast_branches_parity.sql b/tests/queries/0_stateless/04205_jit_decimal_cast_branches_parity.sql index ca1d5926f8309..ac79ea0cb10b5 100644 --- a/tests/queries/0_stateless/04205_jit_decimal_cast_branches_parity.sql +++ b/tests/queries/0_stateless/04205_jit_decimal_cast_branches_parity.sql @@ -40,11 +40,9 @@ -- the analyzer promotes the result type to `Variant` because -- `use_variant_as_common_type = 1` by default, and `canBeNativeType` -- excludes `Variant`. `FunctionIfBase::isCompilableImpl` returns `false` --- and the helper is never called. The helper still has correct code paths --- for these (the `pow10_fp_const` helper avoids 64-bit narrowing via --- `APFloat::convertFromAPInt` so high-bit-width `Decimal128`/`Decimal256` --- factors stay accurate) so it is forward-compatible if the analyzer ever --- stops promoting to `Variant`. +-- and the helper is never called. `Float` -> `Decimal` is asserted against +-- rather than supported: an out-of-range float to integer conversion has no +-- defined result once compiled. -- -- * `Decimal` -> `Decimal` with different scales (scale increase OR decrease): -- `FunctionIf::executeImpl` rejects this with `NOT_IMPLEMENTED: diff --git a/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.reference b/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.reference index 67a5306eb758e..102257aae8b59 100644 --- a/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.reference +++ b/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.reference @@ -1,18 +1,5 @@ --- JIT --- ---- Float -> 128-bit / 256-bit integers --- -2 -2 -2 -2 -2 -2 -2 -2 ---- 128-bit / 256-bit integers -> Float --- -2 -2 -2 -2 +--- 128-bit integers -> Float --- 2 2 2 @@ -36,3 +23,4 @@ 2 2 2 +1 diff --git a/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.sql b/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.sql index 54923031e8c2c..71acd02410deb 100644 --- a/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.sql +++ b/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.sql @@ -6,25 +6,15 @@ SELECT '--- JIT ---'; SET compile_expressions = 1, min_count_to_compile_expression = 0; -SELECT '--- Float -> 128-bit / 256-bit integers ---'; -SELECT toInt128 (materialize(1.5) + materialize(0.5)); -SELECT toUInt128(materialize(1.5) + materialize(0.5)); -SELECT toInt256 (materialize(1.5) + materialize(0.5)); -SELECT toUInt256(materialize(1.5) + materialize(0.5)); -SELECT toInt128 (materialize(1.5)::Float32 + materialize(0.5)::Float32); -SELECT toUInt128(materialize(1.5)::Float32 + materialize(0.5)::Float32); -SELECT toInt256 (materialize(1.5)::Float32 + materialize(0.5)::Float32); -SELECT toUInt256(materialize(1.5)::Float32 + materialize(0.5)::Float32); - -SELECT '--- 128-bit / 256-bit integers -> Float ---'; +-- Only the 128-bit integer to float direction is compiled, and it reaches `__floattisf` and +-- `__floattidf`. A float source with an integer destination is declined, and a 256-bit value is not +-- a native JIT type, so both have an interpreted arm only: the first is covered by +-- `05055_jit_float_to_integer_cast`, the second by the block below. +SELECT '--- 128-bit integers -> Float ---'; SELECT toFloat32(materialize(2::Int128) + materialize(0::Int128)); SELECT toFloat64(materialize(2::Int128) + materialize(0::Int128)); SELECT toFloat32(materialize(2::UInt128) + materialize(0::UInt128)); SELECT toFloat64(materialize(2::UInt128) + materialize(0::UInt128)); -SELECT toFloat32(materialize(2::Int256) + materialize(0::Int256)); -SELECT toFloat64(materialize(2::Int256) + materialize(0::Int256)); -SELECT toFloat32(materialize(2::UInt256) + materialize(0::UInt256)); -SELECT toFloat64(materialize(2::UInt256) + materialize(0::UInt256)); SELECT '--- no JIT ---'; SET compile_expressions = 0; @@ -48,3 +38,24 @@ SELECT toFloat32(materialize(2::Int256) + materialize(0::Int256)); SELECT toFloat64(materialize(2::Int256) + materialize(0::Int256)); SELECT toFloat32(materialize(2::UInt256) + materialize(0::UInt256)); SELECT toFloat64(materialize(2::UInt256) + materialize(0::UInt256)); + +-- Every row above compares a compiled value against an interpreted one, so all of them would still +-- pass if the JIT arm stopped compiling. This pins that the 128-bit conversion is compiled wherever +-- the control is, which also holds in a build without the embedded compiler. The control is plain +-- arithmetic, so it stays at 1 even if every conversion stops being compilable. +SELECT toFloat64(materialize(2::Int128) + materialize(0::Int128)) + SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0, log_comment = '04240_int128' FORMAT Null; +SELECT materialize(2.0) + materialize(0.0) + materialize(1.0) + SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0, log_comment = '04240_control' FORMAT Null; + +SYSTEM FLUSH LOGS query_log; + +WITH shapes AS +( + SELECT log_comment, argMax(ProfileEvents['CompiledFunctionExecute'] > 0, event_time_microseconds) AS compiled + FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment LIKE '04240_%' + GROUP BY log_comment +) +SELECT (SELECT compiled FROM shapes WHERE log_comment = '04240_int128') + = (SELECT compiled FROM shapes WHERE log_comment = '04240_control'); diff --git a/tests/queries/0_stateless/04326_aggregation_two_level_decision.sql b/tests/queries/0_stateless/04326_aggregation_two_level_decision.sql index 606da91b2f94a..f74634032d6ce 100644 --- a/tests/queries/0_stateless/04326_aggregation_two_level_decision.sql +++ b/tests/queries/0_stateless/04326_aggregation_two_level_decision.sql @@ -9,6 +9,7 @@ SET max_bytes_before_external_group_by = 1000000000; -- Ensure conversion to two SET max_bytes_ratio_before_external_group_by = 0; SET group_by_two_level_threshold = 100000; SET group_by_two_level_threshold_bytes = 100000; +SET use_aggregation_memory_tracker = 1; -- Small aggregate state over join stays single-level. SELECT r.k AS g, groupArray(l.s) diff --git a/tests/queries/0_stateless/04365_filter_push_down_window_partition_by.reference b/tests/queries/0_stateless/04365_filter_push_down_window_partition_by.reference deleted file mode 100644 index 7d08fbdbbad17..0000000000000 --- a/tests/queries/0_stateless/04365_filter_push_down_window_partition_by.reference +++ /dev/null @@ -1,9 +0,0 @@ -1 -1 -1 -1 -0 -0 -1 -1 5 -1 5 diff --git a/tests/queries/0_stateless/04365_filter_push_down_window_partition_by.sql b/tests/queries/0_stateless/04365_filter_push_down_window_partition_by.sql deleted file mode 100644 index 6fabb5c2dae5d..0000000000000 --- a/tests/queries/0_stateless/04365_filter_push_down_window_partition_by.sql +++ /dev/null @@ -1,97 +0,0 @@ --- Tags: no-parallel-replicas --- ^ EXPLAIN indexes=1 asserts the pushed key predicate becomes the local primary key --- condition; under parallel replicas the table is read on remote replicas, so the --- coordinator plan carries no such condition and the assertions do not hold. --- Filter push down below WindowStep on PARTITION BY columns (issue #110109). --- A predicate referencing only the window PARTITION BY columns is safe to apply before --- the window, so it must reach storage as a primary key condition and enable pruning. - -SET enable_analyzer = 1; -SET query_plan_filter_push_down = 1; - -DROP TABLE IF EXISTS t_04365; -CREATE TABLE t_04365 (key String, ts DateTime, val UInt64) -ENGINE = MergeTree ORDER BY (key, ts) -AS SELECT toString(number % 100) AS key, toDateTime(number) AS ts, number AS val -FROM numbers(100000); -OPTIMIZE TABLE t_04365 FINAL; - --- Helper output: `pushed` = 1 when the partition-key predicate reached ReadFromMergeTree --- as a primary key Condition (so it was pushed below the window). Matching the --- ReadFromMergeTree "Condition:" line is robust to randomized index_granularity / --- plan-shape settings (no arithmetic on the granule counts, which can differ or be --- absent under some randomized settings). - --- QUALIFY form: partition-key conjunct pushed below Window -> PK condition on storage. -SELECT count() > 0 AS pushed FROM ( - EXPLAIN indexes = 1 - SELECT key, ts, row_number() OVER (PARTITION BY key ORDER BY ts) AS rn - FROM t_04365 QUALIFY rn = 1 AND key = '5' -) WHERE explain ILIKE '%Condition:%key%'; - --- Outer WHERE around a windowed subquery: same push down. -SELECT count() > 0 AS pushed FROM ( - EXPLAIN indexes = 1 - SELECT * FROM ( - SELECT key, ts, row_number() OVER (PARTITION BY key ORDER BY ts) AS rn FROM t_04365 - ) WHERE key = '5' -) WHERE explain ILIKE '%Condition:%key%'; - --- Multiple windows sharing the partition key: the common conjunct still reaches storage. -SELECT count() > 0 AS pushed FROM ( - EXPLAIN indexes = 1 - SELECT key, ts, - row_number() OVER (PARTITION BY key ORDER BY ts) AS rn, - sum(val) OVER (PARTITION BY key ORDER BY ts) AS s - FROM t_04365 QUALIFY rn = 1 AND key = '5' -) WHERE explain ILIKE '%Condition:%key%'; - --- Mixed predicate: the partition-key conjunct pushes (PK condition on storage), the --- window-result conjunct stays above. -SELECT count() > 0 AS pushed FROM ( - EXPLAIN indexes = 1 - SELECT key, ts, row_number() OVER (PARTITION BY key ORDER BY ts) AS rn - FROM t_04365 QUALIFY rn = 1 AND key = '5' AND ts > toDateTime(10) -) WHERE explain ILIKE '%Condition:%key%'; - --- Negative: a predicate on the window result only must NOT be pushed (no PK condition). -SELECT count() > 0 AS pushed FROM ( - EXPLAIN indexes = 1 - SELECT key, ts, row_number() OVER (PARTITION BY key ORDER BY ts) AS rn - FROM t_04365 QUALIFY rn = 1 -) WHERE explain ILIKE '%Condition:%key%'; - --- No PARTITION BY: nothing to push, must not crash and must not push a PK condition. -SELECT count() > 0 AS pushed FROM ( - EXPLAIN indexes = 1 - SELECT key, ts, row_number() OVER (ORDER BY ts) AS rn - FROM t_04365 QUALIFY rn = 1 AND key = '5' -) WHERE explain ILIKE '%Condition:%key%'; - --- Negative: a non-deterministic conjunct on the partition key (`rand64(key) % 2 = 0`) --- must NOT be pushed below the window. Pushing it would remove arbitrary rows inside a --- surviving partition before the window runs and could change which row becomes --- row_number() = 1. With prewhere pinned off, a pushed conjunct would appear as a --- separate Filter step below the window (its `Filter column:` referencing rand64), so --- the count of `Filter column:` lines mentioning rand64 must stay 1 (only the QUALIFY --- filter kept above the window); a value of 2 means it was wrongly pushed down. -SELECT countIf(explain ILIKE '%Filter column:%rand64%') = 1 AS not_pushed FROM ( - EXPLAIN actions = 1 - SELECT key, ts, row_number() OVER (PARTITION BY key ORDER BY ts) AS rn - FROM t_04365 QUALIFY rn = 1 AND rand64(key) % 2 = 0 - SETTINGS optimize_move_to_prewhere = 0, query_plan_optimize_prewhere = 0 -); - --- Correctness is preserved: the optimized query returns the same rows as forcing the --- filter before the window explicitly. -SELECT count(), sum(val) FROM ( - SELECT key, ts, val, row_number() OVER (PARTITION BY key ORDER BY ts) AS rn - FROM t_04365 QUALIFY rn = 1 AND key = '5' -); -SELECT count(), sum(val) FROM ( - SELECT key, ts, val, row_number() OVER (PARTITION BY key ORDER BY ts) AS rn - FROM (SELECT * FROM t_04365 WHERE key = '5') QUALIFY rn = 1 -); - -DROP TABLE t_04365; - diff --git a/tests/queries/0_stateless/04366_filter_push_down_limit_by.reference b/tests/queries/0_stateless/04366_filter_push_down_limit_by.reference index 24970f73243fa..fa1f28c186664 100644 --- a/tests/queries/0_stateless/04366_filter_push_down_limit_by.reference +++ b/tests/queries/0_stateless/04366_filter_push_down_limit_by.reference @@ -1,9 +1,17 @@ 1 0 +1 +0 +0 +0 0 0 1 0 1 5 1 5 +1 5 a +false +true +true diff --git a/tests/queries/0_stateless/04366_filter_push_down_limit_by.sql b/tests/queries/0_stateless/04366_filter_push_down_limit_by.sql index 587399f309b54..c26c4b44f01eb 100644 --- a/tests/queries/0_stateless/04366_filter_push_down_limit_by.sql +++ b/tests/queries/0_stateless/04366_filter_push_down_limit_by.sql @@ -34,6 +34,89 @@ FROM ( ) WHERE key = '5' ); +-- Same query with query_plan_filter_push_down_below_limit_by = 0: the conjunct stays above the +-- LIMIT BY, so the primary key gets no condition -> the plan shape used before 26.7. +SELECT countIf(match(explain, 'Condition: \(key in ')) > 0 AS pushed +FROM ( + EXPLAIN indexes = 1 + SELECT * FROM ( + SELECT key, ts, val FROM t_04366 ORDER BY key, ts LIMIT 1 BY key + ) WHERE key = '5' SETTINGS query_plan_filter_push_down_below_limit_by = 0 +); + +-- `filter_above_limitby` = 1 iff a Filter step exists AND its plan line precedes LimitBy's. +-- Absence of a primary key condition alone cannot tell "filter above LimitBy" from "filter +-- pushed below it but index analysis produced nothing", so the OFF path needs this placement +-- oracle too. Same shape as 04512_filter_push_down_limit_by_nondeterministic. +SELECT (has_filter AND fl < ll) AS filter_above_limitby +FROM ( + SELECT countIf(explain LIKE '%Filter (%') > 0 AS has_filter, + minIf(ln, explain LIKE '%Filter (%') AS fl, + minIf(ln, explain LIKE '%LimitBy%') AS ll + FROM ( + SELECT explain, rowNumberInAllBlocks() AS ln + FROM ( + EXPLAIN + SELECT * FROM ( + SELECT key, ts, val FROM t_04366 ORDER BY key, ts LIMIT 1 BY key + ) WHERE key = '5' SETTINGS query_plan_filter_push_down_below_limit_by = 0, enable_parallel_replicas = 0 + ) + ) +); + +-- The push-down pass also runs with `query_plan_filter_push_down = 0` once a JOIN runtime +-- filter has been added, so the setting has to be honoured on that path as well: at its default +-- the key conjunct still reaches below LimitBy (no Filter step above it -> 0), at 0 the conjunct +-- stays above LimitBy (-> 1). The two arms differ only in the new setting. +-- `query_plan_optimize_join_order_randomize = 0` (its default) is pinned in both arms because a +-- non-zero seed feeds the join order optimizer random cardinalities, which can reverse the two +-- join inputs; the reversed plan prints the right-hand branch first and `fl < ll` then compares +-- lines from different branches. +DROP TABLE IF EXISTS t_04366_rhs; +CREATE TABLE t_04366_rhs (key String) ENGINE = MergeTree ORDER BY key AS SELECT '5'; + +SELECT (has_filter AND fl < ll) AS filter_above_limitby +FROM ( + SELECT countIf(explain LIKE '%Filter (%') > 0 AS has_filter, + minIf(ln, explain LIKE '%Filter (%') AS fl, + minIf(ln, explain LIKE '%LimitBy%') AS ll + FROM ( + SELECT explain, rowNumberInAllBlocks() AS ln + FROM ( + EXPLAIN + SELECT l.key FROM ( + SELECT key, ts FROM t_04366 ORDER BY key, ts LIMIT 1 BY key + ) AS l INNER JOIN t_04366_rhs AS r ON l.key = r.key + WHERE l.key = '5' + SETTINGS query_plan_filter_push_down = 0, query_plan_filter_push_down_below_limit_by = 1, + enable_join_runtime_filters = 1, enable_parallel_replicas = 0, + query_plan_optimize_join_order_randomize = 0 + ) + ) +); + +SELECT (has_filter AND fl < ll) AS filter_above_limitby +FROM ( + SELECT countIf(explain LIKE '%Filter (%') > 0 AS has_filter, + minIf(ln, explain LIKE '%Filter (%') AS fl, + minIf(ln, explain LIKE '%LimitBy%') AS ll + FROM ( + SELECT explain, rowNumberInAllBlocks() AS ln + FROM ( + EXPLAIN + SELECT l.key FROM ( + SELECT key, ts FROM t_04366 ORDER BY key, ts LIMIT 1 BY key + ) AS l INNER JOIN t_04366_rhs AS r ON l.key = r.key + WHERE l.key = '5' + SETTINGS query_plan_filter_push_down = 0, query_plan_filter_push_down_below_limit_by = 0, + enable_join_runtime_filters = 1, enable_parallel_replicas = 0, + query_plan_optimize_join_order_randomize = 0 + ) + ) +); + +DROP TABLE t_04366_rhs; + -- LIMIT n OFFSET m BY (m > 0): NOT pushed. A group of size <= m is fully dropped, so a -- pushed throwing key predicate could be evaluated on rows the query never reaches. SELECT countIf(match(explain, 'Condition: \(key in ')) > 0 AS pushed @@ -94,6 +177,11 @@ SELECT count(), sum(val) FROM ( SELECT key, ts, val FROM t_04366 WHERE key = '5' ORDER BY key, ts LIMIT 1 BY key ); +-- Same result with the push-down declined, so the OFF path is not covered by EXPLAIN alone. +SELECT count(), sum(val) FROM ( + SELECT key, ts, val FROM t_04366 ORDER BY key, ts LIMIT 1 BY key +) WHERE key = '5' SETTINGS query_plan_filter_push_down_below_limit_by = 0; + DROP TABLE t_04366; -- Exception-semantics regression: a singleton group '0' dropped by OFFSET 1 must NOT be @@ -129,3 +217,12 @@ SELECT key FROM ( SELECT key, x FROM t_04366_nonkey ORDER BY key, ord LIMIT 1 BY key ) WHERE intDiv(1, x) > 0; DROP TABLE t_04366_nonkey; + +-- `compatibility` with a version before the one the setting was added in turns it off, so the +-- filter stays above the `LIMIT BY`; from that version on, and unset, it is on. +SET compatibility = '26.7'; +SELECT getSetting('query_plan_filter_push_down_below_limit_by'); +SET compatibility = '26.8'; +SELECT getSetting('query_plan_filter_push_down_below_limit_by'); +SET compatibility = ''; +SELECT getSetting('query_plan_filter_push_down_below_limit_by'); diff --git a/tests/queries/0_stateless/04411_ttl_drop_not_vertical.reference b/tests/queries/0_stateless/04411_ttl_drop_not_vertical.reference new file mode 100644 index 0000000000000..876d94d1857a2 --- /dev/null +++ b/tests/queries/0_stateless/04411_ttl_drop_not_vertical.reference @@ -0,0 +1,4 @@ +TTLDropMerge Horizontal 0 +0 +TTLDropMerge Horizontal 0 +0 diff --git a/tests/queries/0_stateless/04411_ttl_drop_not_vertical.sh b/tests/queries/0_stateless/04411_ttl_drop_not_vertical.sh new file mode 100755 index 0000000000000..d34ea2f74dc6a --- /dev/null +++ b/tests/queries/0_stateless/04411_ttl_drop_not_vertical.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -e + +TABLE=t_ttl_drop_not_vertical + +function wait_for_ttl_drop_merge() +{ + for _ in $(seq 1 600); do + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS part_log" + local merge_count + merge_count=$(${CLICKHOUSE_CLIENT} -q " + SELECT count() + FROM system.part_log + WHERE database = currentDatabase() + AND table = '${TABLE}' + AND event_type = 'MergeParts' + AND merge_reason = 'TTLDropMerge'") + + if [ "${merge_count}" -gt "0" ]; then + return + fi + sleep 0.1 + done + + echo "Timed out waiting for a TTLDropMerge of ${TABLE}" >&2 + exit 1 +} + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${TABLE}" + +${CLICKHOUSE_CLIENT} -q " + CREATE TABLE ${TABLE} + ( + id UInt64, + d DateTime DEFAULT '2000-01-01 00:00:00', + c1 UInt64, + c2 UInt64, + c3 UInt64, + c4 UInt64 + ) + ENGINE = MergeTree + ORDER BY id + TTL d + INTERVAL 1 DAY + SETTINGS + ttl_only_drop_parts = 1, + merge_with_ttl_timeout = 0, + min_bytes_for_wide_part = 0, + min_bytes_for_full_part_storage = 0, + enable_block_number_column = 0, + enable_block_offset_column = 0, + vertical_merge_algorithm_min_rows_to_activate = 1, + vertical_merge_algorithm_min_columns_to_activate = 1, + vertical_merge_optimize_ttl_delete = 1, + ratio_of_defaults_for_sparse_serialization = 1.0" + +${CLICKHOUSE_CLIENT} -q "SYSTEM STOP TTL MERGES ${TABLE}" +${CLICKHOUSE_CLIENT} -q "SYSTEM STOP MERGES ${TABLE}" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${TABLE} + SELECT number, '2000-01-01 00:00:00', number, number, number, number + FROM numbers(1000)" +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${TABLE} + SELECT number + 1000, '2000-01-01 00:00:00', number, number, number, number + FROM numbers(1000)" + +${CLICKHOUSE_CLIENT} -q "SYSTEM START TTL MERGES ${TABLE}" +${CLICKHOUSE_CLIENT} -q "SYSTEM START MERGES ${TABLE}" + +wait_for_ttl_drop_merge + +${CLICKHOUSE_CLIENT} -q " + SELECT + merge_reason, + merge_algorithm, + rows + FROM system.part_log + WHERE database = currentDatabase() + AND table = '${TABLE}' + AND event_type = 'MergeParts' + AND merge_reason = 'TTLDropMerge' + ORDER BY event_time_microseconds DESC + LIMIT 1" + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM ${TABLE}" +${CLICKHOUSE_CLIENT} -q "DROP TABLE ${TABLE}" + +TABLE=t_ttl_drop_not_vertical_mixed_ttl + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${TABLE}" + +${CLICKHOUSE_CLIENT} -q " + CREATE TABLE ${TABLE} + ( + id UInt64, + d DateTime DEFAULT '2000-01-01 00:00:00', + c1 UInt64, + c2 UInt64, + c3 UInt64, + c4 UInt64 + ) + ENGINE = MergeTree + ORDER BY id + TTL d + INTERVAL 1 DAY, d + INTERVAL 2 DAY RECOMPRESS CODEC(ZSTD) + SETTINGS + ttl_only_drop_parts = 1, + merge_with_ttl_timeout = 0, + min_bytes_for_wide_part = 0, + min_bytes_for_full_part_storage = 0, + enable_block_number_column = 0, + enable_block_offset_column = 0, + vertical_merge_algorithm_min_rows_to_activate = 1, + vertical_merge_algorithm_min_columns_to_activate = 1, + vertical_merge_optimize_ttl_delete = 1, + ratio_of_defaults_for_sparse_serialization = 1.0" + +${CLICKHOUSE_CLIENT} -q "SYSTEM STOP TTL MERGES ${TABLE}" +${CLICKHOUSE_CLIENT} -q "SYSTEM STOP MERGES ${TABLE}" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${TABLE} + SELECT number, '2000-01-01 00:00:00', number, number, number, number + FROM numbers(1000)" +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${TABLE} + SELECT number + 1000, '2000-01-01 00:00:00', number, number, number, number + FROM numbers(1000)" + +${CLICKHOUSE_CLIENT} -q "SYSTEM START TTL MERGES ${TABLE}" +${CLICKHOUSE_CLIENT} -q "SYSTEM START MERGES ${TABLE}" + +wait_for_ttl_drop_merge + +${CLICKHOUSE_CLIENT} -q " + SELECT + merge_reason, + merge_algorithm, + rows + FROM system.part_log + WHERE database = currentDatabase() + AND table = '${TABLE}' + AND event_type = 'MergeParts' + AND merge_reason = 'TTLDropMerge' + ORDER BY event_time_microseconds DESC + LIMIT 1" + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM ${TABLE}" +${CLICKHOUSE_CLIENT} -q "DROP TABLE ${TABLE}" diff --git a/tests/queries/0_stateless/04491_vector_search_rescoring_lightweight_delete_filter.reference b/tests/queries/0_stateless/04491_vector_search_rescoring_lightweight_delete_filter.reference index d0d1481c95a75..947ccbe14a42f 100644 --- a/tests/queries/0_stateless/04491_vector_search_rescoring_lightweight_delete_filter.reference +++ b/tests/queries/0_stateless/04491_vector_search_rescoring_lightweight_delete_filter.reference @@ -1,4 +1,4 @@ bruteforce reads a surviving granule mate 1 -rescoring row filter excludes the granule mate -0 +the same answer with the index +[0,2,4] diff --git a/tests/queries/0_stateless/04491_vector_search_rescoring_lightweight_delete_filter.sql b/tests/queries/0_stateless/04491_vector_search_rescoring_lightweight_delete_filter.sql index 456d783eee65d..8bd7b8361b4ab 100644 --- a/tests/queries/0_stateless/04491_vector_search_rescoring_lightweight_delete_filter.sql +++ b/tests/queries/0_stateless/04491_vector_search_rescoring_lightweight_delete_filter.sql @@ -34,9 +34,14 @@ FROM SETTINGS use_skip_indexes = 0 ); -SELECT 'rescoring row filter excludes the granule mate'; +-- The row filter restricts the read to the rows the vector index returned, and the index still holds +-- the deleted ones, so applying it while a lightweight delete is unmaterialized would drop candidates +-- with nothing to take their place - the query would return fewer rows than its LIMIT and miss the +-- neighbours a deleted candidate shadowed. The index is not used for such a part, so the part is read in +-- full and the answer is the same as the bruteforce one. +SELECT 'the same answer with the index'; WITH [1.0, 0.0] AS reference_vec -SELECT throwIf(has(groupArray(id), 4), 'Vector search row filter was not applied after lightweight delete') +SELECT arraySort(groupArray(id)) FROM ( SELECT id diff --git a/tests/queries/0_stateless/04510_distributed_final_many_partitions.reference b/tests/queries/0_stateless/04510_distributed_final_many_partitions.reference index 1545bdb2fa433..7caab58d91b66 100644 --- a/tests/queries/0_stateless/04510_distributed_final_many_partitions.reference +++ b/tests/queries/0_stateless/04510_distributed_final_many_partitions.reference @@ -1,3 +1,3 @@ split groups all partitions into the target tasks 1 -distributed plan 15000 112567500 -plain plan 15000 112567500 +distributed plan 6500 21154250 +plain plan 6500 21154250 diff --git a/tests/queries/0_stateless/04510_distributed_final_many_partitions.sh b/tests/queries/0_stateless/04510_distributed_final_many_partitions.sh index 5c4cfb379b5e4..1e4009602a451 100755 --- a/tests/queries/0_stateless/04510_distributed_final_many_partitions.sh +++ b/tests/queries/0_stateless/04510_distributed_final_many_partitions.sh @@ -18,28 +18,28 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # not expose the split, so the check reads the planner's trace message. $CLICKHOUSE_CLIENT --multiquery -q " -SET max_partitions_per_insert_block = 150; --- Statistics play no role here, and writing a statistics blob for each of the 300 tiny parts --- makes the setup exceed the test time limit on sanitizer builds with object storage. +-- Statistics play no role here, and writing a statistics blob for each tiny part slows the setup +-- down on sanitizer builds with object storage. SET materialize_statistics_on_insert = 0; DROP TABLE IF EXISTS t_final_many_partitions; CREATE TABLE t_final_many_partitions (k UInt64, p UInt16, v UInt64, ver UInt64) ENGINE = ReplacingMergeTree(ver) PARTITION BY p ORDER BY k SETTINGS index_granularity = 64, auto_statistics_types = ''; SYSTEM STOP MERGES t_final_many_partitions; --- 150 partitions, far more than the 4 tasks x 16 layers per task the planner aims for. Every key +-- 65 partitions, one more than the 4 tasks x 16 layers per task the planner aims for. Every key -- appears in every partition and must survive per-partition dedup, and its newer version must win. -INSERT INTO t_final_many_partitions SELECT number % 100, intDiv(number, 100), number, 1 FROM numbers(15000); -INSERT INTO t_final_many_partitions SELECT number % 100, intDiv(number, 100), number + 5, 2 FROM numbers(15000); +INSERT INTO t_final_many_partitions SELECT number % 100, intDiv(number, 100), number, 1 FROM numbers(6500); +INSERT INTO t_final_many_partitions SELECT number % 100, intDiv(number, 100), number + 5, 2 FROM numbers(6500); " -SETTINGS="enable_parallel_replicas = 0, automatic_parallel_replicas_mode = 0, max_rows_to_group_by = 0, +# The table is small enough to be broadcast; force the bucketed read regardless of size. +SETTINGS="enable_parallel_replicas = 0, automatic_parallel_replicas_mode = 0, max_rows_to_group_by = 0, distributed_plan_max_rows_to_broadcast = 0, distributed_plan_default_reader_bucket_count = 4, do_not_merge_across_partitions_select_final = 1" echo -n "split groups all partitions into the target tasks " if $CLICKHOUSE_CLIENT --send_logs_level=trace -q " SELECT count(), sum(v) FROM t_final_many_partitions FINAL FORMAT Null SETTINGS make_distributed_plan = 1, $SETTINGS" 2>&1 \ - | grep -q "Distributed FINAL read bucketed: 150 layers in 38 lanes per task make 4 tasks" + | grep -q "Distributed FINAL read bucketed: 65 layers in 17 lanes per task make 4 tasks" then echo 1; else echo 0; fi # Both plans must return identical results. diff --git a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference index eb36b31ef3f96..5aa390bf9bb08 100644 --- a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference +++ b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference @@ -138,7 +138,7 @@ UNION id: 0, union_mode: UNION_ALL CONSTANT id: 14, constant_value: UInt64_1, constant_value_type: UInt8 JOIN TREE IDENTIFIER id: 15, identifier: system.one --- session_token, the Google ADC secrets (google_adc_client_secret, google_adc_refresh_token) and\n-- the extra_credentials assume-role material (external_id) passed to the explicit-url or\n-- named-collection S3 form must be masked like secret_access_key. Every secret value below is tagged\n-- so the final assertion can prove none of them leaks. They used to leak in plaintext in SHOW CREATE\n-- and logged query text.\n\n-- Engine form: SHOW CREATE hides every secret; the non-secret extra_credentials identifiers\n-- (role_arn, role_session_name) stay visible while external_id is hidden.\nDROP TABLE IF EXISTS t_04510; +-- session_token, the Google ADC secrets (google_adc_client_secret, google_adc_refresh_token) and\n-- the extra_credentials assume-role material (external_id) passed to the explicit-url or\n-- named-collection S3 form must be masked like secret_access_key. Every secret value below is tagged\n-- so the final assertion can prove none of them leaks. They used to leak in plaintext in SHOW CREATE\n-- and logged query text.\n\n-- Engine form: SHOW CREATE hides every secret; the non-secret extra_credentials identifier\n-- (role_arn) stays visible while external_id is hidden.\nDROP TABLE IF EXISTS t_04510; CREATE TABLE t_04510 (`x` UInt8) ENGINE = S3(\'http://localhost:11111/test/04510\', \'ak\', \'[HIDDEN]\', session_token = \'[HIDDEN]\', google_adc_client_secret = \'[HIDDEN]\', google_adc_refresh_token = \'[HIDDEN]\', extra_credentials(role_arn = \'visible_role_arn\', external_id = \'[HIDDEN]\'), format = \'TSV\') SHOW CREATE TABLE t_04510 SETTINGS format_display_secrets_in_show_and_select = 0; DROP TABLE t_04510; @@ -197,6 +197,46 @@ BACKUP TABLE nonexistent_04510 TO S3(\'url_bkp_pos\', \'[HIDDEN]\', \'[HIDDEN]\' BACKUP TABLE nonexistent_04510 TO S3(nc_bkp_missing, \'visible_bkp_dir\', \'[HIDDEN]\') BACKUP TABLE nonexistent_04510 TO S3(nc_bkporder_missing, secret_access_key = \'[HIDDEN]\', \'visible_bkp_dir2\') BACKUP TABLE nonexistent_04510 TO S3(\'url_bkp_mixed\', equals(access_key_id, \'ak\'), \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO File(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO Disk(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO Memory(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO File(\'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO `Null`(\'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO Foo(\'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO Null(); +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureQueue(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=[HIDDEN];\', \'cont\', \'visible_04510_dir/b.zip\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'http://localhost:11111/acct\', \'visible_04510_cont\', \'visible_04510_dir/b.zip\'); +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, connection_string = \'DefaultEndpointsProtocol=https;AccountName=visible_04510_acct;AccountKey=[HIDDEN];\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, account_key = \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, equals(storage_account_url, \'http://localhost:11111/visible_04510_url\'), account_key = \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, container = \'visible_04510_c1\', container = \'visible_04510_c2\'); +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'http://localhost:11111/visible_04510_acct5\', \'visible_04510_cont5\', \'visible_04510_blob5\', \'visible_04510_acctname5\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte1 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte2 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte3 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte4 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte5 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte6 (`x` UInt8) ENGINE = AzureBlobStorage(\'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=[HIDDEN];\', \'visible_04510_teco\', \'visible_04510_teblob\') +CREATE TABLE t_04510_azte7 (`x` UInt8) ENGINE = AzureBlobStorage(\'http://localhost:11111/visible_04510_teacct\', \'visible_04510_tec5\', \'visible_04510_teb5\', \'visible_04510_teacctname\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte8 (`x` UInt8) ENGINE = AzureQueue(\'http://localhost:11111/visible_04510_teq/cont/*\', \'[HIDDEN]\') SETTINGS mode = \'unordered\' CREATE DATABASE db_04510_ec ENGINE = Backup(\'\', S3(\'url_dbec\', \'ak\', \'[HIDDEN]\', extra_credentials(external_id = \'[HIDDEN]\'))) CREATE DATABASE db_04510_postok ENGINE = Backup(\'\', S3(\'url_dbpostok\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) CREATE DATABASE db_04510_ncpos ENGINE = Backup(\'\', S3(nc_dbnc_missing, \'visible_dbnc_dir\', \'[HIDDEN]\')) @@ -207,6 +247,21 @@ CREATE DATABASE db_04510_mixed ENGINE = Backup(\'\', S3(\'url_dbmixed\', access_ CREATE DATABASE db_04510_ncurl ENGINE = Backup(\'\', S3(nc_dburl_missing, url = \'[HIDDEN]\')) CREATE DATABASE db_04510_hdr ENGINE = Backup(\'\', S3(\'url_dbhdr\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) CREATE DATABASE db_04510_expr ENGINE = Backup(\'\', S3(\'url_dbexpr\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_quoted ENGINE = Backup(\'\', \'[HIDDEN]\') +CREATE DATABASE db_04510_tail ENGINE = Backup(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE DATABASE db_04510_tmap ENGINE = Backup(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE DATABASE db_04510_lone ENGINE = Backup(\'[HIDDEN]\') +CREATE DATABASE db_04510_swap ENGINE = Backup(\'[HIDDEN]\', \'[HIDDEN]\') +CREATE DATABASE db_04510_ident ENGINE = Backup(\'[HIDDEN]\', \'[HIDDEN]\') +CREATE DATABASE db_04510_ftail ENGINE = Backup(\'src_04510\', File(\'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_fover ENGINE = Backup(\'src_04510\', File(\'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_dover ENGINE = Backup(\'src_04510\', Disk(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_mover ENGINE = Backup(\'src_04510\', Memory(\'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_fvalid ENGINE = Backup(\'src_04510\', File(\'nonexistent_04510\')); +CREATE DATABASE db_04510_dvalid ENGINE = Backup(\'src_04510\', Disk(\'backups\', \'nonexistent_04510\')); +CREATE DATABASE db_04510_eval ENGINE = Backup(\'\', S3(\'http://localhost:11111/test/04510eval\', \'ak\', \'[HIDDEN]\', extra_credentials(external_id = \'[HIDDEN]\'))) +CREATE DATABASE db_04510_nonlit ENGINE = Backup(\'\', S3(\'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_azure ENGINE = Backup(\'\', AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) CREATE DATABASE db_04510_s3pos ENGINE = S3(\'url_dbs3pos\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\') DROP DATABASE IF EXISTS default_1 CREATE DATABASE default_1 ENGINE = S3(\'url_dbenv\', \'ak\', \'[HIDDEN]\', use_environment_credentials = 1) @@ -218,4 +273,4 @@ EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test/04510qt\', NOSIGN, \'TSV\', \'x UInt8\', headers(\'Authorization\' = \'[HIDDEN]\')) EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(nc_04510_missing, url = \'https://[HIDDEN]@localhost:11111/test/04510qt?X-Amz-Signature=[HIDDEN]\', structure = \'x UInt8\') EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test/04510qt\', \'ak\', \'[HIDDEN]\', \'TSV\', \'x UInt8\') UNION ALL SELECT 1 -1 0 +1 0 0 diff --git a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql index 9cded2e5e118a..4f7eee6fb5109 100644 --- a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql +++ b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql @@ -7,8 +7,8 @@ -- so the final assertion can prove none of them leaks. They used to leak in plaintext in SHOW CREATE -- and logged query text. --- Engine form: SHOW CREATE hides every secret; the non-secret extra_credentials identifiers --- (role_arn, role_session_name) stay visible while external_id is hidden. +-- Engine form: SHOW CREATE hides every secret; the non-secret extra_credentials identifier +-- (role_arn) stays visible while external_id is hidden. DROP TABLE IF EXISTS t_04510; CREATE TABLE t_04510 (x UInt8) ENGINE = S3('http://localhost:11111/test/04510', 'ak', 'SEKRIT_SAK', @@ -234,6 +234,138 @@ BACKUP TABLE nonexistent_04510 TO S3(nc_bkporder_missing, BACKUP TABLE nonexistent_04510 TO S3('url_bkp_mixed', access_key_id = 'ak', 'SEKRIT_BKPMIX'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +-- The credential-free engines (Disk, File, Memory, Null) are credential-free only in the shape they +-- read: File and Memory take one argument, Disk two, Null none. A surplus argument is rejected after +-- the statement is logged, so it reaches the log holding whatever was written in it, an argument that is +-- no string is read by none of these engines and can carry a string of its own, and an engine name that +-- is not registered at all has no known shape to trust; all three must be hidden. The last statement is +-- the control: the shape Null does read carries nothing to hide, so it stays visible verbatim and gets +-- as far as resolving the table. +BACKUP TABLE nonexistent_04510 TO File('nonexistent_04510', + 'SEKRIT_TOFILEOVER'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO Disk('backups', 'nonexistent_04510', + 'SEKRIT_TODISKOVER'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO Memory('nonexistent_04510', + 'SEKRIT_TOMEMOVER'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO File(['SEKRIT_TOFILEARR']); -- { serverError BAD_GET } +BACKUP TABLE nonexistent_04510 TO Null('SEKRIT_TONULLOVER'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO Foo('SEKRIT_TOUNKNOWN'); -- { serverError BACKUP_ENGINE_NOT_FOUND } +BACKUP TABLE nonexistent_04510 TO Null(); -- { serverError UNKNOWN_TABLE } + +-- The AzureBlobStorage backup destination reads a different signature than the table engine of the same +-- name: a named collection with an optional filename, three arguments (connection string or account url, +-- container, path), or five (adding account_name and account_key). An argument outside those shapes is +-- rejected only after the statement is logged, and AzureQueue has no backup engine at all. The last +-- two statements are the controls: a connection string hides its AccountKey, and the three-argument +-- shape has nothing to hide, so it stays visible verbatim. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://localhost:11111/acct', 'cont', 'blob', + 'SEKRIT_AZTO4'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, 'dir', + 'SEKRIT_AZTONCPOS'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureQueue('http://localhost:11111/acct', 'cont', 'blob', + 'SEKRIT_AZQTO'); -- { serverError BACKUP_ENGINE_NOT_FOUND } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZTOCSKEY==;', + 'cont', 'blob', 'acct', 'SEKRIT_AZTOCS5'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('DefaultEndpointsProtocol=https;AccountName=a;AccountKey=c2VrcmV0Cg==;', + 'cont', 'visible_04510_dir/b.zip'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://localhost:11111/acct', 'visible_04510_cont', 'visible_04510_dir/b.zip'); -- { serverError BAD_ARGUMENTS } + +-- A named collection can be overridden per statement, and the destination evaluates those overrides as +-- constant expressions. An override this rule cannot read may hold either credential, and hiding a +-- connection string replaces the whole argument, which cannot be combined with hiding account_key, so +-- both shapes hide the locator whole. The last three statements are the controls: a connection string +-- alone still hides only its AccountKey, account_key alone is hidden by itself, and an account url +-- override does not take the replacement path, so it stays visible next to a hidden account_key. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = concat('DefaultEndpointsProtocol=https;AccountName=a;AccountKey=', + 'SEKRIT_AZNCEXPR')); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = 'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=c2VrcmV0Cg==;', + account_key = 'SEKRIT_AZNCBOTH'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + storage_account_url = concat('https://a.blob.core.windows.net/c?sig=', + 'SEKRIT_AZNCURLEXPR')); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = 'DefaultEndpointsProtocol=https;AccountName=visible_04510_acct;AccountKey=SEKRIT_AZNCCS==;'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + account_key = 'SEKRIT_AZNCKEY'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + storage_account_url = 'http://localhost:11111/visible_04510_url', + account_key = 'SEKRIT_AZNCURLKEY'); -- { serverError BAD_ARGUMENTS } + +-- An override key can be an expression too, and one this rule cannot read hides which credential the +-- value is: both the computed key and a malformed override hide the locator whole, as the S3 form above +-- already does for a key it cannot read. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + concat('account_', 'key') = 'SEKRIT_AZNCKEYEXPR'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + equals('account_key', 'SEKRIT_AZNCEQ3', 'surplus')); -- { serverError BAD_ARGUMENTS } + +-- A readable key does not make the override readable: the destination evaluates its value as a constant +-- expression, so a value that is no plain literal or identifier can nest a credential of its own and is +-- formatted verbatim before that evaluation rejects it, as the S3 form above already is for such a value. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + container = headers('Authorization' = 'SEKRIT_AZNCHDR')); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + blob_path = concat('SEKRIT_AZNCPATHEXPR', '/b.zip')); -- { serverError BAD_ARGUMENTS } + +-- connection_string and storage_account_url are mutually exclusive, and the destination reads at most +-- one of them, so an override it never reads holds whatever was written there. The last statement is +-- the control: a key that carries no credential stays visible however often it is repeated. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = 'http://localhost:11111/acct', + storage_account_url = 'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZNCPAIR==;'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = 'http://localhost:11111/acct', + connection_string = 'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZNCDUP==;'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, container = 'visible_04510_c1', container = 'visible_04510_c2'); -- { serverError BAD_ARGUMENTS } + +-- An account url is shown only when it is a plain storage account URL, which is what the destination +-- requires beside explicit credentials: userinfo, a query string (a SAS is a credential) and a fragment +-- each carry a credential of their own, in any shape and under any scheme spelling. A connection value +-- that is no string is read by neither this rule nor the destination, so it is hidden as well. The last +-- statement is the control: a plain url keeps the account url, container, path and account name visible +-- while account_key is hidden, and reaches the account_key decoding that rejects it. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + storage_account_url = 'http://user:SEKRIT_AZNCUSERINFO@localhost:11111/acct'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + storage_account_url = 'HTTPS://localhost:11111/acct?sig=SEKRIT_AZNCSAS'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://user:SEKRIT_AZ3USERINFO@localhost:11111/acct', + 'cont', 'blob'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://localhost:11111/acct#f', 'cont', 'blob', + 'acct', 'SEKRIT_AZTO5KEY'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(['SEKRIT_AZ3ARR'], 'cont', 'blob'); -- { serverError BAD_GET } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://localhost:11111/visible_04510_acct5', + 'visible_04510_cont5', 'visible_04510_blob5', 'visible_04510_acctname5', + 'SEKRIT_AZ5PLAINKEY'); -- { serverError STD_EXCEPTION } + +-- The AzureBlobStorage table engine reads its arguments the same way: an override whose value or key +-- this rule cannot read may hold a credential the engine does read (the parser evaluates both as +-- constant expressions), an account url is shown only when it is a plain storage account url, and a +-- connection value that is no string is read by neither. A connection string is hidden by replacing +-- its whole argument, which cannot be combined with hiding a following account_key. The last two +-- statements are the controls: the connection string alone still hides only its AccountKey, and a +-- plain account url keeps url, container, path and account name visible beside a hidden account_key. +CREATE TABLE t_04510_azte1 (x UInt8) ENGINE = AzureBlobStorage(nc_04510_missing, + connection_string = headers('Authorization' = 'SEKRIT_AZTEHDR')); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } +CREATE TABLE t_04510_azte2 (x UInt8) ENGINE = AzureBlobStorage(nc_04510_missing, + upper('account_key') = 'SEKRIT_AZTEKEY'); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } +CREATE TABLE t_04510_azte3 (x UInt8) ENGINE = AzureBlobStorage(['SEKRIT_AZTEARR'], 'cont', 'blob'); -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_04510_azte4 (x UInt8) ENGINE = AzureBlobStorage('http://localhost:11111/acct?sig=SEKRIT_AZTESAS', + 'cont', 'blob', 'acct', 'SEKRIT_AZTE5KEY'); -- { serverError STD_EXCEPTION } +CREATE TABLE t_04510_azte5 (x UInt8) ENGINE = AzureBlobStorage('DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZTECS==;', + 'cont', 'blob', 'acct', 'SEKRIT_AZTEMIXKEY'); -- { serverError STD_EXCEPTION } +CREATE TABLE t_04510_azte6 (x UInt8) ENGINE = AzureBlobStorage('DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZTECTLCS==;', + 'visible_04510_teco', 'visible_04510_teblob'); -- { serverError STD_EXCEPTION } +CREATE TABLE t_04510_azte7 (x UInt8) ENGINE = AzureBlobStorage('http://localhost:11111/visible_04510_teacct', + 'visible_04510_tec5', 'visible_04510_teb5', 'visible_04510_teacctname', + 'SEKRIT_AZTECTLKEY'); -- { serverError STD_EXCEPTION } + +-- The two-argument signature takes container and path from the url and a shared access signature +-- beside it, so its second argument is a credential wherever the same rule serves the engine. +CREATE TABLE t_04510_azte8 (x UInt8) ENGINE = AzureQueue('http://localhost:11111/visible_04510_teq/cont/*', + 'SEKRIT_AZTESAS2') SETTINGS mode = 'unordered'; -- { serverError UNKNOWN_FORMAT } + -- Backup database engine reconstructs the nested S3 destination; extra_credentials must be masked. CREATE DATABASE db_04510_ec ENGINE = Backup('', S3('url_dbec', 'ak', 'SEKRIT_SAK', extra_credentials(external_id = 'SEKRIT_EID'))); -- { serverError BAD_ARGUMENTS } @@ -272,14 +404,82 @@ CREATE DATABASE db_04510_mixed ENGINE = Backup('', S3('url_dbmixed', CREATE DATABASE db_04510_ncurl ENGINE = Backup('', S3(nc_dburl_missing, url = concat('https://user:SEKRIT_PW@', 'localhost/x?X-Amz-Signature=SEKRIT_SIG'))); -- { serverError BAD_ARGUMENTS } --- The reconstructor must fail closed on an unsupported tail (headers), not emit it verbatim. +-- The reconstructor must fail closed on an unsupported tail (headers), not emit it verbatim, and the +-- rejection message must not echo it either: a nested map's values are hidden only by its parent's +-- formatter, so such a node formatted on its own carries them in plaintext. CREATE DATABASE db_04510_hdr ENGINE = Backup('', S3('url_dbhdr', 'ak', 'SEKRIT_SAK', - headers('X-Auth' = 'SEKRIT_HDR'))); -- { serverError BAD_ARGUMENTS } + headers('X-Auth' = 'SEKRIT_DBHDR'))); -- { serverError BAD_ARGUMENTS } -- The reconstructor must also fail closed on a constant-expression extra_credentials key. CREATE DATABASE db_04510_expr ENGINE = Backup('', S3('url_dbexpr', 'ak', 'SEKRIT_SAK', extra_credentials(concat('extern', 'al_id') = 'SEKRIT_EXPR'))); -- { serverError BAD_ARGUMENTS } +-- A locator held in a string literal is text this finder cannot parse, and that text can carry an +-- access key or a presigned URL. The `Backup` engine accepts that form only while replaying its own +-- metadata, but a statement carrying it is formatted - by `PARALLEL WITH`, by the distributed DDL +-- queue, by `query_log` - before the engine rejects it, so it must be masked whole. +CREATE DATABASE db_04510_quoted ENGINE = Backup('', + 'S3(\'https://user:SEKRIT_PW@localhost:11111/x?X-Amz-Signature=SEKRIT_SIG\', \'ak\', \'SEKRIT_QUOTED\')'); -- { serverError BAD_ARGUMENTS } + +-- `Backup(database_name, locator)` is the only valid shape. The statement is logged before the arity +-- is validated, so a credential parked in a surplus argument must be hidden as well. +CREATE DATABASE db_04510_tail ENGINE = Backup('', S3('url_dbtail', 'ak', 'SEKRIT_SAK'), + 'SEKRIT_DBTAIL'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_tmap ENGINE = Backup('', S3('url_dbtmap', 'ak', 'SEKRIT_SAK'), + extra_credentials(external_id = 'SEKRIT_DBTMAP')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_lone ENGINE = Backup(S3('url_dblone', 'ak', + 'SEKRIT_DBLONE')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +-- A locator is also a credential carrier in the database-name position, where the argument is neither +-- masked by the S3 rule below nor safe to echo: the message that rejects it formats it standalone. +CREATE DATABASE db_04510_swap ENGINE = Backup(S3('url_dbswap', 'ak', 'SEKRIT_DBSWAP'), + ''); -- { serverError BAD_ARGUMENTS } + +-- An identifier is not a literal either, and the parsers evaluate one as a literal, so the +-- database-name position must hold a literal before the rest of the shape is trusted. +CREATE DATABASE db_04510_ident ENGINE = Backup(SEKRIT_DBIDENT, + S3('url_dbident', 'ak', 'SEKRIT_SAK')); -- { serverError BAD_ARGUMENTS } + +-- A credential-free locator (Disk, File, Memory, Null) names its destination with literals and holds +-- no credential, but it keeps a named override or nested map that it never reads, and that can carry +-- one. Both the logged text and the not-found message identify the locator, so both must hide it. +CREATE DATABASE db_04510_ftail ENGINE = Backup('src_04510', + File('nonexistent_04510', extra_credentials(external_id = 'SEKRIT_DBFTAIL'))); -- { serverError BACKUP_NOT_FOUND } + +-- A surplus literal argument reaches the same locator through the shape check instead of the tail, and +-- it is rejected only after the statement is logged, so it has to be hidden as well. The two controls +-- that follow hold the argument counts these engines do read (File one, Disk two), where the locator +-- names a destination only: nothing is masked, so both the logged text and the not-found message keep +-- it visible verbatim, and the transcript records the statement as sent rather than a re-formatted AST. +CREATE DATABASE db_04510_fover ENGINE = Backup('src_04510', + File('nonexistent_04510', 'SEKRIT_DBFILEOVER')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_dover ENGINE = Backup('src_04510', + Disk('backups', 'nonexistent_04510', 'SEKRIT_DBDISKOVER')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_mover ENGINE = Backup('src_04510', + Memory('nonexistent_04510', 'SEKRIT_DBMEMOVER')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_fvalid ENGINE = Backup('src_04510', File('nonexistent_04510')); -- { serverError BACKUP_NOT_FOUND } +CREATE DATABASE db_04510_dvalid ENGINE = Backup('src_04510', Disk('backups', 'nonexistent_04510')); -- { serverError BACKUP_NOT_FOUND } + +-- A credential value given as an expression is hidden in the logged text, but evaluating it can fail +-- with a message that quotes its input, so the rejection must not carry that message either. The url +-- has to parse for the locator to reach credential evaluation at all. +CREATE DATABASE db_04510_eval ENGINE = Backup('', S3('http://localhost:11111/test/04510eval', 'ak', 'SEKRIT_SAK', + extra_credentials(external_id = toUInt64('SEKRIT_DBEVAL')))); -- { serverError BAD_ARGUMENTS } + +-- A non-tail argument that is neither a literal nor `key = value` cannot be reconstructed either. It +-- is rejected by a different message than the quoted locator above, and that message used to echo the +-- offending argument verbatim, so it needs its own tag. +CREATE DATABASE db_04510_nonlit ENGINE = Backup('', S3(concat('SEKRIT_NONLIT', 'x'), + 'url_dbnonlit')); -- { serverError BAD_ARGUMENTS } + +-- S3 is not the only backup engine whose locator carries credentials: AzureBlobStorage takes an +-- account_key and accepts connection strings and named-collection overrides that carry one too. +-- Only S3 is reconstructed above, so an Azure locator keeps its engine name and argument count +-- (neither is a secret) and every argument is hidden. The url carries a query string, which the +-- engine rejects before it reaches the network. +CREATE DATABASE db_04510_azure ENGINE = Backup('', AzureBlobStorage('http://localhost:11111/acct?sig=x', + 'cont', 'blob', 'account', 'SEKRIT_AZUREKEY')); -- { serverError BAD_ARGUMENTS } + -- The S3 database engine accepts no positional beyond secret_access_key; an extra positional must -- be masked in the logged query text. CREATE DATABASE db_04510_s3pos ENGINE = S3('url_dbs3pos', 'ak', 'SEKRIT_SAK', @@ -338,7 +538,15 @@ ORDER BY event_time_microseconds; -- logs each DDL from the replay worker, which re-masks a rewritten AST independently, so assert -- the masking property over every row this test produced, replay rows included. count() > 0 keeps -- an empty row set from passing vacuously. -SELECT count() > 0, countIf(query LIKE '%SEKRIT%') +-- The third column covers the recorded exception messages of the seven unreconstructible-locator +-- statements above, whose rejections used to echo the locator verbatim - one tag per message, since +-- they are thrown at different sites. It is scoped to those tags because widening it to every +-- deliberately-failing statement here would report unrelated pre-existing echoes. +SELECT count() > 0, countIf(query LIKE '%SEKRIT%'), + countIf(exception LIKE '%SEKRIT_QUOTED%' OR exception LIKE '%SEKRIT_NONLIT%' + OR exception LIKE '%SEKRIT_DBHDR%' OR exception LIKE '%SEKRIT_DBSWAP%' + OR exception LIKE '%SEKRIT_DBEVAL%' OR exception LIKE '%SEKRIT_DBFTAIL%' + OR exception LIKE '%SEKRIT_DBIDENT%') FROM system.query_log WHERE current_database = currentDatabase() AND type != 'QueryStart' diff --git a/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.reference b/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.reference new file mode 100644 index 0000000000000..6cc8f3258bf9e --- /dev/null +++ b/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.reference @@ -0,0 +1,3 @@ +1 +TOO_LARGE_ARRAY_SIZE +TOO_LARGE_ARRAY_SIZE diff --git a/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.sh b/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.sh new file mode 100755 index 0000000000000..627efc72b50a2 --- /dev/null +++ b/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Tests that a RowBinaryWithNames[AndTypes] header with a suspiciously large column +# count is rejected with TOO_LARGE_ARRAY_SIZE instead of amplifying into a huge +# allocation. See https://github.com/ClickHouse/clickhouse-private/issues/69219 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_rowbinary_header" +${CLICKHOUSE_CLIENT} -q "CREATE TABLE t_rowbinary_header (x UInt64) ENGINE = Memory" + +# LEB128 encoder. +varint() { + python3 -c ' +import sys +n = int(sys.argv[1]) +out = bytearray() +while True: + b = n & 0x7F + n >>= 7 + if n: + b |= 0x80 + out.append(b) + if not n: + break +sys.stdout.buffer.write(bytes(out)) +' "$1" +} + +# Negative control: a well-formed one-column header (name "x", type "UInt64") plus one +# 8-byte UInt64 row inserts fine. +{ varint 1; printf '\x01x\x06UInt64'; printf '\x01\x00\x00\x00\x00\x00\x00\x00'; } \ + | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+t_rowbinary_header+FORMAT+RowBinaryWithNamesAndTypes" --data-binary @- +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM t_rowbinary_header" + +# Attack: a header claiming far more columns than the ceiling (1'000'000). +# The tiny body must be rejected up-front, not amplified. +varint 2000000 \ + | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+t_rowbinary_header+FORMAT+RowBinaryWithNamesAndTypes" --data-binary @- 2>&1 \ + | grep -o "TOO_LARGE_ARRAY_SIZE" | head -n1 + +# Same for the WithNames variant (the names header is a separate read path). +varint 2000000 \ + | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+t_rowbinary_header+FORMAT+RowBinaryWithNames" --data-binary @- 2>&1 \ + | grep -o "TOO_LARGE_ARRAY_SIZE" | head -n1 + +${CLICKHOUSE_CLIENT} -q "DROP TABLE t_rowbinary_header" diff --git a/tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.reference b/tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.reference new file mode 100644 index 0000000000000..394d98eaf66f3 --- /dev/null +++ b/tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.reference @@ -0,0 +1,14 @@ +0 1000 0 +1 1000 1 +[[1000],[1000]] +[2000,2000] +['a'] [2000] ['b','c'] [2000,2000] +[2000] [2000,2000] +[2000] +['a','b','c'] [0,1,1] [2000,0,0] ['x','y'] [0,0] +[1000,1000] [1000,1000] +[999,1000] [999,1000] +[1000,1000] [1000,1000] +[1000,1000] [1000,1000] +[1000,1000] [1000,1000] +[1000,1000] [1000,1000] diff --git a/tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.sql b/tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.sql new file mode 100644 index 0000000000000..8fc70d3859534 --- /dev/null +++ b/tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.sql @@ -0,0 +1,102 @@ +-- Tags: no-parallel +-- no-parallel -- enables a process-global failpoint that fires in every -State finalization + +-- A combinator that transfers several sub-states in one insertResultInto call aliases them into the +-- result column one by one. When the transfer throws part-way through, the aggregator destroys the +-- whole place while the result column still owns the already-transferred sub-states, so they are +-- freed twice. The failpoint throws from the -State transfer once the destination column already +-- holds an aliased state, which is exactly that partial transfer. +-- Each sub-state below holds more than 4096 bytes of groupArray data, so it is allocated outside the +-- arena and the second destroy reaches a real deallocation. + +-- The second failpoint throws at the start of a -Tuple element whose predecessor transferred in full, +-- so the child being undone is a COMPLETED one rather than the partially transferred innermost -State. + +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw_after_child; + +-- Plain -State: the transfer is atomic, so this must throw cleanly on every version. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT number % 2 AS k, groupArrayState(number) FROM numbers(2000) GROUP BY k ORDER BY k SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT k, length(finalizeAggregation(s)), finalizeAggregation(s)[1] FROM (SELECT number % 2 AS k, groupArrayState(number) AS s FROM numbers(2000) GROUP BY k) ORDER BY k SETTINGS max_threads = 1; + +-- -Resample over -ForEach, one element per bucket: the throw is in the second bucket, so -Resample +-- has to undo a completed -ForEach child. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateForEachResample(0, 2, 1)([number], number % 2) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> arrayMap(y -> length(finalizeAggregation(y)), x), groupArrayStateForEachResample(0, 2, 1)([number], number % 2)) FROM numbers(2000) SETTINGS max_threads = 1; + +-- -ForEach alone: the throw is in its own loop, with the first element already aliased. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateForEach([number, number + 1]) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), groupArrayStateForEach([number, number + 1])) FROM numbers(2000) SETTINGS max_threads = 1; + +-- -Tuple over -Map with String keys: the first element completes, the second throws on its second +-- key, so -Tuple has to undo a completed -Map child in a different subcolumn. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateMapTuple((map('a', number), map('b', number, 'c', number))) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT mapKeys(t.1), arrayMap(x -> length(finalizeAggregation(x)), mapValues(t.1)), mapKeys(t.2), arrayMap(x -> length(finalizeAggregation(x)), mapValues(t.2)) FROM (SELECT groupArrayStateMapTuple((map('a', number), map('b', number, 'c', number))) AS t FROM numbers(2000)) SETTINGS max_threads = 1; + +-- -Tuple over -ForEach, first element shorter than the second. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateForEachTuple(([number], [number, number])) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), t.1), arrayMap(x -> length(finalizeAggregation(x)), t.2) FROM (SELECT groupArrayStateForEachTuple(([number], [number, number])) AS t FROM numbers(2000)) SETTINGS max_threads = 1; + +-- Exactly one bucket: the -State transfer cannot throw (the column is still empty), so the throw +-- comes from -Resample appending its own offset with the single sub-state already aliased. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateResample(0, 1, 1)(number, 0) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), groupArrayStateResample(0, 1, 1)(number, 0)) FROM numbers(2000) SETTINGS max_threads = 1; + +-- A -Map whose value rows are not uniform: the key with non-NULL values aliases a sub-state, the +-- all-NULL keys make the null adapter insert a default the column itself owns. Undoing such a map in +-- any order other than the reverse of the sorted-key append order applies the wrong row's semantics. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateTupleMapTuple((CAST(map('b', NULL, 'a', (number, number + 1), 'c', NULL), 'Map(String, Nullable(Tuple(UInt64, UInt64)))'), CAST(map('x', (number, number + 1), 'y', (number + 2, number + 3)), 'Map(String, Nullable(Tuple(UInt64, UInt64)))'))) FROM numbers(2000) SETTINGS max_threads = 1, enable_nullable_tuple_type = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT mapKeys(t.1), arrayMap(x -> isNull(x), mapValues(t.1)), arrayMap(x -> length(finalizeAggregation(assumeNotNull(x).1)), mapValues(t.1)), mapKeys(t.2), arrayMap(x -> isNull(x), mapValues(t.2)) FROM (SELECT groupArrayStateTupleMapTuple((CAST(map('b', NULL, 'a', (number, number + 1), 'c', NULL), 'Map(String, Nullable(Tuple(UInt64, UInt64)))'), CAST(map('x', (number, number + 1), 'y', (number + 2, number + 3)), 'Map(String, Nullable(Tuple(UInt64, UInt64)))'))) AS t FROM numbers(2000)) SETTINGS max_threads = 1, enable_nullable_tuple_type = 1; + +-- -Tuple over -Resample. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateResampleTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2)) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), t.1), arrayMap(x -> length(finalizeAggregation(x)), t.2) FROM (SELECT groupArrayStateResampleTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2)) AS t FROM numbers(2000)) SETTINGS max_threads = 1; + +-- The same, through a transparent -If wrapper. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateResampleIfTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2), (number > 0, number > 0)) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), t.1), arrayMap(x -> length(finalizeAggregation(x)), t.2) FROM (SELECT groupArrayStateResampleIfTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2), (number > 0, number > 0)) AS t FROM numbers(2000)) SETTINGS max_threads = 1; + +-- -OrNull around a Tuple of -State results: Tuple can be inside Nullable, so the transfer goes into +-- the nested column of a ColumnNullable. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateResampleTupleOrNullTuple(0, 2, 1)(((number, number + 1), (number + 2, number + 3)), ((number % 2, number % 2), (number % 2, number % 2))) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), assumeNotNull(t.1).1), arrayMap(x -> length(finalizeAggregation(x)), assumeNotNull(t.2).2) FROM (SELECT groupArrayStateResampleTupleOrNullTuple(0, 2, 1)(((number, number + 1), (number + 2, number + 3)), ((number % 2, number % 2), (number % 2, number % 2))) AS t FROM numbers(2000)) SETTINGS max_threads = 1; + +-- The same window through the implicit null adapter instead of -OrNull, plus a -Distinct wrapper. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateResampleTupleDistinctTuple(0, 2, 1)(CAST(((number, number + 1), (number + 2, number + 3)), 'Tuple(Nullable(Tuple(UInt64, UInt64)), Nullable(Tuple(UInt64, UInt64)))'), ((number % 2, number % 2), (number % 2, number % 2))) FROM numbers(2000) SETTINGS max_threads = 1, enable_nullable_tuple_type = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), assumeNotNull(t.1).1), arrayMap(x -> length(finalizeAggregation(x)), assumeNotNull(t.2).2) FROM (SELECT groupArrayStateResampleTupleDistinctTuple(0, 2, 1)(CAST(((number, number + 1), (number + 2, number + 3)), 'Tuple(Nullable(Tuple(UInt64, UInt64)), Nullable(Tuple(UInt64, UInt64)))'), ((number % 2, number % 2), (number % 2, number % 2))) AS t FROM numbers(2000)) SETTINGS max_threads = 1, enable_nullable_tuple_type = 1; + +-- A completed -Resample child undone by its parent: with the second failpoint the first tuple element +-- transfers all of its buckets, and the throw lands before the second element. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw_after_child; +SELECT groupArrayStateResampleTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2)) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw_after_child; +SELECT arrayMap(x -> length(finalizeAggregation(x)), t.1), arrayMap(x -> length(finalizeAggregation(x)), t.2) FROM (SELECT groupArrayStateResampleTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2)) AS t FROM numbers(2000)) SETTINGS max_threads = 1; + +-- The same completed child behind an -OrNull that forwards: Array cannot be inside Nullable, so -OrFill +-- passes the array column straight through and has to forward the undo as well. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw_after_child; +SELECT groupArrayStateResampleOrNullTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2)) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw_after_child; +SELECT arrayMap(x -> length(finalizeAggregation(x)), t.1), arrayMap(x -> length(finalizeAggregation(x)), t.2) FROM (SELECT groupArrayStateResampleOrNullTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2)) AS t FROM numbers(2000)) SETTINGS max_threads = 1; diff --git a/tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.reference b/tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.reference new file mode 100644 index 0000000000000..b170576ac2c00 --- /dev/null +++ b/tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.reference @@ -0,0 +1,19 @@ +1 +1 +1 +[10] +[10] +[10] +[10] +1 +1 +1 +1 +1 +1 +1 +1 +[10] +[10] +1 +1 diff --git a/tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.sql b/tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.sql new file mode 100644 index 0000000000000..4bb4cb0727345 --- /dev/null +++ b/tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.sql @@ -0,0 +1,131 @@ +-- The `optimize_and_compare_chain` / `optimize_redundant_comparisons` passes prune and fold an +-- AND-chain of comparisons. Both skip a Nullable-typed AND, because a comparison over a Nullable +-- operand cannot participate in range pruning. That guard checks only the AND's own result type, +-- which is unsound: when one AND operand is `Nothing`-typed, the function resolver collapses the +-- whole AND's result type to bare `Nothing` (`Nothing::isNullable()` is false), so the guard passes +-- even though another operand is a comparison over a directly-Nullable expression. That operand then +-- reached the pruning path and hit `chassert(!raw_type->isNullable())`. + +SET enable_analyzer = 1; + +-- 1) No logical error. These queries are otherwise invalid (a `Nothing`-typed value cannot be +-- materialized), so they must fail with a normal handled exception, never a logical error. +-- optimize_and_compare_chain path (chassert reached via the seed loop of tryOptimizeAndCompareChain): +SELECT tuple((materialize(toNullable(NULL)) = 1) AND (assumeNotNull(materialize(toNullable(NULL))) = 2)) SETTINGS optimize_and_compare_chain = 1; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((materialize(toNullable(1::Int32)) = 1) AND (assumeNotNull(materialize(toNullable(NULL))) = 2)) SETTINGS optimize_and_compare_chain = 1; -- { serverError ILLEGAL_COLUMN } +-- optimize_redundant_comparisons path (tryOptimizeAndCompareNotEqualsChain), independent of the chain setting: +SELECT tuple((materialize(toNullable(1::Int32)) = 1) AND (assumeNotNull(materialize(toNullable(NULL))) = 2)) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((materialize(toNullable(1::Int32)) != 1) AND (assumeNotNull(materialize(toNullable(NULL))) != 2)) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } + +-- 2) The optimizer still runs on such a `Nothing`-collapsed AND: the directly-Nullable operand is +-- kept as-is (the new opaque-filter fallback) while a redundant non-null comparison in the SAME +-- AND is still folded. In `x > 3 AND x > 5`, `x > 3` is redundant and pruned, so exactly one +-- `greater` survives when enabled against both when disabled. Pin both counts: a relative +-- comparison also holds when the predicates disappear entirely. +SELECT count() = 1 FROM (EXPLAIN QUERY TREE SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = 2)) SETTINGS optimize_redundant_comparisons = 1) WHERE explain LIKE '%function_name: greater,%'; +SELECT count() = 2 FROM (EXPLAIN QUERY TREE SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = 2)) SETTINGS optimize_redundant_comparisons = 0) WHERE explain LIKE '%function_name: greater,%'; +-- The exact predicate that used to hit the assertion is `equals(...) -> Nullable(Nothing)`; assert it +-- survives (exactly one such node) rather than being silently dropped. The chain's other `equals` +-- returns `Nothing`, so match the `Nullable(Nothing)` result type specifically. +SELECT count() = 1 FROM (EXPLAIN QUERY TREE SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = 2)) SETTINGS optimize_redundant_comparisons = 1) WHERE explain LIKE '%function_name: equals, function_type: ordinary, result_type: Nullable(Nothing)%'; + +-- 3) Correctness guard: a genuine (non-collapsed) AND-compare chain over a Nullable column must still +-- be pruned/folded correctly, i.e. the optimization result matches the unoptimized one, and NULLs +-- are excluded by the comparisons. `optimize_and_compare_chain` / `optimize_redundant_comparisons` +-- must not change results. +DROP TABLE IF EXISTS t_and_chain_nullable; +CREATE TABLE t_and_chain_nullable (x Nullable(Int32)) ENGINE = Memory; +INSERT INTO t_and_chain_nullable VALUES (1), (5), (NULL), (10); +SELECT groupArray(x) FROM (SELECT x FROM t_and_chain_nullable WHERE (x > 3) AND (x > 5) ORDER BY x SETTINGS optimize_and_compare_chain = 1); +SELECT groupArray(x) FROM (SELECT x FROM t_and_chain_nullable WHERE (x > 3) AND (x > 5) ORDER BY x SETTINGS optimize_and_compare_chain = 0); +SELECT groupArray(x) FROM (SELECT x FROM t_and_chain_nullable WHERE (x != 1) AND (x != 5) ORDER BY x SETTINGS optimize_redundant_comparisons = 1); +SELECT groupArray(x) FROM (SELECT x FROM t_and_chain_nullable WHERE (x != 1) AND (x != 5) ORDER BY x SETTINGS optimize_redundant_comparisons = 0); +DROP TABLE t_and_chain_nullable; +-- The checks above compare results, which stay equal even if `optimize_and_compare_chain` stops +-- deriving anything. Pin the derivation itself with exact node counts: `a < b AND b < 5` gains the +-- transitive `a < 5`, so the enabled tree holds exactly 3 `less` nodes against 2 when disabled. +SELECT count() = 3 FROM (EXPLAIN QUERY TREE SELECT a, b FROM values('a Int32, b Int32', (1, 2), (4, 9)) WHERE (a < b) AND (b < 5) SETTINGS optimize_and_compare_chain = 1) WHERE explain ILIKE '%function_name: less,%'; +SELECT count() = 2 FROM (EXPLAIN QUERY TREE SELECT a, b FROM values('a Int32, b Int32', (1, 2), (4, 9)) WHERE (a < b) AND (b < 5) SETTINGS optimize_and_compare_chain = 0) WHERE explain ILIKE '%function_name: less,%'; + +-- 4) In a `Nothing`-collapsed AND, an operand may be a comparison whose constant side is itself +-- NULL-valued (e.g. `expr = NULL`), found by the AST fuzzer mutating case 2's `= 2` to `= NULL`. +-- A NULL-valued constant carries no comparable value, so `tryOptimizeAndCompareNotEqualsChain` +-- must not treat it as the constant side (it used to hit `chassert(!literal->getValue().isNull())`). +-- The whole AND is still `Nothing`-typed, so these queries fail with a normal handled exception. +-- RHS-NULL constant: +SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = NULL)) SETTINGS optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (assumeNotNull(materialize(toNullable(NULL))) != NULL)) SETTINGS optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +-- LHS-NULL constant (the other assertion branch): +SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (NULL = assumeNotNull(materialize(toNullable(NULL))))) SETTINGS optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +-- Independent of the pruning setting (the classification loop runs unconditionally): +SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (assumeNotNull(materialize(toNullable(NULL))) = NULL)) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 0; -- { serverError ILLEGAL_COLUMN } + +-- 5) Liveness of the keep-as-is mechanism for the NULL-valued-constant operand: the optimizer must +-- still run on this `Nothing`-collapsed AND (so a redundant sibling is pruned) while keeping the +-- NULL-valued `equals` operands rather than skipping classification entirely. In the tree with +-- pruning on, `x > 3` is redundant given `x > 5`, so exactly one `greater` node survives; with +-- pruning off both survive; both NULL-valued `equals` operands are kept in either case. +SELECT count() = 1 FROM (EXPLAIN QUERY TREE SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = NULL)) SETTINGS optimize_redundant_comparisons = 1) WHERE explain ILIKE '%function_name: greater,%'; +SELECT count() = 2 FROM (EXPLAIN QUERY TREE SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = NULL)) SETTINGS optimize_redundant_comparisons = 0) WHERE explain ILIKE '%function_name: greater,%'; +SELECT count() = 2 FROM (EXPLAIN QUERY TREE SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = NULL)) SETTINGS optimize_redundant_comparisons = 1) WHERE explain ILIKE '%function_name: equals,%'; + +-- 6) A comparison whose RESULT is nullable must be kept as-is too, even when the raw operand type does +-- not report `isNullable`: `LowCardinality(Nullable(T))` (nested nullability) and the NULL-capable +-- carriers `Dynamic` / `Variant` all yield a nullable comparison result. They used to slip past the +-- guard, fold the contradictory `x = 1 AND x = 2`, and change the handled exception depending on the +-- setting. The error must now be the same regardless of the pruning / chain settings. +SET allow_suspicious_low_cardinality_types = 1; +SET allow_experimental_dynamic_type = 1; +SET allow_experimental_variant_type = 1; +SELECT tuple((x = 1) AND (x = 2) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x LowCardinality(Nullable(Int32))', NULL) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 0; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((x = 1) AND (x = 2) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x LowCardinality(Nullable(Int32))', NULL) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((x = 1) AND (x = 2) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x Dynamic', NULL) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 0; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((x = 1) AND (x = 2) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x Dynamic', NULL) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((x = 1) AND (x = 2) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x Variant(Int32, String)', NULL) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 0; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((x = 1) AND (x = 2) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x Variant(Int32, String)', NULL) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +-- Liveness for the LC(Nullable) operand: a redundant NON-nullable sibling (`> 3` given `> 5`) is still +-- pruned to one `greater` node while the LC-nullable `equals` operand is kept, proving the optimizer +-- runs and keeps only that operand opaque rather than declining the whole collapsed AND. +SELECT count() = 1 FROM (EXPLAIN QUERY TREE SELECT tuple((x = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x LowCardinality(Nullable(Int32))', NULL) SETTINGS optimize_redundant_comparisons = 1) WHERE explain ILIKE '%function_name: greater,%'; +SELECT count() = 2 FROM (EXPLAIN QUERY TREE SELECT tuple((x = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x LowCardinality(Nullable(Int32))', NULL) SETTINGS optimize_redundant_comparisons = 0) WHERE explain ILIKE '%function_name: greater,%'; +SELECT count() = 1 FROM (EXPLAIN QUERY TREE SELECT tuple((x = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x LowCardinality(Nullable(Int32))', NULL) SETTINGS optimize_redundant_comparisons = 1) WHERE explain ILIKE '%function_name: equals,%'; +-- A non-collapsed `LowCardinality(Nullable(T))` chain must still return correct results, unchanged by +-- the optimization (NULL excluded, `1`/`5` filtered out; only `10` remains). +SELECT groupArray(x) FROM (SELECT x FROM values('x LowCardinality(Nullable(Int32))', 1, 5, NULL, 10) WHERE (x != 1) AND (x != 5) ORDER BY x SETTINGS optimize_redundant_comparisons = 1); +SELECT groupArray(x) FROM (SELECT x FROM values('x LowCardinality(Nullable(Int32))', 1, 5, NULL, 10) WHERE (x != 1) AND (x != 5) ORDER BY x SETTINGS optimize_redundant_comparisons = 0); + +-- 7) The carrier the AST fuzzer keeps rediscovering on master (`Logical error: +-- '!raw_type->isNullable()'`, STID `2508-50fe`, e.g. +-- https://s3.amazonaws.com/clickhouse-test-reports/json.html?REF=master&sha=d469feea5f342065ffe8b2384d4ddf354dae3978&name_0=MasterCI&name_1=Stress%20test%20%28arm_asan_ubsan%29 ). +-- It reaches the same `addComparisonFilter` through a different route than the cases above: the +-- `Nothing`-typed operand comes from `ARRAY JOIN []` (whose element type is `Nothing`) instead of +-- `assumeNotNull(materialize(toNullable(NULL)))`, and the collapsed AND sits in a `JOIN ON` +-- section. `JOIN ON` is load-bearing - the same AND in a `WHERE` is rejected earlier, so only the +-- join expression lets a `Nothing`-typed AND reach the optimizer. +DROP TABLE IF EXISTS t_and_chain_array_join; +CREATE TABLE t_and_chain_array_join (c0 Int32) ENGINE = MergeTree() ORDER BY tuple(); +INSERT INTO t_and_chain_array_join VALUES (1), (2); + +-- `t2.c0 = a0` is `Nothing`-typed and collapses the AND's own result type, while +-- `toNullable(t2.c0) > 0` stays `Nullable(UInt8)` and reaches the pruning path. `ARRAY JOIN []` +-- produces no rows, so the query is valid and returns nothing; assert the empty result on every +-- combination of the two settings, since either entry point alone reaches the assertion. +SELECT 1 FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join AS t2 ON (t2.c0 = a0) AND (toNullable(t2.c0) > 0) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 1; +SELECT 1 FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join AS t2 ON (t2.c0 = a0) AND (toNullable(t2.c0) > 0) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 0; +SELECT 1 FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join AS t2 ON (t2.c0 = a0) AND (toNullable(t2.c0) > 0) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 1; +SELECT 1 FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join AS t2 ON (t2.c0 = a0) AND (toNullable(t2.c0) > 0) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 0; +-- A result-only check stays green if the queries stop running the optimizer at all, so pin the +-- pruning as well: given `> 5`, the sibling `> 3` is redundant and folded away, leaving the +-- Nullable-result `greater` plus one surviving constant comparison. Both settings are pinned on +-- every query below because `clickhouse-test` randomizes `optimize_and_compare_chain`. +SELECT count() = 2 FROM (EXPLAIN QUERY TREE SELECT 1 FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join AS t2 ON (t2.c0 = a0) AND (toNullable(t2.c0) > 0) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 1) WHERE explain ILIKE '%function_name: greater,%'; +SELECT count() = 3 FROM (EXPLAIN QUERY TREE SELECT 1 FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join AS t2 ON (t2.c0 = a0) AND (toNullable(t2.c0) > 0) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 0) WHERE explain ILIKE '%function_name: greater,%'; +-- The exact fuzzer query: here the Nullable-result comparison is over a correlated scalar +-- subquery rather than a column. `optimize_and_compare_chain` does not gate this one - +-- `tryOptimizeAndCompareChain` skips a chain holding a correlated subquery, while +-- `tryOptimizeAndCompareNotEqualsChain` has no such guard - so it arrives only via +-- `optimize_redundant_comparisons`. A correlated subquery is not supported in a join expression, +-- so the query must report that handled exception instead of aborting. +SELECT 1 AS x FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join ON (t_and_chain_array_join.c0 = a0) AND (t_and_chain_array_join.c0 != a0) AND (0 > (SELECT t_and_chain_array_join.c0)) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 1; -- { serverError NOT_IMPLEMENTED } +SELECT 1 AS x FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join ON (t_and_chain_array_join.c0 = a0) AND (t_and_chain_array_join.c0 != a0) AND (0 > (SELECT t_and_chain_array_join.c0)) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 0; -- { serverError NOT_IMPLEMENTED } +DROP TABLE t_and_chain_array_join; diff --git a/tests/queries/0_stateless/04626_s3_base_setting.reference b/tests/queries/0_stateless/04626_s3_base_setting.reference index 6b40f3b84623a..97a52a1f9a5cb 100644 --- a/tests/queries/0_stateless/04626_s3_base_setting.reference +++ b/tests/queries/0_stateless/04626_s3_base_setting.reference @@ -20,3 +20,5 @@ 5 --- s3_base without a scheme is an error must contain a scheme +must contain a scheme +0 diff --git a/tests/queries/0_stateless/04626_s3_base_setting.sh b/tests/queries/0_stateless/04626_s3_base_setting.sh index e8793ce77cab7..88ef59858adda 100755 --- a/tests/queries/0_stateless/04626_s3_base_setting.sh +++ b/tests/queries/0_stateless/04626_s3_base_setting.sh @@ -61,3 +61,10 @@ ${CLICKHOUSE_CLIENT} -q "DROP TABLE test_s3_base_nc" && ${CLICKHOUSE_CLIENT} -q echo '--- s3_base without a scheme is an error' ${CLICKHOUSE_CLIENT} -q "SELECT * FROM s3('${FILE}', 'test', 'testtest', 'TSV', 'n UInt32, s String') SETTINGS s3_base = 'localhost:11111/test/'" 2>&1 | grep -oF 'must contain a scheme' | head -1 + +# The rejected value is not echoed back: it can carry a credential, and the message reaches the client, +# the exception column of the query log and the server log, none of which the display-secrets setting +# gates. (last line counts the leaks; clickhouse-local is used because the client also prints back the +# query it was given, which is the caller's own input rather than something the message disclosed) +${CLICKHOUSE_LOCAL} -q "SELECT * FROM s3('${FILE}', 'test', 'testtest', 'TSV', 'n UInt32, s String') SETTINGS s3_base = 'user:SEKRIT_PW@localhost:11111/test/'" 2>&1 | grep -oF 'must contain a scheme' | head -1 +${CLICKHOUSE_LOCAL} -q "SELECT * FROM s3('${FILE}', 'test', 'testtest', 'TSV', 'n UInt32, s String') SETTINGS s3_base = 'user:SEKRIT_PW@localhost:11111/test/'" 2>&1 | grep -c SEKRIT_PW ||: diff --git a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference index a6800ceefe7f8..078656c7dc9d3 100644 --- a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference +++ b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference @@ -1,5 +1,5 @@ -- Current defaults -false 1000 1 +false 0 1 -- compatibility = 26.6 restores the legacy defaults true 0 0 0 diff --git a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql index cf37018c792d0..1fe196405fc4c 100644 --- a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql +++ b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql @@ -3,10 +3,10 @@ -- no-replicated-database: named collections are server-global, not database-scoped -- ============================================================================= --- Three AI function defaults were flipped: `ai_function_allow_insecure_endpoint` from 1 to 0 --- and `ai_function_max_api_calls_per_query` from 0 (unlimited) to 1000 in 26.8, and --- `ai_function_max_retries` from 0 to 1 in 26.9. `compatibility = 26.6` predates all three --- and restores them, which pins the previous_value/new_value pairs in `SettingsChangesHistory`. +-- Two AI function defaults were flipped in 26.8: `ai_function_allow_insecure_endpoint` from 1 +-- to 0 and `ai_function_max_retries` from 0 to 1. `ai_function_max_api_calls_per_query` stays 0 +-- (unlimited). `compatibility = 26.6` predates the flips and restores them, which pins the +-- previous_value/new_value pairs in `SettingsChangesHistory`. -- -- The endpoint check runs in `resolveAIParams`, before the zero-row early return -- in `executeImpl`, so an empty source table exercises it without any real HTTP diff --git a/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.reference b/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.reference new file mode 100644 index 0000000000000..f40b28dde3758 --- /dev/null +++ b/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.reference @@ -0,0 +1,200 @@ +-- numeric array, dynamic offset +1 0 9 [0,1] 10 +2 1 8 [1,2] 10 +5 4 5 [4,5] 10 +100 0 0 [] 10 +1 0 10 [0,1] 11 +2 1 9 [1,2] 11 +5 4 6 [4,5] 11 +100 0 0 [] 11 +1 0 9 [0,1] 10 +2 1 8 [1,2] 10 +5 4 5 [4,5] 10 +100 0 0 [] 10 +1 0 10 [0,1] 11 +2 1 9 [1,2] 11 +5 4 6 [4,5] 11 +100 0 0 [] 11 +-- string array, constant and dynamic slices +1 ['s1','s2','s3'] ['s3','s4'] ['s0','s1','s2','s3','s4'] ['s0'] s1 +-1 ['s1','s2','s3'] ['s3','s4'] ['s4'] [] s1 +0 ['s1','s2','s3'] ['s3','s4'] [] [] s1 +3 ['s1','s2','s3'] ['s3','s4'] ['s2','s3','s4'] ['s2','s3','s4'] s1 +1 ['s2','s3','s4'] ['s4','s5'] ['s1','s2','s3','s4','s5'] ['s1'] s2 +-1 ['s2','s3','s4'] ['s4','s5'] ['s5'] [] s2 +0 ['s2','s3','s4'] ['s4','s5'] [] [] s2 +3 ['s2','s3','s4'] ['s4','s5'] ['s3','s4','s5'] ['s3','s4','s5'] s2 +1 ['s1','s2','s3'] ['s3','s4'] ['s0','s1','s2','s3','s4'] ['s0'] s1 +-1 ['s1','s2','s3'] ['s3','s4'] ['s4'] [] s1 +0 ['s1','s2','s3'] ['s3','s4'] [] [] s1 +3 ['s1','s2','s3'] ['s3','s4'] ['s2','s3','s4'] ['s2','s3','s4'] s1 +1 ['s2','s3','s4'] ['s4','s5'] ['s1','s2','s3','s4','s5'] ['s1'] s2 +-1 ['s2','s3','s4'] ['s4','s5'] ['s5'] [] s2 +0 ['s2','s3','s4'] ['s4','s5'] [] [] s2 +3 ['s2','s3','s4'] ['s4','s5'] ['s3','s4','s5'] ['s3','s4','s5'] s2 +-- Array(LowCardinality(String)) elements and slices +1 Array(LowCardinality(String)) lc_0 lc_1 ['lc_0','lc_1'] ['lc_1','lc_2','lc_0'] ['lc_0','lc_1'] +2 Array(LowCardinality(String)) lc_1 lc_1 ['lc_1','lc_2'] ['lc_1','lc_2','lc_0'] ['lc_0','lc_1'] +5 Array(LowCardinality(String)) lc_1 lc_1 ['lc_1'] ['lc_1','lc_2','lc_0'] ['lc_0','lc_1'] +-1 Array(LowCardinality(String)) lc_1 lc_1 ['lc_1'] ['lc_1','lc_2','lc_0'] ['lc_0','lc_1'] +1 Array(LowCardinality(String)) lc_1 lc_2 ['lc_1','lc_2'] ['lc_2','lc_0','lc_1'] ['lc_1','lc_2'] +2 Array(LowCardinality(String)) lc_2 lc_2 ['lc_2','lc_0'] ['lc_2','lc_0','lc_1'] ['lc_1','lc_2'] +5 Array(LowCardinality(String)) lc_2 lc_2 ['lc_2'] ['lc_2','lc_0','lc_1'] ['lc_1','lc_2'] +-1 Array(LowCardinality(String)) lc_2 lc_2 ['lc_2'] ['lc_2','lc_0','lc_1'] ['lc_1','lc_2'] +1 Array(LowCardinality(String)) lc_0 lc_1 ['lc_0','lc_1'] ['lc_1','lc_2','lc_0'] ['lc_0','lc_1'] +2 Array(LowCardinality(String)) lc_1 lc_1 ['lc_1','lc_2'] ['lc_1','lc_2','lc_0'] ['lc_0','lc_1'] +5 Array(LowCardinality(String)) lc_1 lc_1 ['lc_1'] ['lc_1','lc_2','lc_0'] ['lc_0','lc_1'] +-1 Array(LowCardinality(String)) lc_1 lc_1 ['lc_1'] ['lc_1','lc_2','lc_0'] ['lc_0','lc_1'] +1 Array(LowCardinality(String)) lc_1 lc_2 ['lc_1','lc_2'] ['lc_2','lc_0','lc_1'] ['lc_1','lc_2'] +2 Array(LowCardinality(String)) lc_2 lc_2 ['lc_2','lc_0'] ['lc_2','lc_0','lc_1'] ['lc_1','lc_2'] +5 Array(LowCardinality(String)) lc_2 lc_2 ['lc_2'] ['lc_2','lc_0','lc_1'] ['lc_1','lc_2'] +-1 Array(LowCardinality(String)) lc_2 lc_2 ['lc_2'] ['lc_2','lc_0','lc_1'] ['lc_1','lc_2'] +-- nullable elements +1 \N \N [NULL,1] +2 1 1 [1,NULL] +7 \N \N [] +-1 5 5 [5] +1 \N \N [NULL,2] +2 2 2 [2,NULL] +7 \N \N [] +-1 6 6 [6] +1 \N \N [NULL,1] +2 1 1 [1,NULL] +7 \N \N [] +-1 5 5 [5] +1 \N \N [NULL,2] +2 2 2 [2,NULL] +7 \N \N [] +-1 6 6 [6] +-- negative and nullable offsets, negative length +1 [5,6] [1,2,3,4,5,6] [0,1] +2 [5,6] [1,2,3,4,5,6] [0,1] +3 [5,6] [1,2,3,4,5,6] [2,3] +-2 [5,6] [1,2,3,4,5,6] [0,1] +1 [6,7] [1,2,3,4,5,6,7] [0,1] +2 [6,7] [1,2,3,4,5,6,7] [0,1] +3 [6,7] [1,2,3,4,5,6,7] [2,3] +-2 [6,7] [1,2,3,4,5,6,7] [0,1] +1 [7,8] [1,2,3,4,5,6,7,8] [0,1] +2 [7,8] [1,2,3,4,5,6,7,8] [0,1] +3 [7,8] [1,2,3,4,5,6,7,8] [2,3] +-2 [7,8] [1,2,3,4,5,6,7,8] [0,1] +1 [5,6] [1,2,3,4,5,6] [0,1] +2 [5,6] [1,2,3,4,5,6] [0,1] +3 [5,6] [1,2,3,4,5,6] [2,3] +-2 [5,6] [1,2,3,4,5,6] [0,1] +1 [6,7] [1,2,3,4,5,6,7] [0,1] +2 [6,7] [1,2,3,4,5,6,7] [0,1] +3 [6,7] [1,2,3,4,5,6,7] [2,3] +-2 [6,7] [1,2,3,4,5,6,7] [0,1] +1 [7,8] [1,2,3,4,5,6,7,8] [0,1] +2 [7,8] [1,2,3,4,5,6,7,8] [0,1] +3 [7,8] [1,2,3,4,5,6,7,8] [2,3] +-2 [7,8] [1,2,3,4,5,6,7,8] [0,1] +-- tuple elements and replicated empty arrays +1 (0,'t0') [(0,'t0')] 0 [] +4 (3,'t3') [(0,'t0'),(1,'t1'),(2,'t2'),(3,'t3')] 0 [] +5 (0,'') [(0,'t0'),(1,'t1'),(2,'t2'),(3,'t3')] 0 [] +1 (1,'t0') [(1,'t0')] 0 [0] +4 (4,'t3') [(1,'t0'),(2,'t1'),(3,'t2'),(4,'t3')] 0 [0] +5 (0,'') [(1,'t0'),(2,'t1'),(3,'t2'),(4,'t3')] 0 [0] +1 (0,'t0') [(0,'t0')] 0 [] +4 (3,'t3') [(0,'t0'),(1,'t1'),(2,'t2'),(3,'t3')] 0 [] +5 (0,'') [(0,'t0'),(1,'t1'),(2,'t2'),(3,'t3')] 0 [] +1 (1,'t0') [(1,'t0')] 0 [0] +4 (4,'t3') [(1,'t0'),(2,'t1'),(3,'t2'),(4,'t3')] 0 [0] +5 (0,'') [(1,'t0'),(2,'t1'),(3,'t2'),(4,'t3')] 0 [0] +-- array produced by lazy JOIN replication +1 s0 ['s0','s1'] +2 s1 ['s1','s2'] +3 s2 ['s2','s3'] +4 s3 ['s3','s4'] +1 s0 ['s0','s1'] +2 s1 ['s1','s2'] +3 s2 ['s2','s3'] +4 s3 ['s3','s4'] +-- replication indexes of type UInt16 (more than 255 rows in a block) +90000 1800 90000 +90000 1800 90000 +-- arrayElement with a non-constant Nullable index +1 s0 s0 +2 \N \N +5 s4 s4 +-1 s4 s4 +1 s1 s1 +2 \N \N +5 s5 s5 +-1 s5 s5 +1 s0 s0 +2 \N \N +5 s4 s4 +-1 s4 s4 +1 s1 s1 +2 \N \N +5 s5 s5 +-1 s5 s5 +-- arrayElement with a lazily replicated Nullable index (AST fuzzer) +45150 0 0 +45150 0 0 +-- array and index replicated by the same expansion +1 1 [1,2] 1 +2 1 [1,2] 1 +5 1 [1,2] 1 +1 2 [2,3] 2 +2 2 [2,3] 2 +5 2 [2,3] 2 +1 1 [1,2] 1 +2 1 [1,2] 1 +5 1 [1,2] 1 +1 2 [2,3] 2 +2 2 [2,3] 2 +5 2 [2,3] 2 +-- unused nested rows are compacted away before the recursion on nested columns +1 0 1 [0,1] +2 0 1 [0,1] +1 1 1 [1,2] +2 1 1 [1,2] +1 0 1 [0,1] +2 0 1 [0,1] +1 1 1 [1,2] +2 1 1 [1,2] +-- out-of-order replication indexes (JOIN probe order) with unused nested rows +4 0 1 3 [0,1] +4 0 1 3 [0,1] +7 0 1 6 [0,1] +9 2 1 8 [2,3] +4 0 1 3 [0,1] +4 0 1 3 [0,1] +7 0 1 6 [0,1] +9 2 1 8 [2,3] +-- constant index over a replicated array that survives a LIMIT (const-index compute-on-nested branch) +1 10 +1 10 +1 10 +1 10 +1 10 +1 10 +-- array and index lazily indexed by a JOIN under LIMIT (shared-indexes compute-on-nested branch) +0 1 +0 1 +2 1 +0 1 +0 1 +2 1 +-- QBit sliced above a lazy ARRAY JOIN expansion +[9,10,11,12,13,14,15,16] QBit(Float32, 8) [2,3,4,5,6,7,8,9,10,11] QBit(Float32, 10) [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32] 3 +[9,10,11,12,13,14,15,16] QBit(Float32, 8) [2,3,4,5,6,7,8,9,10,11] QBit(Float32, 10) [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32] 3 +[9,10,11,12,13,14,15,16] QBit(Float32, 8) [2,3,4,5,6,7,8,9,10,11] QBit(Float32, 10) [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32] 3 +1 [9,10,11,12,13,14,15,16] [2,3,4,5,6,7,8,9,10,11] +1 [9,10,11,12,13,14,15,16] [2,3,4,5,6,7,8,9,10,11] +2 \N \N +2 \N \N +[9,10,11,12,13,14,15,16] QBit(Float32, 8) [2,3,4,5,6,7,8,9,10,11] QBit(Float32, 10) [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32] 3 +[9,10,11,12,13,14,15,16] QBit(Float32, 8) [2,3,4,5,6,7,8,9,10,11] QBit(Float32, 10) [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32] 3 +[9,10,11,12,13,14,15,16] QBit(Float32, 8) [2,3,4,5,6,7,8,9,10,11] QBit(Float32, 10) [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32] 3 +1 [9,10,11,12,13,14,15,16] [2,3,4,5,6,7,8,9,10,11] +1 [9,10,11,12,13,14,15,16] [2,3,4,5,6,7,8,9,10,11] +2 \N \N +2 \N \N +-- large array is not materialized per joined row +1 diff --git a/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.sql.j2 b/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.sql.j2 new file mode 100644 index 0000000000000..51b799aaa7e5f --- /dev/null +++ b/tests/queries/0_stateless/04628_lazy_replicated_array_slice_element.sql.j2 @@ -0,0 +1,183 @@ +-- arraySlice and arrayElement consume a lazily replicated array produced by ARRAY_JOIN or JOINs +-- Results must be identical with lazy replication disabled and enabled. + +SELECT '-- numeric array, dynamic offset'; +{% for i in range(2) -%} +SELECT x, big[x], big[-x], arraySlice(big, x, 2), length(big) +FROM (SELECT range(10 + number) AS big, [1, 2, 5, 100] AS r FROM numbers(2)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- string array, constant and dynamic slices'; +{% for i in range(2) -%} +SELECT x, arraySlice(big, 2, 3), arraySlice(big, -2), arraySlice(big, x), arraySlice(big, x, x), big[2] +FROM (SELECT arrayMap(i -> 's' || toString(i + number), range(5)) AS big, [1, -1, 0, 3] AS r FROM numbers(2)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- Array(LowCardinality(String)) elements and slices'; +{% for i in range(2) -%} +SELECT x, toTypeName(big), big[x], big[2], arraySlice(big, x, 2), arraySlice(big, 2, 3), arraySlice(big, -2) +FROM (SELECT arrayMap(i -> 'lc_' || toString((i + number) % 3), range(5))::Array(LowCardinality(String)) AS big, [1, 2, 5, -1] AS r FROM numbers(2)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- nullable elements'; +{% for i in range(2) -%} +SELECT x, big[x], arrayElementOrNull(big, x), arraySlice(big, x, 2) +FROM (SELECT arrayMap(i -> if(i % 2 = 0, NULL, i + number), range(6))::Array(Nullable(UInt64)) AS big, [1, 2, 7, -1] AS r FROM numbers(2)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- negative and nullable offsets, negative length'; +{% for i in range(2) -%} +SELECT x, arraySlice(big, -3, 2), arraySlice(big, 2, -1), arraySlice(big, if(x % 2 = 0, NULL, x)::Nullable(Int64), 2) +FROM (SELECT range(8 + number % 3) AS big, [1, 2, 3, -2] AS r FROM numbers(3)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- tuple elements and replicated empty arrays'; +{% for i in range(2) -%} +SELECT x, big[x], arraySlice(big, 1, x), small_arr[x], arraySlice(small_arr, 1, 2) +FROM (SELECT arrayMap(i -> (i + number, 't' || toString(i)), range(4)) AS big, range(number % 2) AS small_arr, [1, 4, 5] AS r FROM numbers(2)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- array produced by lazy JOIN replication'; +{% for i in range(2) -%} +SELECT x, big[x], arraySlice(big, x, 2) +FROM (SELECT 42 AS k, number + 1 AS x FROM numbers(4)) AS l +INNER JOIN (SELECT 42 AS k, materialize(arrayMap(i -> 's' || toString(i), range(5))) AS big) AS r ON l.k = r.k +ORDER BY x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- replication indexes of type UInt16 (more than 255 rows in a block)'; +{% for i in range(2) -%} +SELECT sum(big[x + 1]), sum(length(arraySlice(big, x + 1, 3))), sum(arraySlice(big, x + 1, 3)[1]) +FROM (SELECT range(number, number + 10) AS big, range(2) AS r FROM numbers(300)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- arrayElement with a non-constant Nullable index'; +{% for i in range(2) -%} +SELECT x, big[if(x % 2 = 0, NULL, x)::Nullable(Int64)], arrayElementOrNull(big, if(x % 2 = 0, NULL, x)::Nullable(Int64)) +FROM (SELECT arrayMap(i -> 's' || toString(i + number), range(5)) AS big, [1, 2, 5, -1] AS r FROM numbers(2)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- arrayElement with a lazily replicated Nullable index (AST fuzzer)'; +{% for i in range(2) -%} +SELECT DISTINCT sum(big[2 + x]), quantileTDigestDistinct(notEmpty(arraySlice(big, positiveModuloOrNull(x, 65535), 3))), sum(arraySlice(big, NULL + x, 3)[materialize(toNullable(100))]) +FROM (SELECT DISTINCT range(number, number + 2) AS big, range(2) AS r FROM numbers(300)) LEFT ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}, max_threads = 1; +{% endfor %} + +SELECT '-- array and index replicated by the same expansion'; +-- `big` and `idx` are both non-joined columns, so lazy ARRAY JOIN wraps them in ColumnReplicated +-- sharing one indexes column: `big[idx]` takes the compute-on-nested fast path and stays lazy. +-- `range(x + 3)[idx]` mixes a full array (built from the expanded column) with a replicated index, +-- which pins the materialize fallback for an index replicated independently of the array. +{% for i in range(2) -%} +SELECT x, big[idx], arraySlice(big, idx, 2), range(x + 3)[idx] +FROM (SELECT range(10 + number) AS big, number + 2 AS idx, [1, 2, 5] AS r FROM numbers(2)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- unused nested rows are compacted away before the recursion on nested columns'; +-- Only 2 of the 20 source rows have a non-empty `r`, so the shared replication indexes reference +-- only those nested rows: the compute-on-nested path must compact the unused ones away +-- and adjust the indexes, for both a replicated (`big[idx]`) and a constant (`big[2]`) index. +{% for i in range(2) -%} +SELECT x, big[idx], big[2], arraySlice(big, idx, 2) +FROM (SELECT range(10 + number) AS big, intDiv(number, 10) + 1 AS idx, if(number % 10 = 0, [1, 2], []) AS r FROM numbers(20)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}; +{% endfor %} + +SELECT '-- out-of-order replication indexes (JOIN probe order) with unused nested rows'; +-- A hash JOIN references build-side rows in probe order: left keys [9, 4, 7, 4] give a +-- non-monotonic index sequence with a repeat (e.g. [8, 3, 6, 3]) over 10 nested rows, +-- 7 of which are never referenced. The used rows are not a prefix, so the compaction must +-- remap the surviving indexes to new positions ([2, 0, 1, 0]). This covers index remapping +-- for the shared-indexes (`big[idx]`) and constant-index (`big[2]`) paths, and the +-- random-access gather (`big[l.k]`: full index over a replicated array). +{% for i in range(2) -%} +SELECT l.k, big[idx], big[2], big[l.k], arraySlice(big, idx, 2) +FROM (SELECT arrayJoin([9, 4, 7, 4]) AS k) AS l +INNER JOIN (SELECT number + 1 AS k, range(number + 3) AS big, number % 3 + 1 AS idx FROM numbers(10)) AS r ON l.k = r.k +ORDER BY ALL +SETTINGS enable_lazy_columns_replication = {{ i }}, max_threads = 1, join_algorithm = 'hash', query_plan_join_swap_table = 0; +{% endfor %} + +SELECT '-- constant index over a replicated array that survives a LIMIT (const-index compute-on-nested branch)'; +-- Source-only expressions are normally computed below the expansion, so `big[2]` never sees a +-- replicated array there. The LIMIT barrier keeps the projection above the ARRAY JOIN, and `big` +-- crosses it as `ColumnReplicated`. `compile_expressions = 0` keeps the call out of the JIT +-- (the harness may randomize `min_count_to_compile_expression` to 0). +{% for i in range(2) -%} +SELECT big[2], length(big) +FROM (SELECT big FROM (SELECT range(10 + number) AS big, if(number % 10 = 0, [1, 2, 3], []) AS r FROM numbers(20)) ARRAY JOIN r AS x LIMIT 3) +SETTINGS enable_lazy_columns_replication = {{ i }}, max_threads = 1, enable_parallel_replicas = 0, compile_expressions = 0; +{% endfor %} + +SELECT '-- array and index lazily indexed by a JOIN under LIMIT (shared-indexes compute-on-nested branch)'; +-- A LIMIT (<= query_plan_max_limit_for_join_lazy_indexing) above a hash JOIN with >= 3 probe-side +-- columns makes the join wrap all probe columns in `ColumnReplicated` sharing one indexes column +-- (`optimizeJoinLazyIndexing`), including the fixed-size `idx` that the type-based usefulness +-- heuristic would otherwise materialize. Only 3 of the 20 probe rows are matched, so `big[idx]` +-- (shared-indexes branch) and `big[2]` (constant-index branch) must both compact 17 unused nested +-- rows before the recursion on the nested columns. +DROP TABLE IF EXISTS t04628_left; +DROP TABLE IF EXISTS t04628_right; +CREATE TABLE t04628_left (k UInt64, big Array(UInt64), idx UInt64) ENGINE = Memory; +CREATE TABLE t04628_right (k UInt64) ENGINE = Memory; +INSERT INTO t04628_left SELECT number + 1, range(number + 3), number % 3 + 1 FROM numbers(20); +INSERT INTO t04628_right VALUES (9), (4), (7); +{% for i in range(2) -%} +SELECT big[idx], big[2] +FROM t04628_left AS l INNER JOIN t04628_right AS r ON l.k = r.k +LIMIT 100 +SETTINGS enable_lazy_columns_replication = {{ i }}, max_threads = 1, join_algorithm = 'hash', query_plan_join_swap_table = 0, enable_parallel_replicas = 0, compile_expressions = 0, query_plan_optimize_join_order_limit = 10, query_plan_optimize_join_order_randomize = 0, enable_join_runtime_filters = 0; +{% endfor %} +DROP TABLE t04628_left; +DROP TABLE t04628_right; + +SELECT '-- QBit sliced above a lazy ARRAY JOIN expansion'; +-- `arraySlice` opts out of the framework's replicated materialization and dispatches on the QBit type +-- before any wrapper is peeled, so its QBit branch receives the `ColumnReplicated` the expansion built: +-- `Bad cast from type DB::ColumnReplicated to DB::ColumnQBit` in `executeQBitSlice`. +-- A QBit slice needs constant offset and length, so only the LIMIT barrier keeps it above the expansion. +DROP TABLE IF EXISTS t04628_qbit; +DROP TABLE IF EXISTS t04628_qbit_nullable; +CREATE TABLE t04628_qbit (id UInt32, vec QBit(Float32, 32, 8)) ENGINE = Memory; +CREATE TABLE t04628_qbit_nullable (id UInt32, vec Nullable(QBit(Float32, 32, 8))) ENGINE = Memory; +INSERT INTO t04628_qbit SELECT 1, arrayMap(i -> toFloat32(i + 1), range(32))::QBit(Float32, 32, 8); +INSERT INTO t04628_qbit_nullable SELECT 1, arrayMap(i -> toFloat32(i + 1), range(32))::QBit(Float32, 32, 8); +INSERT INTO t04628_qbit_nullable SELECT 2, NULL; +{% for i in range(2) -%} +SELECT arraySlice(vec, 9, 8) AS aligned, toTypeName(aligned), arraySlice(vec, 2, 10) AS repacked, toTypeName(repacked), arraySlice(vec, 1) AS whole, vec[3] +FROM (SELECT vec FROM t04628_qbit ARRAY JOIN [1, 2, 3] AS z LIMIT 3) +SETTINGS enable_lazy_columns_replication = {{ i }}, max_threads = 1, enable_parallel_replicas = 0, compile_expressions = 0; +SELECT id, arraySlice(vec, 9, 8) AS aligned, arraySlice(vec, 2, 10) AS repacked +FROM (SELECT id, vec FROM t04628_qbit_nullable ARRAY JOIN [1, 2] AS z LIMIT 4) +ORDER BY id +SETTINGS enable_lazy_columns_replication = {{ i }}, max_threads = 1, enable_parallel_replicas = 0, compile_expressions = 0; +{% endfor %} +DROP TABLE t04628_qbit; +DROP TABLE t04628_qbit_nullable; + +SELECT '-- large array is not materialized per joined row'; +{% for i in range(2) -%} +SELECT sum(length(arraySlice(big, x, 2))) + sum(length(big[x])) +FROM (SELECT arrayMap(i -> repeat('a', 100) || toString(i + number), range(1000)) AS big, range(50) AS r FROM numbers(100)) ARRAY JOIN r AS x +SETTINGS enable_lazy_columns_replication = {{ i }}, max_threads = 1, max_block_size = 65536, log_queries = 1, log_comment = '04628_replicated_{{ ['eager', 'lazy'][i] }}' +FORMAT Null; +{% endfor %} + +SYSTEM FLUSH LOGS query_log; + +-- With lazy replication the big array stays compact (one copy per source row) and only the +-- requested slices/elements are copied, so the peak memory usage must be much lower. +SELECT + (SELECT max(memory_usage) FROM system.query_log WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment = '04628_replicated_lazy') + < 0.5 * (SELECT max(memory_usage) FROM system.query_log WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment = '04628_replicated_eager'); diff --git a/tests/queries/0_stateless/04648_mysql_tls_credentials.reference b/tests/queries/0_stateless/04648_mysql_tls_credentials.reference index cd9aa6da303ce..daa2aafc0de8f 100644 --- a/tests/queries/0_stateless/04648_mysql_tls_credentials.reference +++ b/tests/queries/0_stateless/04648_mysql_tls_credentials.reference @@ -12,6 +12,14 @@ SELECT * FROM mysql('127.0.0.1:3306', 'db', 't', 'u', '[HIDDEN]', ssl_ca_pem = ' CREATE TABLE t (`x` Int32) ENGINE = MySQL('127.0.0.1:3306', 'db', 't', 'u', '[HIDDEN]', ssl_cert_pem = '[HIDDEN]') --- MySQL database engine, positional arguments CREATE DATABASE d ENGINE = MySQL('127.0.0.1:3306', 'db', 'u', '[HIDDEN]', ssl_ca_pem = '[HIDDEN]') +--- mysql table function, credentials before the positional arguments +SELECT * FROM mysql(ssl_key_pem = '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') +--- MySQL database engine, credentials before the positional arguments +CREATE DATABASE d ENGINE = MySQL(ssl_ca_pem = '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') +--- mysql table function, named password without a collection +SELECT * FROM mysql('127.0.0.1:3306', 'db', 't', password = '[HIDDEN]') +--- MySQL database engine, named password without a collection +CREATE DATABASE d ENGINE = MySQL('127.0.0.1:3306', 'db', password = '[HIDDEN]') --- key given as a constant expression SELECT * FROM mysql(creds, concat('ssl_ca', '_pem') = '[HIDDEN]', `table` = 't') --- key given as a constant expression, positional arguments diff --git a/tests/queries/0_stateless/04648_mysql_tls_credentials.sh b/tests/queries/0_stateless/04648_mysql_tls_credentials.sh index 0fcb5994fd45e..0bd7978918bb6 100755 --- a/tests/queries/0_stateless/04648_mysql_tls_credentials.sh +++ b/tests/queries/0_stateless/04648_mysql_tls_credentials.sh @@ -49,6 +49,22 @@ format "MySQL table engine, positional arguments" \ format "MySQL database engine, positional arguments" \ "CREATE DATABASE d ENGINE = MySQL('127.0.0.1:3306', 'db', 'u', '${SECRET}', ssl_ca_pem = '${SECRET}')" +# The same credentials written before the positional arguments: the call is invalid, but it is +# formatted for logging before it is rejected, so the scan must not start past the first argument, +# and the positional password must be located among the positional arguments rather than at a fixed +# argument index, which a named argument written first moves. +format "mysql table function, credentials before the positional arguments" \ + "SELECT * FROM mysql(ssl_key_pem = '${SECRET}', '127.0.0.1:3306', 'db', 't', 'u', '${SECRET}')" +format "MySQL database engine, credentials before the positional arguments" \ + "CREATE DATABASE d ENGINE = MySQL(ssl_ca_pem = '${SECRET}', '127.0.0.1:3306', 'db', 'u', '${SECRET}')" + +# A named `password` override with no named collection: the parsers reject the mix, but the statement is +# formatted for logging first, and the key is readable, so only the named scan can hide this value. +format "mysql table function, named password without a collection" \ + "SELECT * FROM mysql('127.0.0.1:3306', 'db', 't', password = '${SECRET}')" +format "MySQL database engine, named password without a collection" \ + "CREATE DATABASE d ENGINE = MySQL('127.0.0.1:3306', 'db', password = '${SECRET}')" + # The key of a named argument is not required to be a plain identifier or literal: the named # collection parser evaluates it as a constant expression, so `concat('ssl_ca', '_pem')` names a TLS # credential too. The formatter cannot evaluate it, so it hides the value of every argument whose key diff --git a/tests/queries/0_stateless/04651_system_sync_transaction_log_access.reference b/tests/queries/0_stateless/04651_system_sync_transaction_log_access.reference new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/queries/0_stateless/04651_system_sync_transaction_log_access.sql b/tests/queries/0_stateless/04651_system_sync_transaction_log_access.sql new file mode 100644 index 0000000000000..44814f2c966e6 --- /dev/null +++ b/tests/queries/0_stateless/04651_system_sync_transaction_log_access.sql @@ -0,0 +1,20 @@ +-- Tags: no-parallel, zookeeper +-- zookeeper: SYSTEM SYNC TRANSACTION LOG reads the transaction log from Keeper after access is granted. +-- no-parallel: creates a temporary user; avoid concurrent GRANT/DROP races on the same name pattern. + +DROP USER IF EXISTS user_test_04651; +CREATE USER user_test_04651; + +-- Before the fix, local SYSTEM SYNC TRANSACTION LOG never called checkAccess, so a user +-- without SYSTEM SYNC TRANSACTION LOG could still run the command (only the feature flag +-- allow_experimental_transactions was checked). ON CLUSTER already required the privilege. +-- Without the privilege, expect ACCESS_DENIED (checked before the transactions feature flag). +EXECUTE AS user_test_04651 SYSTEM SYNC TRANSACTION LOG; -- { serverError ACCESS_DENIED } + +-- Grant the privilege that ON CLUSTER already required. +GRANT SYSTEM SYNC TRANSACTION LOG ON *.* TO user_test_04651; + +-- With the privilege, the command must get past access control and sync the transaction log. +EXECUTE AS user_test_04651 SYSTEM SYNC TRANSACTION LOG; + +DROP USER user_test_04651; diff --git a/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.reference b/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.reference index 515699bb5704e..c3c10ae1556fc 100644 --- a/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.reference +++ b/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.reference @@ -34,13 +34,13 @@ partition_record_count_null_count: [('{\'eu\'}',1,0),('{\'eu\'}',1,1),('{\'us\'} 0 --- per-entry pairing against the referenced Parquet file --- id=1 rows=1/1 score_nulls=0/0 paired=yes lower=[(1,1),(2,10)] upper=[(1,1),(2,10)] -id=2 rows=1/1 score_nulls=1/1 paired=yes lower=[] upper=[] -id=3 rows=1/1 score_nulls=1/1 paired=yes lower=[] upper=[] +id=2 rows=1/1 score_nulls=1/1 paired=yes lower=[(1,2)] upper=[(1,2)] +id=3 rows=1/1 score_nulls=1/1 paired=yes lower=[(1,3)] upper=[(1,3)] --- per-entry bounds describe only their own file --- id=1 lower=[(1,1),(2,100)] upper=[(1,1),(2,100)] bounds_are_own_row=yes id=2 lower=[(1,2),(2,200)] upper=[(1,2),(2,200)] bounds_are_own_row=yes id=3 lower=[(1,3),(2,300)] upper=[(1,3),(2,300)] bounds_are_own_row=yes --- per-entry column_sizes describe only their own file --- -id=1 width=4 id_size=4 s_size=12 -id=2 width=8 id_size=4 s_size=16 -id=3 width=12 id_size=4 s_size=20 +id=1 width=10000 id_size_positive=yes s_size_follows_own_width=yes sizes_fit_own_file=yes +id=2 width=100000 id_size_positive=yes s_size_follows_own_width=yes sizes_fit_own_file=yes +id=3 width=1000000 id_size_positive=yes s_size_follows_own_width=yes sizes_fit_own_file=yes diff --git a/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.sh b/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.sh index dc05b1e1ec7eb..58fcd64bfd73b 100755 --- a/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.sh +++ b/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.sh @@ -153,11 +153,10 @@ ${CLICKHOUSE_CLIENT} --query " echo '--- per-entry pairing against the referenced Parquet file ---' for manifest in $(find "${PAIRED_PATH}/metadata" -maxdepth 1 -name '*.avro' -not -name 'snap-*.avro' -type f | sort); do # `lower_bounds`/`upper_bounds` hold raw little-endian bytes (`dumpValue` in `IcebergWrites.cpp`), - # so they are decoded with `reinterpretAsInt32`; both key columns are `Int32` here. An entry whose - # `score` is all-NULL legitimately has NO bounds at all: `canWriteStatistics` is all-or-nothing - # across the entry's columns and `ColumnNullable::getExtremes` yields NULL extremes for an - # all-NULL column, which `canDumpIcebergStats` rejects. That is pre-existing behaviour, so it is - # asserted here rather than fixed. + # so they are decoded with `reinterpretAsInt32`; both key columns are `Int32` here. Bounds are + # per-field: an entry keeps a bound for every column whose extreme is serializable and omits the + # rest. `ColumnNullable::getExtremes` yields NULL extremes for an all-NULL column, which + # `canDumpIcebergStats` rejects, so an all-NULL `score` contributes no bound while `id` still does. ${CLICKHOUSE_CLIENT} --query " WITH entries AS ( SELECT @@ -190,10 +189,9 @@ for manifest in $(find "${PAIRED_PATH}/metadata" -maxdepth 1 -name '*.avro' -not " done -# The bounds above are present on one entry only, because the two all-NULL files legitimately carry no -# bounds at all. This companion has no nullable column, so every entry carries bounds and the decoded -# value of each is checked against the single row its own file holds. Without it the bounds half of -# this change would be pinned on a single entry. +# Only one entry above carries a `score` bound, because the other two files hold an all-NULL `score`. +# This companion has no nullable column, so every entry carries a bound for both of its columns and the +# decoded value of each is checked against the single row its own file holds. ${CLICKHOUSE_CLIENT} --query " ${ONE_ROW_PER_FILE} CREATE TABLE bounded (id Int32, v Int32) @@ -235,16 +233,16 @@ done # The two scenarios above pin `record_count`, `null_value_counts` and the bounds to the file each # entry names, but not `column_sizes`, whose only other assertion is a cross-table `max()` inequality # that constrains no individual entry. Both of their fixtures also give every file the same per-file -# sizes, so a permutation cannot move a size value there. This scenario gives each file a `String` of -# a different width, which makes the sizes distinct and therefore permutation-sensitive. The values -# are in-memory `IColumn::byteSize` sums, not Parquet file sizes, so they are deterministic: -# `ColumnString::byteSize` is `chars.size() + offsets.size() * sizeof(offsets[0])`, and `insertData` -# appends exactly `length` bytes with no terminator. +# sizes, so a permutation cannot move a size value there. This scenario gives each file an +# incompressible `String` of a different order of magnitude, which makes the sizes distinct and +# therefore permutation-sensitive. `column_sizes` holds the post-compression size of the column chunk +# inside the Parquet file, so the exact byte count is a property of the encoder; what is asserted is +# that each entry's size follows the width of the row its own file holds and fits inside that file. ${CLICKHOUSE_CLIENT} --query " ${ONE_ROW_PER_FILE} CREATE TABLE sized (id Int32, s String) ENGINE = IcebergLocal('${SIZED_PATH}', 'Parquet') ORDER BY (id); - INSERT INTO sized SELECT number + 1, repeat('x', (number + 1) * 4) FROM numbers(3); + INSERT INTO sized SELECT number + 1, randomPrintableASCII(toUInt32(pow(10, number + 4))) FROM numbers(3); " echo '--- per-entry column_sizes describe only their own file ---' @@ -252,8 +250,9 @@ for manifest in $(find "${SIZED_PATH}/metadata" -maxdepth 1 -name '*.avro' -not ${CLICKHOUSE_CLIENT} --query " WITH entries AS ( SELECT - replaceRegexpOne(tupleElement(data_file, 'file_path'), '^.*/', '') AS base, - CAST(tupleElement(data_file, 'column_sizes'), 'Map(Int32, Int64)') AS entry_sizes + replaceRegexpOne(tupleElement(data_file, 'file_path'), '^.*/', '') AS base, + CAST(tupleElement(data_file, 'column_sizes'), 'Map(Int32, Int64)') AS entry_sizes, + tupleElement(data_file, 'file_size_in_bytes') AS entry_file_size FROM file('${manifest}', Avro) ), files AS ( @@ -267,8 +266,11 @@ for manifest in $(find "${SIZED_PATH}/metadata" -maxdepth 1 -name '*.avro' -not SELECT 'id=' || toString(f.own_id) || ' width=' || toString(f.own_width) - || ' id_size=' || toString(e.entry_sizes[1]) - || ' s_size=' || toString(e.entry_sizes[2]) AS entry + || ' id_size_positive=' || if(e.entry_sizes[1] > 0, 'yes', 'no') + || ' s_size_follows_own_width=' || if( + e.entry_sizes[2] > f.own_width / 2 AND e.entry_sizes[2] < f.own_width * 2, 'yes', 'no') + || ' sizes_fit_own_file=' || if( + e.entry_sizes[1] + e.entry_sizes[2] <= e.entry_file_size, 'yes', 'no') AS entry FROM entries AS e INNER JOIN files AS f ON e.base = f.base ORDER BY f.own_id FORMAT TSV; diff --git a/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.reference b/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.reference index 670c62cb8acba..0259a7a6a9049 100644 --- a/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.reference +++ b/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.reference @@ -30,10 +30,10 @@ -- an explicit PREWHERE through both views keeps working -- 10 10 --- re-attached tables are lazy proxies -- -lazy_inner TableProxy +-- only the re-attached MergeTree is a lazy proxy -- +lazy_inner Merge lazy_leaf TableProxy -lazy_outer TableProxy +lazy_outer Merge -- the proxy must still reject the mismatched column, not abort -- -- a matching column still supports PREWHERE through the proxy -- 10 diff --git a/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.sql b/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.sql index 412183279c357..b0e1d21115efb 100644 --- a/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.sql +++ b/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.sql @@ -133,7 +133,7 @@ ATTACH DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; -- as the style check requires (a `{...:String}` parameter is not recognized by it). USE {CLICKHOUSE_DATABASE_1:Identifier}; -SELECT '-- re-attached tables are lazy proxies --'; +SELECT '-- only the re-attached MergeTree is a lazy proxy --'; SELECT name, engine FROM system.tables WHERE database = currentDatabase() ORDER BY name; SELECT '-- the proxy must still reject the mismatched column, not abort --'; diff --git a/tests/queries/0_stateless/04653_merging_aggregated_deserialized_thread_count.sql b/tests/queries/0_stateless/04653_merging_aggregated_deserialized_thread_count.sql index efb4a79a398af..d514602e88b4c 100644 --- a/tests/queries/0_stateless/04653_merging_aggregated_deserialized_thread_count.sql +++ b/tests/queries/0_stateless/04653_merging_aggregated_deserialized_thread_count.sql @@ -22,6 +22,9 @@ SET max_parallel_replicas = 3; SET parallel_replicas_plan_based = 1; SET parallel_replicas_for_non_replicated_merge_tree = 1; SET cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost'; +-- The test verifies a plan packet on a same-version local cluster. Hedged connections retain +-- the SQL fallback for a future, unverified rolling-upgrade peer, so avoid that unrelated route. +SET use_hedged_requests = 0; -- The parallel replicas optimizer rewrite builds a `MergingAggregatedStep` from the params of a -- deserialized `AggregatingStep`, which carry the "resolve locally later" sentinel 0 as the thread @@ -34,6 +37,11 @@ GROUP BY k ORDER BY k SETTINGS serialize_query_plan = 1, distributed_aggregation_memory_efficient = 0, enable_memory_bound_merging_of_aggregation_results = 0, + -- This test asserts the serialized-plan path itself. Keep plan-level limits at their + -- defaults: a hedged connection must deliberately use SQL instead when a future replica + -- could be an older peer that cannot receive serialized execution limits. + max_threads = 0, + use_concurrency_control = 0, -- Required by the second firing oracle below. It defaults true, but it flipped false to -- true in the 24.3 block of `SettingsChangesHistory.cpp`, so a `compatibility` draw -- below 24.3 turns it off and that oracle would silently read 0. diff --git a/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.reference b/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.reference index 0fb27808c3ca1..73b717c7d2550 100644 --- a/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.reference +++ b/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.reference @@ -1,6 +1,6 @@ TOO_DEEP_AST TOO_DEEP_AST -Structured Field value exceeds maximum AST depth limit +Structured Field value rejected Field dump payload exceeds maximum AST depth limit 1 TOO_DEEP_AST diff --git a/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.sh b/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.sh index 455c50c4a2d5f..67c08b16c6ba5 100755 --- a/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.sh +++ b/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.sh @@ -27,14 +27,16 @@ ${CLICKHOUSE_CLIENT} --max_ast_depth 0 --param_json "$BRACKET_BOMB" \ --query "SELECT formatQueryFromJSON({json:String})" 2>&1 | grep -om1 'TOO_DEEP_AST' # 3. A deeply nested structured `Field` value adds no AST nodes and stays under the bracket -# budget, so only the `Field` depth bound rejects it. +# budget, so it is rejected by its own depth bound or by the recursion's stack check. Which of +# the two reports it depends on the build's stack budget, so assert only that it is rejected. ${CLICKHOUSE_CLIENT} --max_ast_depth 0 --query " SELECT formatQueryFromJSON(concat( '{\"type\":\"Literal\",\"value\":', repeat('{\"field_type\":\"Array\",\"value\":[', 2000), '{\"field_type\":\"UInt64\",\"value\":1}', repeat(']}', 2000), '}'))" 2>&1 | - grep -om1 'Structured Field value exceeds maximum AST depth limit' + grep -qE 'Structured Field value exceeds maximum AST depth limit|TOO_DEEP_RECURSION' && + echo 'Structured Field value rejected' # 4. A deeply nested `Field` dump hides its nesting inside a JSON string, so the bracket # pre-scan does not see it either; `Field::restoreFromDump` must not recurse unbounded. diff --git a/tests/queries/0_stateless/04665_valueless_setting_ast_json_and_secret_parts.reference b/tests/queries/0_stateless/04665_valueless_setting_ast_json_and_secret_parts.reference index 209f37b46ed43..f726717f90dba 100644 --- a/tests/queries/0_stateless/04665_valueless_setting_ast_json_and_secret_parts.reference +++ b/tests/queries/0_stateless/04665_valueless_setting_ast_json_and_secret_parts.reference @@ -8,7 +8,7 @@ TYPE_MISMATCH 1 SELECT 1 SETTINGS format_avro_schema_registry_url = \'http://user:pass@localhost\' TYPE_MISMATCH -SELECT 1 SETTINGS format_avro_schema_registry_url = \'http://user:[HIDDEN]@localhost\' 1 +SELECT 1 SETTINGS format_avro_schema_registry_url = \'http://[HIDDEN]@localhost\' 1 TYPE_MISMATCH TYPE_MISMATCH SELECT 1 SETTINGS format_avro_schema_registry_url = \'http://user:pass@localhost\' diff --git a/tests/queries/0_stateless/04673_cross_to_inner_join_non_deterministic.reference b/tests/queries/0_stateless/04673_cross_to_inner_join_non_deterministic.reference new file mode 100644 index 0000000000000..6ed281c757a96 --- /dev/null +++ b/tests/queries/0_stateless/04673_cross_to_inner_join_non_deterministic.reference @@ -0,0 +1,2 @@ +1 +1 diff --git a/tests/queries/0_stateless/04673_cross_to_inner_join_non_deterministic.sql b/tests/queries/0_stateless/04673_cross_to_inner_join_non_deterministic.sql new file mode 100644 index 0000000000000..e315f579392bd --- /dev/null +++ b/tests/queries/0_stateless/04673_cross_to_inner_join_non_deterministic.sql @@ -0,0 +1,26 @@ +SET enable_analyzer = 1; +SET query_plan_merge_filter_into_join_condition = 0; + +DROP TABLE IF EXISTS l; +DROP TABLE IF EXISTS r; + +CREATE TABLE l (a UInt64) ENGINE = Log; +CREATE TABLE r (a UInt64) ENGINE = Log; + +INSERT INTO l SELECT number % 16 FROM numbers(500); +INSERT INTO r SELECT number % 16 FROM numbers(500); + +-- A comma join with `rand` must keep the equality in `WHERE`. +-- Before this fix, the rewrite moved it into the join key, where `rand` was evaluated again +-- per row of each side, so the query returned rows with `r.a` other than 3. +SELECT uniqExact(r.a) = 1 AND min(r.a) = 3 AND max(r.a) = 3 AND count() > 0 +FROM l, r WHERE rand(l.a) % 16 = r.a AND rand(l.a) % 16 = 3 +SETTINGS cross_to_inner_join_rewrite = 1; + +-- An explicit `CROSS` join must keep the equality in `WHERE` too; it went wrong the same way. +SELECT uniqExact(r.a) = 1 AND min(r.a) = 3 AND max(r.a) = 3 AND count() > 0 +FROM l CROSS JOIN r WHERE rand(l.a) % 16 = r.a AND rand(l.a) % 16 = 3 +SETTINGS cross_to_inner_join_rewrite = 1; + +DROP TABLE l; +DROP TABLE r; diff --git a/tests/queries/0_stateless/04704_cross_to_inner_join_show_certificate.reference b/tests/queries/0_stateless/04704_cross_to_inner_join_show_certificate.reference new file mode 100644 index 0000000000000..d00491fd7e5bb --- /dev/null +++ b/tests/queries/0_stateless/04704_cross_to_inner_join_show_certificate.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/04704_cross_to_inner_join_show_certificate.sql b/tests/queries/0_stateless/04704_cross_to_inner_join_show_certificate.sql new file mode 100644 index 0000000000000..44ddf7a2f57f0 --- /dev/null +++ b/tests/queries/0_stateless/04704_cross_to_inner_join_show_certificate.sql @@ -0,0 +1,24 @@ +-- Tags: no-fasttest + +SET enable_analyzer = 1; +SET query_plan_merge_filter_into_join_condition = 0; + +DROP TABLE IF EXISTS l; +DROP TABLE IF EXISTS r; + +CREATE TABLE l (a UInt64) ENGINE = Log; +CREATE TABLE r (a UInt64) ENGINE = Log; + +-- `showCertificate` must not become a join key: its value is the executing node's own +-- certificate, so the two sides of a distributed join can disagree. Before this fix, the +-- rewrite turned this comma join into an `INNER` join keyed on the `concat` expression. +SELECT count() = 0 FROM ( +EXPLAIN QUERY TREE run_passes = 1 +SELECT count() FROM l, r +WHERE concat(toString(l.a), showCertificate()['version']) + = concat(toString(r.a), showCertificate()['version']) +SETTINGS cross_to_inner_join_rewrite = 1 +) WHERE explain ILIKE '%kind: INNER%'; + +DROP TABLE l; +DROP TABLE r; diff --git a/tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.reference b/tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.reference new file mode 100644 index 0000000000000..1a595af2079a5 --- /dev/null +++ b/tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.reference @@ -0,0 +1,39 @@ +--- WHERE on the evolved column --- +4 4 +5 five +--- PREWHERE on the evolved column --- +4 4 +5 five +--- PREWHERE IS NULL / IS NOT NULL --- +none +6 +--- PREWHERE on an untouched column --- +4 4 +--- declared type and full scan --- +7 2 Nullable(Int64) +0 0 +1 1 +2 2 +3 3 +4 4 +5 five +\N none +--- MODIFY COLUMN to Nullable plus RENAME COLUMN --- +4 4 Nullable(Int64) +5 five Nullable(Int64) +--- widening plus nullability (int required to long optional) --- +4 Nullable(Int64) +--- optional to required is still rejected --- +Iceberg spec doesn't allow change type from nullable to non-nullable +--- String column made Nullable --- +4 Nullable(String) +--- externally authored optional to required stays a passthrough --- +id Int64 +s String +none +one +three +none +one +three +three diff --git a/tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.sh b/tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.sh new file mode 100755 index 0000000000000..064a0c290e549 --- /dev/null +++ b/tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel-replicas +# `no-parallel-replicas`: see comment in `04071_iceberg_orc_prewhere_crash.sh`. +# `StorageObjectStorageCluster` (used when `parallel_replicas_for_cluster_engines = 1`, +# default) does not delegate `supportsPrewhere` to its underlying configuration. +# +# Regression test for issue #85029: filtering a column that `ALTER TABLE ... MODIFY COLUMN` +# made `Nullable` fails on the Iceberg data files written before the `ALTER`. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# `optimize_move_to_prewhere=1` + `query_plan_optimize_prewhere=1` are pinned on every +# discriminating statement: the failure only appears when the predicate is pushed into the +# reader, the runner randomizes both, and with either one off the pre-fix result is already +# correct, so the test would stop exercising the fix. +PREWHERE_SETTINGS="--optimize_move_to_prewhere=1 --query_plan_optimize_prewhere=1" + +TABLE="t_null_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_REN="t_ren_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_WID="t_wid_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_REJ="t_rej_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_STR="t_str_${CLICKHOUSE_DATABASE}_${RANDOM}" + +drop_table() { + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS $1" + rm -rf "${USER_FILES_PATH}/$1/" +} + +# Rows 0..4 are written while `id` is still required, rows 5 and NULL after the `ALTER`, so the +# table mixes pre- and post-evolution data files. Only the pre-evolution ones carry the defect. +create_mixed_nullability_table() { + local table="$1" + local table_path="${USER_FILES_PATH}/${table}/" + rm -rf "${table_path}" + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${table}" + ${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${table} (id Int64, s String) ENGINE = IcebergLocal('${table_path}', 'Parquet')" + ${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${table} SELECT number, toString(number) FROM numbers(5)" + ${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "ALTER TABLE ${table} MODIFY COLUMN id Nullable(Int64)" + ${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${table} SELECT * FROM values('id Nullable(Int64), s String', (5, 'five'), (NULL, 'none'))" +} + +create_mixed_nullability_table "${TABLE}" + +echo "--- WHERE on the evolved column ---" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT id, s FROM ${TABLE} WHERE id > 3 ORDER BY id" + +# Explicit PREWHERE probes the reader-side path directly, without depending on the +# WHERE->PREWHERE mover. +echo "--- PREWHERE on the evolved column ---" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT id, s FROM ${TABLE} PREWHERE id > 3 ORDER BY id" + +echo "--- PREWHERE IS NULL / IS NOT NULL ---" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT s FROM ${TABLE} PREWHERE id IS NULL ORDER BY s" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT count() FROM ${TABLE} PREWHERE id IS NOT NULL" + +echo "--- PREWHERE on an untouched column ---" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT id, s FROM ${TABLE} PREWHERE s = '4' ORDER BY id" + +echo "--- declared type and full scan ---" +${CLICKHOUSE_CLIENT} --query "SELECT count(), countIf(id > 3), toTypeName(any(id)) FROM ${TABLE}" +${CLICKHOUSE_CLIENT} --query "SELECT id, s FROM ${TABLE} ORDER BY id, s" + +# Composition with a rename: the transform must apply the new name and the new nullability in one +# node. Before the fix the rename branch renamed the column and dropped the type change. +echo "--- MODIFY COLUMN to Nullable plus RENAME COLUMN ---" +create_mixed_nullability_table "${TABLE_REN}" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "ALTER TABLE ${TABLE_REN} RENAME COLUMN id TO idx" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT idx, s, toTypeName(idx) FROM ${TABLE_REN} PREWHERE idx > 3 ORDER BY idx" + +# Widening and nullability at once already took the type-conversion branch and was already +# correct; assert it stays correct. +echo "--- widening plus nullability (int required to long optional) ---" +rm -rf "${USER_FILES_PATH}/${TABLE_WID}/" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${TABLE_WID} (id Int32, s String) ENGINE = IcebergLocal('${USER_FILES_PATH}/${TABLE_WID}/', 'Parquet')" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${TABLE_WID} SELECT toInt32(number), toString(number) FROM numbers(5)" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "ALTER TABLE ${TABLE_WID} MODIFY COLUMN id Nullable(Int64)" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT id, toTypeName(id) FROM ${TABLE_WID} PREWHERE id > 3 ORDER BY id" + +# The reverse direction is not legal evolution and must keep being rejected. +echo "--- optional to required is still rejected ---" +rm -rf "${USER_FILES_PATH}/${TABLE_REJ}/" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${TABLE_REJ} (id Nullable(Int64), s String) ENGINE = IcebergLocal('${USER_FILES_PATH}/${TABLE_REJ}/', 'Parquet')" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${TABLE_REJ} SELECT number, toString(number) FROM numbers(3)" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "ALTER TABLE ${TABLE_REJ} MODIFY COLUMN id Int64" 2>&1 \ + | grep -oF "Iceberg spec doesn't allow change type from nullable to non-nullable" | head -1 + +# A non-numeric type reaches the same branch through a different comparison function. +echo "--- String column made Nullable ---" +rm -rf "${USER_FILES_PATH}/${TABLE_STR}/" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${TABLE_STR} (v String, s String) ENGINE = IcebergLocal('${USER_FILES_PATH}/${TABLE_STR}/', 'Parquet')" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${TABLE_STR} SELECT toString(number), toString(number) FROM numbers(5)" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "ALTER TABLE ${TABLE_STR} MODIFY COLUMN v Nullable(String)" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT v, toTypeName(v) FROM ${TABLE_STR} PREWHERE v = '4' ORDER BY v" + +# The reverse direction (optional -> required) cannot be produced by `ALTER`, which rejects it +# above, so the passthrough branch is reached only through metadata written by another engine. +# Appending a schema leaves schema 0 byte-identical, so the schema-id immutability check still +# passes, and the read selects the pair with `iceberg_metadata_file_path`. +echo "--- externally authored optional to required stays a passthrough ---" +TABLE_REV="t_rev_${CLICKHOUSE_DATABASE}_${RANDOM}" +REV_PATH="${USER_FILES_PATH}/${TABLE_REV}/" +rm -rf "${REV_PATH}" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${TABLE_REV} (id Nullable(Int64), s String) ENGINE = IcebergLocal('${REV_PATH}', 'Parquet')" +# The NULL row is what makes the assertion non-vacuous: a cast to the required type only +# misbehaves when a NULL actually has to pass through it. +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${TABLE_REV} SELECT * FROM values('id Nullable(Int64), s String', (1, 'one'), (NULL, 'none'), (3, 'three'))" + +LATEST_REV=$(ls "${REV_PATH}metadata/" | grep -E '^v[0-9]+\.metadata\.json$' | sort -t v -k2 -n | tail -1) +REV_META=$(python3 - "${REV_PATH}metadata" "${LATEST_REV}" <<'PYEOF' +import copy, json, os, re, sys + +metadata_dir, latest_file = sys.argv[1], sys.argv[2] + +with open(os.path.join(metadata_dir, latest_file)) as fh: + metadata = json.load(fh) + +current = next(s for s in metadata["schemas"] if s["schema-id"] == metadata["current-schema-id"]) +tightened = copy.deepcopy(current) +tightened["schema-id"] = max(s["schema-id"] for s in metadata["schemas"]) + 1 +for field in tightened["fields"]: + if field["name"] == "id": + field["required"] = True + +metadata["schemas"].append(tightened) +metadata["current-schema-id"] = tightened["schema-id"] +metadata["last-updated-ms"] = metadata.get("last-updated-ms", 0) + 60000 + +version = int(re.match(r"v(\d+)\.metadata\.json", latest_file).group(1)) + 1 +tmp_file = os.path.join(metadata_dir, ".tmp_next") +with open(tmp_file, "w") as fh: + json.dump(metadata, fh) +os.rename(tmp_file, os.path.join(metadata_dir, f"v{version}.metadata.json")) +print(f"metadata/v{version}.metadata.json") +PYEOF +) + +REV_TF="icebergLocal('${REV_PATH}', 'Parquet', SETTINGS iceberg_metadata_file_path = '${REV_META}')" +# The tightened schema is what the reader resolves against. +${CLICKHOUSE_CLIENT} --query "DESCRIBE ${REV_TF}" | cut -f1,2 +# Each of these three shapes fails if the transform casts the old optional column to the new +# required type instead of passing it through. +${CLICKHOUSE_CLIENT} --query "SELECT s FROM ${REV_TF} ORDER BY s" +${CLICKHOUSE_CLIENT} --query "SELECT s FROM ${REV_TF} WHERE id IS NOT NULL ORDER BY s" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT s FROM ${REV_TF} PREWHERE id > 1 ORDER BY s" + +drop_table "${TABLE}" +drop_table "${TABLE_REN}" +drop_table "${TABLE_WID}" +drop_table "${TABLE_REJ}" +drop_table "${TABLE_STR}" +drop_table "${TABLE_REV}" diff --git a/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.reference b/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.reference new file mode 100644 index 0000000000000..415eb3c1f59a8 --- /dev/null +++ b/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.reference @@ -0,0 +1,8 @@ +--- A: key Int32 + arr Array(Int32) +lower_ids=[1] upper_ids=[1] key_lower=[0] key_upper=[4] +--- B: key Int32 + opt Nullable(Int32) all NULL +lower_ids=[1] upper_ids=[1] key_lower=[0] key_upper=[4] +--- C: key Int32 + val String (control, all supported) +lower_ids=[1,2] upper_ids=[1,2] key_lower=[0] key_upper=[4] +--- D: key Int32 + f Nullable(Float64) with values +lower_ids=[1] upper_ids=[1] key_lower=[0] key_upper=[4] diff --git a/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.sh b/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.sh new file mode 100755 index 0000000000000..8acdb1edb91bc --- /dev/null +++ b/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CUR_DIR"/../shell_config.sh + +PREFIX="t_${CLICKHOUSE_DATABASE}_${RANDOM}" +ROOT="${USER_FILES_PATH}/${PREFIX}" + +cleanup() +{ + for suffix in a b c d; do + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${PREFIX}_${suffix}" + done + rm -rf "${ROOT}" +} +trap cleanup EXIT + +# Prints the field ids present in lower_bounds / upper_bounds, plus the decoded Int32 bounds of +# field id 1 (the `key` column, always Int32 and always serializable here). +report() +{ + local table_dir="$1" + for manifest in $(find "${table_dir}/metadata" -maxdepth 1 -name '*.avro' \ + -not -name 'snap-*.avro' -type f | sort); do + ${CLICKHOUSE_CLIENT} --query " + SELECT + 'lower_ids=' || toString(arraySort(arrayMap(x -> x.1, tupleElement(data_file, 'lower_bounds')))), + 'upper_ids=' || toString(arraySort(arrayMap(x -> x.1, tupleElement(data_file, 'upper_bounds')))), + 'key_lower=' || toString(arrayMap(x -> reinterpretAsInt32(x.2), + arrayFilter(x -> x.1 = 1, tupleElement(data_file, 'lower_bounds')))), + 'key_upper=' || toString(arrayMap(x -> reinterpretAsInt32(x.2), + arrayFilter(x -> x.1 = 1, tupleElement(data_file, 'upper_bounds')))) + FROM file('${manifest}', Avro) + ORDER BY 1, 2, 3, 4 + " + done +} + +# Case A: an unsupported column type (Array(Int32)) must not suppress the bounds of `key`. +echo '--- A: key Int32 + arr Array(Int32)' +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${PREFIX}_a (key Int32, arr Array(Int32)) + ENGINE = IcebergLocal('${ROOT}/a/') +" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query \ + "INSERT INTO ${PREFIX}_a SELECT number::Int32, [number, number] FROM numbers(5)" +report "${ROOT}/a" + +# Case B: an entirely-NULL Nullable column yields a Null bound, which must not suppress `key` either. +echo '--- B: key Int32 + opt Nullable(Int32) all NULL' +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${PREFIX}_b (key Int32, opt Nullable(Int32)) + ENGINE = IcebergLocal('${ROOT}/b/') +" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query \ + "INSERT INTO ${PREFIX}_b SELECT number::Int32, NULL FROM numbers(5)" +report "${ROOT}/b" + +# Case C: control. All columns serializable, so both field ids keep their bounds exactly as before. +echo '--- C: key Int32 + val String (control, all supported)' +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${PREFIX}_c (key Int32, val String) + ENGINE = IcebergLocal('${ROOT}/c/') +" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query \ + "INSERT INTO ${PREFIX}_c SELECT number::Int32, 'val-' || toString(number) FROM numbers(5)" +report "${ROOT}/c" + +# Case D: wrapper composition with a non-NULL value. Float64 is unsupported for bounds even though +# the byte dumper handles it, so the filter must gate on the bounds predicate and skip only `f`. +echo '--- D: key Int32 + f Nullable(Float64) with values' +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${PREFIX}_d (key Int32, f Nullable(Float64)) + ENGINE = IcebergLocal('${ROOT}/d/') +" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query \ + "INSERT INTO ${PREFIX}_d SELECT number::Int32, number + 0.5 FROM numbers(5)" +report "${ROOT}/d" diff --git a/tests/queries/0_stateless/04824_handler_safe_method_session_state.sh b/tests/queries/0_stateless/04824_handler_safe_method_session_state.sh index 5d3ebaf796bda..6fdfccbcebf05 100755 --- a/tests/queries/0_stateless/04824_handler_safe_method_session_state.sh +++ b/tests/queries/0_stateless/04824_handler_safe_method_session_state.sh @@ -6,9 +6,9 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # Session-mutating statements (`SET`, `SET ROLE`, `USE`, `BEGIN` / `COMMIT` / `ROLLBACK` / # `SET TRANSACTION SNAPSHOT`) run under the `readonly = 2` mode that the HTTP execution path sets for safe -# methods, so the runtime `readonly` enforcement cannot fence them off. But they change session or transaction -# state that persists across requests when `session_id` is in use, and HTTP requires safe methods to be -# side-effect-free: a handler declared for GET is also served for HEAD, where the suppressed response body +# methods, so the runtime `readonly` enforcement cannot be relied on to fence them off. But they change session +# or transaction state that persists across requests when `session_id` is in use, and HTTP requires safe methods +# to be side-effect-free: a handler declared for GET is also served for HEAD, where the suppressed response body # would hide the effect entirely. So such a handler must list only mutating methods (POST, PUT, DELETE) - # both at CREATE and at ALTER time. diff --git a/tests/queries/0_stateless/04833_injective_plus_minus_type_gate.reference b/tests/queries/0_stateless/04833_injective_plus_minus_type_gate.reference new file mode 100644 index 0000000000000..48531a4780d35 --- /dev/null +++ b/tests/queries/0_stateless/04833_injective_plus_minus_type_gate.reference @@ -0,0 +1,95 @@ +-- { echo } + +-- --------------------------------------------------------------------------- +-- Correctness: per-partition evaluation must agree with the merged evaluation. Each arm +-- prints the forced count and then the merged count; they must be equal. The two settings +-- apply to the outer query, so they reach the aggregation under test. +-- --------------------------------------------------------------------------- + +-- Date + INTERVAL MONTH collapses the 29th, 30th and 31st of a month into one key. 2001 and +-- 2002 are not leap years, so all three days of each map to February 28. +DROP TABLE IF EXISTS t_month; +CREATE TABLE t_month (d Date, x UInt32) ENGINE = MergeTree ORDER BY d PARTITION BY d; +INSERT INTO t_month SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-01-', toString(29 + (intDiv(number, 10) % 3)))) AS d, number FROM numbers(60); +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM t_month GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +2 +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM t_month GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +2 +SELECT count() FROM (SELECT DISTINCT d + INTERVAL 1 MONTH AS k FROM t_month) SETTINGS force_distinct_partitions_independently = 1; +2 +SELECT count() FROM (SELECT DISTINCT d + INTERVAL 1 MONTH AS k FROM t_month) SETTINGS allow_distinct_partitions_independently = 0; +2 +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k FROM t_month LIMIT 1 BY k) SETTINGS allow_limit_by_partitions_independently = 1; +2 +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k FROM t_month LIMIT 1 BY k) SETTINGS allow_limit_by_partitions_independently = 0; +2 +DROP TABLE t_month; +-- the minus direction collapses the same way, going back into a shorter month +DROP TABLE IF EXISTS t_month_minus; +CREATE TABLE t_month_minus (d Date, x UInt32) ENGINE = MergeTree ORDER BY d PARTITION BY d; +INSERT INTO t_month_minus SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-03-', toString(29 + (intDiv(number, 10) % 3)))) AS d, number FROM numbers(60); +SELECT count() FROM (SELECT d - INTERVAL 1 MONTH AS k, count() FROM t_month_minus GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +2 +SELECT count() FROM (SELECT d - INTERVAL 1 MONTH AS k, count() FROM t_month_minus GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +2 +DROP TABLE t_month_minus; +-- DateTime + INTERVAL DAY collapses across a DST spring-forward, so the interval kind alone +-- does not decide safety +DROP TABLE IF EXISTS t_dst; +CREATE TABLE t_dst (t DateTime('Europe/Moscow'), x UInt32) ENGINE = MergeTree ORDER BY t PARTITION BY t; +INSERT INTO t_dst SELECT toDateTime('2010-03-27 00:00:00', 'Europe/Moscow') + (intDiv(number, 100) * 1800) AS t, number FROM numbers_mt(600); +SELECT count() FROM (SELECT t + INTERVAL 1 DAY AS k, count() FROM t_dst GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +4 +SELECT count() FROM (SELECT t + INTERVAL 1 DAY AS k, count() FROM t_dst GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +4 +DROP TABLE t_dst; +-- a Float64 addend collapses by mantissa rounding; a float column reaches the optimizer +-- through a non-float partition key +DROP TABLE IF EXISTS t_float; +CREATE TABLE t_float (f Float64, x UInt32) ENGINE = MergeTree ORDER BY f PARTITION BY toUInt64(f - 1e16); +INSERT INTO t_float SELECT 1e16 + intDiv(number, 100) AS f, number FROM numbers_mt(600); +SELECT count() FROM (SELECT f + 1.0 AS k, count() FROM t_float GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +2 +SELECT count() FROM (SELECT f + 1.0 AS k, count() FROM t_float GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +2 +-- the collapse the arm above depends on: fewer distinct sums than distinct addends +SELECT uniqExact(f), uniqExact(f + 1.0) FROM t_float; +3 2 +DROP TABLE t_float; +-- a constant of a narrower date type narrows the result: every multiple of 65536 maps to +-- 1970-01-01. A NULL constant maps every value to NULL. +DROP TABLE IF EXISTS t_narrow; +CREATE TABLE t_narrow (x UInt32, v UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x; +INSERT INTO t_narrow SELECT intDiv(number, 100) * 65536 AS x, number FROM numbers_mt(600); +SELECT count() FROM (SELECT x + toDate(0) AS k, count() FROM t_narrow GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +1 +SELECT count() FROM (SELECT x + toDate(0) AS k, count() FROM t_narrow GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +1 +SELECT count() FROM (SELECT DISTINCT x + CAST(NULL, 'Nullable(UInt32)') AS k FROM t_narrow) SETTINGS force_distinct_partitions_independently = 1; +1 +SELECT count() FROM (SELECT DISTINCT x + CAST(NULL, 'Nullable(UInt32)') AS k FROM t_narrow) SETTINGS allow_distinct_partitions_independently = 0; +1 +SELECT uniqExact(x), uniqExact(x + toDate(0)) FROM t_narrow; +6 1 +DROP TABLE t_narrow; +-- a Decimal constant rescales the varying operand: every multiple of 2^32 maps to one +-- Decimal(9, 1) +DROP TABLE IF EXISTS t_decimal; +CREATE TABLE t_decimal (x UInt64, v UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x; +INSERT INTO t_decimal SELECT intDiv(number, 100) * 4294967296 AS x, number FROM numbers_mt(400); +SELECT count() FROM (SELECT x + toDecimal32(0, 1) AS k, count() FROM t_decimal GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +1 +SELECT count() FROM (SELECT x + toDecimal32(0, 1) AS k, count() FROM t_decimal GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +1 +SELECT uniqExact(x), uniqExact(x + toDecimal32(0, 1)) FROM t_decimal; +4 1 +DROP TABLE t_decimal; +-- two varying operands are not injective either: many pairs share one sum +DROP TABLE IF EXISTS t_two_cols; +CREATE TABLE t_two_cols (x UInt32, y UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x; +INSERT INTO t_two_cols SELECT intDiv(number, 100) AS x, number % 100 AS y FROM numbers_mt(1000); +SELECT count() FROM (SELECT x + y AS k, count() FROM t_two_cols GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +109 +SELECT count() FROM (SELECT x + y AS k, count() FROM t_two_cols GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +109 +DROP TABLE t_two_cols; diff --git a/tests/queries/0_stateless/04833_injective_plus_minus_type_gate.sql b/tests/queries/0_stateless/04833_injective_plus_minus_type_gate.sql new file mode 100644 index 0000000000000..cf5290884f8db --- /dev/null +++ b/tests/queries/0_stateless/04833_injective_plus_minus_type_gate.sql @@ -0,0 +1,97 @@ +-- Tags: no-random-settings, no-random-merge-tree-settings +-- no-random-settings, no-random-merge-tree-settings: randomized settings and part counts +-- change both the plans and the values these arms count. + +-- max_threads is pinned because the cost heuristic accepts a fixture only when its partition +-- count is at least max_threads / 2; arms that must not depend on the heuristic force it instead. +SET max_threads = 8; +SET enable_parallel_replicas = 0; +SET max_rows_in_distinct = 0; +SET max_bytes_in_distinct = 0; +-- The stateless CI profile sets these to 10G, and a nonzero limit disables per-partition +-- evaluation outright. +SET max_rows_to_group_by = 0; +SET max_rows_to_sort = 0; +SET max_bytes_to_sort = 0; +SET optimize_use_implicit_projections = 0; +-- The values and plans below are the analyzer's, so pin it. +SET enable_analyzer = 1; + +-- { echo } + +-- --------------------------------------------------------------------------- +-- Correctness: per-partition evaluation must agree with the merged evaluation. Each arm +-- prints the forced count and then the merged count; they must be equal. The two settings +-- apply to the outer query, so they reach the aggregation under test. +-- --------------------------------------------------------------------------- + +-- Date + INTERVAL MONTH collapses the 29th, 30th and 31st of a month into one key. 2001 and +-- 2002 are not leap years, so all three days of each map to February 28. +DROP TABLE IF EXISTS t_month; +CREATE TABLE t_month (d Date, x UInt32) ENGINE = MergeTree ORDER BY d PARTITION BY d; +INSERT INTO t_month SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-01-', toString(29 + (intDiv(number, 10) % 3)))) AS d, number FROM numbers(60); +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM t_month GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM t_month GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +SELECT count() FROM (SELECT DISTINCT d + INTERVAL 1 MONTH AS k FROM t_month) SETTINGS force_distinct_partitions_independently = 1; +SELECT count() FROM (SELECT DISTINCT d + INTERVAL 1 MONTH AS k FROM t_month) SETTINGS allow_distinct_partitions_independently = 0; +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k FROM t_month LIMIT 1 BY k) SETTINGS allow_limit_by_partitions_independently = 1; +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k FROM t_month LIMIT 1 BY k) SETTINGS allow_limit_by_partitions_independently = 0; +DROP TABLE t_month; + +-- the minus direction collapses the same way, going back into a shorter month +DROP TABLE IF EXISTS t_month_minus; +CREATE TABLE t_month_minus (d Date, x UInt32) ENGINE = MergeTree ORDER BY d PARTITION BY d; +INSERT INTO t_month_minus SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-03-', toString(29 + (intDiv(number, 10) % 3)))) AS d, number FROM numbers(60); +SELECT count() FROM (SELECT d - INTERVAL 1 MONTH AS k, count() FROM t_month_minus GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT d - INTERVAL 1 MONTH AS k, count() FROM t_month_minus GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +DROP TABLE t_month_minus; + +-- DateTime + INTERVAL DAY collapses across a DST spring-forward, so the interval kind alone +-- does not decide safety +DROP TABLE IF EXISTS t_dst; +CREATE TABLE t_dst (t DateTime('Europe/Moscow'), x UInt32) ENGINE = MergeTree ORDER BY t PARTITION BY t; +INSERT INTO t_dst SELECT toDateTime('2010-03-27 00:00:00', 'Europe/Moscow') + (intDiv(number, 100) * 1800) AS t, number FROM numbers_mt(600); +SELECT count() FROM (SELECT t + INTERVAL 1 DAY AS k, count() FROM t_dst GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT t + INTERVAL 1 DAY AS k, count() FROM t_dst GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +DROP TABLE t_dst; + +-- a Float64 addend collapses by mantissa rounding; a float column reaches the optimizer +-- through a non-float partition key +DROP TABLE IF EXISTS t_float; +CREATE TABLE t_float (f Float64, x UInt32) ENGINE = MergeTree ORDER BY f PARTITION BY toUInt64(f - 1e16); +INSERT INTO t_float SELECT 1e16 + intDiv(number, 100) AS f, number FROM numbers_mt(600); +SELECT count() FROM (SELECT f + 1.0 AS k, count() FROM t_float GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT f + 1.0 AS k, count() FROM t_float GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +-- the collapse the arm above depends on: fewer distinct sums than distinct addends +SELECT uniqExact(f), uniqExact(f + 1.0) FROM t_float; +DROP TABLE t_float; + +-- a constant of a narrower date type narrows the result: every multiple of 65536 maps to +-- 1970-01-01. A NULL constant maps every value to NULL. +DROP TABLE IF EXISTS t_narrow; +CREATE TABLE t_narrow (x UInt32, v UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x; +INSERT INTO t_narrow SELECT intDiv(number, 100) * 65536 AS x, number FROM numbers_mt(600); +SELECT count() FROM (SELECT x + toDate(0) AS k, count() FROM t_narrow GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT x + toDate(0) AS k, count() FROM t_narrow GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +SELECT count() FROM (SELECT DISTINCT x + CAST(NULL, 'Nullable(UInt32)') AS k FROM t_narrow) SETTINGS force_distinct_partitions_independently = 1; +SELECT count() FROM (SELECT DISTINCT x + CAST(NULL, 'Nullable(UInt32)') AS k FROM t_narrow) SETTINGS allow_distinct_partitions_independently = 0; +SELECT uniqExact(x), uniqExact(x + toDate(0)) FROM t_narrow; +DROP TABLE t_narrow; + +-- a Decimal constant rescales the varying operand: every multiple of 2^32 maps to one +-- Decimal(9, 1) +DROP TABLE IF EXISTS t_decimal; +CREATE TABLE t_decimal (x UInt64, v UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x; +INSERT INTO t_decimal SELECT intDiv(number, 100) * 4294967296 AS x, number FROM numbers_mt(400); +SELECT count() FROM (SELECT x + toDecimal32(0, 1) AS k, count() FROM t_decimal GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT x + toDecimal32(0, 1) AS k, count() FROM t_decimal GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +SELECT uniqExact(x), uniqExact(x + toDecimal32(0, 1)) FROM t_decimal; +DROP TABLE t_decimal; + +-- two varying operands are not injective either: many pairs share one sum +DROP TABLE IF EXISTS t_two_cols; +CREATE TABLE t_two_cols (x UInt32, y UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x; +INSERT INTO t_two_cols SELECT intDiv(number, 100) AS x, number % 100 AS y FROM numbers_mt(1000); +SELECT count() FROM (SELECT x + y AS k, count() FROM t_two_cols GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT x + y AS k, count() FROM t_two_cols GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +DROP TABLE t_two_cols; diff --git a/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.reference b/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.reference new file mode 100644 index 0000000000000..74b069ccd930a --- /dev/null +++ b/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.reference @@ -0,0 +1,13 @@ +-- a valid frame still executes (proves the arms below fail for the intended reason) +1 +-- a codec that stores data uncompressed must not lie about the uncompressed size +1 +-- and neither may the other verbatim codec +1 +-- nor may it understate the uncompressed size +1 +-- engines keep working on ordinary data +1000 100000 +1000 100000 +1 +1 diff --git a/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.sh b/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.sh new file mode 100755 index 0000000000000..a0248bf8eb067 --- /dev/null +++ b/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +set -o pipefail + +# Wire layout of a compressed frame: +# [16B checksum][1B method][4B LE size_compressed][4B LE size_decompressed][payload] +# size_compressed counts the 9-byte header. The checksum is an unkeyed CityHash128 of everything +# after it, so a peer can compute it: these frames are accepted at default settings, and +# http_native_compression_disable_checksumming_on_decompress is deliberately NOT used. +# +# checksum-for-compressed-block prints CityHash128 of every single-bit mutation of its input, so +# feeding it the body with bit 0 flipped yields CityHash128(body) on the line labelled "0, 0". The +# wire order is low64 then high64, each little-endian, i.e. the reverse of the printed hex. +emit() { # $1 = frame body as hex, checksum excluded + local checksum + checksum=$(python3 -c " +import sys +b = bytearray.fromhex(sys.argv[1]); b[0] ^= 1 +sys.stdout.buffer.write(bytes(b)) +" "$1" | $CLICKHOUSE_BINARY checksum-for-compressed-block | awk -F'\t' '$2 == "0, 0" { print $1; exit }') + python3 -c " +import sys +sys.stdout.buffer.write(bytearray.fromhex(sys.argv[1])[::-1] + bytearray.fromhex(sys.argv[2])) +" "$checksum" "$1" +} + +# Method bytes are from CompressionInfo.h: NONE is 0x02 = 2 and Quantized is 0x9e = 158. +frame() { # $1 = method byte (decimal), $2 = size_decompressed, $3 = payload + emit "$(python3 -c " +import struct, sys +payload = sys.argv[3].encode() +sys.stdout.buffer.write(bytes([int(sys.argv[1])]) + struct.pack('&1 | grep -c '(8) does not match size_decompressed (999)' +# Quantized is the other codec reporting isNone(), so it takes the same shortcut. The read path +# builds it from the method byte alone, without the enable_quantized_codec setting its DDL requires. +echo '-- and neither may the other verbatim codec' +frame 158 999 'SELECT 1' | post 2>&1 | grep -c '(8) does not match size_decompressed (999)' + +# The comparison is for inequality, so arms that only ever declare more than the body pin one side of +# it: narrowed to "body shorter than declared", both frames above would still be refused. This one +# declares less instead, and without the check it is not refused at all, it executes its real body. +echo '-- nor may it understate the uncompressed size' +frame 2 3 'SELECT 1' | post 2>&1 | grep -c '(8) does not match size_decompressed (3)' + +# No-regression control: only a NONE-coded frame takes the shortcut, so the column, the marks and +# the primary key are all stored uncompressed here. StripeLog is absent because it compresses its +# whole stream with the default codec and ignores the column codec, so its reads never take it. +echo '-- engines keep working on ordinary data' +${CLICKHOUSE_CLIENT} --query " + DROP TABLE IF EXISTS t_log_bound; + DROP TABLE IF EXISTS t_mt_bound; + + CREATE TABLE t_log_bound (s String CODEC(NONE)) ENGINE = Log; + CREATE TABLE t_mt_bound (k UInt64, s String CODEC(NONE), INDEX idx_s s TYPE minmax GRANULARITY 1) + ENGINE = MergeTree ORDER BY k + SETTINGS index_granularity = 8, compress_marks = 1, compress_primary_key = 1, + min_bytes_for_wide_part = 0, packed_skip_index_max_bytes = 0, + marks_compression_codec = 'NONE', primary_key_compression_codec = 'NONE'; + + INSERT INTO t_log_bound SELECT repeat('a', 100) FROM numbers(1000); + INSERT INTO t_mt_bound SELECT number, repeat('a', 100) FROM numbers(1000); + + SELECT count(), sum(length(s)) FROM t_log_bound; + SELECT count(), sum(length(s)) FROM t_mt_bound WHERE s LIKE '%a%'; + SELECT count() FROM t_mt_bound WHERE k = 42; + -- packed_skip_index_max_bytes = 0 keeps the index in its own file, which is the skip-index class + -- checkDataPart iterates over; a packed one carries no per-file checksum entry to visit. + CHECK TABLE t_mt_bound SETTINGS check_query_single_value_result = 1; + + DROP TABLE t_log_bound; + DROP TABLE t_mt_bound; +" diff --git a/tests/queries/0_stateless/04854_ie_join_on_array_join_residual.reference b/tests/queries/0_stateless/04854_ie_join_on_array_join_residual.reference new file mode 100644 index 0000000000000..8ecb43a2ae458 --- /dev/null +++ b/tests/queries/0_stateless/04854_ie_join_on_array_join_residual.reference @@ -0,0 +1,9 @@ +1 +1 +1 +2 +2 +2 +3 +3 +3 diff --git a/tests/queries/0_stateless/04854_ie_join_on_array_join_residual.sql b/tests/queries/0_stateless/04854_ie_join_on_array_join_residual.sql new file mode 100644 index 0000000000000..8b9e37e855a05 --- /dev/null +++ b/tests/queries/0_stateless/04854_ie_join_on_array_join_residual.sql @@ -0,0 +1,25 @@ +-- Tags: no-old-analyzer +-- `ie_join` is not in the default `join_algorithm`, so both queries pin the algorithm themselves. + +DROP TABLE IF EXISTS ie_l; +DROP TABLE IF EXISTS ie_r; + +CREATE TABLE ie_l (id Int32, lo Int32, hi Int32, price Float64) ENGINE = MergeTree ORDER BY id; +CREATE TABLE ie_r (id Int32, lo Int32, hi Int32, bid Float64) ENGINE = MergeTree ORDER BY id; + +INSERT INTO ie_l VALUES (1, 10, 20, 150.0), (2, 20, 30, 151.0), (3, 30, 40, 380.0); +INSERT INTO ie_r VALUES (1, 10, 30, 149.5), (2, 30, 40, 150.5), (3, 40, 50, 379.0); + +-- A cross-side `arrayJoin` lands in the condition that `ie_join` evaluates during the join. +SELECT l.id FROM ie_l AS l SEMI LEFT JOIN ie_r AS r ON (l.lo < r.hi) AND (r.lo < l.hi) + AND (300 > minus(l.price + r.bid, arrayJoin(range(2)))) ORDER BY ALL + SETTINGS join_algorithm = 'ie_join,hash', join_use_nulls = 1; -- { serverError INVALID_JOIN_ON_EXPRESSION } + +-- A one-sided `arrayJoin` is extracted into a filter before the join, so it stays accepted. +SELECT l.id FROM ie_l AS l SEMI LEFT JOIN ie_r AS r ON (l.lo < r.hi) AND (r.lo < l.hi) + AND (l.price > arrayJoin(range(3))) ORDER BY ALL + SETTINGS join_algorithm = 'ie_join,hash', join_use_nulls = 1, + query_plan_split_filter = 1; + +DROP TABLE ie_l; +DROP TABLE ie_r; diff --git a/tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.reference b/tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.reference new file mode 100644 index 0000000000000..3350bf0bdf334 --- /dev/null +++ b/tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.reference @@ -0,0 +1,12 @@ +reported greedy: +0 Join_1_Value_0 0 Join_2_Value_0 0 Join_3_Value_0 +reported dpsub: +0 Join_1_Value_0 0 Join_2_Value_0 0 Join_3_Value_0 +filter deep relation greedy: +0 Join_2_Value_0 0 Join_3_Value_0 0 Join_1_Value_0 +filter deep relation dpsub: +0 Join_2_Value_0 0 Join_3_Value_0 0 Join_1_Value_0 +four way filter last greedy: +0 0 0 0 Join_4_Value_0 +four way filter last dpsub: +0 0 0 0 Join_4_Value_0 diff --git a/tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.sql b/tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.sql new file mode 100644 index 0000000000000..b0657008aedb2 --- /dev/null +++ b/tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.sql @@ -0,0 +1,86 @@ +-- Tests that the DPsub join-order algorithm keeps single-table filter predicates that live in a +-- join's ON clause (e.g. `... JOIN t ON t.a = u.a AND t.b = 'x'`). DPsub attaches such a predicate +-- at the join that introduces its relation. Two earlier placement bugs silently dropped the +-- predicate, letting extra rows through: +-- 1. attaching only at two-relation joins dropped it when the relation was introduced against an +-- already-multi-relation subplan (e.g. `t1 JOIN (t2 JOIN t3)`); +-- 2. gating on fromLeft()/fromRight() dropped it for any filter on a relation whose id is >= 2, +-- because those helpers test relation ids 0 and 1 specifically. +-- For each shape we print the result with the 'greedy' algorithm (the reference, which places the +-- predicate correctly) and with 'dpsub'; the two must be identical row-for-row. +-- +-- `query_plan_enable_optimizations = 0` is required to expose the bug and does NOT disable join +-- reordering (that is controlled by `query_plan_optimize_join_order_algorithm`). It disables the +-- general plan-optimization passes, notably filter push-down; with push-down enabled the +-- single-table filter is applied independently of the join and masks the dropped ON-clause +-- conjunct, so the wrong-result would not surface. + +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; +DROP TABLE IF EXISTS t4; + +CREATE TABLE t1 (id UInt64, value String) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t2 (id UInt64, value String) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t3 (id UInt64, value String) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t4 (id UInt64, value String) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t1 VALUES (0, 'Join_1_Value_0'), (1, 'Join_1_Value_1'), (2, 'Join_1_Value_2'); +INSERT INTO t2 VALUES (0, 'Join_2_Value_0'), (1, 'Join_2_Value_1'), (3, 'Join_2_Value_3'); +INSERT INTO t3 VALUES (0, 'Join_3_Value_0'), (1, 'Join_3_Value_1'), (4, 'Join_3_Value_4'); +INSERT INTO t4 VALUES (0, 'Join_4_Value_0'), (1, 'Join_4_Value_1'), (5, 'Join_4_Value_5'); + +SET enable_analyzer = 1; +SET query_plan_optimize_join_order_limit = 10; + +-- The reported case: filter on the first relation, INNER then LEFT join. +SELECT 'reported greedy:'; +SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value +FROM t1 INNER JOIN t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' +LEFT JOIN t3 ON t2.id = t3.id ORDER BY ALL +SETTINGS query_plan_optimize_join_order_algorithm = 'greedy', + query_plan_enable_optimizations = 0; + +SELECT 'reported dpsub:'; +SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value +FROM t1 INNER JOIN t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' +LEFT JOIN t3 ON t2.id = t3.id ORDER BY ALL +SETTINGS query_plan_optimize_join_order_algorithm = 'dpsub', + query_plan_enable_optimizations = 0; + +-- Filter on a relation that is introduced last, against an already-joined subplan (relation id >= 2). +SELECT 'filter deep relation greedy:'; +SELECT t2.id, t2.value, t3.id, t3.value, t1.id, t1.value +FROM t2 INNER JOIN t3 ON t2.id = t3.id +INNER JOIN t1 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' ORDER BY ALL +SETTINGS query_plan_optimize_join_order_algorithm = 'greedy', + query_plan_enable_optimizations = 0; + +SELECT 'filter deep relation dpsub:'; +SELECT t2.id, t2.value, t3.id, t3.value, t1.id, t1.value +FROM t2 INNER JOIN t3 ON t2.id = t3.id +INNER JOIN t1 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' ORDER BY ALL +SETTINGS query_plan_optimize_join_order_algorithm = 'dpsub', + query_plan_enable_optimizations = 0; + +-- Four-way all-inner chain with the filter on the last relation (id = 3). +SELECT 'four way filter last greedy:'; +SELECT t1.id, t2.id, t3.id, t4.id, t4.value +FROM t1 INNER JOIN t2 ON t1.id = t2.id +INNER JOIN t3 ON t2.id = t3.id +INNER JOIN t4 ON t3.id = t4.id AND t4.value = 'Join_4_Value_0' ORDER BY ALL +SETTINGS query_plan_optimize_join_order_algorithm = 'greedy', + query_plan_enable_optimizations = 0; + +SELECT 'four way filter last dpsub:'; +SELECT t1.id, t2.id, t3.id, t4.id, t4.value +FROM t1 INNER JOIN t2 ON t1.id = t2.id +INNER JOIN t3 ON t2.id = t3.id +INNER JOIN t4 ON t3.id = t4.id AND t4.value = 'Join_4_Value_0' ORDER BY ALL +SETTINGS query_plan_optimize_join_order_algorithm = 'dpsub', + query_plan_enable_optimizations = 0; + +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +DROP TABLE t4; diff --git a/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.reference b/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.reference index 494e4eb3e2075..1809b85a1aac3 100644 --- a/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.reference +++ b/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.reference @@ -1,5 +1,5 @@ t_lazy TableProxy -t_lazy_memory TableProxy +t_lazy_log TableProxy CREATE TABLE default_1.t_lazy\n(\n `key` UInt64,\n `v` Float64 STATISTICS(tdigest),\n `s` String TTL d + toIntervalMonth(1),\n `d` DateTime\n)\nENGINE = MergeTree\nORDER BY key\nSETTINGS index_granularity = 8192 TableProxy CREATE TABLE default_1.t_lazy\n(\n `key` UInt64,\n `v` Float64 STATISTICS(tdigest),\n `s` String TTL d + toIntervalMonth(1),\n `d` DateTime,\n `v2` Float64 STATISTICS(uniq)\n)\nENGINE = MergeTree\nORDER BY key\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.sql b/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.sql index de3fda3e11857..80ff346f1857b 100644 --- a/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.sql +++ b/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.sql @@ -14,7 +14,7 @@ DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; CREATE DATABASE {CLICKHOUSE_DATABASE_1:Identifier} ENGINE = Atomic SETTINGS lazy_load_tables = 1; CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t_lazy (key UInt64, v Float64, s String, d DateTime) ENGINE = MergeTree ORDER BY key; -CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t_lazy_memory (x UInt64) ENGINE = Memory; +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t_lazy_log (x UInt64) ENGINE = Log; DETACH DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; ATTACH DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; @@ -38,7 +38,7 @@ SHOW CREATE TABLE t_lazy; -- The proxy forwards the nested answer rather than a blanket `true`: a lazily loaded engine without -- statistics support is still rejected. -ALTER TABLE t_lazy_memory MODIFY COLUMN x UInt64 STATISTICS(tdigest); -- { serverError NOT_IMPLEMENTED } -ALTER TABLE t_lazy_memory ADD COLUMN y UInt64 STATISTICS(tdigest); -- { serverError NOT_IMPLEMENTED } +ALTER TABLE t_lazy_log MODIFY COLUMN x UInt64 STATISTICS(tdigest); -- { serverError NOT_IMPLEMENTED } +ALTER TABLE t_lazy_log ADD COLUMN y UInt64 STATISTICS(tdigest); -- { serverError NOT_IMPLEMENTED } DROP DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; diff --git a/tests/queries/0_stateless/04909_client_hints_history_navigation.python b/tests/queries/0_stateless/04909_client_hints_history_navigation.python new file mode 100644 index 0000000000000..54cd43293f00c --- /dev/null +++ b/tests/queries/0_stateless/04909_client_hints_history_navigation.python @@ -0,0 +1,392 @@ +import multiprocessing +import os +import pty +import re +import select +import shlex +import sys +import time + +TIMEOUT_SECONDS = 30 + +# How long to keep reading after the last keystroke while waiting for the expected output. The +# committed-query case only sees the result once the query has executed, which can take longer +# than the inter-keystroke drain on slow (sanitizer) builds. +FINAL_WAIT_SECONDS = 20 + +# How long to wait for the client to print the next `:) ` prompt after a committed query. This is +# the time a query of this test may take, and it is generous because the query does not have to be +# fast for the test to mean anything: in an MSan build a trivial `SELECT` that fails to resolve an +# identifier has been seen taking 36 seconds. +PROMPT_WAIT_SECONDS = 120 + +# Idle window after every keystroke, long enough for `replxx` to repaint the line. +DRAIN_IDLE_SECONDS = 0.4 + +# Extra slack for the supervisor on top of everything the worker itself may legitimately wait for. +SUPERVISOR_SLACK_SECONDS = 15 + +# A pseudo-keystroke: wait for the next prompt instead of writing anything. Sending the keys of the +# next line while the previous query is still running would let the terminal driver echo them +# instead of `replxx` interpreting them - a bracketed paste would then appear verbatim as +# `^[[200~...^[[201~` in the output and never reach the line editor. +WAIT_FOR_PROMPT = object() + +# `clickhouse-test` dumps this file into the failure report, but only under the name it derives +# from the testcase (the `.sh` wrapper), so take the path from there. +DEBUG_LOG = os.environ["CLICKHOUSE_TEST_DEBUG_LOG"] + +# The gray SGR sequence replxx uses for the as-you-type hint (Color::GRAY -> "0;90"). +HINT_COLOR = "\x1b[0;90m" + + +def make_history_file(name, entries): + """Write a replxx-format history file with the given entries, oldest first.""" + path = os.path.join( + os.environ["CLICKHOUSE_TMP"], + os.path.splitext(os.path.basename(os.path.abspath(__file__)))[0] + + f".{name}.history", + ) + with open(path, "w") as f: + for i, entry in enumerate(entries): + f.write(f"### 2020-01-01 00:00:{i:02}.000\n{entry}\n") + return path + + +def read_until(master, debug_log_fd, predicate, timeout): + output = "" + deadline = time.time() + timeout + while time.time() < deadline: + r, _, _ = select.select([master], [], [], 0.3) + if not r: + if predicate(output): + return output + continue + try: + chunk = os.read(master, 4096) + except OSError: + break + debug_log_fd.write(repr(chunk) + "\n") + debug_log_fd.flush() + output += chunk.decode(errors="replace") + if predicate(output): + return output + return output + + +def drain(master, debug_log_fd, idle): + output = "" + while True: + r, _, _ = select.select([master], [], [], idle) + if not r: + break + try: + chunk = os.read(master, 4096) + except OSError: + break + debug_log_fd.write(repr(chunk) + "\n") + debug_log_fd.flush() + output += chunk.decode(errors="replace") + return output + + +def report_failure(name, shell_pid, output, debug_log_fd): + """A PTY case that fails prints just `FAIL`, which says nothing about why. Record what the + terminal actually showed and whether the client is still alive - the client dying (the read + loop hitting EOF) looks exactly like the expected text never being printed.""" + try: + reaped_pid, wait_status = os.waitpid(shell_pid, os.WNOHANG) + except ChildProcessError: + reaped_pid, wait_status = shell_pid, -1 + running = reaped_pid == 0 + banner = ( + f"=== {name}: FAIL ===\n" + f"client still running: {running}, wait status: {wait_status}\n" + f"terminal output: {output[-8000:]!r}\n" + ) + debug_log_fd.write(banner) + debug_log_fd.flush() + sys.stderr.write(banner) + sys.stderr.flush() + + +def run_case(program, argv, name, keystrokes, check, state=None): + """Fork a PTY, wait for the prompt, send keystrokes (with a small pause between + them so replxx repaints), then run `check` over the collected output.""" + shell_pid, master = pty.fork() + if shell_pid == 0: + os.environ["TERM"] = "xterm" + os.execv(program, argv) + return + + debug_log_fd = open(DEBUG_LOG, "a") + try: + read_until(master, debug_log_fd, lambda o: ":)" in o, TIMEOUT_SECONDS) + + output = "" + # Only what arrived after the last keystroke - a prompt seen earlier in `output` says + # nothing about the prompt this step is waiting for. + since_last_keystroke = "" + for keys in keystrokes: + if keys is WAIT_FOR_PROMPT: + extra = read_until( + master, + debug_log_fd, + lambda more, seen=since_last_keystroke: ":)" in seen + more, + PROMPT_WAIT_SECONDS, + ) + output += extra + since_last_keystroke += extra + # `read_until` returns on its deadline as well as on a match, and going on from a + # deadline would write the next keys into a terminal that is still running the + # previous query: the driver echoes them instead of `replxx` interpreting them, so + # a bracketed paste ends up in the output verbatim and the case fails on its + # content, saying nothing about the prompt that never came. Stop here instead. + if ":)" not in since_last_keystroke: + print(f"{name}: FAIL") + report_failure(name, shell_pid, output, debug_log_fd) + state.value = 1 + return + continue + os.write(master, keys) + since_last_keystroke = drain(master, debug_log_fd, DRAIN_IDLE_SECONDS) + output += since_last_keystroke + + # The expected output may arrive only after the committed query executes, which can + # outlast the inter-keystroke drain on slow builds. Keep reading until the check passes + # instead of giving up after a single drain window. + if not check(output): + output += read_until( + master, + debug_log_fd, + lambda extra: check(output + extra), + FINAL_WAIT_SECONDS, + ) + + ok = check(output) + print(f"{name}: {'OK' if ok else 'FAIL'}") + if not ok: + report_failure(name, shell_pid, output, debug_log_fd) + state.value = 0 if ok else 1 + finally: + os.close(master) + debug_log_fd.close() + + +def worker_budget(keystrokes): + """Everything `run_case` may legitimately spend: the wait for the first prompt, the idle window + after every keystroke, every explicit prompt wait, and the final wait for the expected output. + The supervisor must not expire before that, or ordinary slowness looks like a failure.""" + prompt_waits = sum(1 for keys in keystrokes if keys is WAIT_FOR_PROMPT) + return ( + TIMEOUT_SECONDS + + PROMPT_WAIT_SECONDS * prompt_waits + + DRAIN_IDLE_SECONDS * len(keystrokes) + + FINAL_WAIT_SECONDS + + SUPERVISOR_SLACK_SECONDS + ) + + +def run_with_timeout(program, argv, name, history, keystrokes, check): + """Run a case with a history file holding the `history` entries. The file is written anew for + every attempt: an attempt killed after its final Enter may already have appended the query to + it, and a retry starting from that tail would recall a different entry than the case intends + while still printing the expected result.""" + for attempt in range(5): + history_file = make_history_file(name, history) + state = multiprocessing.Value("i", -1) + process = multiprocessing.Process( + target=run_case, + args=(program, argv + [f"--history_file={history_file}"], name, keystrokes, check), + kwargs={"state": state}, + ) + process.start() + process.join(worker_budget(keystrokes)) + if process.is_alive(): + process.terminate() + if state.value in (0, 1): + return + # transient timeout on a loaded machine - retry + print(f"{name}: FAIL") + sys.stderr.write(f"=== {name}: FAIL === the case timed out on every attempt\n") + + +def main(): + program = os.environ["CLICKHOUSE_LOCAL"] + base = shlex.split(program) + args = base + ["--wait_for_suggestions_to_load", "--hints", "1"] + + # A line displayed programmatically (recalled from history, pasted) must not pop the + # as-you-type hints by itself: with hints visible, the next Up/Down press would navigate + # the hints instead of the history. An edit shows the hints again. + + # 1. Up recalls an entry; no hint appears at its end ("ive" would be the ghost suffix of + # "concatAssumeInjective"). + run_with_timeout( + base[0], + args, + "up_recall_no_hints", + ["SELECT concatAssumeInject"], + [b"\x1b[A"], + lambda o: "concatAssumeInject" in o and (HINT_COLOR + "ive") not in o, + ) + + # Meta-Up is replxx's direct history-previous binding. It must use the same suppression as + # plain Up, rather than leaving the default action to regenerate hints. + run_with_timeout( + base[0], + args, + "meta_up_recall_no_hints", + ["SELECT concatAssumeInject"], + [b"\x1b\x1b[A"], + lambda o: "concatAssumeInject" in o and (HINT_COLOR + "ive") not in o, + ) + + # Ctrl-G restores the currently recalled history entry after an edit. It is another whole + # line display, so Down must return to the scratch line rather than navigate the hint list. + run_with_timeout( + base[0], + args, + "restore_current_down_navigates_history", + ["SELECT concatAssumeInjec"], + [b"SELECT 111 AS rand", b"\x1b[A", b"t", b"\x07", b"\x1b[B", b"\r"], + lambda o: re.search(r"(?:│\s*111\s*│|\x1b\[\?25h111\r?\n)", o) is not None, + ) + + # 2. Up, Up, Down walks the history (the entries end in "rand", which has many hint + # matches - Down must not step into the hint list); Enter then runs the recalled entry, + # so its result (222) is printed. The result can use either Pretty or TabSeparated output + # depending on the client's default format. + run_with_timeout( + base[0], + args, + "down_navigates_history", + ["SELECT 111 AS rand", "SELECT 222 AS rand"], + [b"\x1b[A", b"\x1b[A", b"\x1b[B", b"\r"], + lambda o: re.search(r"(?:│\s*222\s*│|\x1b\[\?25h222\r?\n)", o) is not None, + ) + + # 3. Recalling a history entry with the same text as the scratch line is still a programmatic + # display. Down must return from that entry to the scratch line instead of selecting a hint. + run_with_timeout( + base[0], + args, + "same_text_recall_down_navigates_history", + ["SELECT 222 AS rand", "SELECT 111 AS rand"], + [b"SELECT 111 AS rand", b"\x1b[A", b"\x1b[B", b"\r"], + lambda o: re.search(r"(?:│\s*111\s*│|\x1b\[\?25h111\r?\n)", o) is not None, + ) + + # 3b. An incremental history search (Ctrl-T, the ClickHouse binding for the regular non-fuzzy + # reverse search) that accepts an entry is another whole-line display: it goes through a + # different `replxx` code path than the plain history recall, so it needs its own proof that + # Down keeps walking the history. Ctrl-T, then the search text, then Ctrl-E leaves the search + # keeping the found line ("SELECT 111 AS rand") and puts the cursor at its end, where the hints + # would render. The accepted entry is committed as the most recent recall, so the first Down + # re-displays it (`replxx` emulates the Windows down-arrow there) and the second one moves on + # to "SELECT 222 AS rand", which Enter then runs. + run_with_timeout( + base[0], + args, + "incremental_search_down_navigates_history", + ["SELECT 111 AS rand", "SELECT 222 AS rand"], + [b"\x14", b"111", b"\x05", b"\x1b[B", b"\x1b[B", b"\r"], + lambda o: re.search(r"(?:│\s*222\s*│|\x1b\[\?25h222\r?\n)", o) is not None, + ) + + # 4. Editing a recalled entry shows the hints again: after Up, typing the "t" completes the + # prefix "concatAssumeInject" and the ghost "ive" appears. + run_with_timeout( + base[0], + args, + "edit_after_recall_shows_hints", + ["SELECT concatAssumeInjec"], + [b"\x1b[A", b"t"], + lambda o: (HINT_COLOR + "ive") in o, + ) + + # 5. A (bracketed) paste is also a whole new line displayed at once - no hint at its end. + run_with_timeout( + base[0], + args, + "paste_no_hints", + [], + [b"\x1b[200~SELECT concatAssumeInject\x1b[201~"], + lambda o: "concatAssumeInject" in o and (HINT_COLOR + "ive") not in o, + ) + + # 5b. Pasting the exact text that carried a visible hint on an earlier prompt: replxx caches + # the hints by the buffer text and the cache outlives the prompt, so without invalidating it + # the paste would redisplay the stale ghost without ever asking our hint callback. The first + # line is typed (so the hints for it are generated and cached) and committed - it is not a + # valid query, which is irrelevant here, only the cache state matters - and then the very same + # text is pasted on the next prompt, where no hint may appear. Only the output after the last + # prompt is examined: the first, typed line legitimately shows the ghost. + run_with_timeout( + base[0], + args, + "same_text_paste_no_hints", + [], + [ + b"SELECT concatAssumeInject", + b"\r", + WAIT_FOR_PROMPT, + b"\x1b[200~SELECT concatAssumeInject\x1b[201~", + ], + lambda o: "concatAssumeInject" in o.rsplit(":)", 1)[-1] + and (HINT_COLOR + "ive") not in o.rsplit(":)", 1)[-1], + ) + + # 6. An unchanged editor round-trip redisplays the line programmatically. It must invalidate + # replxx's cached hints, so Down remains history navigation. + editor = os.path.join(os.environ["CLICKHOUSE_TMP"], "client_hints_unchanged_editor.sh") + with open(editor, "w") as f: + f.write("#!/bin/sh\nexit 0\n") + os.chmod(editor, 0o755) + previous_editor = os.environ.get("EDITOR") + os.environ["EDITOR"] = editor + try: + run_with_timeout( + base[0], + args, + "unchanged_editor_down_navigates_history", + [], + [b"SELECT 111 AS rand", b"\x1bE", b"\x1b[B", b"\r"], + lambda o: re.search(r"(?:│\s*111\s*│|\x1b\[\?25h111\r?\n)", o) is not None, + ) + finally: + if previous_editor is None: + del os.environ["EDITOR"] + else: + os.environ["EDITOR"] = previous_editor + + # 7. An editor that exits unsuccessfully brings the original line back - also a whole-line + # programmatic display, and one that happens while the hints of the line typed before the + # editor was opened are still live. Down must remain history navigation there too. + editor = os.path.join(os.environ["CLICKHOUSE_TMP"], "client_hints_failing_editor.sh") + with open(editor, "w") as f: + f.write("#!/bin/sh\nexit 1\n") + os.chmod(editor, 0o755) + previous_editor = os.environ.get("EDITOR") + os.environ["EDITOR"] = editor + try: + run_with_timeout( + base[0], + args, + "failing_editor_down_navigates_history", + [], + [b"SELECT 111 AS rand", b"\x1bE", b"\x1b[B", b"\r"], + lambda o: re.search(r"(?:│\s*111\s*│|\x1b\[\?25h111\r?\n)", o) is not None, + ) + finally: + if previous_editor is None: + del os.environ["EDITOR"] + else: + os.environ["EDITOR"] = previous_editor + + +if __name__ == "__main__": + # The check lambdas are not picklable, so the subprocesses must be forked (not spawned, + # which became the default on newer Python). + multiprocessing.set_start_method("fork") + main() diff --git a/tests/queries/0_stateless/04909_client_hints_history_navigation.reference b/tests/queries/0_stateless/04909_client_hints_history_navigation.reference new file mode 100644 index 0000000000000..bcd45174864a0 --- /dev/null +++ b/tests/queries/0_stateless/04909_client_hints_history_navigation.reference @@ -0,0 +1,11 @@ +up_recall_no_hints: OK +meta_up_recall_no_hints: OK +restore_current_down_navigates_history: OK +down_navigates_history: OK +same_text_recall_down_navigates_history: OK +incremental_search_down_navigates_history: OK +edit_after_recall_shows_hints: OK +paste_no_hints: OK +same_text_paste_no_hints: OK +unchanged_editor_down_navigates_history: OK +failing_editor_down_navigates_history: OK diff --git a/tests/queries/0_stateless/04909_client_hints_history_navigation.sh b/tests/queries/0_stateless/04909_client_hints_history_navigation.sh new file mode 100755 index 0000000000000..04878aed03896 --- /dev/null +++ b/tests/queries/0_stateless/04909_client_hints_history_navigation.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Tags: long, no-debug + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `clickhouse-test` dumps `$CLICKHOUSE_TMP/.debuglog` into the failure report, and the +# testcase name is the name of this shell script - the Python part cannot derive it from its own +# file name, so pass the path down. +export CLICKHOUSE_TEST_DEBUG_LOG="$CLICKHOUSE_TMP/$(basename "${BASH_SOURCE[0]}").debuglog" + +python3 "$CUR_DIR"/04909_client_hints_history_navigation.python diff --git a/tests/queries/0_stateless/04952_projection_argumentless_function_json.reference b/tests/queries/0_stateless/04952_projection_argumentless_function_json.reference new file mode 100644 index 0000000000000..15a67caf6921e --- /dev/null +++ b/tests/queries/0_stateless/04952_projection_argumentless_function_json.reference @@ -0,0 +1,2 @@ +NUMBER_OF_ARGUMENTS_DOESNT_MATCH +1 diff --git a/tests/queries/0_stateless/04952_projection_argumentless_function_json.sh b/tests/queries/0_stateless/04952_projection_argumentless_function_json.sh new file mode 100755 index 0000000000000..37b6ea150204a --- /dev/null +++ b/tests/queries/0_stateless/04952_projection_argumentless_function_json.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Malformed JSON AST: a function without an `arguments` list inside a projection expression slot +# (`query` for a SELECT projection, `index` for an INDEX projection) must not crash the server: +# `count` accepts no arguments, `plus` fails with NUMBER_OF_ARGUMENTS_DOESNT_MATCH. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# $1: projection definition, $2: name of the function whose `arguments` are removed +function check() +{ + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_04952 SYNC" + + # The valid statement round-trips through the JSON dialect. + JSON=$(${CLICKHOUSE_CLIENT} -q "SELECT parseQueryToJSON('CREATE TABLE t_04952 (x UInt8, PROJECTION $1) ENGINE = MergeTree ORDER BY x') FORMAT TSVRaw") + ${CLICKHOUSE_CLIENT} --enable_json_ast_dialect 1 --dialect clickhouse_json -q "$JSON" + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_04952 SYNC" + + JSON_BAD=$(printf '%s' "$JSON" | python3 -c ' +import json, sys + +def strip_args(node, name): + if isinstance(node, dict): + if node.get("type") == "Function" and node.get("name") == name: + node.pop("arguments", None) + for v in node.values(): + strip_args(v, name) + elif isinstance(node, list): + for v in node: + strip_args(v, name) + +ast = json.load(sys.stdin) +strip_args(ast, sys.argv[1]) +print(json.dumps(ast)) +' "$2") + + OUT=$(${CLICKHOUSE_CLIENT} --enable_json_ast_dialect 1 --dialect clickhouse_json -q "$JSON_BAD" 2>&1 || true) + echo "$OUT" | grep -oE 'NUMBER_OF_ARGUMENTS_DOESNT_MATCH' | head -1 + + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_04952 SYNC" +} + +check "p (SELECT count() GROUP BY x)" count +check "p INDEX x + 1 TYPE basic" plus + +# The server is still alive to serve a plain query. +${CLICKHOUSE_CLIENT} -q 'SELECT 1' diff --git a/tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.reference b/tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.reference new file mode 100644 index 0000000000000..a19952f23e73d --- /dev/null +++ b/tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.reference @@ -0,0 +1,42 @@ +saturate +2149-06-06 1970-01-01 2106-02-07 06:28:15 1970-01-01 00:00:00 +2149-06-06 2149-06-06 2149-06-06 +2149-06-06 2149-06-06 +ignore +2149-06-06 1970-01-01 2106-02-07 06:28:15 1970-01-01 00:00:00 +throw +throw, in range +2149-06-06 1970-01-01 2106-02-07 06:28:15 1970-01-01 00:00:00 +2299-12-31 1900-01-01 2299-12-31 23:59:59.999 +throw, OrNull and OrZero still fall back +\N 1970-01-01 \N 1970-01-01 00:00:00 +throw, input formats +2149-06-06 +2106-02-07 06:28:15 +throw, tentative parsers must not accept a clamped value +2150-12-31 \N 2150-12-31 +2149-06-06 +throw, a digit-only timestamp is text too +2106-02-07 06:28:15 2023-11-14 22:13:20 \N 1970-01-01 00:00:00 +throw, Date32 rejects what it cannot represent instead of substituting a default +\N 2299-12-31 1900-01-01 +2000-13-01 +throw, an unquoted numeric token is checked too +2023-11-14 22:13:20 +2106-02-07 06:28:15 +2023-12-23 20:37:33 +throw, the last second keeps its fractional ticks +9999-12-31 23:59:59.500 +9999-12-31 23:59:59.999 +9999-12-31 23:59:59.500 +throw, the range is checked after the timezone offset is applied +2106-02-07 06:28:15 1970-01-01 00:00:00 +2106-02-07 06:28:15 2106-02-07 06:28:15 +throw, typed JSON columns are checked like declared ones +2149-06-06 +2299-12-31 23:59:59.999 +throw, JSONExtract keeps returning a default or NULL +1970-01-01 \N 2149-06-06 +throw, an explicitly written year 0000 is not silently replaced +\N 1970-01-01 00:00:00 +3 diff --git a/tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.sql b/tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.sql new file mode 100644 index 0000000000000..95e7f651a5034 --- /dev/null +++ b/tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.sql @@ -0,0 +1,105 @@ +-- date_time_overflow_behavior was ignored when the value came from text instead of a typed column +SET session_timezone = 'UTC'; + +SELECT 'saturate'; +SET date_time_overflow_behavior = 'saturate'; +SELECT toDate('9999-12-31'), toDate('1969-12-31'), toDateTime('2106-02-07 06:28:16'), toDateTime('1969-12-31 23:59:59'); +SELECT CAST('9999-12-31' AS Date), CAST(materialize('9999-12-31') AS Date), CAST('9999-12-31'::FixedString(10) AS Date); +SELECT toDateOrNull('9999-12-31'), toDateOrZero('9999-12-31'); + +SELECT 'ignore'; +SET date_time_overflow_behavior = 'ignore'; +SELECT toDate('9999-12-31'), toDate('1969-12-31'), toDateTime('2106-02-07 06:28:16'), toDateTime('1969-12-31 23:59:59'); + +SELECT 'throw'; +SET date_time_overflow_behavior = 'throw'; +SELECT toDate('2149-06-07'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT toDate('1969-12-31'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT toDateTime('2106-02-07 06:28:16'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT toDateTime('1969-12-31 23:59:59'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT CAST('2149-06-07' AS Date); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT CAST(materialize('2149-06-07') AS Date); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT CAST('2149-06-07'::FixedString(10) AS Date); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT CAST(materialize('2106-02-07 06:28:16') AS DateTime); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } + +SELECT 'throw, in range'; +SELECT toDate('2149-06-06'), toDate('1970-01-01'), toDateTime('2106-02-07 06:28:15'), toDateTime('1970-01-01 00:00:00'); +-- Date32 and DateTime64 are not covered yet: a high-scale DateTime64 text parse still raises DECIMAL_OVERFLOW +-- in every mode, because the tick range is only checked in DecimalUtils +SELECT toDate32('2299-12-31'), toDate32('1900-01-01'), toDateTime64('2299-12-31 23:59:59.999', 3); + +SELECT 'throw, OrNull and OrZero still fall back'; +SELECT toDateOrNull('2149-06-07'), toDateOrZero('2149-06-07'), toDateTimeOrNull('2106-02-07 06:28:16'), toDateTimeOrZero('1969-12-31 23:59:59'); + +SELECT 'throw, input formats'; +SELECT * FROM format(CSV, 'v Date', '2150-12-31'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(CSV, 'v Date', '1960-01-01'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(TSV, 'v Date', '2150-12-31'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(JSONEachRow, 'v Date', '{"v":"2150-12-31"}'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(CSV, 'v DateTime', '2106-02-07 06:28:16'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(TSV, 'v DateTime', '1960-01-01 00:00:00'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":"2106-02-07 06:28:16"}'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(CSV, 'v Date', '2149-06-06'); +SELECT * FROM format(TSV, 'v DateTime', '2106-02-07 06:28:15'); + +SELECT 'throw, tentative parsers must not accept a clamped value'; +SELECT v, variantElement(v, 'Date') AS d, variantElement(v, 'String') AS s +FROM format(CSV, 'v Variant(Date, String)', '2150-12-31') SETTINGS allow_experimental_variant_type = 1; +SELECT v FROM format(CSV, 'v Variant(Date, String)', '2149-06-06') SETTINGS allow_experimental_variant_type = 1; + +SELECT 'throw, a digit-only timestamp is text too'; +SELECT toDateTime('4294967296'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT CAST('4294967296' AS DateTime); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT CAST(materialize('4294967296') AS DateTime); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":"4294967296"}'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(Values, 'v DateTime', '(\'4294967296\')'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT toDateTime('4294967295'), toDateTime('1700000000'), toDateTimeOrNull('4294967296'), toDateTimeOrZero('4294967296'); + +SELECT 'throw, Date32 rejects what it cannot represent instead of substituting a default'; +SELECT toDate32('2000-13-01'); -- { serverError CANNOT_PARSE_DATE } +SELECT toDate32('99999999'); -- { serverError CANNOT_PARSE_DATE } +SELECT CAST(materialize('2000-13-01') AS Date32); -- { serverError CANNOT_PARSE_DATE } +SELECT * FROM format(CSV, 'v Date32', '2000-13-01'); -- { serverError CANNOT_PARSE_DATE } +SELECT * FROM format(TSV, 'v Date32', '2000-13-01'); -- { serverError CANNOT_PARSE_DATE } +SELECT * FROM format(JSONEachRow, 'v Date32', '{"v":"2000-13-01"}'); -- { serverError CANNOT_PARSE_DATE } +SELECT toDate32OrNull('2000-13-01'), toDate32('2299-12-31'), toDate32('1900-01-01'); +SELECT v FROM format(CSV, 'v Variant(Date32, String)', '2000-13-01') SETTINGS allow_experimental_variant_type = 1; + +SELECT 'throw, an unquoted numeric token is checked too'; +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":4294967296}'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":-1}'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(Values, 'v DateTime', '(4294967296)'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":1700000000}'); +SELECT * FROM format(Values, 'v DateTime', '(4294967295)'); +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":1703363853.5}'); + +SELECT 'throw, the last second keeps its fractional ticks'; +SELECT * FROM format(JSONEachRow, 'v DateTime64(3)', '{"v":253402300799.5}'); +SELECT * FROM format(JSONEachRow, 'v DateTime64(3)', '{"v":253402300799.999}'); +SELECT * FROM format(Values, 'v DateTime64(3)', '(253402300799.5)'); + +SELECT 'throw, the range is checked after the timezone offset is applied'; +SELECT parseDateTimeBestEffort('2106-02-07 07:28:15+01:00', 'UTC'), parseDateTimeBestEffort('1969-12-31 23:00:00-01:00', 'UTC'); +SELECT parseDateTimeBestEffortOrNull('2106-02-07 07:28:15+01:00', 'UTC'), toDateTime('2106-02-07 07:28:15+01:00', 'UTC'); +SELECT parseDateTimeBestEffort('2106-02-07 08:28:15+01:00', 'UTC'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } + +SELECT 'throw, typed JSON columns are checked like declared ones'; +SELECT j.d FROM format(JSONEachRow, 'j JSON(d Date)', '{"j":{"d":"2150-12-31"}}'); -- { serverError INCORRECT_DATA } +SELECT j.d FROM format(JSONEachRow, 'j JSON(d DateTime)', '{"j":{"d":"2106-02-07 06:28:16"}}'); -- { serverError INCORRECT_DATA } +SELECT j.d FROM format(JSONEachRow, 'j JSON(d DateTime)', '{"j":{"d":"2106-02-07 06:28:16"}}') SETTINGS date_time_input_format = 'basic'; -- { serverError INCORRECT_DATA } +SELECT j.d FROM format(JSONEachRow, 'j JSON(d DateTime)', '{"j":{"d":4294967296}}'); -- { serverError INCORRECT_DATA } +SELECT j.d FROM format(JSONEachRow, 'j JSON(d Date)', '{"j":{"d":"2149-06-06"}}'); +SELECT j.d FROM format(JSONEachRow, 'j JSON(d DateTime64(3))', '{"j":{"d":"2299-12-31 23:59:59.999"}}'); + +SELECT 'throw, JSONExtract keeps returning a default or NULL'; +SELECT JSONExtract('{"d":"2150-12-31"}', 'd', 'Date'), JSONExtract('{"d":"2150-12-31"}', 'd', 'Nullable(Date)'), JSONExtract('{"d":"2149-06-06"}', 'd', 'Date'); + +SELECT 'throw, an explicitly written year 0000 is not silently replaced'; +SELECT parseDateTimeBestEffort('0000-01-01 00:00:00'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT parseDateTimeBestEffort('00000101'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT parseDateTimeBestEffortUS('01/01/0000'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT toDateTime('0000-01-01 00:00:00'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":"0000-01-01 00:00:00"}'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT parseDateTimeBestEffortOrNull('0000-01-01 00:00:00'), parseDateTimeBestEffortOrZero('0000-01-01 00:00:00'); +-- An absent year is a documented best-effort feature, not an overflow +SELECT toMonth(parseDateTimeBestEffort('Mar 3 01:33:48')); diff --git a/tests/queries/0_stateless/05025_client_hints_prepopulated_query.python b/tests/queries/0_stateless/05025_client_hints_prepopulated_query.python new file mode 100644 index 0000000000000..072594e6baef4 --- /dev/null +++ b/tests/queries/0_stateless/05025_client_hints_prepopulated_query.python @@ -0,0 +1,338 @@ +"""A query prepopulated into the next input line (the `??` AI SQL generation flow) is displayed +programmatically, so - like a recalled or pasted line - it must not pop the as-you-type hints: +with hints visible, the next Up/Down press would navigate the hints instead of the history. + +The AI provider is a local HTTP server speaking just enough of the OpenAI chat completion protocol +to answer with a canned query, so the test needs no network and no API key. +""" + +import http.server +import json +import multiprocessing +import os +import pty +import select +import shlex +import sys +import threading +import time + +TIMEOUT_SECONDS = 60 + +# How long to keep reading after the last keystroke while waiting for the expected output. +FINAL_WAIT_SECONDS = 30 + +# How long an explicit `wait_for` of a keystroke may take. It is generous because it covers the +# wait for the next `:) ` prompt after a committed query, and a query does not have to be fast for +# this test to mean anything: in an MSan build a trivial `SELECT` that fails to resolve an +# identifier has been seen taking 36 seconds. +STEP_WAIT_SECONDS = 120 + +# Idle window after a keystroke that has no explicit `wait_for`, long enough for `replxx` to +# repaint the line. +DRAIN_IDLE_SECONDS = 0.4 + +# Extra slack for the supervisor on top of everything the worker itself may legitimately wait for. +SUPERVISOR_SLACK_SECONDS = 15 + +# `clickhouse-test` dumps this file into the failure report, but only under the name it derives +# from the testcase (the `.sh` wrapper), so take the path from there. +DEBUG_LOG = os.environ["CLICKHOUSE_TEST_DEBUG_LOG"] + +# The gray SGR sequence replxx uses for the as-you-type hint (Color::GRAY -> "0;90"). +HINT_COLOR = "\x1b[0;90m" + +# The query the fake provider "generates". Its last word is a prefix of `concatAssumeInjective`, +# so an unsuppressed hint would render the gray ghost suffix "ive" right after it. +GENERATED_QUERY = "SELECT concatAssumeInject" + +# The prepopulated line is syntax-highlighted, so only its last word survives as a plain substring +# of the terminal output. +GENERATED_QUERY_LAST_WORD = "concatAssumeInject" + + +class CompletionHandler(http.server.BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + self.rfile.read(length) + body = json.dumps( + { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 0, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": f"{GENERATED_QUERY}", + }, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args): + pass + + +def write_config(port): + path = os.path.join( + os.environ["CLICKHOUSE_TMP"], + os.path.splitext(os.path.basename(os.path.abspath(__file__)))[0] + ".config.xml", + ) + with open(path, "w") as f: + f.write( + "\n" + " \n" + " openai\n" + " test\n" + f" http://127.0.0.1:{port}\n" + " test-model\n" + " false\n" + " \n" + "\n" + ) + return path + + +def make_history_file(): + path = os.path.join( + os.environ["CLICKHOUSE_TMP"], + os.path.splitext(os.path.basename(os.path.abspath(__file__)))[0] + ".history", + ) + with open(path, "w") as f: + pass + return path + + +def read_until(master, debug_log_fd, predicate, timeout): + """Read until the predicate holds, the deadline passes, or the client is gone. Returns whether + the predicate matched along with what was read: a deadline is not a match, and treating it as + one lets a case go on from a state it never reached.""" + output = "" + deadline = time.time() + timeout + while time.time() < deadline: + r, _, _ = select.select([master], [], [], 0.3) + if not r: + if predicate(output): + return True, output + continue + try: + chunk = os.read(master, 4096) + except OSError: + break + debug_log_fd.write(repr(chunk) + "\n") + debug_log_fd.flush() + output += chunk.decode(errors="replace") + if predicate(output): + return True, output + return predicate(output), output + + +def drain(master, debug_log_fd, idle): + output = "" + while True: + r, _, _ = select.select([master], [], [], idle) + if not r: + break + try: + chunk = os.read(master, 4096) + except OSError: + break + debug_log_fd.write(repr(chunk) + "\n") + debug_log_fd.flush() + output += chunk.decode(errors="replace") + return output + + +def report_failure(name, shell_pid, output, debug_log_fd): + """A PTY case that fails prints just `FAIL`, which says nothing about why. Record what the + terminal actually showed and whether the client is still alive - the client dying (the read + loop hitting EOF) looks exactly like the expected text never being printed.""" + try: + reaped_pid, wait_status = os.waitpid(shell_pid, os.WNOHANG) + except ChildProcessError: + reaped_pid, wait_status = shell_pid, -1 + running = reaped_pid == 0 + banner = ( + f"=== {name}: FAIL ===\n" + f"client still running: {running}, wait status: {wait_status}\n" + f"terminal output: {output[-8000:]!r}\n" + ) + debug_log_fd.write(banner) + debug_log_fd.flush() + sys.stderr.write(banner) + sys.stderr.flush() + + +def run_case(program, argv, name, keystrokes, check, state=None, check_from=0): + """Each keystroke is a pair of the bytes to send and an optional predicate to wait for before + sending the next one (in place of a fixed idle drain). Only the output produced from the + keystroke with index `check_from` on is passed to `check`, so the earlier keystrokes may + legitimately pop hints without failing a "no hints" check.""" + shell_pid, master = pty.fork() + if shell_pid == 0: + os.environ["TERM"] = "xterm" + os.execv(program, argv) + return + + debug_log_fd = open(DEBUG_LOG, "a") + try: + matched, transcript = read_until( + master, debug_log_fd, lambda o: ":)" in o, TIMEOUT_SECONDS + ) + if not matched: + print(f"{name}: FAIL") + report_failure(name, shell_pid, transcript, debug_log_fd) + state.value = 1 + return + + # Everything the terminal showed, for the failure transcript - `output` holds only what + # the check is allowed to look at. + output = "" + for index, (keys, wait_for) in enumerate(keystrokes): + os.write(master, keys) + if wait_for is not None: + matched, segment = read_until( + master, debug_log_fd, wait_for, STEP_WAIT_SECONDS + ) + else: + matched, segment = True, drain(master, debug_log_fd, DRAIN_IDLE_SECONDS) + transcript += segment + if index >= check_from: + output += segment + # A `wait_for` that expired means the state the next keystroke assumes was never + # reached: the hint that arms the stale cache never appeared, or the prompt never came + # back and the next line would be typed into a still running query, where the terminal + # driver echoes the bytes instead of `replxx` interpreting them. Either way the case + # would prove nothing while still being able to pass, so end it here with its + # transcript. + if not matched: + print(f"{name}: FAIL") + report_failure(name, shell_pid, transcript, debug_log_fd) + state.value = 1 + return + + # The prepopulated line only appears after the (faked) generation round trip, which can + # outlast the inter-keystroke drain on slow builds. + if not check(output): + _, extra = read_until( + master, + debug_log_fd, + lambda more: check(output + more), + FINAL_WAIT_SECONDS, + ) + output += extra + transcript += extra + + ok = check(output) + print(f"{name}: {'OK' if ok else 'FAIL'}") + if not ok: + report_failure(name, shell_pid, transcript, debug_log_fd) + state.value = 0 if ok else 1 + finally: + os.close(master) + debug_log_fd.close() + + +def worker_budget(keystrokes): + """Everything `run_case` may legitimately spend: the wait for the first prompt, every explicit + `wait_for`, the idle window after every other keystroke, and the final wait for the expected + output. The supervisor must not expire before that, or ordinary slowness looks like a + failure.""" + waits = sum(1 for _, wait_for in keystrokes if wait_for is not None) + return ( + TIMEOUT_SECONDS + + STEP_WAIT_SECONDS * waits + + DRAIN_IDLE_SECONDS * (len(keystrokes) - waits) + + FINAL_WAIT_SECONDS + + SUPERVISOR_SLACK_SECONDS + ) + + +def run_with_timeout(program, argv, name, keystrokes, check, check_from=0): + for attempt in range(5): + state = multiprocessing.Value("i", -1) + process = multiprocessing.Process( + target=run_case, + args=(program, argv, name, keystrokes, check), + kwargs={"state": state, "check_from": check_from}, + ) + process.start() + process.join(worker_budget(keystrokes)) + if process.is_alive(): + process.terminate() + if state.value in (0, 1): + return + # transient timeout on a loaded machine - retry + print(f"{name}: FAIL") + sys.stderr.write(f"=== {name}: FAIL === the case timed out on every attempt\n") + + +def main(): + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), CompletionHandler) + server.daemon_threads = True + threading.Thread(target=server.serve_forever, daemon=True).start() + + config = write_config(server.server_address[1]) + history = make_history_file() + + program = os.environ["CLICKHOUSE_LOCAL"] + base = shlex.split(program) + args = base + [ + "--wait_for_suggestions_to_load", + "--hints", + "1", + f"--config-file={config}", + f"--history_file={history}", + ] + + no_hints = lambda o: GENERATED_QUERY_LAST_WORD in o and (HINT_COLOR + "ive") not in o + + try: + run_with_timeout( + base[0], + args, + "prepopulated_query_no_hints", + [(b"?? show me a query\r", None)], + no_hints, + ) + # Same-text regression: replxx keeps the hint cache across prompts, so when the generated + # query is byte-for-byte equal to the previously displayed line, the hint callback may not + # even be called - the stale cached hint must not be reused either. Type the query manually + # first (and wait for the hint ghost, which poisons the cache), execute it, then have the + # AI "generate" the very same text. The `??` request is kept as short as possible so that + # replxx's rapid-refresh throttling (1 ms) can swallow its keystrokes without regenerating + # the hint seed - only then does the preloaded line hit the stale cache. Only the output + # after the `??` keystroke is checked, as the manually typed line legitimately shows the + # hint. + run_with_timeout( + base[0], + args, + "same_text_prepopulated_query_no_hints", + [ + (GENERATED_QUERY.encode(), lambda o: (HINT_COLOR + "ive") in o), + (b"\r", lambda o: ":)" in o), + (b"??q\r", None), + ], + no_hints, + check_from=2, + ) + finally: + server.shutdown() + + +if __name__ == "__main__": + # The check lambdas are not picklable, so the subprocesses must be forked (not spawned, + # which became the default on newer Python). + multiprocessing.set_start_method("fork") + main() diff --git a/tests/queries/0_stateless/05025_client_hints_prepopulated_query.reference b/tests/queries/0_stateless/05025_client_hints_prepopulated_query.reference new file mode 100644 index 0000000000000..fccb0eff5b813 --- /dev/null +++ b/tests/queries/0_stateless/05025_client_hints_prepopulated_query.reference @@ -0,0 +1,2 @@ +prepopulated_query_no_hints: OK +same_text_prepopulated_query_no_hints: OK diff --git a/tests/queries/0_stateless/05025_client_hints_prepopulated_query.sh b/tests/queries/0_stateless/05025_client_hints_prepopulated_query.sh new file mode 100755 index 0000000000000..d48ace2e154ef --- /dev/null +++ b/tests/queries/0_stateless/05025_client_hints_prepopulated_query.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Tags: long, no-debug, no-fasttest +# no-fasttest: needs the AI SQL generator (`ENABLE_CLIENT_AI`), which is not built in the fast test. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `clickhouse-test` dumps `$CLICKHOUSE_TMP/.debuglog` into the failure report, and the +# testcase name is the name of this shell script - the Python part cannot derive it from its own +# file name, so pass the path down. +export CLICKHOUSE_TEST_DEBUG_LOG="$CLICKHOUSE_TMP/$(basename "${BASH_SOURCE[0]}").debuglog" + +python3 "$CUR_DIR"/05025_client_hints_prepopulated_query.python diff --git a/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.reference b/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.reference index dff69852207d2..ba2bb9215f919 100644 --- a/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.reference +++ b/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.reference @@ -1,13 +1,13 @@ ---- default: aws_external_id hidden, role identifiers visible +--- default: aws_external_id and aws_role_session_name hidden, aws_role_arn visible aws_external_id = '[HIDDEN]' aws_access_key_id = '[HIDDEN]' aws_secret_access_key = '[HIDDEN]' aws_role_arn = 'arn:aws:iam::1:role/r' -aws_role_session_name = 'sess' +aws_role_session_name = '[HIDDEN]' OK: no secret in formatted query ---- show_secrets: aws_external_id visible +--- show_secrets: aws_external_id and aws_role_session_name visible aws_external_id = 'SECRET_THAT_MUST_NOT_LEAK' aws_access_key_id = 'SECRET_THAT_MUST_NOT_LEAK' aws_secret_access_key = 'SECRET_THAT_MUST_NOT_LEAK' aws_role_arn = 'arn:aws:iam::1:role/r' -aws_role_session_name = 'sess' +aws_role_session_name = 'SECRET_THAT_MUST_NOT_LEAK' diff --git a/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.sh b/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.sh index f33348fb92d45..d846eca5fa41e 100755 --- a/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.sh +++ b/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash -# Regression test: aws_external_id is the shared secret of the AWS AssumeRole triple, so it must be -# redacted as [HIDDEN] when a DataLakeCatalog CREATE query is formatted (system.databases.engine_full, -# SHOW CREATE DATABASE), while aws_role_arn and aws_role_session_name are non-secret identifiers that -# stay visible. Uses clickhouse-format so it needs no live catalog and is safe to run in parallel. +# Regression test: aws_external_id is the shared secret of the AWS AssumeRole triple, and +# aws_role_session_name can be one too (a trust policy can require a specific value through the +# sts:RoleSessionName condition), so both must be redacted as [HIDDEN] when a DataLakeCatalog CREATE +# query is formatted (system.databases.engine_full, SHOW CREATE DATABASE), while aws_role_arn is a +# non-secret identifier that stays visible. Uses clickhouse-format so it needs no live catalog and is +# safe to run in parallel. CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -17,7 +19,7 @@ aws_access_key_id = '${SECRET}', aws_secret_access_key = '${SECRET}', aws_external_id = '${SECRET}', aws_role_arn = 'arn:aws:iam::1:role/r', -aws_role_session_name = 'sess'" +aws_role_session_name = '${SECRET}'" show_settings() { local formatted="$1" @@ -32,9 +34,9 @@ show_settings() { done } -# Arm A: at the default the secret is redacted; arms C (non-secret identifiers stay visible) and +# Arm A: at the default the secrets are redacted; arms C (the non-secret role ARN stays visible) and # D (sibling AWS keys still redacted) are asserted by the same output. -echo "--- default: aws_external_id hidden, role identifiers visible" +echo "--- default: aws_external_id and aws_role_session_name hidden, aws_role_arn visible" formatted=$(echo "$query" | $CLICKHOUSE_FORMAT --oneline) show_settings "$formatted" if echo "$formatted" | grep -q "$SECRET"; then @@ -44,6 +46,6 @@ else fi # Arm B: an authorized caller can still retrieve the value, so the fix redacts rather than destroys. -echo "--- show_secrets: aws_external_id visible" +echo "--- show_secrets: aws_external_id and aws_role_session_name visible" formatted=$(echo "$query" | $CLICKHOUSE_FORMAT --oneline --show_secrets) show_settings "$formatted" diff --git a/tests/queries/0_stateless/05038_query_plan_limits_hedged_sql_fallback.reference b/tests/queries/0_stateless/05038_query_plan_limits_hedged_sql_fallback.reference new file mode 100644 index 0000000000000..3f5bade2d42ee --- /dev/null +++ b/tests/queries/0_stateless/05038_query_plan_limits_hedged_sql_fallback.reference @@ -0,0 +1,4 @@ +499500 1000 +499500 1000 +1 +0 diff --git a/tests/queries/0_stateless/05038_query_plan_limits_hedged_sql_fallback.sql b/tests/queries/0_stateless/05038_query_plan_limits_hedged_sql_fallback.sql new file mode 100644 index 0000000000000..f943a7cb3e782 --- /dev/null +++ b/tests/queries/0_stateless/05038_query_plan_limits_hedged_sql_fallback.sql @@ -0,0 +1,72 @@ +-- Tags: no-darwin +-- `HedgedConnections` are compiled only under `OS_LINUX`, so on macOS `use_hedged_requests` is a no-op +-- and the initiator never has to fall back to SQL. + +-- Regression test: a distributed query with `serialize_query_plan = 1` over hedged connections +-- must fall back to sending SQL when the plan carries non-default execution limits and a later +-- hedge could select a replica whose query-plan serialization version is unverified. Plan-level +-- `max_threads` and `concurrency_control` are serialized only since plan serialization version 10; +-- shipping the plan to an unverified peer would silently drop them during a rolling upgrade. + +DROP TABLE IF EXISTS t_plan_limits_hedged_fallback; + +CREATE TABLE t_plan_limits_hedged_fallback (a UInt64) ENGINE = MergeTree ORDER BY a; +INSERT INTO t_plan_limits_hedged_fallback SELECT number FROM numbers(1000); + +-- `serialize_query_plan` requires the analyzer; pin it against `compatibility` randomization. +SET enable_analyzer = 1; +-- The ordinary distributed path is under test; parallel replicas gate plan shipping differently. +SET enable_parallel_replicas = 0; +SET serialize_query_plan = 1; +-- Non-default plan-level execution limits, pinned against CI randomization: these are what the +-- fallback protects during a rolling upgrade. +SET max_threads = 4; +SET use_concurrency_control = 1; +-- Force a real remote connection: with the default the initiator would execute the only shard +-- locally and never open a connection, hedged or not. +SET prefer_localhost_replica = 0; + +-- Hedged connections: the first replica is established up front, but a later hedge may select any +-- remaining replica of the pool, whose version is unknown at send time. With three replicas in the +-- pool only one is verified, so the initiator must choose the SQL fallback. +SELECT sum(a), count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), t_plan_limits_hedged_fallback) +SETTINGS use_hedged_requests = 1, log_comment = '05038_hedged_plan_limits_fallback'; + +-- Control: with hedged connections off every connected replica is verified (same-version local +-- cluster), so the plan is shipped and the fallback must stay silent. +SELECT sum(a), count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), t_plan_limits_hedged_fallback) +SETTINGS use_hedged_requests = 0, log_comment = '05038_hedged_plan_limits_no_fallback'; + +SYSTEM FLUSH LOGS query_log, text_log; + +-- The firing oracle: the hedged query logged the SQL fallback on the initiator. Resolve the newest +-- matching initial query by `log_comment` (CI reuses one database across executions in some jobs, +-- so aggregate over history could be satisfied by an earlier run). +SELECT count() > 0 +FROM system.text_log +WHERE query_id = ( + SELECT argMax(query_id, event_time_microseconds) + FROM system.query_log + WHERE current_database = currentDatabase() + AND log_comment = '05038_hedged_plan_limits_fallback' + AND type = 'QueryFinish' + AND is_initial_query + AND event_date >= yesterday() AND event_time >= now() - 600) + AND message LIKE 'Sending query as SQL because a replica does not support query-plan serialization version%' + AND event_date >= yesterday() AND event_time >= now() - 600; + +-- The control oracle: the non-hedged query did not fall back. +SELECT count() +FROM system.text_log +WHERE query_id = ( + SELECT argMax(query_id, event_time_microseconds) + FROM system.query_log + WHERE current_database = currentDatabase() + AND log_comment = '05038_hedged_plan_limits_no_fallback' + AND type = 'QueryFinish' + AND is_initial_query + AND event_date >= yesterday() AND event_time >= now() - 600) + AND message LIKE 'Sending query as SQL because a replica does not support query-plan serialization version%' + AND event_date >= yesterday() AND event_time >= now() - 600; + +DROP TABLE t_plan_limits_hedged_fallback; diff --git a/tests/queries/0_stateless/05043_direct_dictionary_in_merge.reference b/tests/queries/0_stateless/05043_direct_dictionary_in_merge.reference new file mode 100644 index 0000000000000..72c3262a309d8 --- /dev/null +++ b/tests/queries/0_stateless/05043_direct_dictionary_in_merge.reference @@ -0,0 +1 @@ +Hello 1 diff --git a/tests/queries/0_stateless/05043_direct_dictionary_in_merge.sh b/tests/queries/0_stateless/05043_direct_dictionary_in_merge.sh new file mode 100755 index 0000000000000..8385d6f97658c --- /dev/null +++ b/tests/queries/0_stateless/05043_direct_dictionary_in_merge.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# A `direct` dictionary reads through the source's own pipeline, and `merge` wraps every child in +# `materialize`, which rejects a header that carries rows. +$CLICKHOUSE_CLIENT -q " + CREATE DICTIONARY dict (word String, counter UInt32) + PRIMARY KEY word + SOURCE(HTTP(url '${CLICKHOUSE_URL}&query=SELECT+%27Hello%27,1+FORMAT+CSV' format 'CSV')) + LAYOUT(DIRECT())" + +$CLICKHOUSE_CLIENT -q "SELECT * FROM merge(currentDatabase(), '^dict\$')" diff --git a/tests/queries/0_stateless/05045_merge_prefilter_lazy_load_tables.reference b/tests/queries/0_stateless/05045_merge_prefilter_lazy_load_tables.reference index 9f607859a6e8f..586c62c104083 100644 --- a/tests/queries/0_stateless/05045_merge_prefilter_lazy_load_tables.reference +++ b/tests/queries/0_stateless/05045_merge_prefilter_lazy_load_tables.reference @@ -1,4 +1,4 @@ -TableProxy +Distributed 3 t05045_leaf 3 diff --git a/tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.reference b/tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.reference new file mode 100644 index 0000000000000..d93a3042cfa81 --- /dev/null +++ b/tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.reference @@ -0,0 +1,97 @@ +-- { echo } + +-- per-partition set building reads the same predicate. The set fill deduplicates across +-- partitions anyway, so the merged answer stays correct and only the plan shape shows the +-- decline; the bare-key arm is the control that the fixture reaches the optimization. +-- 2001 and 2002 are not leap years, so all three days of each map to February 28. +DROP TABLE IF EXISTS t_set_month; +CREATE TABLE t_set_month (d Date, x UInt32) ENGINE = MergeTree ORDER BY d PARTITION BY d; +SYSTEM STOP MERGES t_set_month; +INSERT INTO t_set_month SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-01-', toString(29 + (intDiv(number, 10) % 3)))) AS d, number FROM numbers(60); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT count() FROM numbers(100) WHERE toDate('2001-02-28') + number IN (SELECT d + INTERVAL 1 MONTH FROM t_set_month) SETTINGS allow_creating_set_partitions_independently = 1) WHERE explain LIKE '%Pre-distinct%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT count() FROM numbers(100) WHERE toDate('2001-02-28') + number IN (SELECT d FROM t_set_month) SETTINGS allow_creating_set_partitions_independently = 1) WHERE explain LIKE '%Pre-distinct%' OR explain LIKE '%Read each partition through separate port%'; +Pre-distinct: 1 +Read each partition through separate port: 1 +DROP TABLE t_set_month; +-- an integer key keeps per-partition set building; the partition key is a function of the set's +-- own output column, which the interval arm above cannot use because its key is the collapsing one +DROP TABLE IF EXISTS t_set_int; +CREATE TABLE t_set_int (a UInt32, b UInt32) ENGINE = MergeTree ORDER BY tuple() PARTITION BY a % 8; +SYSTEM STOP MERGES t_set_int; +INSERT INTO t_set_int SELECT number % 64, number FROM numbers_mt(400); +INSERT INTO t_set_int SELECT number % 64, number FROM numbers_mt(400); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT count() FROM numbers(100) WHERE number IN (SELECT a + 1 FROM t_set_int) SETTINGS allow_creating_set_partitions_independently = 1) WHERE explain LIKE '%Pre-distinct%' OR explain LIKE '%Read each partition through separate port%'; +Pre-distinct: 1 +Read each partition through separate port: 1 +SELECT (SELECT count() FROM numbers(100) WHERE number IN (SELECT a + 1 FROM t_set_int) SETTINGS allow_creating_set_partitions_independently = 0) = (SELECT count() FROM numbers(100) WHERE number IN (SELECT a + 1 FROM t_set_int) SETTINGS allow_creating_set_partitions_independently = 1); +1 +DROP TABLE t_set_int; +-- --------------------------------------------------------------------------- +-- The distributed sharding-key consumer reaches the same predicate through its own rejection +-- loop and its own direct call, so it gets its own arms. Dropping the merge step is only +-- correct when the group key determines the shard: a key that collapses distinct shard-key +-- values leaves each shard's partial groups unmerged, so the same key is returned twice. +-- Each view filters itself by shardNum() so the two shards hold the disjoint rows the +-- declared key implies - a declared key alone does not redistribute rows on a read, and +-- without the filter every shard holds every row and even a sound merge drop doubles the +-- answer. The first arm of each pair counts merge steps (1 = kept, 0 = dropped) and the +-- second compares the answer against the unoptimized one; the integer pair is the control +-- that the optimization still fires where it is sound. +-- --------------------------------------------------------------------------- + +SELECT shardNum() AS s, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY s ORDER BY s; +1 10 +2 20 +SELECT count() FROM (EXPLAIN SELECT d + INTERVAL 1 MONTH AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1) WHERE explain ILIKE '%MergingAggregated%'; +1 +SELECT (SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 0)) = (SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1)); +1 +SELECT shardNum() AS s, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY s ORDER BY s; +1 15 +2 15 +SELECT count() FROM (EXPLAIN SELECT x + 1 AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1) WHERE explain ILIKE '%MergingAggregated%'; +0 +SELECT (SELECT count() FROM (SELECT x + 1 AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 0)) = (SELECT count() FROM (SELECT x + 1 AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1)); +1 +-- --------------------------------------------------------------------------- +-- The window consumer reaches the same predicate through the stream-disjointness +-- propagation, at two sites: the per-partition read request and the scatter skip above it. +-- INTERVAL MONTH collapses the 29th, 30th and 31st into one key, so one logical window +-- partition spans the table partitions those days live in and must not be evaluated per +-- table partition. The default arm carries no setting: the cost heuristic accepts this +-- fixture, so the answer has to be right without opting out. +-- --------------------------------------------------------------------------- + +DROP TABLE IF EXISTS t_win_month; +CREATE TABLE t_win_month (d Date) ENGINE = MergeTree ORDER BY d PARTITION BY d; +INSERT INTO t_win_month SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-01-', toString(29 + (intDiv(number, 10) % 3)))) AS d FROM numbers(60); +-- the collapse the arms below depend on: more table partitions than window keys +SELECT uniqExact(_partition_id), uniqExact(d + INTERVAL 1 MONTH) FROM t_win_month; +6 2 +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) AS c FROM t_win_month) ORDER BY c SETTINGS force_window_partitions_independently = 1; +30 +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) AS c FROM t_win_month) ORDER BY c SETTINGS allow_window_partitions_independently = 0; +30 +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) AS c FROM t_win_month) ORDER BY c; +30 +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) FROM t_win_month SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Read each partition through separate port: 1%'; +0 +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) FROM t_win_month SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Skip scatter by partition: 1%'; +0 +DROP TABLE t_win_month; +-- the integer control: an injective addend keeps both sites firing, so the arms above +-- attribute to the operand type and not to the window shape or the fixture +DROP TABLE IF EXISTS t_win_int; +CREATE TABLE t_win_int (x UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x % 8; +INSERT INTO t_win_int SELECT number % 8 FROM numbers_mt(800); +SELECT uniqExact(_partition_id), uniqExact(x + 1) FROM t_win_int; +8 8 +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY x + 1) AS c FROM t_win_int) ORDER BY c SETTINGS force_window_partitions_independently = 1; +100 +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY x + 1) AS c FROM t_win_int) ORDER BY c SETTINGS allow_window_partitions_independently = 0; +100 +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY x + 1) FROM t_win_int SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Read each partition through separate port: 1%'; +1 +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY x + 1) FROM t_win_int SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Skip scatter by partition: 1%'; +1 +DROP TABLE t_win_int; diff --git a/tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.sql b/tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.sql new file mode 100644 index 0000000000000..509e8a3e3d5e7 --- /dev/null +++ b/tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.sql @@ -0,0 +1,97 @@ +-- Tags: distributed, no-random-settings, no-random-merge-tree-settings +-- no-random-settings, no-random-merge-tree-settings: randomized settings and part counts +-- change both the plans and the values these arms count. + +SET explain_query_plan_default = 'legacy'; +-- max_threads is pinned because the cost heuristic accepts a fixture only when its partition +-- count is at least max_threads / 2; arms that must not depend on the heuristic force it instead. +SET max_threads = 8; +SET enable_parallel_replicas = 0; +SET max_rows_in_distinct = 0; +SET max_bytes_in_distinct = 0; +-- The stateless CI profile sets these to 10G, and a nonzero limit disables per-partition +-- evaluation outright. +SET max_rows_to_group_by = 0; +SET max_rows_to_sort = 0; +SET max_bytes_to_sort = 0; +SET optimize_use_implicit_projections = 0; +-- The values and plans below are the analyzer's, so pin it. +SET enable_analyzer = 1; + +-- { echo } + +-- per-partition set building reads the same predicate. The set fill deduplicates across +-- partitions anyway, so the merged answer stays correct and only the plan shape shows the +-- decline; the bare-key arm is the control that the fixture reaches the optimization. +-- 2001 and 2002 are not leap years, so all three days of each map to February 28. +DROP TABLE IF EXISTS t_set_month; +CREATE TABLE t_set_month (d Date, x UInt32) ENGINE = MergeTree ORDER BY d PARTITION BY d; +SYSTEM STOP MERGES t_set_month; +INSERT INTO t_set_month SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-01-', toString(29 + (intDiv(number, 10) % 3)))) AS d, number FROM numbers(60); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT count() FROM numbers(100) WHERE toDate('2001-02-28') + number IN (SELECT d + INTERVAL 1 MONTH FROM t_set_month) SETTINGS allow_creating_set_partitions_independently = 1) WHERE explain LIKE '%Pre-distinct%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT count() FROM numbers(100) WHERE toDate('2001-02-28') + number IN (SELECT d FROM t_set_month) SETTINGS allow_creating_set_partitions_independently = 1) WHERE explain LIKE '%Pre-distinct%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_set_month; + +-- an integer key keeps per-partition set building; the partition key is a function of the set's +-- own output column, which the interval arm above cannot use because its key is the collapsing one +DROP TABLE IF EXISTS t_set_int; +CREATE TABLE t_set_int (a UInt32, b UInt32) ENGINE = MergeTree ORDER BY tuple() PARTITION BY a % 8; +SYSTEM STOP MERGES t_set_int; +INSERT INTO t_set_int SELECT number % 64, number FROM numbers_mt(400); +INSERT INTO t_set_int SELECT number % 64, number FROM numbers_mt(400); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT count() FROM numbers(100) WHERE number IN (SELECT a + 1 FROM t_set_int) SETTINGS allow_creating_set_partitions_independently = 1) WHERE explain LIKE '%Pre-distinct%' OR explain LIKE '%Read each partition through separate port%'; +SELECT (SELECT count() FROM numbers(100) WHERE number IN (SELECT a + 1 FROM t_set_int) SETTINGS allow_creating_set_partitions_independently = 0) = (SELECT count() FROM numbers(100) WHERE number IN (SELECT a + 1 FROM t_set_int) SETTINGS allow_creating_set_partitions_independently = 1); +DROP TABLE t_set_int; + +-- --------------------------------------------------------------------------- +-- The distributed sharding-key consumer reaches the same predicate through its own rejection +-- loop and its own direct call, so it gets its own arms. Dropping the merge step is only +-- correct when the group key determines the shard: a key that collapses distinct shard-key +-- values leaves each shard's partial groups unmerged, so the same key is returned twice. +-- Each view filters itself by shardNum() so the two shards hold the disjoint rows the +-- declared key implies - a declared key alone does not redistribute rows on a read, and +-- without the filter every shard holds every row and even a sound merge drop doubles the +-- answer. The first arm of each pair counts merge steps (1 = kept, 0 = dropped) and the +-- second compares the answer against the unoptimized one; the integer pair is the control +-- that the optimization still fires where it is sound. +-- --------------------------------------------------------------------------- + +SELECT shardNum() AS s, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY s ORDER BY s; +SELECT count() FROM (EXPLAIN SELECT d + INTERVAL 1 MONTH AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1) WHERE explain ILIKE '%MergingAggregated%'; +SELECT (SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 0)) = (SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1)); +SELECT shardNum() AS s, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY s ORDER BY s; +SELECT count() FROM (EXPLAIN SELECT x + 1 AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1) WHERE explain ILIKE '%MergingAggregated%'; +SELECT (SELECT count() FROM (SELECT x + 1 AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 0)) = (SELECT count() FROM (SELECT x + 1 AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1)); + +-- --------------------------------------------------------------------------- +-- The window consumer reaches the same predicate through the stream-disjointness +-- propagation, at two sites: the per-partition read request and the scatter skip above it. +-- INTERVAL MONTH collapses the 29th, 30th and 31st into one key, so one logical window +-- partition spans the table partitions those days live in and must not be evaluated per +-- table partition. The default arm carries no setting: the cost heuristic accepts this +-- fixture, so the answer has to be right without opting out. +-- --------------------------------------------------------------------------- + +DROP TABLE IF EXISTS t_win_month; +CREATE TABLE t_win_month (d Date) ENGINE = MergeTree ORDER BY d PARTITION BY d; +INSERT INTO t_win_month SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-01-', toString(29 + (intDiv(number, 10) % 3)))) AS d FROM numbers(60); +-- the collapse the arms below depend on: more table partitions than window keys +SELECT uniqExact(_partition_id), uniqExact(d + INTERVAL 1 MONTH) FROM t_win_month; +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) AS c FROM t_win_month) ORDER BY c SETTINGS force_window_partitions_independently = 1; +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) AS c FROM t_win_month) ORDER BY c SETTINGS allow_window_partitions_independently = 0; +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) AS c FROM t_win_month) ORDER BY c; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) FROM t_win_month SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Read each partition through separate port: 1%'; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) FROM t_win_month SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Skip scatter by partition: 1%'; +DROP TABLE t_win_month; + +-- the integer control: an injective addend keeps both sites firing, so the arms above +-- attribute to the operand type and not to the window shape or the fixture +DROP TABLE IF EXISTS t_win_int; +CREATE TABLE t_win_int (x UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x % 8; +INSERT INTO t_win_int SELECT number % 8 FROM numbers_mt(800); +SELECT uniqExact(_partition_id), uniqExact(x + 1) FROM t_win_int; +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY x + 1) AS c FROM t_win_int) ORDER BY c SETTINGS force_window_partitions_independently = 1; +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY x + 1) AS c FROM t_win_int) ORDER BY c SETTINGS allow_window_partitions_independently = 0; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY x + 1) FROM t_win_int SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Read each partition through separate port: 1%'; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY x + 1) FROM t_win_int SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Skip scatter by partition: 1%'; +DROP TABLE t_win_int; diff --git a/tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.reference b/tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.reference new file mode 100644 index 0000000000000..d695e73641ae3 --- /dev/null +++ b/tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.reference @@ -0,0 +1,98 @@ +-- { echo } + +-- --------------------------------------------------------------------------- +-- Preservation: integer arithmetic with an integer constant stays injective, so the +-- optimization must keep firing. A correctness-only test would not notice this. Each arm +-- prints two plan lines; an arm printing nothing has lost the optimization. +-- --------------------------------------------------------------------------- + +DROP TABLE IF EXISTS t_int; +CREATE TABLE t_int (a UInt32) ENGINE = MergeTree ORDER BY a PARTITION BY intDiv(a, 2) * 2 + 1; +INSERT INTO t_int SELECT number FROM numbers_mt(32); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT intDiv(a, 2) + 1 AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +-- subtraction widens to a signed result, which is a different branch of the integer test +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT intDiv(a, 2) - 1 AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +-- the constant may be either operand, so both positions have to be recognized +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT 7 - intDiv(a, 2) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT 1 + intDiv(a, 2) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +-- negate and tuple are not named by the query-plan consumer: they answer isInjective themselves, so +-- the optimization has to keep firing for them. +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT -intDiv(a, 2) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT tuple(intDiv(a, 2)) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_int; +-- wide integers are exact, so the gate must not be narrowed to native widths +DROP TABLE IF EXISTS t_int128; +CREATE TABLE t_int128 (a Int128) ENGINE = MergeTree ORDER BY a PARTITION BY a % 8; +INSERT INTO t_int128 SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT a + 7 AS a1 FROM t_int128 SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_int128; +DROP TABLE IF EXISTS t_uint256; +CREATE TABLE t_uint256 (b UInt256) ENGINE = MergeTree ORDER BY b PARTITION BY b % 8; +INSERT INTO t_uint256 SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT b + 7 AS b1 FROM t_uint256 SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_uint256; +-- both type-wrapper orders must be stripped before the integer test. Contrast with the NULL +-- constant arm above: a Nullable column keeps the optimization, a NULL constant loses it. +DROP TABLE IF EXISTS t_nullable; +CREATE TABLE t_nullable (n Nullable(UInt32)) ENGINE = MergeTree ORDER BY n PARTITION BY n % 8 SETTINGS allow_nullable_key = 1; +INSERT INTO t_nullable SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT n + 1 AS n1 FROM t_nullable SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_nullable; +DROP TABLE IF EXISTS t_lc; +CREATE TABLE t_lc (l LowCardinality(UInt32)) ENGINE = MergeTree ORDER BY l PARTITION BY l % 8; +INSERT INTO t_lc SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT l + 1 AS l1 FROM t_lc SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_lc; +DROP TABLE IF EXISTS t_lc_nullable; +CREATE TABLE t_lc_nullable (ln LowCardinality(Nullable(UInt32))) ENGINE = MergeTree ORDER BY ln PARTITION BY ln % 8 SETTINGS allow_nullable_key = 1; +INSERT INTO t_lc_nullable SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT ln + 1 AS ln1 FROM t_lc_nullable SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_lc_nullable; +-- --------------------------------------------------------------------------- +-- Accepted narrowings. These three expressions are injective, but every type they would +-- admit also admits a non-injective case above that cannot be told apart: an Interval on +-- Date is not separable from one on DateTime by kind, a Date operand re-admits the +-- narrowing constant, and a Decimal operand re-admits the rescaling constant. Each arm +-- prints nothing; the bare-key arm after it is the control showing the fixture does reach +-- the optimization. +-- --------------------------------------------------------------------------- + +DROP TABLE IF EXISTS t_date; +CREATE TABLE t_date (d Date) ENGINE = MergeTree ORDER BY d PARTITION BY toDayOfWeek(d); +INSERT INTO t_date SELECT toDate('2001-01-01') + number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT d + INTERVAL 1 DAY AS d1 FROM t_date SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT d + 1 AS d1 FROM t_date SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT toDayOfWeek(d) AS d1 FROM t_date SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_date; +DROP TABLE IF EXISTS t_dec; +CREATE TABLE t_dec (dec Decimal64(2)) ENGINE = MergeTree ORDER BY dec PARTITION BY toUInt32(dec) % 8; +INSERT INTO t_dec SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT dec + 1 AS dec1 FROM t_dec SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT toUInt32(dec) AS dec1 FROM t_dec SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_dec; diff --git a/tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.sql b/tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.sql new file mode 100644 index 0000000000000..cb93bc55fbcc5 --- /dev/null +++ b/tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.sql @@ -0,0 +1,100 @@ +-- Tags: no-random-settings, no-random-merge-tree-settings +-- no-random-settings, no-random-merge-tree-settings: randomized settings and part counts +-- change both the plans and the values these arms count. + +SET explain_query_plan_default = 'legacy'; +-- max_threads is pinned because the cost heuristic accepts a fixture only when its partition +-- count is at least max_threads / 2; arms that must not depend on the heuristic force it instead. +SET max_threads = 8; +SET enable_parallel_replicas = 0; +SET max_rows_in_distinct = 0; +SET max_bytes_in_distinct = 0; +-- The stateless CI profile sets these to 10G, and a nonzero limit disables per-partition +-- evaluation outright. +SET max_rows_to_group_by = 0; +SET max_rows_to_sort = 0; +SET max_bytes_to_sort = 0; +SET optimize_use_implicit_projections = 0; +SET allow_suspicious_low_cardinality_types = 1; +-- The EXPLAIN QUERY TREE arms below are analyzer-only; old-analyzer jobs would error on them. +SET enable_analyzer = 1; + +-- { echo } + +-- --------------------------------------------------------------------------- +-- Preservation: integer arithmetic with an integer constant stays injective, so the +-- optimization must keep firing. A correctness-only test would not notice this. Each arm +-- prints two plan lines; an arm printing nothing has lost the optimization. +-- --------------------------------------------------------------------------- + +DROP TABLE IF EXISTS t_int; +CREATE TABLE t_int (a UInt32) ENGINE = MergeTree ORDER BY a PARTITION BY intDiv(a, 2) * 2 + 1; +INSERT INTO t_int SELECT number FROM numbers_mt(32); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT intDiv(a, 2) + 1 AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +-- subtraction widens to a signed result, which is a different branch of the integer test +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT intDiv(a, 2) - 1 AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +-- the constant may be either operand, so both positions have to be recognized +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT 7 - intDiv(a, 2) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT 1 + intDiv(a, 2) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +-- negate and tuple are not named by the query-plan consumer: they answer isInjective themselves, so +-- the optimization has to keep firing for them. +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT -intDiv(a, 2) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT tuple(intDiv(a, 2)) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_int; + +-- wide integers are exact, so the gate must not be narrowed to native widths +DROP TABLE IF EXISTS t_int128; +CREATE TABLE t_int128 (a Int128) ENGINE = MergeTree ORDER BY a PARTITION BY a % 8; +INSERT INTO t_int128 SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT a + 7 AS a1 FROM t_int128 SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_int128; + +DROP TABLE IF EXISTS t_uint256; +CREATE TABLE t_uint256 (b UInt256) ENGINE = MergeTree ORDER BY b PARTITION BY b % 8; +INSERT INTO t_uint256 SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT b + 7 AS b1 FROM t_uint256 SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_uint256; + +-- both type-wrapper orders must be stripped before the integer test. Contrast with the NULL +-- constant arm above: a Nullable column keeps the optimization, a NULL constant loses it. +DROP TABLE IF EXISTS t_nullable; +CREATE TABLE t_nullable (n Nullable(UInt32)) ENGINE = MergeTree ORDER BY n PARTITION BY n % 8 SETTINGS allow_nullable_key = 1; +INSERT INTO t_nullable SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT n + 1 AS n1 FROM t_nullable SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_nullable; + +DROP TABLE IF EXISTS t_lc; +CREATE TABLE t_lc (l LowCardinality(UInt32)) ENGINE = MergeTree ORDER BY l PARTITION BY l % 8; +INSERT INTO t_lc SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT l + 1 AS l1 FROM t_lc SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_lc; + +DROP TABLE IF EXISTS t_lc_nullable; +CREATE TABLE t_lc_nullable (ln LowCardinality(Nullable(UInt32))) ENGINE = MergeTree ORDER BY ln PARTITION BY ln % 8 SETTINGS allow_nullable_key = 1; +INSERT INTO t_lc_nullable SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT ln + 1 AS ln1 FROM t_lc_nullable SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_lc_nullable; + +-- --------------------------------------------------------------------------- +-- Accepted narrowings. These three expressions are injective, but every type they would +-- admit also admits a non-injective case above that cannot be told apart: an Interval on +-- Date is not separable from one on DateTime by kind, a Date operand re-admits the +-- narrowing constant, and a Decimal operand re-admits the rescaling constant. Each arm +-- prints nothing; the bare-key arm after it is the control showing the fixture does reach +-- the optimization. +-- --------------------------------------------------------------------------- + +DROP TABLE IF EXISTS t_date; +CREATE TABLE t_date (d Date) ENGINE = MergeTree ORDER BY d PARTITION BY toDayOfWeek(d); +INSERT INTO t_date SELECT toDate('2001-01-01') + number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT d + INTERVAL 1 DAY AS d1 FROM t_date SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT d + 1 AS d1 FROM t_date SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT toDayOfWeek(d) AS d1 FROM t_date SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_date; + +DROP TABLE IF EXISTS t_dec; +CREATE TABLE t_dec (dec Decimal64(2)) ENGINE = MergeTree ORDER BY dec PARTITION BY toUInt32(dec) % 8; +INSERT INTO t_dec SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT dec + 1 AS dec1 FROM t_dec SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT toUInt32(dec) AS dec1 FROM t_dec SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_dec; diff --git a/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.reference b/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.reference new file mode 100644 index 0000000000000..19219d2f6180d --- /dev/null +++ b/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.reference @@ -0,0 +1,8 @@ +Granules: 2/10 +Granules: 2/10 +Granules: 2/10 +3 15 +3 6 +6 105 +6 105 +9 120 diff --git a/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.sql b/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.sql new file mode 100644 index 0000000000000..aae7082ce17b0 --- /dev/null +++ b/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.sql @@ -0,0 +1,33 @@ +-- The monotonicity of a conversion does not depend on the `Nullable` wrapper of its argument, so a +-- `toDate`/`toDateTime` predicate over a `Nullable` key must still be usable for index analysis. + +SET session_timezone = 'UTC'; + +DROP TABLE IF EXISTS t_nullable_key; +CREATE TABLE t_nullable_key (x Nullable(DateTime64(6)), y Int64) +ENGINE = MergeTree ORDER BY x SETTINGS index_granularity = 4, allow_nullable_key = 1; + +INSERT INTO t_nullable_key +SELECT if(number % 7 = 0, NULL, toDateTime64('2026-03-01 00:00:00', 6) + INTERVAL number * 6 HOUR), number +FROM numbers(40); + +SELECT extract(explain, 'Granules: \\d+/\\d+') AS granules FROM ( + EXPLAIN indexes = 1 SELECT count() FROM t_nullable_key WHERE toDate(x) = toDate('2026-03-02') +) WHERE granules != ''; + +SELECT extract(explain, 'Granules: \\d+/\\d+') AS granules FROM ( + EXPLAIN indexes = 1 SELECT count() FROM t_nullable_key WHERE toDate32(x) = toDate32('2026-03-02') +) WHERE granules != ''; + +SELECT extract(explain, 'Granules: \\d+/\\d+') AS granules FROM ( + EXPLAIN indexes = 1 SELECT count() FROM t_nullable_key WHERE toDateTime(x) = toDateTime('2026-03-02 06:00:00') +) WHERE granules != ''; + +-- The rows the index analysis keeps must be the rows the predicate selects, `NULL`s included. +SELECT count(), sum(y) FROM t_nullable_key WHERE toDate(x) = toDate('2026-03-02'); +SELECT count(), sum(y) FROM t_nullable_key WHERE toDate(x) < toDate('2026-03-02'); +SELECT count(), sum(y) FROM t_nullable_key WHERE toDate(x) IS NULL; +SELECT count(), sum(y) FROM t_nullable_key WHERE toDate(x) IN (toDate('2026-03-02'), toDate('2026-03-08')); +SELECT count(), sum(y) FROM t_nullable_key WHERE isNull(x) OR toDate(x) = toDate('2026-03-02'); + +DROP TABLE t_nullable_key; diff --git a/tests/queries/0_stateless/05055_clear_column_ephemeral_materialized.reference b/tests/queries/0_stateless/05055_clear_column_ephemeral_materialized.reference new file mode 100644 index 0000000000000..9ad7af9de8a73 --- /dev/null +++ b/tests/queries/0_stateless/05055_clear_column_ephemeral_materialized.reference @@ -0,0 +1,3 @@ +1 8 108 2 +0 8 108 1 +1 0 8 diff --git a/tests/queries/0_stateless/05055_clear_column_ephemeral_materialized.sql b/tests/queries/0_stateless/05055_clear_column_ephemeral_materialized.sql new file mode 100644 index 0000000000000..352f695101204 --- /dev/null +++ b/tests/queries/0_stateless/05055_clear_column_ephemeral_materialized.sql @@ -0,0 +1,50 @@ +-- CLEAR COLUMN recomputes the MATERIALIZED columns that read the cleared one. A MATERIALIZED column +-- derived from an EPHEMERAL column cannot be recomputed outside INSERT, so it must be left alone: +-- recomputing it fails to resolve the EPHEMERAL name and the whole mutation dies with +-- "There is no column or subcolumn e in table". + +SET mutations_sync = 2; + +DROP TABLE IF EXISTS t_clear_ephemeral; + +CREATE TABLE t_clear_ephemeral +( + x Int32, + y Int32, + e Int32 EPHEMERAL 0, + me Int32 MATERIALIZED x + e, + me2 Int32 MATERIALIZED me + 100, + mk Int32 MATERIALIZED x + 1 +) +ENGINE = MergeTree ORDER BY tuple() PARTITION BY tuple() +-- The runner randomizes both of these together with `min_bytes_for_wide_part`, and on a Wide part +-- with either block column on the mutation does not recompute MATERIALIZED columns at all, so `mk` +-- would keep its pre-clear value and the recompute this test is about would never run. +SETTINGS enable_block_number_column = 0, enable_block_offset_column = 0; + +INSERT INTO t_clear_ephemeral (x, y, e) VALUES (1, 0, 7); +SELECT x, me, me2, mk FROM t_clear_ephemeral; + +-- `mk` is recomputed from the cleared `x`. `me` reads the EPHEMERAL `e`, so it cannot be recomputed +-- and keeps its stored value. `me2` reads only `me`, so it keeps its value too. +ALTER TABLE t_clear_ephemeral CLEAR COLUMN x IN PARTITION tuple(); +SELECT x, me, me2, mk FROM t_clear_ephemeral; + +-- A cleared column that no MATERIALIZED column reads must not start a recompute at all. +DROP TABLE IF EXISTS t_clear_unrelated; + +CREATE TABLE t_clear_unrelated +( + x Int32, + y Int32, + e Int32 EPHEMERAL 0, + me Int32 MATERIALIZED x + e +) +ENGINE = MergeTree ORDER BY tuple() PARTITION BY tuple(); + +INSERT INTO t_clear_unrelated (x, y, e) VALUES (1, 5, 7); +ALTER TABLE t_clear_unrelated CLEAR COLUMN y IN PARTITION tuple(); +SELECT x, y, me FROM t_clear_unrelated; + +DROP TABLE t_clear_ephemeral; +DROP TABLE t_clear_unrelated; diff --git a/tests/queries/0_stateless/05055_jit_float_to_integer_cast.reference b/tests/queries/0_stateless/05055_jit_float_to_integer_cast.reference new file mode 100644 index 0000000000000..1c4357e61ffd8 --- /dev/null +++ b/tests/queries/0_stateless/05055_jit_float_to_integer_cast.reference @@ -0,0 +1,8 @@ +230 1 230 +250 1 250 +true true +true true +true true +true true +0 +1 1 1 diff --git a/tests/queries/0_stateless/05055_jit_float_to_integer_cast.sql b/tests/queries/0_stateless/05055_jit_float_to_integer_cast.sql new file mode 100644 index 0000000000000..26e66cc4e70c8 --- /dev/null +++ b/tests/queries/0_stateless/05055_jit_float_to_integer_cast.sql @@ -0,0 +1,88 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/117442 +SET compile_expressions = 1; +SET min_count_to_compile_expression = 0; + +DROP TABLE IF EXISTS t_jit_float_cast; +CREATE TABLE t_jit_float_cast (c0 UInt8) ENGINE = Memory; +INSERT INTO t_jit_float_cast VALUES (230), (250); + +-- In-range conversions are exact and identical compiled or interpreted. +SELECT c0, + toFloat64(c0) / 10 <= CAST(toFloat64(c0) AS UInt8) AS in_range_by_cast, + CAST(toFloat64(c0) AS Decimal32(2)) AS in_range_decimal +FROM t_jit_float_cast +ORDER BY c0; + +-- A value the destination cannot hold raises. The non-finite one is built from the column so that +-- it is not constant folded before execution. +SELECT CAST(-toFloat64(c0) * 1e9 AS Decimal32(2)) FROM t_jit_float_cast; -- { serverError DECIMAL_OVERFLOW } +SELECT toDecimal32(-toFloat64(c0) * 1e9, 2) FROM t_jit_float_cast; -- { serverError DECIMAL_OVERFLOW } +SELECT toUInt8(toFloat64(c0) / (toFloat64(c0) - toFloat64(c0))) FROM t_jit_float_cast; -- { serverError CANNOT_CONVERT_TYPE } +SELECT CAST(toFloat64(c0) / (toFloat64(c0) - toFloat64(c0)) AS UInt8) FROM t_jit_float_cast; -- { serverError CANNOT_CONVERT_TYPE } + +-- `Bool` is the one float to integer destination that stays compiled, because it is lowered as +-- `value != 0` rather than as a conversion, which is exact for every value. +SELECT CAST(-toFloat64(c0) * 1e30 AS Bool), + CAST(toFloat64(c0) / (toFloat64(c0) - toFloat64(c0)) AS Bool) +FROM t_jit_float_cast +ORDER BY c0; + +SET compile_expressions = 0; +SELECT CAST(-toFloat64(c0) * 1e30 AS Bool), + CAST(toFloat64(c0) / (toFloat64(c0) - toFloat64(c0)) AS Bool) +FROM t_jit_float_cast +ORDER BY c0; +SET compile_expressions = 1, min_count_to_compile_expression = 0; + +-- The value an out-of-range float converts to is not defined by the language, so compare the compiled +-- and the interpreted evaluation of the same expression instead of pinning a literal. +CREATE TABLE t_jit_float_cast_arms (c0 UInt8, lte UInt8) ENGINE = Memory; + +INSERT INTO t_jit_float_cast_arms +SELECT c0, toFloat64(c0) / 10 <= CAST(-toFloat64(c0) AS UInt8) FROM t_jit_float_cast; + +SET compile_expressions = 0; +INSERT INTO t_jit_float_cast_arms +SELECT c0, toFloat64(c0) / 10 <= CAST(-toFloat64(c0) AS UInt8) FROM t_jit_float_cast; +SET compile_expressions = 1, min_count_to_compile_expression = 0; + +SELECT count() FROM (SELECT c0 FROM t_jit_float_cast_arms GROUP BY c0 HAVING uniqExact(lte) > 1); + +DROP TABLE t_jit_float_cast_arms; + +-- Every row above is a value oracle, so all of them would still pass if the conversions silently +-- stopped or started being compiled. The shapes below pin which of them compiles. Each has one +-- compilable child, so once its `CAST` is declined nothing is left to compile: a declined shape +-- reaches zero even where the control compiles. `CompiledFunctionExecute` counts executions of an +-- already-compiled node, so a warm compiled cache does not change any of them. +SELECT CAST(toFloat64(number) AS Bool) FROM numbers(2) + SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0, log_comment = '05055_bool' FORMAT Null; +SELECT CAST(toFloat64(number) AS UInt8) FROM numbers(2) + SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0, log_comment = '05055_declined' FORMAT Null; +SELECT toFloat64(number) + 1 FROM numbers(2) + SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0, log_comment = '05055_control' FORMAT Null; +-- The reported shape: a comparison whose right side is a declined conversion. The conversion becomes +-- an input to the compiled expression, so the comparison around it must still compile. Neither side +-- can compile on its own, so the counter here belongs to the comparison and to nothing else. +SELECT toFloat64(number) <= CAST(toFloat64(number) AS UInt8) FROM numbers(2) + SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0, log_comment = '05055_parent' FORMAT Null; + +SYSTEM FLUSH LOGS query_log; + +WITH shapes AS +( + SELECT log_comment, argMax(ProfileEvents['CompiledFunctionExecute'] > 0, event_time_microseconds) AS compiled + FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment LIKE '05055_%' + GROUP BY log_comment +) +-- The control keeps the first column honest in a build without the embedded compiler, where every +-- shape is interpreted and an absolute assertion would go green on nothing being compiled. +SELECT + (SELECT compiled FROM shapes WHERE log_comment = '05055_bool') + = (SELECT compiled FROM shapes WHERE log_comment = '05055_control'), + (SELECT compiled FROM shapes WHERE log_comment = '05055_declined') = 0, + (SELECT compiled FROM shapes WHERE log_comment = '05055_parent') + = (SELECT compiled FROM shapes WHERE log_comment = '05055_control'); + +DROP TABLE t_jit_float_cast; diff --git a/tests/queries/0_stateless/05055_json_flattened_native_truncated.reference b/tests/queries/0_stateless/05055_json_flattened_native_truncated.reference new file mode 100644 index 0000000000000..68961ec9d0d83 --- /dev/null +++ b/tests/queries/0_stateless/05055_json_flattened_native_truncated.reference @@ -0,0 +1,9 @@ +t_json_flattened: accepted truncated blocks 0 +0 +12 18 6 51 +t_json_array_flattened: accepted truncated blocks 0 +0 +12 12 2 +t_dynamic_flattened: accepted truncated blocks 0 +0 +12 6 27 str8 diff --git a/tests/queries/0_stateless/05055_json_flattened_native_truncated.sh b/tests/queries/0_stateless/05055_json_flattened_native_truncated.sh new file mode 100755 index 0000000000000..7577adec5191a --- /dev/null +++ b/tests/queries/0_stateless/05055_json_flattened_native_truncated.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Tags: no-old-analyzer +# no-old-analyzer: the queries building the payloads read JSON/Dynamic subcolumns, which it cannot resolve. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A flattened JSON/Dynamic Native block must be rejected when a stream ends before the declared row count. +# The data starts with rows that have no dynamic paths, so the Dynamic indexes stream starts with a run of +# NULL markers: truncated inside that run it demands no values at all and passes every per-type check. + +DATA_FILE=$CLICKHOUSE_TMP/flattened_native_$CLICKHOUSE_DATABASE.bin + +function check_truncated_blocks() +{ + local table=$1 + local query=$2 + + $CLICKHOUSE_CLIENT --output_format_native_use_flattened_dynamic_and_json_serialization=1 -q "$query FORMAT Native" > "$DATA_FILE" + + local size + size=$(stat -c%s "$DATA_FILE") + local accepted=0 + local n + # Sent over HTTP: a client process per truncation is what made this test time out under sanitizers. + for ((n = 1; n < size; n += 5)); do + if head -c "$n" "$DATA_FILE" \ + | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+$table+FORMAT+Native" --data-binary @- \ + | grep -q "Exception" + then + continue + fi + accepted=$((accepted + 1)) + done + + echo "$table: accepted truncated blocks $accepted" + $CLICKHOUSE_CLIENT -q "SELECT count() FROM $table" + + $CLICKHOUSE_CLIENT -q "INSERT INTO $table FORMAT Native" < "$DATA_FILE" + rm "$DATA_FILE" +} + +$CLICKHOUSE_CLIENT -q " + CREATE TABLE t_json_flattened (json JSON(a UInt64, max_dynamic_paths=2)) ENGINE = Memory; + CREATE TABLE t_json_array_flattened (arr Array(JSON)) ENGINE = Memory; + CREATE TABLE t_dynamic_flattened (dyn Dynamic) ENGINE = Memory; +" + +check_truncated_blocks t_json_flattened " + SELECT (number < 6 ? '{\"a\":1}' : '{\"a\":2,\"b\":\"str' || toString(number) || '\",\"c\":' || toString(number) || '}')::JSON(a UInt64, max_dynamic_paths=2) AS json + FROM numbers(12)" +$CLICKHOUSE_CLIENT -q "SELECT count(), sum(json.a), countIf(json.b IS NULL), sum(json.c.:Int64) FROM t_json_flattened" + +check_truncated_blocks t_json_array_flattened " + SELECT arrayMap(x -> (number < 6 ? '{}' : '{\"a\":' || toString(x) || '}')::JSON, range(number % 3)) AS arr + FROM numbers(12)" +$CLICKHOUSE_CLIENT -q "SELECT count(), sum(length(arr)), sum(arraySum(arrayMap(x -> x.a::UInt64, arr))) FROM t_json_array_flattened" + +check_truncated_blocks t_dynamic_flattened " + SELECT json.a AS dyn + FROM (SELECT (number < 6 ? '{}' : (number % 2 ? '{\"a\":' || toString(number) || '}' : '{\"a\":\"str' || toString(number) || '\"}'))::JSON AS json FROM numbers(12))" +$CLICKHOUSE_CLIENT -q "SELECT count(), countIf(dyn IS NULL), sum(dyn.Int64), max(dyn.String) FROM t_dynamic_flattened" + +$CLICKHOUSE_CLIENT -q " + DROP TABLE t_json_flattened; + DROP TABLE t_json_array_flattened; + DROP TABLE t_dynamic_flattened; +" diff --git a/tests/queries/0_stateless/05055_json_native_flattened_paths_memory_tracking.reference b/tests/queries/0_stateless/05055_json_native_flattened_paths_memory_tracking.reference new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/queries/0_stateless/05055_json_native_flattened_paths_memory_tracking.sql b/tests/queries/0_stateless/05055_json_native_flattened_paths_memory_tracking.sql new file mode 100644 index 0000000000000..526e1f167e6d6 --- /dev/null +++ b/tests/queries/0_stateless/05055_json_native_flattened_paths_memory_tracking.sql @@ -0,0 +1,5 @@ +-- The FLATTENED (version 3) path list is sized from an untrusted count in the Native prefix, so its +-- container must be memory-tracked like the non-flat sibling (version 2) below. +-- Frame: 1 column, 1 row, name "j", type "JSON", 8-byte LE structure version, VarUInt path count 1000000. +SELECT * FROM format(Native, 'j JSON', unhex('0101016A044A534F4E0300000000000000C0843D')) SETTINGS max_memory_usage = 10000000; -- { serverError MEMORY_LIMIT_EXCEEDED } +SELECT * FROM format(Native, 'j JSON', unhex('0101016A044A534F4E0200000000000000C0843D')) SETTINGS max_memory_usage = 10000000; -- { serverError MEMORY_LIMIT_EXCEEDED } diff --git a/tests/queries/0_stateless/05055_native_protocol_collection_limits.python b/tests/queries/0_stateless/05055_native_protocol_collection_limits.python new file mode 100644 index 0000000000000..df47b8da1ff3b --- /dev/null +++ b/tests/queries/0_stateless/05055_native_protocol_collection_limits.python @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 + +# The server allocates from counts and sizes that a client puts on the wire, before the corresponding +# data arrives, so a collection read from the protocol has to be bounded by what the data can +# plausibly hold. In interserver mode some of these fields are read before the secret hash of the +# query is checked. + +import os +import socket +import uuid + +CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") +CLICKHOUSE_PORT = int(os.environ.get("CLICKHOUSE_PORT_TCP", "900000")) +CLICKHOUSE_DATABASE = os.environ.get("CLICKHOUSE_DATABASE", "default") +CLIENT_NAME = "05055 native protocol" +CLIENT_REVISION = 54459 # the first revision with query parameters + + +def writeVarUInt(x, ba): + for _ in range(0, 9): + byte = x & 0x7F + if x > 0x7F: + byte |= 0x80 + + ba.append(byte) + + x >>= 7 + if x == 0: + return + + +def writeStringBinary(s, ba): + b = bytes(s, "utf-8") + writeVarUInt(len(b), ba) + ba.extend(b) + + +def readStrict(s, size=1): + res = bytearray() + while size: + cur = s.recv(size) + if not cur: + raise RuntimeError("Socket is closed") + size -= len(cur) + res.extend(cur) + + return res + + +def readUInt(s, size=1): + res = readStrict(s, size) + val = 0 + for i in range(len(res)): + val += res[i] << (i * 8) + return val + + +def readVarUInt(s): + x = 0 + for i in range(9): + byte = readStrict(s)[0] + x |= (byte & 0x7F) << (7 * i) + + if not byte & 0x80: + return x + + return x + + +def readStringBinary(s): + size = readVarUInt(s) + return readStrict(s, size).decode("utf-8", "replace") + + +def connect(): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(30) + s.connect((CLICKHOUSE_HOST, CLICKHOUSE_PORT)) + + ba = bytearray() + writeVarUInt(0, ba) # Hello + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(21, ba) + writeVarUInt(9, ba) + writeVarUInt(CLIENT_REVISION, ba) + writeStringBinary(CLICKHOUSE_DATABASE, ba) + writeStringBinary("default", ba) + writeStringBinary("", ba) + s.sendall(ba) + + assert readVarUInt(s) == 0 # Hello + readStringBinary(s) # server name + readVarUInt(s) # version major + readVarUInt(s) # version minor + readVarUInt(s) # revision + readStringBinary(s) # timezone + readStringBinary(s) # display name + readVarUInt(s) # version patch + + ba = bytearray() + writeStringBinary("", ba) # addendum: quota key + s.sendall(ba) + return s + + +def serializeClientInfo(ba, query_id, os_user_declared_size=None): + writeStringBinary("default", ba) # initial_user + writeStringBinary(query_id, ba) # initial_query_id + writeStringBinary("127.0.0.1:9000", ba) # initial_address + ba.extend([0] * 8) # initial_query_start_time_microseconds + ba.append(1) # TCP + if os_user_declared_size is None: + writeStringBinary("os_user", ba) # os_user + else: + writeVarUInt(os_user_declared_size, ba) # os_user, but the value never follows + return + writeStringBinary("client_hostname", ba) # client_hostname + writeStringBinary(CLIENT_NAME, ba) # client_name + writeVarUInt(21, ba) + writeVarUInt(9, ba) + writeVarUInt(CLIENT_REVISION, ba) + writeStringBinary("", ba) # quota_key + writeVarUInt(0, ba) # distributed_depth + writeVarUInt(1, ba) # client_version_patch + ba.append(0) # no telemetry + writeVarUInt(0, ba) # collaborate_with_initiator + writeVarUInt(0, ba) # count_participating_replicas + writeVarUInt(0, ba) # number_of_current_replica + + +def sendQuery( + s, + query, + num_settings=0, + os_user_declared_size=None, + setting_name_declared_size=None, + parameter_name_declared_size=None, +): + ba = bytearray() + query_id = uuid.uuid4().hex + writeVarUInt(1, ba) # Query + writeStringBinary(query_id, ba) + ba.append(1) # INITIAL_QUERY + serializeClientInfo(ba, query_id, os_user_declared_size) + if os_user_declared_size is not None: + s.sendall(ba) + return + + if setting_name_declared_size is not None: + writeVarUInt( + setting_name_declared_size, ba + ) # a setting name that never follows + s.sendall(ba) + return + + for _ in range(num_settings): + writeStringBinary("max_threads", ba) + writeVarUInt(0, ba) # flags + writeStringBinary("1", ba) + writeStringBinary("", ba) # end of settings + + writeStringBinary("", ba) # no interserver secret + writeVarUInt(2, ba) # stage: Complete + ba.append(0) # no compression + writeStringBinary(query, ba) + + if parameter_name_declared_size is not None: + writeVarUInt( + parameter_name_declared_size, ba + ) # a parameter name that never follows + s.sendall(ba) + return + + writeStringBinary("", ba) # end of query parameters + s.sendall(ba) + + +def sendTablesStatusRequest(s, num_tables, database_declared_size=None): + ba = bytearray() + writeVarUInt(5, ba) # TablesStatusRequest + writeVarUInt(num_tables, ba) + if database_declared_size is not None: + writeVarUInt( + database_declared_size, ba + ) # a database name whose value never follows + s.sendall(ba) + + +def sendEmptyBlock(s): + ba = bytearray() + writeVarUInt(2, ba) # Data + writeStringBinary("", ba) + writeVarUInt(1, ba) + ba.append(0) # is_overflows + writeVarUInt(2, ba) + writeVarUInt(0, ba) + ba.extend([0] * 4) # bucket_num + writeVarUInt(0, ba) # columns + writeVarUInt(0, ba) # rows + s.sendall(ba) + + +def readException(s): + readUInt(s, 4) # code + readStringBinary(s) # name + message = readStringBinary(s) + readStringBinary(s) # stack trace + readUInt(s) # has_nested + for known in ( + "Too many settings received", + "Too large string size", + "Too large collection size", + "Attempt to read after eof", + "Cannot read all data", + ): + if known in message: + return known + return "UNEXPECTED " + message[:120] + + +def outcome(s): + """The name of the exception the server answered with, or how the exchange ended.""" + try: + while True: + packet_type = readVarUInt(s) + if packet_type == 2: # Exception + return readException(s) + if packet_type == 5: # EndOfStream + return "end of stream" + if packet_type == 1: # Data + return "data" + if packet_type == 3: # Progress + continue + return "unexpected packet {}".format(packet_type) + except socket.timeout: + return "TIMEOUT" + except RuntimeError: + return "closed" + + +def rejected(s): + """A `TablesStatusRequest` is read before there is a query state to attach an exception to, so + the server may either answer with one or simply close the connection; either way it stops + reading instead of waiting for data that never comes.""" + result = outcome(s) + if result in ("closed", "Too large collection size", "Too large string size"): + return "rejected" + return result + + +def endOfStream(s): + """The client closed its side in the middle of a packet: the server may either answer with the + exception about the end of the stream or simply close the connection.""" + result = outcome(s) + if result in ("closed", "Attempt to read after eof", "Cannot read all data"): + return "end of stream" + return result + + +def main(): + # A settings sequence longer than any real query. + s = connect() + sendQuery(s, "SELECT 1", num_settings=65537) + print("too many settings:", outcome(s)) + s.close() + + # A `ClientInfo` string that declares a gigabyte and never sends it. It is read growing as the + # bytes arrive, so the server waits for them instead of allocating the declared size, and stops + # at the end of the stream. + s = connect() + sendQuery(s, "SELECT 1", os_user_declared_size=1 << 30) + s.shutdown(socket.SHUT_WR) + print("client info string of a gigabyte:", endOfStream(s)) + s.close() + + # A setting name that declares a gigabyte and never sends it: the count bound alone does not + # cover a single pair. + s = connect() + sendQuery(s, "SELECT 1", setting_name_declared_size=1 << 30) + print("setting name of a gigabyte:", outcome(s)) + s.close() + + # The same for a query parameter name. + s = connect() + sendQuery(s, "SELECT 1", parameter_name_declared_size=1 << 30) + print("query parameter name of a gigabyte:", outcome(s)) + s.close() + + # A `TablesStatusRequest` about more tables than a server can hold. + s = connect() + sendTablesStatusRequest(s, 1 << 30) + print("too many tables in a tables status request:", rejected(s)) + s.close() + + # A `TablesStatusRequest` whose database name declares a gigabyte and never sends it. + s = connect() + sendTablesStatusRequest(s, 1, database_declared_size=1 << 30) + print("table status identifier of a gigabyte:", rejected(s)) + s.close() + + # A well-formed query still works. + s = connect() + sendQuery(s, "SELECT 1", num_settings=1) + sendEmptyBlock(s) + print("well-formed query:", outcome(s)) + s.close() + + +if __name__ == "__main__": + main() diff --git a/tests/queries/0_stateless/05055_native_protocol_collection_limits.reference b/tests/queries/0_stateless/05055_native_protocol_collection_limits.reference new file mode 100644 index 0000000000000..4fc7c418dc5b5 --- /dev/null +++ b/tests/queries/0_stateless/05055_native_protocol_collection_limits.reference @@ -0,0 +1,7 @@ +too many settings: Too many settings received +client info string of a gigabyte: end of stream +setting name of a gigabyte: Too large string size +query parameter name of a gigabyte: Too large string size +too many tables in a tables status request: rejected +table status identifier of a gigabyte: rejected +well-formed query: data diff --git a/tests/queries/0_stateless/05055_native_protocol_collection_limits.sh b/tests/queries/0_stateless/05055_native_protocol_collection_limits.sh new file mode 100755 index 0000000000000..dc01e3f6e8b3d --- /dev/null +++ b/tests/queries/0_stateless/05055_native_protocol_collection_limits.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +# NOTE: this sh wrapper is required because of shell_config + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +python3 "$CUR_DIR"/05055_native_protocol_collection_limits.python diff --git a/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.reference b/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.reference new file mode 100644 index 0000000000000..d50220788b0b6 --- /dev/null +++ b/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.reference @@ -0,0 +1,20 @@ +--- A1 control: the boundary the injection bypasses is enforced when the path is named directly +PATH_ACCESS_DENIED +--- A2 injection through the remote table name +no such table +objects created in the anchor: none +canary read from outside user_files: none +objects written outside user_files: none +--- A3 injection through a column name +no column named +objects created in the anchor: none +canary read from outside user_files: none +objects written outside user_files: none +--- A4 control: a table and column name that legitimately contain a double quote round-trip +NO_ERROR +11 +--- A5 control: a table name that legitimately contains a backslash round-trips +NO_ERROR +22 +--- A6 a NUL in the remote table name fails loudly, it is never silently truncated +unrecognized token diff --git a/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.sh b/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.sh new file mode 100755 index 0000000000000..9ec9abb0b9bae --- /dev/null +++ b/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: Fast tests don't build external libraries (SQLite) + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +ANCHOR="${USER_FILES_PATH}/05055_anchor_${CLICKHOUSE_DATABASE}.db" +# Outside the user_files confinement. +OUTSIDE="${CLICKHOUSE_TMP}/05055_outside_${CLICKHOUSE_DATABASE}.db" + +cleanup() +{ + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_name_inj" + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_col_inj" + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_quote_id" + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_backslash_id" + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_nul_name" + rm -f "${ANCHOR}" "${OUTSIDE}" +} +trap cleanup EXIT +cleanup + +# Odd names: the old backslash escaping produced them, so the injected script's first statement succeeds and the rest runs. +sqlite3 "${ANCHOR}" 'CREATE TABLE "target\"(id INTEGER)' +sqlite3 "${ANCHOR}" 'CREATE TABLE col_anchor("c\" INTEGER)' +sqlite3 "${ANCHOR}" 'CREATE TABLE "ta""ble" ("c""1" INTEGER)' +sqlite3 "${ANCHOR}" 'CREATE TABLE "a\b" (x INTEGER)' +chmod ugo+rw "${ANCHOR}" + +# A database outside user_files, holding a canary the user must never be able to reach. +sqlite3 "${OUTSIDE}" "CREATE TABLE secrets(s TEXT); INSERT INTO secrets VALUES ('CANARY')" +chmod ugo+rw "${OUTSIDE}" + +# One stable token per failure, so the reference embeds no path and no server version. +classify() +{ + local out + out=$(${CLICKHOUSE_CLIENT} --query="$1" 2>&1 \ + | grep -oF -e 'no such table' -e 'no column named' -e 'unrecognized token' \ + -e 'syntax error' -e 'PATH_ACCESS_DENIED' \ + | sed -n 1p) + echo "${out:-NO_ERROR}" +} + +# Read out of band: the ClickHouse read path cannot address these names, so an in-band readback would be a false negative. +injected_state() +{ + sqlite3 "${ANCHOR}" "SELECT 'objects created in the anchor: ' || coalesce((SELECT group_concat(name) FROM (SELECT name FROM sqlite_master WHERE name LIKE '%\_marker' ESCAPE '\' ORDER BY name)), 'none')" + sqlite3 "${ANCHOR}" "SELECT 'canary read from outside user_files: ' || coalesce((SELECT group_concat(s) FROM stolen_marker), 'none')" 2>/dev/null \ + || echo 'canary read from outside user_files: none' + sqlite3 "${OUTSIDE}" "SELECT 'objects written outside user_files: ' || coalesce((SELECT group_concat(name) FROM (SELECT name FROM sqlite_master WHERE name LIKE '%\_marker' ESCAPE '\' ORDER BY name)), 'none')" +} + +echo '--- A1 control: the boundary the injection bypasses is enforced when the path is named directly' +classify "CREATE TABLE t_ctl (s String) ENGINE = SQLite('${OUTSIDE}', 'secrets')" + +echo '--- A2 injection through the remote table name' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_name_inj (id UInt32) ENGINE = SQLite('${ANCHOR}', \$\$target\" (id) VALUES (999); ATTACH DATABASE '${OUTSIDE}' AS v; CREATE TABLE stolen_marker AS SELECT s FROM v.secrets; CREATE TABLE v.written_marker(z); DETACH v; --\$\$)" +classify "INSERT INTO t_name_inj VALUES (0)" +injected_state + +echo '--- A3 injection through a column name' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_col_inj (\`c') VALUES (1); CREATE TABLE col_marker(z); --\` UInt32) ENGINE = SQLite('${ANCHOR}', 'col_anchor')" +classify "INSERT INTO t_col_inj VALUES (0)" +injected_state + +echo '--- A4 control: a table and column name that legitimately contain a double quote round-trip' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_quote_id (\`c\"1\` UInt32) ENGINE = SQLite('${ANCHOR}', 'ta\"ble')" +classify "INSERT INTO t_quote_id VALUES (11)" +sqlite3 "${ANCHOR}" 'SELECT "c""1" FROM "ta""ble"' + +echo '--- A5 control: a table name that legitimately contains a backslash round-trips' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_backslash_id (x UInt32) ENGINE = SQLite('${ANCHOR}', 'a\\\\b')" +classify "INSERT INTO t_backslash_id VALUES (22)" +sqlite3 "${ANCHOR}" 'SELECT x FROM "a\b"' + +echo '--- A6 a NUL in the remote table name fails loudly, it is never silently truncated' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_nul_name (id UInt32) ENGINE = SQLite('${ANCHOR}', 'a\0b')" +classify "INSERT INTO t_nul_name VALUES (0)" diff --git a/tests/queries/0_stateless/05056_json_dynamic_native_malformed_structure.reference b/tests/queries/0_stateless/05056_json_dynamic_native_malformed_structure.reference new file mode 100644 index 0000000000000..2c297afeb42e3 --- /dev/null +++ b/tests/queries/0_stateless/05056_json_dynamic_native_malformed_structure.reference @@ -0,0 +1,12 @@ +object_v3: Version 4 of Object structure serialization is written only into MergeTree data parts and is not allowed in Native format. (INCORRECT_DATA) +dynamic_v3: Version 4 of Dynamic structure serialization is written only into MergeTree data parts and is not allowed in Native format. (INCORRECT_DATA) +dynamic_duplicate_types: Duplicate type Int64 in the list of types of a flattened Dynamic column. (INCORRECT_DATA) +dynamic_nothing_type: Type Nothing is not allowed in the list of types of a flattened Dynamic column. (INCORRECT_DATA) +object_duplicate_flattened_path: Duplicate path x in the list of paths of a flattened Object column. (INCORRECT_DATA) +object_duplicate_dynamic_path: Duplicate path in the list of dynamic paths of an Object column. (INCORRECT_DATA) +object_typed_path_collision_flattened: Path x of an Object column is stored as a dynamic path, but it is a typed path in the type of the column. (INCORRECT_DATA) +object_typed_path_collision: Path x of an Object column is stored as a dynamic path, but it is a typed path in the type of the column. (INCORRECT_DATA) +object_duplicate_path_after_realloc: Duplicate path p0 in the list of paths of a flattened Object column. (INCORRECT_DATA) +0 +0 +0 diff --git a/tests/queries/0_stateless/05056_json_dynamic_native_malformed_structure.sh b/tests/queries/0_stateless/05056_json_dynamic_native_malformed_structure.sh new file mode 100755 index 0000000000000..1078ca8f04e0e --- /dev/null +++ b/tests/queries/0_stateless/05056_json_dynamic_native_malformed_structure.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Malformed structure prefixes of JSON/Dynamic columns in a Native block. Each of them used to be +# accepted and then read out of bounds, or to escape as an internal error instead of a data error. + +DATA_DIR=$CLICKHOUSE_TMP/native_structure_$CLICKHOUSE_DATABASE +rm -rf "$DATA_DIR" +mkdir -p "$DATA_DIR" + +python3 - "$DATA_DIR" <<'EOF' +import struct +import sys + +data_dir = sys.argv[1] + +def s(b): + return bytes([len(b)]) + b + +def block(column_type, tail, rows=2): + """One Native block with a single column and a hand-built serialization prefix.""" + return bytes([1, rows]) + s(b'j') + s(column_type) + tail + +def varuint(n): + out = b'' + while True: + byte = n & 0x7f + n >>= 7 + out += bytes([byte | 0x80]) if n else bytes([byte]) + if not n: + return out + +def object_prefix(version, paths): + return struct.pack('&1 | head -10 + fi +} + +function read_part() +{ + $CLICKHOUSE_LOCAL --path "$DATA_DIR" -q "SELECT count(), sum(length(toString(json))) FROM t" 2>&1 \ + | grep -m1 -oE "ATTEMPT_TO_READ_AFTER_EOF|CANNOT_READ_ALL_DATA" +} + +create_part +truncate_streams "paths indexes stream" "$PART"/*object_shared_data*paths_indexes.bin +read_part + +create_part +truncate_streams "structure stream" "$PART"/*object_shared_data*structure.bin +read_part + +rm -rf "$DATA_DIR" diff --git a/tests/queries/0_stateless/05070_merge_expression_into_join_nondeterministic.reference b/tests/queries/0_stateless/05070_merge_expression_into_join_nondeterministic.reference new file mode 100644 index 0000000000000..e59286f7f0c94 --- /dev/null +++ b/tests/queries/0_stateless/05070_merge_expression_into_join_nondeterministic.reference @@ -0,0 +1,10 @@ +0 1000 +0 1000 +deterministic +0 1000 +merged +0 1 +stateful +0 1 +sleep +0 1 diff --git a/tests/queries/0_stateless/05070_merge_expression_into_join_nondeterministic.sql b/tests/queries/0_stateless/05070_merge_expression_into_join_nondeterministic.sql new file mode 100644 index 0000000000000..23821a031d2f6 --- /dev/null +++ b/tests/queries/0_stateless/05070_merge_expression_into_join_nondeterministic.sql @@ -0,0 +1,70 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/116848 +-- `query_plan_merge_expression_into_join` merges an expression step into the join graph, where +-- reordering can leave it computed twice from the raw inputs - once for the join key that decides +-- matching and once for the output column. A non-deterministic expression then draws independently +-- in the two places, and the returned rows violate the query's own `JOIN ON` condition. + +SET query_plan_optimize_join_order_limit = 64; +SET query_plan_optimize_join_order_randomize = 0; +SET enable_analyzer = 1; + +DROP TABLE IF EXISTS t_merge_expr_1; +DROP TABLE IF EXISTS t_merge_expr_2; +DROP TABLE IF EXISTS t_merge_expr_3; +CREATE TABLE t_merge_expr_1 (a UInt64) ENGINE = MergeTree ORDER BY a AS SELECT number FROM numbers(1000); +CREATE TABLE t_merge_expr_2 (b UInt64) ENGINE = MergeTree ORDER BY b AS SELECT number FROM numbers(1000); +CREATE TABLE t_merge_expr_3 (k UInt8) ENGINE = MergeTree ORDER BY k AS SELECT number FROM numbers(2); + +-- The subquery emits 1000 rows with r in {0, 1} and t_merge_expr_3 holds exactly {0, 1}, so the join +-- must return exactly 1000 rows and every one of them must satisfy r = k. +SELECT countIf(r != k) AS on_violations, count() AS total +FROM (SELECT t_merge_expr_1.a AS a, rand() % 2 AS r FROM t_merge_expr_1 JOIN t_merge_expr_2 ON t_merge_expr_1.a = t_merge_expr_2.b) s +JOIN t_merge_expr_3 ON s.r = t_merge_expr_3.k; + +SELECT countIf(r != k) AS on_violations, count() AS total +FROM (SELECT t_merge_expr_1.a AS a, rand() % 2 AS r FROM t_merge_expr_1 JOIN t_merge_expr_2 ON t_merge_expr_1.a = t_merge_expr_2.b) s +JOIN t_merge_expr_3 ON s.r = t_merge_expr_3.k +SETTINGS query_plan_merge_expression_into_join = 0; + +-- A deterministic expression is still merged. +SELECT 'deterministic'; +SELECT countIf(r != k) AS on_violations, count() AS total +FROM (SELECT t_merge_expr_1.a AS a, t_merge_expr_1.a % 2 AS r FROM t_merge_expr_1 JOIN t_merge_expr_2 ON t_merge_expr_1.a = t_merge_expr_2.b) s +JOIN t_merge_expr_3 ON s.r = t_merge_expr_3.k; + +-- The gate must decide by determinism rather than reject every expression, so assert both +-- directions. Whether the merge happened shows up as the plan changing when the setting is toggled; +-- comparing the two plans to each other keeps the assertion independent of how a plan is printed. +SELECT 'merged'; +SELECT + (SELECT groupArray(explain) FROM (EXPLAIN SELECT * FROM (SELECT t_merge_expr_1.a AS a, rand() % 2 AS r FROM t_merge_expr_1 JOIN t_merge_expr_2 ON t_merge_expr_1.a = t_merge_expr_2.b) s JOIN t_merge_expr_3 ON s.r = t_merge_expr_3.k SETTINGS query_plan_merge_expression_into_join = 1)) + != (SELECT groupArray(explain) FROM (EXPLAIN SELECT * FROM (SELECT t_merge_expr_1.a AS a, rand() % 2 AS r FROM t_merge_expr_1 JOIN t_merge_expr_2 ON t_merge_expr_1.a = t_merge_expr_2.b) s JOIN t_merge_expr_3 ON s.r = t_merge_expr_3.k SETTINGS query_plan_merge_expression_into_join = 0)) AS nondeterministic_merged, + (SELECT groupArray(explain) FROM (EXPLAIN SELECT * FROM (SELECT t_merge_expr_1.a AS a, t_merge_expr_1.a % 2 AS r FROM t_merge_expr_1 JOIN t_merge_expr_2 ON t_merge_expr_1.a = t_merge_expr_2.b) s JOIN t_merge_expr_3 ON s.r = t_merge_expr_3.k SETTINGS query_plan_merge_expression_into_join = 1)) + != (SELECT groupArray(explain) FROM (EXPLAIN SELECT * FROM (SELECT t_merge_expr_1.a AS a, t_merge_expr_1.a % 2 AS r FROM t_merge_expr_1 JOIN t_merge_expr_2 ON t_merge_expr_1.a = t_merge_expr_2.b) s JOIN t_merge_expr_3 ON s.r = t_merge_expr_3.k SETTINGS query_plan_merge_expression_into_join = 0)) AS deterministic_merged; + +-- The gate must also reject expressions that are deterministic in the scope of the query but whose +-- evaluation count matters: a stateful function such as `timeSeriesExtractTag` reports +-- `isDeterministicInScopeOfQuery` but reads per-query state and must not run on rows the original +-- join order discards. It is paired with a same-shape expression that is still merged, so the +-- assertion cannot pass by rejecting everything. +SELECT 'stateful'; +SELECT + (SELECT groupArray(explain) FROM (EXPLAIN SELECT * FROM (SELECT t_merge_expr_1.a AS a, length(ifNull(timeSeriesExtractTag(t_merge_expr_1.a, 'x'), '')) % 2 AS r FROM t_merge_expr_1 JOIN t_merge_expr_2 ON t_merge_expr_1.a = t_merge_expr_2.b) s JOIN t_merge_expr_3 ON s.r = t_merge_expr_3.k SETTINGS query_plan_merge_expression_into_join = 1)) + != (SELECT groupArray(explain) FROM (EXPLAIN SELECT * FROM (SELECT t_merge_expr_1.a AS a, length(ifNull(timeSeriesExtractTag(t_merge_expr_1.a, 'x'), '')) % 2 AS r FROM t_merge_expr_1 JOIN t_merge_expr_2 ON t_merge_expr_1.a = t_merge_expr_2.b) s JOIN t_merge_expr_3 ON s.r = t_merge_expr_3.k SETTINGS query_plan_merge_expression_into_join = 0)) AS stateful_merged, + (SELECT groupArray(explain) FROM (EXPLAIN SELECT * FROM (SELECT t_merge_expr_1.a AS a, length(ifNull(toString(t_merge_expr_1.a), '')) % 2 AS r FROM t_merge_expr_1 JOIN t_merge_expr_2 ON t_merge_expr_1.a = t_merge_expr_2.b) s JOIN t_merge_expr_3 ON s.r = t_merge_expr_3.k SETTINGS query_plan_merge_expression_into_join = 1)) + != (SELECT groupArray(explain) FROM (EXPLAIN SELECT * FROM (SELECT t_merge_expr_1.a AS a, length(ifNull(toString(t_merge_expr_1.a), '')) % 2 AS r FROM t_merge_expr_1 JOIN t_merge_expr_2 ON t_merge_expr_1.a = t_merge_expr_2.b) s JOIN t_merge_expr_3 ON s.r = t_merge_expr_3.k SETTINGS query_plan_merge_expression_into_join = 0)) AS stateless_analogue_merged; + +-- `sleep` is deterministic in the scope of the query and returns the same value for every row, but how +-- much time it spends, whether it throws `TOO_SLOW` and what it accounts depend on how many times and on +-- how many rows it runs, so it must not be merged either. Paired with `materialize`, which has the same +-- shape - a function that is not constant-folded either - and is still merged. +SELECT 'sleep'; +SELECT + (SELECT groupArray(explain) FROM (EXPLAIN SELECT * FROM (SELECT t_merge_expr_1.a AS a, (t_merge_expr_1.a + sleep(0)) % 2 AS r FROM t_merge_expr_1 JOIN t_merge_expr_2 ON t_merge_expr_1.a = t_merge_expr_2.b) s JOIN t_merge_expr_3 ON s.r = t_merge_expr_3.k SETTINGS query_plan_merge_expression_into_join = 1)) + != (SELECT groupArray(explain) FROM (EXPLAIN SELECT * FROM (SELECT t_merge_expr_1.a AS a, (t_merge_expr_1.a + sleep(0)) % 2 AS r FROM t_merge_expr_1 JOIN t_merge_expr_2 ON t_merge_expr_1.a = t_merge_expr_2.b) s JOIN t_merge_expr_3 ON s.r = t_merge_expr_3.k SETTINGS query_plan_merge_expression_into_join = 0)) AS sleep_merged, + (SELECT groupArray(explain) FROM (EXPLAIN SELECT * FROM (SELECT t_merge_expr_1.a AS a, (t_merge_expr_1.a + materialize(toUInt8(0))) % 2 AS r FROM t_merge_expr_1 JOIN t_merge_expr_2 ON t_merge_expr_1.a = t_merge_expr_2.b) s JOIN t_merge_expr_3 ON s.r = t_merge_expr_3.k SETTINGS query_plan_merge_expression_into_join = 1)) + != (SELECT groupArray(explain) FROM (EXPLAIN SELECT * FROM (SELECT t_merge_expr_1.a AS a, (t_merge_expr_1.a + materialize(toUInt8(0))) % 2 AS r FROM t_merge_expr_1 JOIN t_merge_expr_2 ON t_merge_expr_1.a = t_merge_expr_2.b) s JOIN t_merge_expr_3 ON s.r = t_merge_expr_3.k SETTINGS query_plan_merge_expression_into_join = 0)) AS materialize_analogue_merged; + +DROP TABLE t_merge_expr_1; +DROP TABLE t_merge_expr_2; +DROP TABLE t_merge_expr_3; diff --git a/tests/queries/0_stateless/05081_text_index_fixed_string_constant.reference b/tests/queries/0_stateless/05081_text_index_fixed_string_constant.reference new file mode 100644 index 0000000000000..97ca403d83b52 --- /dev/null +++ b/tests/queries/0_stateless/05081_text_index_fixed_string_constant.reference @@ -0,0 +1,112 @@ +Ground truth +1 +Text Index +-- ngrams tokenizer on a String column +---- FixedString comparison +2 +2 +2 +2 +-- ngrams tokenizer on a FixedString column +---- FixedString comparison +2 +2 +2 +2 +-- splitByNonAlpha tokenizer on a String column +---- FixedString comparison +2 +2 +2 +2 +-- splitByNonAlpha tokenizer on a FixedString column +---- FixedString comparison +2 +2 +2 +2 +-- array tokenizer on a String column +---- FixedString comparison +2 +2 +2 +2 +-- array tokenizer on a FixedString column +---- FixedString comparison +2 +2 +2 +2 +-- sparseGrams tokenizer on a String column +---- FixedString comparison +2 +2 +2 +2 +-- sparseGrams tokenizer on a FixedString column +---- FixedString comparison +2 +2 +2 +2 +Bloom Filter Index +-- tokenbf_v1 on a String column +---- FixedString comparison +2 +2 +2 +2 +-- tokenbf_v1 on a FixedString column +---- FixedString comparison +2 +2 +2 +2 +-- ngrambf_v1 on a String column +---- FixedString comparison +2 +2 +2 +2 +-- ngrambf_v1 on a FixedString column +---- FixedString comparison +2 +2 +2 +2 +Functions +-- hasAny and hasAll ignore the padding +2 +2 +2 +2 +2 +2 +2 +2 +-- has keeps the padding +0 +0 +2 +2 +-- mapContainsKey and mapContainsValue keep the padding +0 +0 +0 +0 +2 +2 +-- startsWith and endsWith keep the padding +0 +0 +0 +0 +2 +2 +2 +2 +-- an unpadded constant still prunes +2 +0 +2 +0 diff --git a/tests/queries/0_stateless/05081_text_index_fixed_string_constant.sql b/tests/queries/0_stateless/05081_text_index_fixed_string_constant.sql new file mode 100644 index 0000000000000..11ef86b681cb4 --- /dev/null +++ b/tests/queries/0_stateless/05081_text_index_fixed_string_constant.sql @@ -0,0 +1,403 @@ +-- Fixes https://github.com/ClickHouse/ClickHouse/issues/117021. + +-- `String = FixedString(N)` ignores the constant's trailing zero padding, but the index terms were +-- extracted from the padded bytes, so every granule looked unmatched and matching rows disappeared. + +SELECT 'Ground truth'; +SELECT 'hello' = toFixedString('hello', 10); + +SELECT 'Text Index'; + +SELECT '-- ngrams tokenizer on a String column'; + +DROP TABLE IF EXISTS tab_string; +CREATE TABLE tab_string +( + id UInt32, + s String, + INDEX idx s TYPE text(tokenizer = ngrams(3)) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'), (4, 'foobar'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_string; + +SELECT '-- ngrams tokenizer on a FixedString column'; + +DROP TABLE IF EXISTS tab_fixed_string; +CREATE TABLE tab_fixed_string +( + id UInt32, + s FixedString(6), + INDEX idx s TYPE text(tokenizer = ngrams(3)) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_fixed_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_fixed_string; + +SELECT '-- splitByNonAlpha tokenizer on a String column'; + +DROP TABLE IF EXISTS tab_string; +CREATE TABLE tab_string +( + id UInt32, + s String, + INDEX idx s TYPE text(tokenizer = splitByNonAlpha) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_string; + +SELECT '-- splitByNonAlpha tokenizer on a FixedString column'; + +DROP TABLE IF EXISTS tab_fixed_string; +CREATE TABLE tab_fixed_string +( + id UInt32, + s FixedString(6), + INDEX idx s TYPE text(tokenizer = splitByNonAlpha) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_fixed_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_fixed_string; + +SELECT '-- array tokenizer on a String column'; + +DROP TABLE IF EXISTS tab_string; +CREATE TABLE tab_string +( + id UInt32, + s String, + INDEX idx s TYPE text(tokenizer = array) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_string; + +SELECT '-- array tokenizer on a FixedString column'; + +-- The `array` tokenizer stores the whole padded value as one term, so the constant keeps its padding +-- and only a constant as wide as the column matches. +DROP TABLE IF EXISTS tab_fixed_string; +CREATE TABLE tab_fixed_string +( + id UInt32, + s FixedString(6), + INDEX idx s TYPE text(tokenizer = array) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_fixed_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 6); +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 6) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 6)); +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 6)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_fixed_string; + +SELECT '-- sparseGrams tokenizer on a String column'; + +DROP TABLE IF EXISTS tab_string; +CREATE TABLE tab_string +( + id UInt32, + s String, + INDEX idx s TYPE text(tokenizer = sparseGrams(3)) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_string; + +SELECT '-- sparseGrams tokenizer on a FixedString column'; + +DROP TABLE IF EXISTS tab_fixed_string; +CREATE TABLE tab_fixed_string +( + id UInt32, + s FixedString(6), + INDEX idx s TYPE text(tokenizer = sparseGrams(3)) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_fixed_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_fixed_string; + +SELECT 'Bloom Filter Index'; + +SELECT '-- tokenbf_v1 on a String column'; + +DROP TABLE IF EXISTS tab_string; +CREATE TABLE tab_string +( + id UInt32, + s String, + INDEX idx s TYPE tokenbf_v1(512, 2, 0) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'), (4, 'foobar'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_string; + +SELECT '-- tokenbf_v1 on a FixedString column'; + +DROP TABLE IF EXISTS tab_fixed_string; +CREATE TABLE tab_fixed_string +( + id UInt32, + s FixedString(6), + INDEX idx s TYPE tokenbf_v1(512, 2, 0) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_fixed_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'), (4, 'foobar'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_fixed_string; + +SELECT '-- ngrambf_v1 on a String column'; + +DROP TABLE IF EXISTS tab_string; +CREATE TABLE tab_string +( + id UInt32, + s String, + INDEX idx s TYPE ngrambf_v1(3, 512, 2, 0) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'), (4, 'foobar'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_string; + +SELECT '-- ngrambf_v1 on a FixedString column'; + +DROP TABLE IF EXISTS tab_fixed_string; +CREATE TABLE tab_fixed_string +( + id UInt32, + s FixedString(6), + INDEX idx s TYPE ngrambf_v1(3, 512, 2, 0) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_fixed_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'), (4, 'foobar'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_fixed_string; + +SELECT 'Functions'; + +DROP TABLE IF EXISTS tab_string; +DROP TABLE IF EXISTS tab_string_ngrambf; +DROP TABLE IF EXISTS tab_array; +DROP TABLE IF EXISTS tab_array_ngrambf; +DROP TABLE IF EXISTS tab_map; + +CREATE TABLE tab_string +( + id UInt32, + s String, + INDEX idx s TYPE text(tokenizer = ngrams(3)) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +CREATE TABLE tab_string_ngrambf +( + id UInt32, + s String, + INDEX idx s TYPE ngrambf_v1(3, 512, 2, 0) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +CREATE TABLE tab_array +( + id UInt32, + arr Array(String), + INDEX idx arr TYPE text(tokenizer = array) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +CREATE TABLE tab_array_ngrambf +( + id UInt32, + arr Array(String), + INDEX idx arr TYPE ngrambf_v1(3, 512, 2, 0) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +CREATE TABLE tab_map +( + id UInt32, + m Map(String, String), + INDEX idx_keys mapKeys(m) TYPE text(tokenizer = array), + INDEX idx_values mapValues(m) TYPE text(tokenizer = array) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'), (4, 'foobar'); +INSERT INTO tab_string_ngrambf VALUES (1, 'hello'), (2, 'world'), (3, 'hello'), (4, 'foobar'); +INSERT INTO tab_array VALUES (1, ['hello']), (2, ['world']), (3, ['hello']); +INSERT INTO tab_array_ngrambf VALUES (1, ['hello']), (2, ['world']), (3, ['hello']); +INSERT INTO tab_map VALUES (1, map('hello', 'world')), (2, map('foo', 'bar')), (3, map('hello', 'world')); + +SELECT '-- hasAny and hasAll ignore the padding'; +SELECT count() FROM tab_array WHERE hasAny(arr, [toFixedString('hello', 10)]); +SELECT count() FROM tab_array WHERE hasAny(arr, [toFixedString('hello', 10)]) SETTINGS use_skip_indexes = 0, query_plan_direct_read_from_text_index = 0; +SELECT count() FROM tab_array WHERE hasAll(arr, [toFixedString('hello', 10)]); +SELECT count() FROM tab_array WHERE hasAll(arr, [toFixedString('hello', 10)]) SETTINGS use_skip_indexes = 0, query_plan_direct_read_from_text_index = 0; +SELECT count() FROM tab_array_ngrambf WHERE hasAny(arr, [toFixedString('hello', 10)]); +SELECT count() FROM tab_array_ngrambf WHERE hasAny(arr, [toFixedString('hello', 10)]) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_array_ngrambf WHERE hasAll(arr, [toFixedString('hello', 10)]); +SELECT count() FROM tab_array_ngrambf WHERE hasAll(arr, [toFixedString('hello', 10)]) SETTINGS use_skip_indexes = 0; + +-- The functions below compare the raw padded bytes, so their terms must keep the padding. +-- `text(tokenizer = array)` answers them by exact direct read, where a stripped term would return +-- rows the predicate rejects. +SELECT '-- has keeps the padding'; +SELECT count() FROM tab_array WHERE has(arr, toFixedString('hello', 10)); +SELECT count() FROM tab_array WHERE has(arr, toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0, query_plan_direct_read_from_text_index = 0; +SELECT count() FROM tab_array WHERE has(arr, 'hello'); +SELECT count() FROM tab_array WHERE has(arr, 'hello') SETTINGS use_skip_indexes = 0, query_plan_direct_read_from_text_index = 0; + +SELECT '-- mapContainsKey and mapContainsValue keep the padding'; +SELECT count() FROM tab_map WHERE mapContainsKey(m, toFixedString('hello', 10)); +SELECT count() FROM tab_map WHERE mapContainsKey(m, toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0, query_plan_direct_read_from_text_index = 0; +SELECT count() FROM tab_map WHERE mapContainsValue(m, toFixedString('world', 10)); +SELECT count() FROM tab_map WHERE mapContainsValue(m, toFixedString('world', 10)) SETTINGS use_skip_indexes = 0, query_plan_direct_read_from_text_index = 0; +SELECT count() FROM tab_map WHERE mapContainsKey(m, 'hello'); +SELECT count() FROM tab_map WHERE mapContainsKey(m, 'hello') SETTINGS use_skip_indexes = 0, query_plan_direct_read_from_text_index = 0; + +SELECT '-- startsWith and endsWith keep the padding'; +SELECT count() FROM tab_string WHERE startsWith(s, toFixedString('hel', 10)); +SELECT count() FROM tab_string WHERE startsWith(s, toFixedString('hel', 10)) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_string WHERE endsWith(s, toFixedString('llo', 10)); +SELECT count() FROM tab_string WHERE endsWith(s, toFixedString('llo', 10)) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_string WHERE startsWith(s, 'hel'); +SELECT count() FROM tab_string WHERE startsWith(s, 'hel') SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_string WHERE endsWith(s, 'llo'); +SELECT count() FROM tab_string WHERE endsWith(s, 'llo') SETTINGS use_skip_indexes = 0; + +SELECT '-- an unpadded constant still prunes'; +SELECT count() FROM tab_string WHERE s = 'hello'; +SELECT count() FROM tab_string WHERE s = 'nosuch'; +SELECT count() FROM tab_string_ngrambf WHERE s = 'hello'; +SELECT count() FROM tab_string_ngrambf WHERE s = 'nosuch'; + +DROP TABLE tab_string; +DROP TABLE tab_string_ngrambf; +DROP TABLE tab_array; +DROP TABLE tab_array_ngrambf; +DROP TABLE tab_map; diff --git a/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.reference b/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.reference new file mode 100644 index 0000000000000..e9efacb33c69d --- /dev/null +++ b/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.reference @@ -0,0 +1,12 @@ +KILL TRANSACTION -> KILL TRANSACTION ON *.* +KILL PART_MOVE_TO_SHARD -> SELECT ON system.part_moves_between_shards +SYSTEM STOP THREAD FUZZER -> SYSTEM THREAD FUZZER ON *.* +SYSTEM START THREAD FUZZER -> SYSTEM THREAD FUZZER ON *.* +SYSTEM RESET COVERAGE -> SYSTEM ON *.* +KILL TRANSACTION without system.transactions -> SELECT ON system.transactions +KILL PART_MOVE_TO_SHARD without move privileges -> ALTER MOVE PARTITION, MOVE PARTITION BETWEEN SHARDS ON *.* +no CLUSTER grant -> CLUSTER ON *.* +KILL TRANSACTION local -> allowed +KILL TRANSACTION on cluster -> allowed +KILL PART_MOVE_TO_SHARD local -> allowed +KILL PART_MOVE_TO_SHARD on cluster -> allowed diff --git a/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.sh b/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.sh new file mode 100755 index 0000000000000..9f246c25da6d2 --- /dev/null +++ b/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Holds CLUSTER and nothing else. +cluster_user="cluster_$CLICKHOUSE_TEST_UNIQUE_NAME" +# Holds every statement privilege but not CLUSTER. +no_cluster_user="no_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME" +# Hold the complete privilege set of one statement each. +kill_txn_user="kill_txn_$CLICKHOUSE_TEST_UNIQUE_NAME" +move_user="move_$CLICKHOUSE_TEST_UNIQUE_NAME" +# Hold all but one privilege of one statement each. +partial_txn_user="partial_txn_$CLICKHOUSE_TEST_UNIQUE_NAME" +partial_move_user="partial_move_$CLICKHOUSE_TEST_UNIQUE_NAME" + +function cleanup() +{ + $CLICKHOUSE_CLIENT -mq " + DROP USER IF EXISTS $cluster_user; + DROP USER IF EXISTS $no_cluster_user; + DROP USER IF EXISTS $kill_txn_user; + DROP USER IF EXISTS $move_user; + DROP USER IF EXISTS $partial_txn_user; + DROP USER IF EXISTS $partial_move_user; + " +} +cleanup +trap cleanup EXIT + +# CLUSTER is a global privilege, so it cannot share a GRANT with a table-scoped one. +$CLICKHOUSE_CLIENT -mq " + CREATE USER $cluster_user, $no_cluster_user, $kill_txn_user, $move_user, + $partial_txn_user, $partial_move_user IDENTIFIED WITH no_password; + + GRANT CLUSTER ON *.* TO $cluster_user; + + GRANT KILL TRANSACTION, SYSTEM THREAD FUZZER, SYSTEM ON *.* TO $no_cluster_user; + + GRANT CLUSTER, KILL TRANSACTION ON *.* TO $kill_txn_user; + GRANT SELECT ON system.transactions TO $kill_txn_user; + + GRANT CLUSTER, ALTER MOVE PARTITION, MOVE PARTITION BETWEEN SHARDS ON *.* TO $move_user; + GRANT SELECT ON system.part_moves_between_shards TO $move_user; + + GRANT CLUSTER, KILL TRANSACTION ON *.* TO $partial_txn_user; + + GRANT CLUSTER ON *.* TO $partial_move_user; + GRANT SELECT ON system.part_moves_between_shards TO $partial_move_user; +" + +cluster="test_shard_localhost" +# Match no live transaction and no live part move, so every allowed arm below is a no-op. The +# transaction predicate deliberately does not compare `tid`: its tuple shape is not the same in +# every build, so a literal tuple fails type analysis before the access check this test is about. +txn_predicate="tid_hash = 0" +task_uuid="'00000000-0000-0000-0000-000000000000'" + +# Report the privilege the server asked for and let the reference hold the expected mapping. The +# privilege name is what discriminates: an ACCESS_DENIED-only assertion would also pass when the +# CLUSTER check, which runs before the required-access check, is what fired. +required_privilege() { + $CLICKHOUSE_CLIENT --distributed_ddl_output_mode none --user "$1" --query "$2" 2>&1 | + sed -n "/necessary to have the grant/{s/.*grant \(.*\)\. (ACCESS_DENIED).*/\1/p;q;}" +} + +# Each statement must name over ON CLUSTER the same privilege its local spelling names. +while IFS= read -r statement; do + echo "${statement%% ON CLUSTER*} -> $(required_privilege "$cluster_user" "$statement")" +done < $(required_privilege "$partial_txn_user" "KILL TRANSACTION ON CLUSTER $cluster WHERE $txn_predicate")" +echo "KILL PART_MOVE_TO_SHARD without move privileges -> $(required_privilege "$partial_move_user" "KILL PART_MOVE_TO_SHARD ON CLUSTER $cluster WHERE task_uuid = $task_uuid")" + +# In-range control: holding the statement privileges without CLUSTER is refused by the earlier check, +# so a mapping that refuses everything would not produce the five lines above. +echo "no CLUSTER grant -> $(required_privilege "$no_cluster_user" "KILL TRANSACTION ON CLUSTER $cluster WHERE $txn_predicate")" + +# The two statements whose privileges can be granted in full are allowed in both spellings, which +# proves the new elements are a gate rather than an unconditional refusal. The local half is asserted +# too: a user who is cluster-allowed while locally refused is the bypass this test exists to catch. +# The thread fuzzer and coverage statements get no allowed arm, because executing them would change +# server-global state that concurrent tests read and neither takes an argument that lets the host +# reject them harmlessly. +allowed() { + local out + out=$($CLICKHOUSE_CLIENT --distributed_ddl_output_mode none --user "$1" --query "$2" 2>&1) + if [ -z "$out" ]; then + echo "$3 -> allowed" + else + echo "$3 -> FAIL: $out" + fi +} + +allowed "$kill_txn_user" "KILL TRANSACTION WHERE $txn_predicate" "KILL TRANSACTION local" +allowed "$kill_txn_user" "KILL TRANSACTION ON CLUSTER $cluster WHERE $txn_predicate" "KILL TRANSACTION on cluster" +allowed "$move_user" "KILL PART_MOVE_TO_SHARD WHERE task_uuid = $task_uuid" "KILL PART_MOVE_TO_SHARD local" +allowed "$move_user" "KILL PART_MOVE_TO_SHARD ON CLUSTER $cluster WHERE task_uuid = $task_uuid" "KILL PART_MOVE_TO_SHARD on cluster" diff --git a/tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.reference b/tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.reference new file mode 100644 index 0000000000000..23a4dd0460b91 --- /dev/null +++ b/tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.reference @@ -0,0 +1,2 @@ +Map(String, UInt64) {'k':1} Map(String, Tuple(a UInt64)) {'k':(2)} +1 2 diff --git a/tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.sql b/tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.sql new file mode 100644 index 0000000000000..b6da283b5a30f --- /dev/null +++ b/tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.sql @@ -0,0 +1,17 @@ +DROP TABLE IF EXISTS json_shared_bucketed_map; +CREATE TABLE json_shared_bucketed_map (j JSON(max_dynamic_paths = 0)) +ENGINE = MergeTree ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 0, + object_shared_data_serialization_version_for_zero_level_parts = 'advanced', + map_serialization_version_for_zero_level_parts = 'with_buckets', + propagate_types_serialization_versions_to_nested_types = 1; + +-- Preserve runtime `Map` types in shared data instead of parsing them as `JSON` objects. +INSERT INTO json_shared_bucketed_map +SELECT * FROM format(RowBinary, 'j JSON(max_dynamic_paths = 0)', concat( + unhex('02016d'), formatRowNoNewline('RowBinary', CAST(map('k', toUInt64(1)) AS Dynamic)), + unhex('0174'), formatRowNoNewline('RowBinary', CAST(map('k', CAST(tuple(toUInt64(2)) AS Tuple(a UInt64))) AS Dynamic)))) +SETTINGS input_format_binary_read_json_as_string = 0; +SELECT dynamicType(j.m), j.m, dynamicType(j.t), j.t FROM json_shared_bucketed_map; +SELECT j.m.:`Map(String, UInt64)`['k'], j.t.:`Map(String, Tuple(a UInt64))`['k'].a FROM json_shared_bucketed_map; +DROP TABLE json_shared_bucketed_map; diff --git a/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.reference b/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.reference new file mode 100644 index 0000000000000..ca36e0892c1a6 --- /dev/null +++ b/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.reference @@ -0,0 +1,9 @@ +CREATE DICTIONARY default.d_05141\n(\n `id` UInt64,\n `v` String\n)\nPRIMARY KEY id\nSOURCE(HTTP(URL \'http://localhost:11111/x.tsv\' FORMAT \'TabSeparated\' CREDENTIALS (USER \'user\' PASSWORD \'[HIDDEN]\') HEADERS (\'[HIDDEN]\')))\nLIFETIME(MIN 0 MAX 0)\nLAYOUT(FLAT()) +0 0 1 +d_05141_flat HEADERS \'[HIDDEN]\')) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_foo HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_key HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_nested HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_nobr HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_typo HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +2 14 0 diff --git a/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.sql b/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.sql new file mode 100644 index 0000000000000..0e1468fdee93c --- /dev/null +++ b/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.sql @@ -0,0 +1,68 @@ +-- The values of custom HTTP headers of an `HTTP` dictionary source often carry credentials, +-- so they must be hidden in `SHOW CREATE DICTIONARY`, `system.tables` and `system.query_log`, +-- the same way as the password. They are hidden as a whole, header names included. + +SET format_display_secrets_in_show_and_select = 0; + +DROP DICTIONARY IF EXISTS d_05141; +CREATE DICTIONARY d_05141 (id UInt64, v String) +PRIMARY KEY id +SOURCE(HTTP( + url 'http://localhost:11111/x.tsv' + format 'TabSeparated' + credentials(user 'user' password 'SEKRIT_PW') + headers( + header(name 'API-KEY' value 'SEKRIT_TOKEN_1') + header(name 'X-Other' value 'SEKRIT_TOKEN_2') + ) +)) +LIFETIME(0) LAYOUT(FLAT()); + +SHOW CREATE DICTIONARY d_05141; + +SELECT create_table_query LIKE concat('%', 'SEKRIT', '%'), create_table_query LIKE '%API-KEY%', create_table_query LIKE '%HEADERS (\'[HIDDEN]\')%' +FROM system.tables WHERE database = currentDatabase() AND name = 'd_05141'; + +DROP DICTIONARY d_05141; + +-- The query is logged before the dictionary source validates its structure, so malformed header +-- definitions must not leak either. +CREATE DICTIONARY d_05141_typo (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(name 'API-KEY' vaule 'SEKRIT_TYPO')))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_key (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(secret 'SEKRIT_KEY')))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_nested (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(name(foo 'SEKRIT_NESTED'))))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_func (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(name concat('X-', 'SEKRIT_FUNC') value 'SEKRIT_FUNC_VALUE')))) +LIFETIME(0) LAYOUT(FLAT()); -- { serverError INCORRECT_DICTIONARY_DEFINITION } +CREATE DICTIONARY d_05141_array (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(name ['SEKRIT_ARRAY'])))) +LIFETIME(0) LAYOUT(FLAT()); -- { serverError BAD_ARGUMENTS } +CREATE DICTIONARY d_05141_nobr (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header 'SEKRIT_NOBR'))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_foo (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(foo 'SEKRIT_FOO'))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_flat (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers 'SEKRIT_HEADERS')) +LIFETIME(0) LAYOUT(FLAT()); + +SELECT name, extract(create_table_query, 'HEADERS.*$') +FROM system.tables WHERE database = currentDatabase() AND name LIKE 'd\\_05141\\_%' ORDER BY name; + +SYSTEM FLUSH LOGS query_log; +SELECT countIf(query LIKE '%d\\_05141 %'), countIf(query LIKE '%d\\_05141\\_%'), countIf(query LIKE concat('%', 'SEKRIT', '%')) +FROM system.query_log +WHERE current_database = currentDatabase() AND query_kind = 'Create' AND event_date >= yesterday(); + +DROP DICTIONARY d_05141_typo; +DROP DICTIONARY d_05141_key; +DROP DICTIONARY d_05141_nested; +DROP DICTIONARY d_05141_nobr; +DROP DICTIONARY d_05141_foo; +DROP DICTIONARY d_05141_flat; diff --git a/tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.reference b/tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.reference new file mode 100644 index 0000000000000..f7c7e2e153250 --- /dev/null +++ b/tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.reference @@ -0,0 +1,3 @@ +HEADERS (\'[HIDDEN]\'))) +CREDENTIALS (USER \'user\' PASSWORD \'[HIDDEN]\') HEADERS (\'[HIDDEN]\'))) +1 0 diff --git a/tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.sh b/tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.sh new file mode 100755 index 0000000000000..11eae7fc24b49 --- /dev/null +++ b/tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + + +# The HTTP headers of an `HTTP` dictionary source must also be hidden when the query is submitted +# as a JSON AST (`dialect = 'clickhouse_json'`). + +CLICKHOUSE_CLIENT_JSON="${CLICKHOUSE_CLIENT} --enable_json_ast_dialect 1 --dialect clickhouse_json" + +function to_json() +{ + ${CLICKHOUSE_CLIENT} --query "SELECT parseQueryToJSON(\$\$$1\$\$) FORMAT TSVRaw" +} + +# A definition submitted as a JSON AST is masked like the SQL one. +JSON=$(to_json "CREATE DICTIONARY ${CLICKHOUSE_DATABASE}.d_05142 (id UInt64, v String) PRIMARY KEY id + SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(name 'API-KEY' value 'SEKRIT_JSON')))) + LIFETIME(0) LAYOUT(FLAT())") +${CLICKHOUSE_CLIENT_JSON} --query "$JSON" +${CLICKHOUSE_CLIENT} --query "SELECT extract(create_table_query, 'HEADERS.*\\)\\)\\)') FROM system.tables WHERE database = currentDatabase() AND name = 'd_05142' SETTINGS format_display_secrets_in_show_and_select = 0" + +# The SQL parser lower-cases the keys, but a JSON AST can spell them in any case; the secret keys must +# be recognized anyway. +JSON=$(to_json "CREATE DICTIONARY ${CLICKHOUSE_DATABASE}.d_05142_case (id UInt64, v String) PRIMARY KEY id + SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' credentials(user 'user' password 'SEKRIT_JSON_CASE_PW') + headers(header(name 'API-KEY' value 'SEKRIT_JSON_CASE')))) + LIFETIME(0) LAYOUT(FLAT())") +for key in headers header value password +do + JSON=${JSON//\"first\":\"$key\"/\"first\":\"${key^^}\"} +done +${CLICKHOUSE_CLIENT_JSON} --query "$JSON" +${CLICKHOUSE_CLIENT} --query "SELECT extract(create_table_query, 'CREDENTIALS.*\\)\\)\\)') FROM system.tables WHERE database = currentDatabase() AND name = 'd_05142_case' SETTINGS format_display_secrets_in_show_and_select = 0" + +# The JSON queries are logged without the headers and the password. +${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS query_log" +${CLICKHOUSE_CLIENT} --query " + SELECT count() > 0, countIf(query LIKE concat('%', 'SEKRIT', '%')) + FROM system.query_log + WHERE current_database = currentDatabase() AND query_kind = 'Create' AND query LIKE '%d_05142%' AND event_date >= yesterday()" + +${CLICKHOUSE_CLIENT} --query "DROP DICTIONARY d_05142" +${CLICKHOUSE_CLIENT} --query "DROP DICTIONARY d_05142_case" diff --git a/tests/queries/0_stateless/05148_database_backup_modify_comment.reference b/tests/queries/0_stateless/05148_database_backup_modify_comment.reference new file mode 100644 index 0000000000000..b3863579a9540 --- /dev/null +++ b/tests/queries/0_stateless/05148_database_backup_modify_comment.reference @@ -0,0 +1,4 @@ +10 +10 +a comment +refused diff --git a/tests/queries/0_stateless/05148_database_backup_modify_comment.sh b/tests/queries/0_stateless/05148_database_backup_modify_comment.sh new file mode 100755 index 0000000000000..4af56ac37b84b --- /dev/null +++ b/tests/queries/0_stateless/05148_database_backup_modify_comment.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Tags: no-encrypted-storage + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +BACKUP_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_backup +ATTACHED_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_attached + +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE IF EXISTS $BACKUP_DATABASE_NAME; +CREATE DATABASE $BACKUP_DATABASE_NAME; + +CREATE TABLE $BACKUP_DATABASE_NAME.test_table (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO $BACKUP_DATABASE_NAME.test_table SELECT number FROM numbers(10); + +BACKUP DATABASE $BACKUP_DATABASE_NAME TO Disk('backups', '$BACKUP_DATABASE_NAME') FORMAT Null; + +DROP DATABASE IF EXISTS $ATTACHED_DATABASE_NAME; +CREATE DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', Disk('backups', '$BACKUP_DATABASE_NAME')); + +SELECT count() FROM $ATTACHED_DATABASE_NAME.test_table; +""" + +# A comment change rewrites the metadata file of the database, so the locator has to survive the +# round trip: loading it back is what the server does at every start. +$CLICKHOUSE_CLIENT -q """ +ALTER DATABASE $ATTACHED_DATABASE_NAME MODIFY COMMENT 'a comment'; +DETACH DATABASE $ATTACHED_DATABASE_NAME; +ATTACH DATABASE $ATTACHED_DATABASE_NAME; + +SELECT count() FROM $ATTACHED_DATABASE_NAME.test_table; +SELECT comment FROM system.databases WHERE name = '$ATTACHED_DATABASE_NAME'; +""" + +# A locator serialized as a string literal is accepted only while loading metadata an older server +# rewrote: in a statement a user writes it must be the function it is, because that is the form the +# secret masker redacts. +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE $ATTACHED_DATABASE_NAME; +CREATE DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', 'Disk(\\'backups\\', \\'$BACKUP_DATABASE_NAME\\')'); +""" 2>&1 | grep -q -F 'Expected function' && echo 'refused' + +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE IF EXISTS $ATTACHED_DATABASE_NAME; +DROP DATABASE $BACKUP_DATABASE_NAME; +""" diff --git a/tests/queries/0_stateless/05153_alias_truncate_lock.reference b/tests/queries/0_stateless/05153_alias_truncate_lock.reference new file mode 100644 index 0000000000000..bb31165c34ab2 --- /dev/null +++ b/tests/queries/0_stateless/05153_alias_truncate_lock.reference @@ -0,0 +1,25 @@ +-- fixture +al_a Alias +al_b Alias +al_c Alias +al_d_inner Alias +al_d_outer Alias +al_f Alias +al_g Alias +al_h Alias +mem_c Memory +mem_g Memory +mt_a MergeTree +mt_d MergeTree +mt_e MergeTree +mt_h MergeTree +rmt_b ReplicatedMergeTree +-- arms +A_alias_to_mergetree blocked=0 rows_after=0 +B_alias_to_replicated_mergetree blocked=0 rows_after=0 +C_alias_to_memory blocked=1 rows_after=300 +D_alias_chain_to_mergetree blocked=0 rows_after=0 +E_direct_mergetree blocked=0 rows_after=0 +G_direct_reader_of_memory_target blocked=1 rows_after=300 +H_direct_reader_of_mergetree_target blocked=0 rows_after=0 +F_dangling_alias err=UNKNOWN_TABLE diff --git a/tests/queries/0_stateless/05153_alias_truncate_lock.sh b/tests/queries/0_stateless/05153_alias_truncate_lock.sh new file mode 100755 index 0000000000000..c0a21d0f4db60 --- /dev/null +++ b/tests/queries/0_stateless/05153_alias_truncate_lock.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-replicated-database, no-shared-merge-tree, memory-engine +# no-replicated-database: there TRUNCATE is enqueued as a DDL entry instead of taking the lock these +# arms measure, so every arm would report blocked=0 regardless of the fix. +# no-shared-merge-tree, memory-engine: the fixture assertion pins each engine name, and the +# ReplicatedMergeTree -> SharedMergeTree and Memory -> MergeTree test-runner +# replacements rewrite two of them. +# +# TRUNCATE through an Alias must choose its lock on, and take it on, the storage that executes the +# truncate. Each arm starts a reader, then issues TRUNCATE with a 3s lock_acquire_timeout and reports +# whether that reader blocked it. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q " + CREATE TABLE mt_a (id UInt64) ENGINE = MergeTree ORDER BY id; + CREATE TABLE rmt_b (id UInt64) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/rmt_b', 'r1') ORDER BY id; + CREATE TABLE mem_c (id UInt64) ENGINE = Memory; + CREATE TABLE mt_d (id UInt64) ENGINE = MergeTree ORDER BY id; + CREATE TABLE mt_e (id UInt64) ENGINE = MergeTree ORDER BY id; + CREATE TABLE mem_g (id UInt64) ENGINE = Memory; + CREATE TABLE mt_h (id UInt64) ENGINE = MergeTree ORDER BY id; + + INSERT INTO mt_a SELECT number FROM numbers(300); + INSERT INTO rmt_b SELECT number FROM numbers(300); + INSERT INTO mem_c SELECT number FROM numbers(300); + INSERT INTO mt_d SELECT number FROM numbers(300); + INSERT INTO mt_e SELECT number FROM numbers(300); + INSERT INTO mem_g SELECT number FROM numbers(300); + INSERT INTO mt_h SELECT number FROM numbers(300); +" + +# al_d_outer is created while al_d_inner is still absent: an Alias whose target already resolves to +# an Alias is rejected at creation, an Alias to a missing name is not. +$CLICKHOUSE_CLIENT -q " + CREATE TABLE al_a ENGINE = Alias(currentDatabase(), 'mt_a'); + CREATE TABLE al_b ENGINE = Alias(currentDatabase(), 'rmt_b'); + CREATE TABLE al_c ENGINE = Alias(currentDatabase(), 'mem_c'); + CREATE TABLE al_d_outer ENGINE = Alias(currentDatabase(), 'al_d_inner'); + CREATE TABLE al_d_inner ENGINE = Alias(currentDatabase(), 'mt_d'); + CREATE TABLE al_f ENGINE = Alias(currentDatabase(), 'no_such_table'); + CREATE TABLE al_g ENGINE = Alias(currentDatabase(), 'mem_g'); + CREATE TABLE al_h ENGINE = Alias(currentDatabase(), 'mt_h'); +" + +# Without this an arm could report the expected value while probing a plain table instead of an alias. +echo '-- fixture' +$CLICKHOUSE_CLIENT -q "SELECT name, engine FROM system.tables WHERE database = currentDatabase() ORDER BY name" + +# arm
... +# Each argument is one file: sorted, comma-separated ISO dates, or a single NULL. +# `no_min` / `no_max` omit only that bound; `none` emits only `numRecords`. +make_table() { + local table=$1 type=$2 stats_mode=$3 + shift 3 + mkdir -p "${ROOT}/${table}/_delta_log" + TABLES+=("${table}") + TABLE_TYPE[$table]="${type}" + TABLE_STATS[$table]="${stats_mode}" + TABLE_DATES[$table]="$*" + local date value + for date in "$@"; do + value="'${date//,/\',\'}'" + [[ "${date}" == NULL ]] && value=NULL + FIXTURE_QUERIES+=" + INSERT INTO FUNCTION file('${ROOT}/${table}/${date}.parquet', Parquet, 'd ${type}') + SELECT CAST(arrayJoin([${value}]), '${type}');" + done +} + +# The second phase of `make_table`: protocol, metaData, then one add action per file. The log is +# hand-written because ClickHouse's Delta writer emits no per-column stats, and it can only be +# written after the fixture batch ran - it embeds the data files' byte sizes. `schemaString` and +# `stats` are JSON documents embedded as JSON strings, hence the escaped quotes. +write_log() { + local table=$1 + local type="${TABLE_TYPE[$table]}" stats_mode="${TABLE_STATS[$table]}" + local nullable=false + [[ "${type}" == Nullable* ]] && nullable=true + local log="${ROOT}/${table}/_delta_log/00000000000000000000.json" + cat > "${log}" <> "${log}" < [settings]: rows with the engine predicate on must match the rows with +# it off, the off run must scan every file, and the on rows and scan count go to the reference. +check() { + ITEMS+=("check ${#CHECK_TABLES[@]}") + CHECK_TABLES+=("$1") + CHECK_WHERES+=("$2") + CHECK_SETTINGS+=("${3:-}") +} + +# expect_error
[settings]: the query must fail with +# the given error identity; scan counts are not pinned after cancellation. +expect_error() { + ITEMS+=("error ${#ERR_HEADERS[@]}") + ERR_HEADERS+=("$1") + ERR_TABLES+=("$2") + ERR_WHERES+=("$3") + ERR_ENABLED+=("$4") + ERR_CODES+=("$5") + ERR_SETTINGS+=("${6:-}") +} + +# The data query of one recorded case, with its per-case settings attached. +data_query() { + printf "SELECT arraySort(groupArray(ifNull(toString(d), 'NULL'))) FROM deltaLakeLocal('%s') WHERE %s%s;" \ + "${ROOT}/$1" "$2" "${3:+ SETTINGS $3}" +} + +# Disable every applicable Parquet pruning path, so only the engine predicate can skip files. +run_batch() { + local enabled=$1 + shift + ${CLICKHOUSE_LOCAL} --date_time_overflow_behavior=ignore --convert_query_to_cnf=0 \ + --short_circuit_function_evaluation=disable \ + --input_format_parquet_filter_push_down=0 --input_format_parquet_page_filter_push_down=0 \ + --input_format_parquet_bloom_filter_push_down=0 --input_format_parquet_dictionary_filter_push_down=0 \ + --delta_lake_enable_engine_predicate="${enabled}" --delta_lake_throw_on_engine_predicate_error=1 "$@" +} + +# Runs every recorded check in one launch. Each data query is followed by a read of the +# cumulative `DeltaLakeScannedFiles` counter; consecutive differences give per-query scan counts. +run_checks() { + local enabled=$1 + local -n rows_out=$2 deltas_out=$3 + local i batch="" out + for i in "${!CHECK_TABLES[@]}"; do + batch+="$(data_query "${CHECK_TABLES[$i]}" "${CHECK_WHERES[$i]}" "${CHECK_SETTINGS[$i]}") + SELECT sumIf(value, event = 'DeltaLakeScannedFiles') FROM system.events;" + done + out=$(run_batch "${enabled}" --query "${batch}") + local -a lines + mapfile -t lines <<< "${out}" + if [[ "${#lines[@]}" != "$(( 2 * ${#CHECK_TABLES[@]} ))" ]]; then + printf 'Expected %s lines from the check batch (engine predicate %s), got %s:\n%s\n' \ + "$(( 2 * ${#CHECK_TABLES[@]} ))" "${enabled}" "${#lines[@]}" "${out}" >&2 + return 1 + fi + local previous=0 cumulative + for i in "${!CHECK_TABLES[@]}"; do + rows_out+=("${lines[2 * i]}") + cumulative="${lines[2 * i + 1]}" + deltas_out+=("$(( cumulative - previous ))") + previous="${cumulative}" + done +} + +# Runs each error case with a `-- { serverError }` hint that consumes the expected exception (no stderr), while its following `system.errors` counter must grow to prove the query failed. +run_errors() { + local enabled=$1 + local i batch="" out + local -a case_ids=() + for i in "${!ERR_HEADERS[@]}"; do + [[ "${ERR_ENABLED[$i]}" == "${enabled}" ]] || continue + case_ids+=("$i") + batch+="$(data_query "${ERR_TABLES[$i]}" "${ERR_WHERES[$i]}" "${ERR_SETTINGS[$i]}") -- { serverError ${ERR_CODES[$i]} } + SELECT sumIf(value, name = '${ERR_CODES[$i]}') FROM system.errors;" + done + out=$(run_batch "${enabled}" --query "${batch}") + local -a lines + mapfile -t lines <<< "${out}" + if [[ "${#lines[@]}" != "${#case_ids[@]}" ]]; then + printf 'Expected %s failing queries (engine predicate %s), got output:\n%s\n' \ + "${#case_ids[@]}" "${enabled}" "${out}" >&2 + return 1 + fi + local j code + declare -A previous=() + for j in "${!case_ids[@]}"; do + i="${case_ids[$j]}" + code="${ERR_CODES[$i]}" + if [[ ! "${lines[$j]}" =~ ^[0-9]+$ ]] || [[ "${lines[$j]}" -le "${previous[$code]:-0}" ]]; then + printf 'Expected %s (engine predicate %s) from: %s\n' "${code}" "${enabled}" "${ERR_WHERES[$i]}" >&2 + return 1 + fi + previous[$code]="${lines[$j]}" + done +} + +# Prints the results in declaration order; the reference interleaves checks and expected errors. +emit_results() { + local item kind idx + for item in "${ITEMS[@]}"; do + read -r kind idx <<< "${item}" + if [[ "${kind}" == error ]]; then + printf '%s\n' "${ERR_HEADERS[$idx]}" + printf '1\n' + continue + fi + local table="${CHECK_TABLES[$idx]}" where="${CHECK_WHERES[$idx]}" settings="${CHECK_SETTINGS[$idx]}" + local files=("${ROOT}/${table}/"*.parquet) + printf '%s: %s%s\n' "${table}" "${where}" "${settings:+ SETTINGS ${settings}}" + # Compare rows, not scan counts. The reference independently checks rows and selected files. + diff -u <(printf '%s\n' "${OFF_ROWS[$idx]}") <(printf '%s\n' "${ON_ROWS[$idx]}") + if [[ "${OFF_DELTAS[$idx]}" != "${#files[@]}" ]]; then + printf 'Unexpected unpruned file count %s for: %s\n' "${OFF_DELTAS[$idx]}" "${where}" >&2 + return 1 + fi + printf '%s\n' "${ON_ROWS[$idx]}" + printf '%s\n' "${ON_DELTAS[$idx]}" + done +} + +run_all() { + ${CLICKHOUSE_LOCAL} --query "${FIXTURE_QUERIES}" + local table + for table in "${TABLES[@]}"; do + write_log "${table}" + done + run_checks 0 OFF_ROWS OFF_DELTAS + run_checks 1 ON_ROWS ON_DELTAS + run_errors 0 + run_errors 1 + emit_results +} + +# `Date` wraps modulo 65536 days: the alias matches equality but stays scanned even for `!=`. +make_table t Date32 full 2026-01-01 2026-01-02 2205-06-08 +check t "d = '2026-01-01'" +check t "CAST(d AS Date) = '2026-01-01'" +check t "d::Date = '2026-01-01'" +check t "toDate(d) = '2026-01-01'" +check t "d::Date32 = '2026-01-01'" +check t "toDate32(d) = toDate32('2026-01-01')" +check t "d::Date != '2026-01-01'" +check t "toDate('2026-01-01') = toDate(d)" +check t "toDate('2026-01-01') != toDate(d)" +check t "toDate(d) = '2026-01-03'" +check t "toDate32(d) = '2026-01-03'" + +make_table lower Date32 full 1969-12-31 1970-01-01 1970-01-02 +check lower "toDate(d) = '1970-01-01'" "date_time_overflow_behavior='saturate'" +check lower "toDate(d) != '1970-01-01'" "date_time_overflow_behavior='saturate'" +make_table upper Date32 full 2149-06-05 2149-06-06 2149-06-07 +check upper "CAST(d AS Date) = '2149-06-06'" "date_time_overflow_behavior='saturate'" +check upper "CAST(d AS Date) != '2149-06-06'" "date_time_overflow_behavior='saturate'" + +make_table negation Date32 full 2026-01-01 2026-01-02 2205-06-09 +check negation "NOT (toDate(d) = '2026-01-01' AND d > toDate32('1970-01-01'))" + +# Aliases at -65536 and +131072 days also lie outside the DateLUT's 1900..2299 fast range. +make_table aliases Date32 full 1846-07-28 2026-01-02 2384-11-12 +check aliases "toDate(d) = '2026-01-01'" +check aliases "toDate(d) != '2026-01-01'" + +make_table valid Date32 full 1970-01-02 2026-01-01 +check valid "toDate(d) != toDateTime('2026-01-01 00:00:01', 'UTC')" +check valid "CAST(d AS Date) = '2026-01-01'" "date_time_overflow_behavior='throw'" +check valid "toDate(d) != '2026-01-01'" "date_time_overflow_behavior='throw'" + +make_table nullable 'Nullable(Date32)' full NULL 2026-01-01 2026-01-02 2205-06-08 +check nullable "toDate(d) = '2026-01-01'" +check nullable "CAST(d AS Nullable(Date)) = toNullable(toDate('2026-01-01'))" +check nullable "CAST(d AS Nullable(Date)) != '2026-01-01'" + +make_table no_stats Date32 none 2026-01-01 2026-01-02 +check no_stats "toDate(d) = '2026-01-01'" + +# A non-nullable cast must still reject the stats-bearing NULL file. +for enabled in 0 1; do + expect_error "nullable: cast rejects NULL (engine predicate ${enabled})" \ + nullable "CAST(d AS Date) = '2026-01-01'" "${enabled}" \ + CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN "cast_keep_nullable=0" +done + +# `groupArray` must not hide the all-null file in the unfiltered fixture check. +check nullable "1" + +# Two files cross the Date boundaries; two safe singletons must still be prunable. +make_table mixed Date32 full 1969-12-31,1970-01-01 2149-06-06,2149-06-07 2026-01-01 2026-01-02 +for mode in ignore saturate; do + for boundary in 1970-01-01 2149-06-06; do + check mixed "toDate(d) = '${boundary}'" "date_time_overflow_behavior='${mode}'" + check mixed "toDate(d) != '${boundary}'" "date_time_overflow_behavior='${mode}'" + done +done + +# Both endpoints wrap to day zero, but the file also contains a matching interior value. +make_table spanning Date32 full 1970-01-01,1970-01-02,2149-06-07 2026-01-01 +check spanning "toDate(d) = '1970-01-02'" +check spanning "NOT (toDate(d) != '1970-01-02')" + +# One missing bound cannot prove that a Date cast is safe. +for mode in no_min no_max; do + make_table "$mode" Date32 "$mode" 2026-01-01 2026-01-02 + check "$mode" "toDate(d) = '2026-01-01'" + check "$mode" "toDate(d) != '2026-01-01'" +done +# Identity conversions can still prune from the remaining bound. +check no_min "toDate32(d) = '2026-01-02'" +check no_max "toDate32(d) = '2026-01-01'" + +# Actual overflows at both bounds must survive equality, inequality, and nested negation. +for table in lower upper; do + for where in "CAST(d AS Date) = '2026-01-01'" "toDate(d) != '2026-01-01'" \ + "NOT (toDate(d) = '2026-01-01' AND d > toDate32('1900-01-01'))"; do + for enabled in 0 1; do + expect_error "${table}: ${where} (engine predicate ${enabled}, throw)" \ + "${table}" "${where}" "${enabled}" VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE \ + "date_time_overflow_behavior='throw'" + done + done +done + +# The optional timezone argument of `toDate` / `toDate32` is inert for a `Date32` source. +check t "toDate(d, 'UTC') = '2026-01-01'" +check t "toDate32(d, 'UTC') = '2026-01-01'" +check t "toDate32(d, 'UTC') != '2026-01-01'" + +# Ordering: identity conversions map exactly; `Date`-narrowing ones are fenced so out-of-domain files stay scanned in either polarity. +check t "toDate32(d) < '2026-01-02'" +check t "toDate32(d) >= '2026-01-02'" +check t "'2026-01-02' > toDate32(d)" +check t "d::Date32 <= '2026-01-01'" +check t "toDate(d) < '2026-01-02'" +check t "toDate(d) >= '2026-01-02'" +check t "'2026-01-02' > toDate(d)" +check t "NOT (toDate(d) < '2026-01-02')" +check t "NOT (toDate(d) > '2026-01-01')" +check aliases "toDate(d) <= '2026-01-01'" +check lower "toDate(d) < '1970-01-02'" "date_time_overflow_behavior='saturate'" +check upper "toDate(d) > '2149-06-05'" "date_time_overflow_behavior='saturate'" +# Identity ordering can still prune from a single remaining bound. +check no_min "toDate32(d) > '2026-01-01'" +check no_max "toDate32(d) < '2026-01-02'" +# An actual overflow must survive an ordering comparison whose in-domain answer is false. +for enabled in 0 1; do + expect_error "upper: toDate(d) < '2026-01-01' (engine predicate ${enabled}, throw)" \ + upper "toDate(d) < '2026-01-01'" "${enabled}" VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE \ + "date_time_overflow_behavior='throw'" +done + +# Mixed `Date` / `Date32` literals convert exactly; one beyond the `Date` domain stays untranslated and scans everything. +check t "toDate32(d) = toDate('2026-01-01')" +check t "toDate(d) = toDate32('2026-01-01')" +check t "toDate(d) != toDate32('2026-01-01')" +check t "toDate32(d) < toDate('2026-01-02')" +check t "toDate(d) = toDate32('2205-06-08')" +check t "toDate(d) != toDate32('2205-06-08')" +check t "toDate(d) < toDate32('2205-06-08')" + +# The snapshot scan must build the engine predicate with the query's overflow settings. A nullable +# source hides the out-of-range RHS parse from zero-row header analysis, so if pruning coerced the +# literal with the default `ignore` it would clamp `'1969-12-31'` to a real day and drop both files, +# suppressing the exception. Both rows stay inside the `Date` domain, so the only overflow is the RHS. +make_table overflow_rhs 'Nullable(Date32)' full 1970-01-02 2026-01-01 +for enabled in 0 1; do + expect_error "overflow_rhs: toDate(d) = '1969-12-31' (engine predicate ${enabled}, throw)" \ + overflow_rhs "toDate(d) = '1969-12-31'" "${enabled}" VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE \ + "date_time_overflow_behavior='throw'" +done + +run_all diff --git a/tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.reference b/tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.reference new file mode 100644 index 0000000000000..c0412f2f07f69 --- /dev/null +++ b/tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.reference @@ -0,0 +1,9 @@ +legacy {'k':1} {'k':(2)} [{'k':3},{'k':4}] {"m":{"k":5}} +types Map(String, UInt64) Map(String, Tuple(a UInt64)) Array(Map(String, UInt64)) JSON(m Map(String, UInt64)) +subcolumns 1 2 +rewritten {'k':1} {'k':(2)} [{'k':3},{'k':4}] {"m":{"k":5}} +rewritten {'k':1} {'k':(2)} [{'k':3},{'k':4}] {"m":{"k":5}} +rewritten_types Map(String, UInt64) Map(String, Tuple(a UInt64)) Array(Map(String, UInt64)) JSON(m Map(String, UInt64)) +rewritten_types Map(String, UInt64) Map(String, Tuple(a UInt64)) Array(Map(String, UInt64)) JSON(m Map(String, UInt64)) +rewritten_subcolumns 1 2 +rewritten_subcolumns 1 2 diff --git a/tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.sh b/tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.sh new file mode 100755 index 0000000000000..250255f61c6c9 --- /dev/null +++ b/tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +set -euo pipefail + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +workdir=$(mktemp -d "${CLICKHOUSE_TMP}/legacy_bucketed_map.XXXXXX") +trap 'rm -rf "$workdir"' EXIT + +part_dir=$($CLICKHOUSE_LOCAL --path "$workdir" --multiquery --query " +CREATE DATABASE legacy ENGINE = Atomic; +CREATE TABLE legacy.maps (j JSON(max_dynamic_paths = 0)) ENGINE = MergeTree ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 0, object_shared_data_buckets_for_wide_part = 1, + object_shared_data_serialization_version = 'advanced', object_shared_data_serialization_version_for_zero_level_parts = 'advanced', + map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets', + propagate_types_serialization_versions_to_nested_types = 1; +SELECT concat(data_paths[1], 'detached') FROM system.tables WHERE database = 'legacy' AND name = 'maps'; +") + +# A real part written by `91ffe225686`, before the `Map` statistics flag fix. +# Its `JSON` paths contain a scalar-valued map, a tuple-valued map, an array of maps, +# and nested `JSON` with a typed map. The original checksums are preserved. +base64 -d <<'PART' | tar -xz -C "$part_dir" +H4sIAAAAAAACA+2dC1xTZR/H/88uCAPGuKggyBnjfnGcjbHVwAuQiKAvKOYFL/M4DjjG2NzFFF8TZ1YWKb4mFZWa19Twfs8LlWaW +mn3MlKQsw8osyxKxlHyfcREbL5Kk6/PG+fP5c7bnPDuDnfN9fv/zPP/nGVVYqJLgHzJWPZlWa00WnUlsnmaG+2kkNrlM1rjFZr+V +SiTxLY+byiVS/ANCEhxgFpOZMuK3h65pt8+6ME9v1FFm4VTaaNLoi5RCGa9GGlHzTsw3PK2HRvpm6VpeWRAHINEJIGJ7bMPERK5F +CLKHe7qo9ZYis+2iQYWvPLrvYsWif6/NHjH6qg/3FhAFYv2kAlptVpkmU0Y6V5VLmSkxKZ6kKdqM6q/nT/vyswn1J9YcSytccXQo +qmYVlK6Nrtnks/jy+uf2Hxq4MDlIrTNqpZKFCeveebVhe2JS9NaNGaPy9IhcYnqqoUqjOtl74OHqd4feXBFmoMyTTSodZdSabEeP +5Y8a+fCVt3Oszz1Xs9DtVGUKGrDph4u3prI2JGbIwumpaet1EbH7L9WlFtef4wVL5jWUplecR2Q4O+XXiidQ6pYX+nHUJxPGy0yW +SSazkaZ0+Ni0meoPu9/6rvTY5+8EdU8jx1jmRKMStE5mrdW+lH/inNPHwaeq30by+dcuxyU2nC8r2MT6IfOx1z9B5Gfe39euNrjt +PNTfOVlYNWhdCD6mRW22GGnxLv2xwWs+OxW8e/uR+m3y0hikfHKxC582+FVQO5c3HK6dfDxs3LqjledWpRSPSTdHxZhX7EbkmuqD +xMhvT7zoOq4+su+aeetD69lffJm+tWfygB9Ljg96a8uyq0+jw/zLyHWPmjv24KrxIp/gyafdw6VB/cOXjebnJWzPKrpILeKOQuTg +fUPklQt+HZRbYTj1oSafE70P7SutOzL0lbjggzOnhJZRMfNRlZPp4rZr+r2/sG5d20bNGu5l6CN96lzx3FTltZGhR7+oEzUs6o/I +0d5HNvxkGLUy3fW9L1eGv3QjWq03TNcU5dLT6MDdt0JHu/SbM2H4FHddlalShTi1vsTwOYMGX3z/2WLvHFVlTOz7F3aeRiO2na6Y +5bv924ANpYhc+tGwZd25I/fWLqwRu1xY9l6wSVMsWh22hS94yn/t4z3XnzlFTV2CnB+pk29OfDOv/KvYHhz2hrAjoT8W0nM5Jfuz +066oImmoy0bknJXcA+IXT6Dw6yn9vjq57EbIVKrQkjvj95k3V30SMS6px2+e/kPY/VBO+tmNF57amPr4LzsneM0LoN8JW7t1EBF7 +xJrWMOXlgzt68MsRmX3deu2DC5l9e6uEmYqjFR498LVGuqQcTRqx+jU9v+bH1Kpbe/6D/CYK8tMjjYIVV13o4Qecglf4Sqd5sA6s +OfjuFf+80yvm9Yh0QeRzr08K+eCL1B17Jpiq4ycVcr1MtFFDFWqKKTPmT1xg0heVIn7Np0OvVl3iZb+gnpWdPPN98MTNlVljMmvU +JrGBUmvp3JnCa+WXE2azo+ZXBhz8RfbYSSCFZSKXg5+wdwtOqKoozazjvDMN3d1HH4ezgzyDz/7+hK9PtwHXdG877dSjEJczEb9Z +vy8vzXvi93OvocWuN+ddmf4E6r3Uj31MMbH3pT5k2sbZXFXkk4fcNxWnb80/vdOTzXd/5v2Zhn9HH3h+9kR3cenKXl9bXCaNfkG8 +txsw9v9mVKv+6wstuqL7r/4d6j8pIePs9F8iJxWM/jtE/5vOehv1l/AkwuZ9St7EgonC9OzMf0XoqGmq3OlFlE6jVjUqbl8yksdA +9E/iX9Ua8tyvpqAj/uXxpB3/MlIuZ/h3JP+tZ72dNkBC3lkHNxfNu4S4cVDyXFoD/ZbQ9s6yPwT/HdewbdrdeUek3/5b3P5L/0SV +Do7V9H52NwJ3+Qttca8Yx6q06W4Vmo7aFB/ftaItVsU1HMF/823cfX+PjvjHEYC9/kvjZQz/jjAJI4GM/jfyn0vnUZZCs0qt1xmM +tMkmAfhxLq3+y21Ch/xLpHb8x8fFkQz/jrCUzEcGpkQMyZFFMix0bf7vEovZelQfIP9Sub3+x0viGf4dYnU//7rwg17Jv+V88/YG +Z/7uN6yv4cIxCEDEAQSTUGB4dw5ygxAAFxiAELAE4OwNiA98AZBaLbsnyPALsoHjDATKAQ6XBiS3lWk9gfsw7CChxFmIkA4fZgpw +AiEcWCnQzRUgA4bhF2q53cDpIcCffi0XurHAQwDunAKYhcK741dRBGJxttmOXaZlMaz+ffw3DsM8OP1vy79MwsT/DrEDLxd3E+5a +Mfdqgzr55bOGp8uCbCcMmsb4hORM/ABwQzCGDbIElBvDjmGQ6Vr82w2sPhD+4xV2/Mvj45n+P4cY73vOl19r3IddMC/32FNzY5WV +wIUDsBOYeo7/WF/wf14AJQGIQYXhv5NhwL3zryCljP47xIZ51c9S7npxcPaOmxR33YWsMuIP+q9rfIJwi8AewQJXphXoovz/j+GP +Px8NdMi/wo5/qUSmiGP4d4Sdqv6uUP3KvKHjL8/3/Djb72NrLC4sadZ/n6gkFAm9NnmAEvqyPME/i4SoXk5CKFnG3I53ef7/dDRw +z/ovlSjimPwfh1gw/WrKXD79rXGO7nPjykR2O/pfwuh/F+W/NVW504MAHfMvt7//jyOZ/H+H2OTPzeneIqeowOvl9XUlZ16wBuJC +JfYDiIMopEN6ZMb898xiUOnq/Hd6EKBT/X/M+J9DbMc2S0jK6vMj4XzGkszKNcnt6L+S0f+uyn9rOnBnA4CO+FfI2/Avk0kZ/h1h +xkM3i1csP5Q/XPkWh+hZ0d/6DWZ8UDeA3wa6eolbcv1vi0B38VTKqKGKzKpcjUltVBmMdJ5mWoQ4yWikpkcMpQwR2Wajpig/Rvjo +4CKzXBYZKZ5kUWtpsy3XNU/P7w9E0EPgorflx5LBInDxJEMgVhIZArwpodKBWnq6KRqIkCjwJMJCwf1naEp/9RZnN16UIwUITYRe +fB8geEmoR2yQOAGBhxK5EAESYLE8USwp6g1sZx4iegcDjyACwcnjJvQguuNfS/uImyYxCNkInoxsvtxns4hoMa6qk+GjhCWAG7sf +ImLCwd05DRHhfcCHiIwAAWscIpKjgMfxQGgGa24LHEvY7AVsQp4CBKQD0VcBxId39Jcn9gNi6W1+6kDE7QusEiAysLP2g9OTkNV4 +N81DHgsgiCPiLsD32Xn481QugN5JtuzfgRwPd0SgBGAdgwXNycIJj4OINZvlUc1R+kWlc2aNsBgK6QgqwMmLCFWCK7uSTUSHgZvz +G2wiLAa8bf+LBysSZeKPl+qDXLMaT1+n+O9sANDx+L99/p9crmD6/x1iVoP/Vs/NG1W/V2uTZebNu8pEf9D/uQAP2fQftwhsARwu +21zKZTSzi+p/p5MAOsz/U9jP/1fI45j7f4fY6C+una5LLR9SFRqgXPW9ppu1EhcOYLeM/7v7gn93AfgHYg/CHon9YeyDsWcIgBgr +gAD/fBL89bhgGvYZ2OdgfxZ7Ka7wvK3COlxhCy7Yh/1d7Eewf4S9Gvun2M/jirUCGBLlgyggAgSQ6B+TBf4yvKsv9gHY07BnYR8m +gMAxAmC6I/42/u89Crhn/ZfiewKGf4fYpde/nm/ps8u4ZvFiQn7959p29B+3CIz+d0H+285VfAD6T8bZ9/8rFBKm/88h9tKM2Rmv +8/38d32ay/Ne9F2dlY8LObYYABCLzfDB8G/P/wPQ/zbj/6REzui/QyxqWEjJI9ffvGB6duYJ3ZGk3Hb6/zlM/3/X5b9xMYO/MAew +Q/5lbfr/pCTT/+cQ26eY8uxDN5555b3JWz/wTq32s3rhQmfsJRwGDob/O/jv/BzAToz/y+RM/O8Q+zAzZc1UN/HhUeQU+TwqJrYd +/Xdm9L/r8t80Bvvg8v8krftu5//FM/G/Qwyd+sFwK+BmTv76uOQfx1zyto7AhTnYt/jbpv6zENKyMf/JeMvhwSLg29YDQFrkAVX+ +d8ztj4BsHS7mBoDBq2nqfiyUMHME/lH8OzL/Ty5l8n8cYnuPT7x16mrGo5+9B6J0oe/GdvQ/h9H/rsP/X8/4v0f+5fbrf8YpSCb+ +d4iVHH80nDtjufbkL2NufHd5A88qxIV+2INt6//8hHAAwEF6LPIImbGqMw1AV+L/ry37cw/82/f/yZj1vxxkVpEwo/zGlZyf3Hzf +9Y38obzt+P8AAFfcFPgBJ4sFWw6mMMz8M/lvmderal7/+f4tBXzv6//GyZj7f8cYySDA8N/If9uvn7lv11hH+b8S+/6/OBwBMPw7 +wmaImlf4FynHjo8RGYx6A5VPmWmVebqBNqn+cFG0KINJZdarimiTmc5tqiZS4pCRjhHd7TUi5QyRjjKIlJIYkalxjhB+ODNG1Lzf +9oSh8W/l3/6LpcBR/Eva5P9LFcz8PwdZyx29V+v5VxWIbU9MLWt9Dmve+rEeL6nuNSZogrKwr0Xv52NNwoUDsSdzQIAP9LObX7vf +EoSCIBg8nYEjgDIms4Axxhhj7O+3/wKOKbQwAHgAAA== +PART + +$CLICKHOUSE_LOCAL --path "$workdir" --multiquery <<'SQL' +ALTER TABLE legacy.maps ATTACH PART 'all_1_1_0'; +SELECT 'legacy', j.m, j.t, j.a, j.o FROM legacy.maps; +SELECT 'types', dynamicType(j.m), dynamicType(j.t), dynamicType(j.a), dynamicType(j.o) FROM legacy.maps; +SELECT 'subcolumns', j.m.:`Map(String, UInt64)`['k'], j.t.:`Map(String, Tuple(a UInt64))`['k'].a FROM legacy.maps; +INSERT INTO legacy.maps SELECT j FROM legacy.maps; +OPTIMIZE TABLE legacy.maps FINAL; +SELECT 'rewritten', j.m, j.t, j.a, j.o FROM legacy.maps; +SELECT 'rewritten_types', dynamicType(j.m), dynamicType(j.t), dynamicType(j.a), dynamicType(j.o) FROM legacy.maps; +SELECT 'rewritten_subcolumns', j.m.:`Map(String, UInt64)`['k'], j.t.:`Map(String, Tuple(a UInt64))`['k'].a FROM legacy.maps; +SQL diff --git a/tests/queries/0_stateless/05175_materialize_statistics_session_gate.reference b/tests/queries/0_stateless/05175_materialize_statistics_session_gate.reference new file mode 100644 index 0000000000000..d033580b60410 --- /dev/null +++ b/tests/queries/0_stateless/05175_materialize_statistics_session_gate.reference @@ -0,0 +1,6 @@ +refused at submission +0 +accepted and finishes +0 +100 +0 diff --git a/tests/queries/0_stateless/05175_materialize_statistics_session_gate.sql b/tests/queries/0_stateless/05175_materialize_statistics_session_gate.sql new file mode 100644 index 0000000000000..328b69a5bc38e --- /dev/null +++ b/tests/queries/0_stateless/05175_materialize_statistics_session_gate.sql @@ -0,0 +1,37 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/115448 +-- `allow_statistics` gates the DDL - whether statistics may be declared - and only the submitting +-- session's value is meaningful for that. It used to be checked again while the mutation ran, under +-- the background context that reads the server-default profile, so a `MATERIALIZE STATISTICS` +-- accepted with a session opt-in failed forever and wedged the table's mutation queue. The full +-- reproducer needs a server profile with `allow_statistics = 0`, which a stateless test cannot set; +-- it lives in `tests/integration/test_statistics_mutation_session_gate`. What is pinned here is that +-- submission-time validation still refuses the statement, and that the statement works with the +-- setting on. + +SET mutations_sync = 2; + +DROP TABLE IF EXISTS t_materialize_statistics_gate; +CREATE TABLE t_materialize_statistics_gate (k UInt64, a Int64) ENGINE = MergeTree ORDER BY k; +INSERT INTO t_materialize_statistics_gate SELECT number, number FROM numbers(100); + +SELECT 'refused at submission'; +ALTER TABLE t_materialize_statistics_gate ADD STATISTICS a TYPE tdigest SETTINGS allow_statistics = 0; -- { serverError INCORRECT_QUERY } +ALTER TABLE t_materialize_statistics_gate MATERIALIZE STATISTICS a SETTINGS allow_statistics = 0; -- { serverError INCORRECT_QUERY } +ALTER TABLE t_materialize_statistics_gate DROP STATISTICS a SETTINGS allow_statistics = 0; -- { serverError INCORRECT_QUERY } + +-- Nothing of the above reached the mutation queue. +SELECT count() FROM system.mutations +WHERE database = currentDatabase() AND table = 't_materialize_statistics_gate'; + +SELECT 'accepted and finishes'; +ALTER TABLE t_materialize_statistics_gate ADD STATISTICS a TYPE tdigest SETTINGS allow_statistics = 1; +ALTER TABLE t_materialize_statistics_gate MATERIALIZE STATISTICS a SETTINGS allow_statistics = 1; +SELECT countIf(is_done = 0) FROM system.mutations +WHERE database = currentDatabase() AND table = 't_materialize_statistics_gate'; +SELECT count() FROM t_materialize_statistics_gate; + +ALTER TABLE t_materialize_statistics_gate DROP STATISTICS a SETTINGS allow_statistics = 1; +SELECT countIf(is_done = 0) FROM system.mutations +WHERE database = currentDatabase() AND table = 't_materialize_statistics_gate'; + +DROP TABLE t_materialize_statistics_gate; diff --git a/tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.reference b/tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.reference new file mode 100644 index 0000000000000..f037c389e431f --- /dev/null +++ b/tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.reference @@ -0,0 +1,10 @@ +Compact +{"m":{"k":1},"t":{"k":{"a":2}}} +Map(String, UInt64) {'k':1} Map(String, Tuple(a UInt64)) {'k':(2)} +1 2 +{"m":{"k":1},"t":{"k":{"a":2}}} +{"m":{"k":1},"t":{"k":{"a":2}}} +Map(String, UInt64) {'k':1} Map(String, Tuple(a UInt64)) {'k':(2)} +Map(String, UInt64) {'k':1} Map(String, Tuple(a UInt64)) {'k':(2)} +1 2 +1 2 diff --git a/tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.sql b/tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.sql new file mode 100644 index 0000000000000..ec084ba7434c1 --- /dev/null +++ b/tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.sql @@ -0,0 +1,27 @@ +DROP TABLE IF EXISTS json_shared_bucketed_map_compact; +CREATE TABLE json_shared_bucketed_map_compact (j JSON(max_dynamic_paths = 0)) +ENGINE = MergeTree ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 1000000000, min_rows_for_wide_part = 1000000000, + write_marks_for_substreams_in_compact_parts = 0, + object_shared_data_serialization_version = 'advanced', + map_serialization_version = 'with_buckets', + object_shared_data_serialization_version_for_zero_level_parts = 'advanced', + map_serialization_version_for_zero_level_parts = 'with_buckets', + propagate_types_serialization_versions_to_nested_types = 1; + +-- Preserve runtime `Map` types in shared data instead of parsing them as `JSON` objects. +INSERT INTO json_shared_bucketed_map_compact +SELECT * FROM format(RowBinary, 'j JSON(max_dynamic_paths = 0)', concat( + unhex('02016d'), formatRowNoNewline('RowBinary', CAST(map('k', toUInt64(1)) AS Dynamic)), + unhex('0174'), formatRowNoNewline('RowBinary', CAST(map('k', CAST(tuple(toUInt64(2)) AS Tuple(a UInt64))) AS Dynamic)))) +SETTINGS input_format_binary_read_json_as_string = 0; +SELECT part_type FROM system.parts WHERE database = currentDatabase() AND table = 'json_shared_bucketed_map_compact' AND active; +SELECT j FROM json_shared_bucketed_map_compact; +SELECT dynamicType(j.m), j.m, dynamicType(j.t), j.t FROM json_shared_bucketed_map_compact; +SELECT j.m.:`Map(String, UInt64)`['k'], j.t.:`Map(String, Tuple(a UInt64))`['k'].a FROM json_shared_bucketed_map_compact; +INSERT INTO json_shared_bucketed_map_compact SELECT j FROM json_shared_bucketed_map_compact; +OPTIMIZE TABLE json_shared_bucketed_map_compact FINAL; +SELECT j FROM json_shared_bucketed_map_compact; +SELECT dynamicType(j.m), j.m, dynamicType(j.t), j.t FROM json_shared_bucketed_map_compact; +SELECT j.m.:`Map(String, UInt64)`['k'], j.t.:`Map(String, Tuple(a UInt64))`['k'].a FROM json_shared_bucketed_map_compact; +DROP TABLE json_shared_bucketed_map_compact; diff --git a/tests/queries/0_stateless/05189_update_cte_out_of_scope_database.reference b/tests/queries/0_stateless/05189_update_cte_out_of_scope_database.reference new file mode 100644 index 0000000000000..88d32f1ade59d --- /dev/null +++ b/tests/queries/0_stateless/05189_update_cte_out_of_scope_database.reference @@ -0,0 +1,9 @@ +2 +2 +2 +2 +2 +7002 +7 +2 +2 diff --git a/tests/queries/0_stateless/05189_update_cte_out_of_scope_database.sql b/tests/queries/0_stateless/05189_update_cte_out_of_scope_database.sql new file mode 100644 index 0000000000000..c7408aab6b1f3 --- /dev/null +++ b/tests/queries/0_stateless/05189_update_cte_out_of_scope_database.sql @@ -0,0 +1,98 @@ +-- A common table expression is expanded before the database is filled into a mutation command, so a +-- table identifier hidden by an expression alias of the same name is left unqualified. With +-- `enable_global_with_statement` disabled the alias is not visible in the subquery, so the identifier +-- names a table there and has to be resolved in the database of the updated table, not of the session. +-- The source table exists in both databases with a different row, so an expression resolved in the +-- database of the session silently reads the wrong row instead of failing. +-- The reference is one `SELECT` deeper than the alias: only a lookup in an enclosing scope is +-- disabled, so a reference in the select that declares the alias reads it either way. +-- The old analyzer resolves a common table expression in a subquery of a mutation as a table, so the +-- analyzer is requested explicitly, as in `04693_merge_table_function_in_mutation`. + +CREATE DATABASE IF NOT EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; + +CREATE TABLE src (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO src VALUES (99); + +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.src (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO {CLICKHOUSE_DATABASE_1:Identifier}.src VALUES (2); + +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.u (id UInt64, v UInt64) ENGINE = MergeTree ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1; +INSERT INTO {CLICKHOUSE_DATABASE_1:Identifier}.u VALUES (1, 0), (2, 0), (3, 0), (4, 0), (5, 0), (6, 0), (8, 0), (99, 0); + +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t (id UInt64, v UInt64) ENGINE = MergeTree ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1; +INSERT INTO {CLICKHOUSE_DATABASE_1:Identifier}.t VALUES (1, 0), (2, 0), (3, 0), (99, 0); + +-- Every row starts at 0 and every expected value is non-zero, so a statement that does not update the +-- row at all also fails. The row `id = 99` exists so that a predicate resolved in the database of the +-- session updates the wrong row rather than no row. + +-- An assignment reads the table of the updated database (2), not the table of the session (99). +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = (WITH src AS (SELECT 7 AS id) SELECT (SELECT max(id) FROM src SETTINGS enable_global_with_statement = 0)) + WHERE id = 1 SETTINGS enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE id = 1; + +-- The `ALTER TABLE ... UPDATE` spelling of the same expression answers the same. +ALTER TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t + UPDATE v = (WITH src AS (SELECT 7 AS id) SELECT (SELECT max(id) FROM src SETTINGS enable_global_with_statement = 0)) + WHERE id = 1 SETTINGS mutations_sync = 2, enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.t WHERE id = 1; + +-- The predicate is expanded by a separate call, so it is asserted separately: it marks the row the +-- updated database names (2), not the one the session database names (99). +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = 11 + WHERE id IN (WITH src AS (SELECT 7 AS id) SELECT (SELECT max(id) FROM src SETTINGS enable_global_with_statement = 0)) + SETTINGS enable_analyzer = 1; +SELECT id FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE v = 11; + +ALTER TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t + UPDATE v = 11 + WHERE id IN (WITH src AS (SELECT 7 AS id) SELECT (SELECT max(id) FROM src SETTINGS enable_global_with_statement = 0)) + SETTINGS mutations_sync = 2, enable_analyzer = 1; +SELECT id FROM {CLICKHOUSE_DATABASE_1:Identifier}.t WHERE v = 11; + +-- The effective value of the setting is read, so a profile that carries it answers the same as the +-- setting written out. +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = (WITH src AS (SELECT 7 AS id) SELECT (SELECT max(id) FROM src SETTINGS compatibility = '20.3')) + WHERE id = 3 SETTINGS enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE id = 3; + +-- A clause is read per arm of a union: the arm carrying it names a table there, resolved in the +-- updated database (2), while the arm without it keeps reading the alias (7 * 1000). The two arms +-- are weighted differently so that reading the clause for the wrong arm, for both arms or for +-- neither answers 2007, 2002 and 7007 rather than the expected value. +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = (WITH src AS (SELECT 7 AS id) SELECT sum(m) FROM ( + SELECT max(id) * 1000 AS m FROM src + UNION ALL + SELECT max(id) AS m FROM src SETTINGS enable_global_with_statement = 0)) + WHERE id = 8 SETTINGS enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE id = 8; + +-- With the setting at its default the alias IS visible in the subquery, so the same reference reads +-- the common table expression (7) and must not be qualified as a table. +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = (WITH src AS (SELECT 7 AS id) SELECT (SELECT max(id) FROM src)) + WHERE id = 4 SETTINGS enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE id = 4; + +-- A reference in the select that declares the alias keeps reading the alias, and the table its body +-- names is resolved in the updated database (2). +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = (WITH c AS (SELECT max(id) AS m FROM src) SELECT m FROM c) + WHERE id = 5 SETTINGS enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE id = 5; + +-- An identifier that no alias hides was always resolved in the updated database (2). +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = (SELECT max(id) FROM src) + WHERE id = 6 SETTINGS enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE id = 6; + +DROP TABLE src; +DROP DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; diff --git a/tests/queries/0_stateless/05205_vector_search_lightweight_delete_limit.reference b/tests/queries/0_stateless/05205_vector_search_lightweight_delete_limit.reference new file mode 100644 index 0000000000000..ea9603c180be9 --- /dev/null +++ b/tests/queries/0_stateless/05205_vector_search_lightweight_delete_limit.reference @@ -0,0 +1,16 @@ +rows left 297 +with the index 3 +without the index 3 +without rescoring 3 +with rescoring 3 +the index answer [1,1,1] +the bruteforce answer [1,1,1] +apply_deleted_mask = 0 uses the index 1 +apply_deleted_mask = 0 [0,0,0] +apply_deleted_mask = 0 and _row_exists uses the index 0 +apply_deleted_mask = 0 and _row_exists 3 +apply_deleted_mask = 0 and _row_exists [1,1,1] +FINAL, apply_deleted_mask = 0 and deferred PREWHERE _row_exists uses the index 0 +a deleted cluster 3 +a pending delete 3 +a delete in a patch part 3 diff --git a/tests/queries/0_stateless/05205_vector_search_lightweight_delete_limit.sql b/tests/queries/0_stateless/05205_vector_search_lightweight_delete_limit.sql new file mode 100644 index 0000000000000..86d90930fb694 --- /dev/null +++ b/tests/queries/0_stateless/05205_vector_search_lightweight_delete_limit.sql @@ -0,0 +1,84 @@ +-- Tags: no-fasttest, no-ordinary-database, no-parallel-replicas +-- no-fasttest: the vector similarity index is not built in the fast test. +-- no-parallel-replicas: vector-search read hints are produced during local index analysis. + +-- A lightweight `DELETE` does not rebuild a vector index, so the index keeps returning the row ids of +-- deleted rows. Restricting the read to exactly those rows then drops the deleted candidates with +-- nothing to take their place: the query returned fewer rows than its `LIMIT` and missed the true +-- neighbours that a deleted candidate shadowed, down to an empty result when a whole cluster of near +-- neighbours is deleted. Such a part is read in full, without the index, until a merge or a mutation +-- rebuilds the index. + +-- The tables are small, with a small HNSW graph, because every index build runs for more than a minute +-- under ASan. Three copies of a 10x10 grid: the three exact matches of the reference vector [5, 5] are the +-- whole result of a `LIMIT 3`, so deleting them used to empty it. Every row is its own granule, so +-- reading only the granules of the index candidates would read nothing but the deleted rows. +-- Direct I/O is disabled because it reads every one-row granule separately and makes the test slow. + +SET enable_analyzer = 1; +SET lightweight_deletes_sync = 2; +SET mutations_sync = 2; +SET parallel_replicas_local_plan = 1; +SET min_bytes_to_use_direct_io = 0; + +DROP TABLE IF EXISTS t_05205; +-- `ReplacingMergeTree` (the ids are unique, so it reads like a `MergeTree`) to check the `FINAL` read as well. +CREATE TABLE t_05205 (id UInt64, v Array(Float32), INDEX vidx v TYPE vector_similarity('hnsw', 'L2Distance', 2, 'f32', 16, 32)) +ENGINE = ReplacingMergeTree ORDER BY id SETTINGS index_granularity = 1, index_granularity_bytes = 10485760; +INSERT INTO t_05205 SELECT number, [toFloat32(number % 10), toFloat32(intDiv(number, 10) % 10)] FROM numbers(300); + +-- The three exact matches of the reference vector. +DELETE FROM t_05205 WHERE v = [5.0, 5.0]; + +SELECT 'rows left', count() FROM t_05205; +SELECT 'with the index', count() FROM (SELECT id FROM t_05205 ORDER BY L2Distance(v, [5.0, 5.0]) LIMIT 3); +SELECT 'without the index', count() FROM (SELECT id FROM t_05205 ORDER BY L2Distance(v, [5.0, 5.0]) LIMIT 3 SETTINGS use_skip_indexes = 0); +SELECT 'without rescoring', count() FROM (SELECT id FROM t_05205 ORDER BY L2Distance(v, [5.0, 5.0]) LIMIT 3 SETTINGS vector_search_with_rescoring = 0); +SELECT 'with rescoring', count() FROM (SELECT id FROM t_05205 ORDER BY L2Distance(v, [5.0, 5.0]) LIMIT 3 SETTINGS vector_search_with_rescoring = 1); + +-- And they are as near as the neighbours the bruteforce scan finds (twelve rows tie at distance 1, so compare distances, not ids). +SELECT 'the index answer', arraySort(groupArray(d)) FROM (SELECT id, L2Distance(v, [5.0, 5.0]) AS d FROM t_05205 ORDER BY L2Distance(v, [5.0, 5.0]) LIMIT 3); +SELECT 'the bruteforce answer', arraySort(groupArray(d)) FROM (SELECT id, L2Distance(v, [5.0, 5.0]) AS d FROM t_05205 ORDER BY L2Distance(v, [5.0, 5.0]) LIMIT 3 SETTINGS use_skip_indexes = 0); + +-- With `apply_deleted_mask = 0` the deleted rows are returned, so the index still matches the data and is used. +SELECT 'apply_deleted_mask = 0 uses the index', count() FROM (EXPLAIN indexes = 1 SELECT id FROM t_05205 ORDER BY L2Distance(v, [5.0, 5.0]) LIMIT 3 SETTINGS apply_deleted_mask = 0, use_skip_indexes_on_data_read = 0) WHERE explain LIKE '% Granules: 3/300'; +SELECT 'apply_deleted_mask = 0', arraySort(groupArray(d)) FROM (SELECT L2Distance(v, [5.0, 5.0]) AS d FROM t_05205 ORDER BY d LIMIT 3 SETTINGS apply_deleted_mask = 0); +-- But an explicit filter by `_row_exists` drops them again, the same as the default masked read, so the index is not used. +SELECT 'apply_deleted_mask = 0 and _row_exists uses the index', count() FROM (EXPLAIN indexes = 1 SELECT id FROM t_05205 WHERE _row_exists ORDER BY L2Distance(v, [5.0, 5.0]) LIMIT 3 SETTINGS apply_deleted_mask = 0, use_skip_indexes_on_data_read = 0) WHERE explain LIKE '% Granules: 3/300'; +SELECT 'apply_deleted_mask = 0 and _row_exists', count() FROM (SELECT id FROM t_05205 WHERE _row_exists ORDER BY L2Distance(v, [5.0, 5.0]) LIMIT 3 SETTINGS apply_deleted_mask = 0); +SELECT 'apply_deleted_mask = 0 and _row_exists', arraySort(groupArray(d)) FROM (SELECT L2Distance(v, [5.0, 5.0]) AS d FROM t_05205 WHERE _row_exists ORDER BY d LIMIT 3 SETTINGS apply_deleted_mask = 0); +-- The same with `FINAL`, where the `PREWHERE` on `_row_exists` is deferred until after the `FINAL` merge. +SELECT 'FINAL, apply_deleted_mask = 0 and deferred PREWHERE _row_exists uses the index', count() FROM (EXPLAIN indexes = 1 SELECT id FROM t_05205 FINAL PREWHERE _row_exists ORDER BY L2Distance(v, [5.0, 5.0]) LIMIT 3 SETTINGS apply_deleted_mask = 0, apply_prewhere_after_final = 1, use_skip_indexes_on_data_read = 0) WHERE explain LIKE '% Granules: 3/300'; +-- Only the plan is checked here: reading with a bare `PREWHERE _row_exists` deferred after `FINAL` fails with +-- `THERE_IS_NO_COLUMN` in 26.8 regardless of the index, fixed on master by https://github.com/ClickHouse/ClickHouse/pull/114578. + +-- Deleting a whole cluster of near neighbours used to empty the result as well. +DELETE FROM t_05205 WHERE L2Distance(v, [5.0, 5.0]) < 1.5; + +SELECT 'a deleted cluster', count() FROM (SELECT id FROM t_05205 ORDER BY L2Distance(v, [5.0, 5.0]) LIMIT 3); + +-- A delete that is still a pending mutation, applied on the fly, has the same effect on the index. +DROP TABLE IF EXISTS t_05205_on_fly; +CREATE TABLE t_05205_on_fly (id UInt64, v Array(Float32), INDEX vidx v TYPE vector_similarity('hnsw', 'L2Distance', 2, 'f32', 16, 32)) +ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 1, index_granularity_bytes = 10485760; +INSERT INTO t_05205_on_fly SELECT number, [toFloat32(number % 10), toFloat32(intDiv(number, 10) % 10)] FROM numbers(300); +SYSTEM STOP MERGES t_05205_on_fly; +ALTER TABLE t_05205_on_fly DELETE WHERE v = [5.0, 5.0] SETTINGS mutations_sync = 0, alter_sync = 0; + +SELECT 'a pending delete', count() FROM (SELECT id FROM t_05205_on_fly ORDER BY L2Distance(v, [5.0, 5.0]) LIMIT 3 SETTINGS apply_mutations_on_fly = 1); + +SYSTEM START MERGES t_05205_on_fly; + +-- A delete written as a patch part (a lightweight update of `_row_exists`) is invisible to the index too. +DROP TABLE IF EXISTS t_05205_patch; +CREATE TABLE t_05205_patch (id UInt64, v Array(Float32), INDEX vidx v TYPE vector_similarity('hnsw', 'L2Distance', 2, 'f32', 16, 32)) +ENGINE = MergeTree ORDER BY id +SETTINGS index_granularity = 1, index_granularity_bytes = 10485760, enable_block_number_column = 1, enable_block_offset_column = 1; +INSERT INTO t_05205_patch SELECT number, [toFloat32(number % 10), toFloat32(intDiv(number, 10) % 10)] FROM numbers(300); +DELETE FROM t_05205_patch WHERE v = [5.0, 5.0] SETTINGS lightweight_delete_mode = 'lightweight_update'; + +SELECT 'a delete in a patch part', count() FROM (SELECT id FROM t_05205_patch ORDER BY L2Distance(v, [5.0, 5.0]) LIMIT 3 SETTINGS apply_patch_parts = 1); + +DROP TABLE t_05205_patch; +DROP TABLE t_05205_on_fly; +DROP TABLE t_05205; diff --git a/tests/queries/0_stateless/05210_iceberg_decimal38_bound_widening_116999.reference b/tests/queries/0_stateless/05210_iceberg_decimal38_bound_widening_116999.reference new file mode 100644 index 0000000000000..ae1d0aec24b02 --- /dev/null +++ b/tests/queries/0_stateless/05210_iceberg_decimal38_bound_widening_116999.reference @@ -0,0 +1,29 @@ +--- A0 a filter on the column returns every row that matches it --- +2 +1 +1 +1 +1 +--- A1 a probe outside one file min/max-prunes it, and only it --- +0 +0 +0 +0 +1 +1 +1 +1 +--- A2 a value inside both shifted bounds prunes nothing --- +0 +0 +--- B0 a bound the writer rounded past the precision of the type still prunes --- +2 +1 +1 +--- B1 the saturated bounds prune the file that cannot hold the probe --- +0 +0 +0 +1 +1 +0 diff --git a/tests/queries/0_stateless/05210_iceberg_decimal38_bound_widening_116999.sh b/tests/queries/0_stateless/05210_iceberg_decimal38_bound_widening_116999.sh new file mode 100755 index 0000000000000..15fbecb66e05d --- /dev/null +++ b/tests/queries/0_stateless/05210_iceberg_decimal38_bound_widening_116999.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# - no-fasttest: requires `IcebergLocal` (USE_AVRO build option) + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +ROOT="${CLICKHOUSE_USER_FILES}/${CLICKHOUSE_DATABASE}_dec38" +rm -rf "${ROOT}" +trap 'rm -rf "${ROOT}"' EXIT + +# Iceberg stores a decimal min/max bound rounded to the integral part, so the reader moves the +# bound one integral unit outwards before pruning on it. At scale 38 that unit is `10^38` and the +# bound lives in an `Int128`, which holds `1.7 * 10^38`: the shift leaves the type for every bound +# of magnitude above roughly 0.7014. `high` holds one such value per data file and `low` holds two +# that the shift keeps inside the type, so the two tables differ only in whether the shift saturates. +# +# One value per `INSERT`, so each table has two data files and a bound on one can exclude it. +# Background Iceberg compaction, which the cloud build runs off a member flag rather than a query +# setting, would rewrite these manifests, so it is pinned off per table. +mk() { # mk
+ cat < saw that probe prune. +# Reading a counter back through `log_comment` keeps a probe from counting itself. +report() { # report ... + local counter=$1 tags="" t + shift + for t in "$@"; do tags="${tags:+${tags}, }'${CLICKHOUSE_DATABASE}_${t}'"; done + ${CLICKHOUSE_CLIENT} --query " + SELECT max(ProfileEvents['${counter}']) + FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment IN (${tags}) + GROUP BY log_comment + ORDER BY indexOf([${tags}], log_comment) + SETTINGS enable_parallel_replicas = 0" +} + +echo '--- A0 a filter on the column returns every row that matches it ---' +# A bound that comes back inverted prunes the data file that holds the row, so these lose rows. +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --query " + SELECT count() FROM high; + SELECT count() FROM high WHERE d = toDecimal128('0.99', 38); + SELECT count() FROM high WHERE d = toDecimal128('-0.99', 38); + SELECT count() FROM high WHERE d > toDecimal128('0.5', 38); + SELECT count() FROM high WHERE d < toDecimal128('-0.5', 38);" + +echo '--- A1 a probe outside one file min/max-prunes it, and only it ---' +# `low` is the control: its shift stays inside `Int128` whatever the reader does with an overflow. +# Each `high` probe sits outside exactly one of the two shifted bounds, so one file is pruned and +# the other is read - the row counts say the surviving file is the one that could hold a match. +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --query " + SELECT count() FROM low WHERE d = toDecimal128('0.95', 38) SETTINGS log_comment = '${CLICKHOUSE_DATABASE}_low_pos'; + SELECT count() FROM low WHERE d = toDecimal128('-0.95', 38) SETTINGS log_comment = '${CLICKHOUSE_DATABASE}_low_neg'; + SELECT count() FROM high WHERE d = toDecimal128('0.5', 38) SETTINGS log_comment = '${CLICKHOUSE_DATABASE}_high_pos'; + SELECT count() FROM high WHERE d = toDecimal128('-0.5', 38) SETTINGS log_comment = '${CLICKHOUSE_DATABASE}_high_neg'; + SYSTEM FLUSH LOGS query_log;" +report IcebergMinMaxIndexPrunedFiles low_pos low_neg high_pos high_neg + +echo '--- A2 a value inside both shifted bounds prunes nothing ---' +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --query " + SELECT count() FROM high WHERE d = toDecimal128('0.0', 38) SETTINGS log_comment = '${CLICKHOUSE_DATABASE}_high_zero'; + SYSTEM FLUSH LOGS query_log;" +report IcebergMinMaxIndexPrunedFiles high_zero + +echo '--- B0 a bound the writer rounded past the precision of the type still prunes ---' +# Spark and Amazon Athena round the bound to the integral part, so a file whose extreme value is +# `0.99` carries `1.0` - `10^38` unscaled, one unit outside what `Decimal(38, 38)` holds. ClickHouse +# writes the exact unscaled value instead, so the rounded form is patched into the manifests here. +# The bound is an uncompressed 16-byte big-endian `fixed`, and both forms need all 16 bytes, so the +# substitution keeps every length in the Avro container intact. +python3 - "${ROOT}/high/metadata" <<'PATCH_MANIFESTS' +import pathlib +import sys + +def big_endian(value): + return (value % (1 << 128)).to_bytes(16, 'big') + +exact, rounded = 99 * 10 ** 36, 10 ** 38 +substitutions = [(big_endian(exact), big_endian(rounded)), (big_endian(-exact), big_endian(-rounded))] + +patched = 0 +for path in pathlib.Path(sys.argv[1]).rglob('*.avro'): + manifest = path.read_bytes() + replaced = manifest + for old, new in substitutions: + replaced = replaced.replace(old, new) + if replaced != manifest: + path.write_bytes(replaced) + patched += 1 + +if patched != 2: + raise SystemExit(f'patched {patched} manifests, expected one per data file') +PATCH_MANIFESTS + +# A table function reads the patched manifests afresh; the `high` table above still holds the ones +# it wrote. Every bound is now `±1.0`, so one data file declares `[0, 1)` and the other `(-1, 0]`. +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --query " + SELECT count() FROM icebergLocal('${ROOT}/high/'); + SELECT count() FROM icebergLocal('${ROOT}/high/') WHERE d = toDecimal128('0.99', 38); + SELECT count() FROM icebergLocal('${ROOT}/high/') WHERE d = toDecimal128('-0.99', 38);" + +echo '--- B1 the saturated bounds prune the file that cannot hold the probe ---' +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --query " + SELECT count() FROM icebergLocal('${ROOT}/high/') WHERE d = toDecimal128('0.5', 38) + SETTINGS log_comment = '${CLICKHOUSE_DATABASE}_rounded_pos'; + SELECT count() FROM icebergLocal('${ROOT}/high/') WHERE d = toDecimal128('-0.5', 38) + SETTINGS log_comment = '${CLICKHOUSE_DATABASE}_rounded_neg'; + SELECT count() FROM icebergLocal('${ROOT}/high/') WHERE d = toDecimal128('0.0', 38) + SETTINGS log_comment = '${CLICKHOUSE_DATABASE}_rounded_zero'; + SYSTEM FLUSH LOGS query_log;" +report IcebergMinMaxIndexPrunedFiles rounded_pos rounded_neg rounded_zero + +${CLICKHOUSE_CLIENT} --query " + DROP TABLE IF EXISTS high SYNC; + DROP TABLE IF EXISTS low SYNC;" diff --git a/tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.reference b/tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.reference new file mode 100644 index 0000000000000..a3a948fe51786 --- /dev/null +++ b/tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.reference @@ -0,0 +1,7 @@ +strict read +ICEBERG_SPECIFICATION_VIOLATION +tolerant compaction +3 +DateTime64(6, \'UTC\') 2024-01-01 00:00:00.000000 1 +DateTime64(6, \'UTC\') 2024-01-02 00:00:00.000000 2 +DateTime64(6, \'UTC\') 2024-01-03 00:00:00.000000 3 diff --git a/tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.sh b/tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.sh new file mode 100755 index 0000000000000..9a72f718081a2 --- /dev/null +++ b/tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel +# - no-fasttest: requires `IcebergLocal` (USE_AVRO build option) +# - no-parallel: uses DETACH/ATTACH which serializes per database + +# A manifest file header keeps a copy of the schema the data was written with, while metadata.json is +# the authoritative source. The conflict below is produced the way broken writers produce it: the +# manifest headers keep the `timestamp` type, while metadata.json binds the same schema-id to +# `timestamptz`. +# +# A read registers the metadata.json schemas before it walks the manifests, so a strict read meets a +# header conflicting with a known metadata.json copy and fails with `ICEBERG_SPECIFICATION_VIOLATION`. +# `OPTIMIZE TABLE ... MANIFEST` walks the current manifests first and registers the metadata.json +# schemas only afterwards, so on a freshly attached table the schemas that came from the manifests are registered before +# metadata.json: the metadata.json copy has to replace them and the compaction has to succeed. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The data is written as `timestamp` (parsed in the session time zone) and read back as `timestamptz` +# (rendered in UTC), so the session time zone has to be UTC for the values to round-trip unchanged. +CLICKHOUSE_CLIENT="${CLICKHOUSE_CLIENT} --session_timezone UTC" + +TABLE="t_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_PATH="${USER_FILES_PATH}/${TABLE}/" + +trap 'rm -rf "${TABLE_PATH}" 2>/dev/null' EXIT + +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${TABLE} (ts DateTime64(6), v Int32) + ENGINE = IcebergLocal('${TABLE_PATH}', 'Parquet') +" + +# One INSERT per manifest, so the table has enough manifests to compact. +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --use_iceberg_metadata_files_cache=0 -m --query " + INSERT INTO ${TABLE} VALUES ('2024-01-01 00:00:00', 1); + INSERT INTO ${TABLE} VALUES ('2024-01-02 00:00:00', 2); + INSERT INTO ${TABLE} VALUES ('2024-01-03 00:00:00', 3); +" + +# Rebind the schema-id in metadata.json to a schema that differs from the schemas that came from the manifests. +LATEST_METADATA=$(ls "${TABLE_PATH}"metadata/v*.metadata.json | sed 's#.*/v##;s#\.metadata.json##' | sort -n | tail -1) +python3 - "${TABLE_PATH}metadata/v${LATEST_METADATA}.metadata.json" <<'PY' +import json, sys +path = sys.argv[1] +meta = json.load(open(path)) +for schema in meta["schemas"]: + for field in schema["fields"]: + if field["type"] == "timestamp": + field["type"] = "timestamptz" +json.dump(meta, open(path, "w")) +PY + +# Drop the in-memory metadata and the shared schema processor, so the read starts from an empty +# processor and registers the metadata.json schemas before it meets the schemas that came from the manifests. +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --query "DETACH TABLE ${TABLE}" +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --send_logs_level=fatal --query "ATTACH TABLE ${TABLE}" + +echo "strict read" +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --iceberg_tolerate_conflicting_manifest_schemas=0 \ + --query "SELECT count() FROM ${TABLE}" 2>&1 \ + | grep -oF 'ICEBERG_SPECIFICATION_VIOLATION' | head -n1 + +# Start from an empty processor again, so the compaction registers the schemas that came from the manifests first +# and the metadata.json schemas replace them. +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --query "DETACH TABLE ${TABLE}" +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --send_logs_level=fatal --query "ATTACH TABLE ${TABLE}" + +echo "tolerant compaction" +${CLICKHOUSE_CLIENT} --allow_experimental_iceberg_compaction=1 --use_iceberg_metadata_files_cache=0 \ + --iceberg_tolerate_conflicting_manifest_schemas=1 --send_logs_level=error \ + --query "OPTIMIZE TABLE ${TABLE} MANIFEST SETTINGS iceberg_manifest_min_count_to_compact=2" + +# The compaction rewrote three data manifests into one, and the data is intact. +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --iceberg_tolerate_conflicting_manifest_schemas=1 --query " + SELECT count() FROM ${TABLE}; + SELECT toTypeName(ts), ts, v FROM ${TABLE} ORDER BY v; +" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${TABLE} SYNC" diff --git a/tests/queries/0_stateless/05213_explain_hide_secrets_across_steps.reference b/tests/queries/0_stateless/05213_explain_hide_secrets_across_steps.reference new file mode 100644 index 0000000000000..0574372757b1d --- /dev/null +++ b/tests/queries/0_stateless/05213_explain_hide_secrets_across_steps.reference @@ -0,0 +1,9 @@ +0 1 +0 +0 1 +0 +0 1 +1 +0 +0 1 +0 1 diff --git a/tests/queries/0_stateless/05213_explain_hide_secrets_across_steps.sql b/tests/queries/0_stateless/05213_explain_hide_secrets_across_steps.sql new file mode 100644 index 0000000000000..17bc2285af964 --- /dev/null +++ b/tests/queries/0_stateless/05213_explain_hide_secrets_across_steps.sql @@ -0,0 +1,72 @@ +-- Tags: no-fasttest +-- no-fasttest: encrypt/decrypt/HMAC require the OpenSSL-based functions, absent in the fast test build. + +-- A secret argument can be bound to a constant that is computed in another plan step: a derived +-- table, or the opposite side of a JOIN. Such an argument is an INPUT while the query is planned, and +-- plan optimizations later replace it with (or inline) the constant. The mask must travel with the +-- argument so the pretty ActionsDAG dump never prints the value. viewExplain lets us assert that no +-- fragment leaks without dumping the config-dependent plan into the reference. + +SET enable_analyzer = 1; +SET format_display_secrets_in_show_and_select = 0; + +-- Key from a derived table, consumed by WHERE: the filter is merged with the projection of the +-- subquery and the masked copy of the constant is deduplicated with an unmasked twin. +SELECT countIf(explain LIKE '%SEKRIT_SUBQKEY%') AS derived_table_leaks, countIf(explain LIKE '%[HIDDEN]%') > 0 AS derived_table_hidden +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1', ( + SELECT number FROM (SELECT number, 'SEKRIT_SUBQKEY' AS k FROM numbers(1)) AS s + WHERE empty(HMAC('sha256', toString(number), s.k)))); + +-- Same, with the key decrypted in the subquery: the plaintext exists only as a folded constant. +SELECT countIf(explain LIKE '%SEKRIT_DECRYPTED%') AS derived_table_decrypt_leaks +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1', ( + SELECT number FROM ( + SELECT number, decrypt('aes-128-ecb', encrypt('aes-128-ecb', 'SEKRIT_DECRYPTED', '0123456789abcdef'), '0123456789abcdef') AS k + FROM numbers(1)) AS s + WHERE empty(HMAC('sha256', toString(number), s.k)))); + +-- Key from the opposite JOIN side: the ON condition is pushed down to the other side and the +-- constant is inlined in place of the unbound input. +SELECT countIf(explain LIKE '%SEKRIT_JOINKEY%') AS join_leaks, countIf(explain LIKE '%[HIDDEN]%') > 0 AS join_hidden +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1', ( + SELECT n.number FROM numbers(1) AS n + INNER JOIN (SELECT 'SEKRIT_JOINKEY' AS k) AS s ON HMAC('sha256', toString(n.number), s.k) = '')); + +SELECT countIf(explain LIKE '%SEKRIT_JOINKEY%') AS join_logical_leaks +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1, keep_logical_steps = 1', ( + SELECT n.number FROM numbers(1) AS n + INNER JOIN (SELECT 'SEKRIT_JOINKEY' AS k) AS s ON HMAC('sha256', toString(n.number), s.k) = '')); + +-- A non-constant key from the opposite JOIN side stays an input; the pretty dump must not expand it +-- into the expression the other side computes it from. The slot is hidden whole, as SHOW CREATE does. +SELECT countIf(explain LIKE '%SEKRIT_MATKEY%') AS join_column_leaks, countIf(explain LIKE '%HMAC(''sha256'', toString(number), [HIDDEN])%') AS join_column_hidden +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1', ( + SELECT n.number FROM numbers(1) AS n + INNER JOIN (SELECT materialize('SEKRIT_MATKEY') AS k) AS s ON HMAC('sha256', toString(n.number), s.k) = '')); + +-- Every argument in a secret position is hidden whole, a column expression included: the secret +-- span of encrypt covers the plaintext as well as the key. +SELECT countIf(explain LIKE '%encrypt(''aes-128-ecb'', [HIDDEN], [HIDDEN])%') AS secret_span_hidden +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1', ( + SELECT encrypt('aes-128-ecb', toString(number), 'SEKRIT_LITERALKEY') FROM numbers(1))); + +-- A literal set inside a secret slot: the IN operator formats its set without going through the +-- per-node rendering, so the slot must be hidden before descending into the argument. +SELECT countIf(explain LIKE '%SEKRIT_IN%') AS in_set_leaks +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1', ( + SELECT number FROM numbers(1) + WHERE empty(HMAC('sha256', toString(number), if(toString(number) IN ('SEKRIT_IN1', 'SEKRIT_IN2'), 'a', 'b'))))); + +-- The exact shape from the report: the HMAC key is a decrypt() of a hex ciphertext with an IV, +-- computed on the opposite JOIN side. +SELECT countIf(explain LIKE '%JOIN_SIDE_SECRET_PLAINTEXT%') AS report_join_leaks, countIf(explain LIKE '%[HIDDEN]%') > 0 AS report_join_hidden +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1', ( + SELECT n.number FROM numbers(1) AS n + INNER JOIN ( + SELECT decrypt('aes-128-cbc', unhex(hex(encrypt('aes-128-cbc', 'JOIN_SIDE_SECRET_PLAINTEXT', '0123456789abcdef', 'abcdef9876543210'))), '0123456789abcdef', 'abcdef9876543210') AS k + ) AS s ON HMAC('sha256', toString(n.number), s.k) = '')); + +-- The report's positive control: a key bound through a WITH alias in the same scope. +SELECT countIf(explain LIKE '%WITH_ALIAS_SECRET%') AS with_alias_leaks, countIf(explain LIKE '%[HIDDEN]%') > 0 AS with_alias_hidden +FROM viewExplain('EXPLAIN PLAN', 'actions = 1, pretty = 1', ( + WITH 'WITH_ALIAS_SECRET' AS k SELECT number FROM numbers(1) WHERE empty(HMAC('sha256', toString(number), k)))); diff --git a/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.reference b/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.reference new file mode 100644 index 0000000000000..a9796c468d97d --- /dev/null +++ b/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.reference @@ -0,0 +1,3 @@ +the maximum size of multipart/form-data +30 +50 diff --git a/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.sh b/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.sh new file mode 100755 index 0000000000000..4d497382264e5 --- /dev/null +++ b/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `http_max_multipart_form_data_size` cannot be set through URL parameters, so it comes from a user. +user="u_${CLICKHOUSE_DATABASE}" +${CLICKHOUSE_CLIENT} -q "CREATE USER $user IDENTIFIED WITH plaintext_password BY 'pw' SETTINGS http_max_multipart_form_data_size = 100" +${CLICKHOUSE_CLIENT} -q "GRANT CREATE TEMPORARY TABLE, SELECT ON *.* TO $user" + +url="${CLICKHOUSE_URL}&user=${user}&password=pw" +part="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_part.tsv" +python3 -c 'import sys; sys.stdout.write("x\n" * 30)' > "$part" + +# Two parts of 60 bytes: each one fits the 100 byte limit on its own, the form as a whole does not. +${CLICKHOUSE_CURL} -sS -F "a=@$part" -F "b=@$part" \ + "${url}&query=SELECT+count()+FROM+a&a_structure=s+String&b_structure=s+String&a_format=TSV&b_format=TSV" \ + | grep -o -m1 'the maximum size of multipart/form-data' + +# One part of the same size stays under the limit. +${CLICKHOUSE_CURL} -sS -F "a=@$part" \ + "${url}&query=SELECT+count()+FROM+a&a_structure=s+String&a_format=TSV" + +# A part that ends exactly at the limit is not over it. +exact="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_exact.tsv" +python3 -c 'import sys; sys.stdout.write("x\n" * 50)' > "$exact" +${CLICKHOUSE_CURL} -sS -F "a=@$exact" \ + "${url}&query=SELECT+count()+FROM+a&a_structure=s+String&a_format=TSV" + +${CLICKHOUSE_CLIENT} -q "DROP USER $user" +rm -f "$part" "$exact" diff --git a/tests/queries/0_stateless/05215_uniq_theta_short_state.reference b/tests/queries/0_stateless/05215_uniq_theta_short_state.reference new file mode 100644 index 0000000000000..f5f31343b08b4 --- /dev/null +++ b/tests/queries/0_stateless/05215_uniq_theta_short_state.reference @@ -0,0 +1,16 @@ +char(8, 0, 1, 3, 0, 0, 0, 0, 0) -> at least 24 bytes expected, actual 8 +char(8, 0, 2, 3, 0, 0, 0, 0, 0) -> seed hash mismatch +char(8, 0, 3, 3, 0, 0, 0, 0, 0) -> at least 16 bytes expected, actual 8 +char(8, 0, 4, 3, 0, 0, 0, 0, 0) -> seed hash mismatch +char(8, 0, 5, 3, 0, 0, 0, 0, 0) -> unsupported serial version 5 +char(8, 2, 2, 3, 0, 0, 0, 204, 147) -> at least 16 bytes expected, actual 8 +char(8, 3, 2, 3, 0, 0, 0, 204, 147) -> at least 24 bytes expected, actual 8 +char(8, 2, 3, 3, 0, 0, 0, 204, 147) -> at least 16 bytes expected, actual 8 +char(8, 3, 3, 3, 0, 0, 0, 204, 147) -> at least 24 bytes expected, actual 8 +char(8, 2, 4, 3, 0, 0, 0, 204, 147) -> at least 16 bytes expected, actual 8 +1000 +1000 +100 +1 0 +17 1 +33 2 diff --git a/tests/queries/0_stateless/05215_uniq_theta_short_state.sh b/tests/queries/0_stateless/05215_uniq_theta_short_state.sh new file mode 100755 index 0000000000000..81eced6b8d120 --- /dev/null +++ b/tests/queries/0_stateless/05215_uniq_theta_short_state.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest -- compiled w/o datasketches + +# `compact_theta_sketch_parser::parse` verifies that the buffer holds 8 bytes and then reads header +# fields that lie beyond them - `num_entries` at offset 8, `theta` at offset 16 - before it checks the +# size again. A state shorter than that, which any `CAST` from a string can produce, was therefore read +# past its end (an ASan heap-buffer-overflow), and the out-of-bounds value decided the size the parser +# reported as missing: the same query answered `at least 86160 bytes expected` on one run and something +# else on the next. Every size below is now the one the parser really needs before it can validate +# anything, and the same on every run. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The outcome of one state, reduced to the part that used to come from outside the buffer. The client +# prints the message of the exception rather than only its code. +outcome() +{ + ${CLICKHOUSE_CLIENT} --query "SELECT finalizeAggregation(CAST($1, 'AggregateFunction(uniqTheta, UInt64)'))" 2>&1 \ + | grep -o -E "at least [0-9]+ bytes expected, actual [0-9]+|seed hash mismatch|unsupported serial version [0-9]+|^[0-9]+$" \ + | head -n 1 +} + +# The leading byte of each `char` is the length of the state that follows it. All of these states are +# eight bytes long - the most the parser checks for before it reads further out. +# +# The four states the fuzzer produced come first, and their seed hash is zero. Serial versions 2 and 4 +# compare that against the one they compute and stop at `check_seed_hash`; serial version 1 computes +# the seed hash itself instead of comparing it, and serial version 3 reads `num_entries` before it +# compares, so those two are refused for their size. Serial version 5 does not exist and is refused +# outright. +for STATE in \ + "char(8, 0, 1, 3, 0, 0, 0, 0, 0)" \ + "char(8, 0, 2, 3, 0, 0, 0, 0, 0)" \ + "char(8, 0, 3, 3, 0, 0, 0, 0, 0)" \ + "char(8, 0, 4, 3, 0, 0, 0, 0, 0)" \ + "char(8, 0, 5, 3, 0, 0, 0, 0, 0)" +do + echo "${STATE} -> $(outcome "${STATE}")" +done + +# `204, 147` at offset 6 is the seed hash of the default seed, so these states get past +# `check_seed_hash` and reach every one of the reads that happen before the size is validated: +# `num_entries` alone for two preamble longs, `num_entries` and `theta` for three. In particular a +# serial version 2 state with two preamble longs is refused rather than taken for an empty sketch on +# the strength of bytes it does not have. +for STATE in \ + "char(8, 2, 2, 3, 0, 0, 0, 204, 147)" \ + "char(8, 3, 2, 3, 0, 0, 0, 204, 147)" \ + "char(8, 2, 3, 3, 0, 0, 0, 204, 147)" \ + "char(8, 3, 3, 3, 0, 0, 0, 204, 147)" \ + "char(8, 2, 4, 3, 0, 0, 0, 204, 147)" +do + echo "${STATE} -> $(outcome "${STATE}")" +done + +# A state that a `uniqTheta` aggregation wrote still reads back. +${CLICKHOUSE_CLIENT} --query " + SELECT uniqTheta(number) FROM numbers(1000); + SELECT uniqThetaMerge(s) FROM (SELECT uniqThetaState(number) AS s FROM numbers(1000)); + SELECT finalizeAggregation(CAST(unhex(hex(uniqThetaState(number))), 'AggregateFunction(uniqTheta, UInt64)')) FROM numbers(100); +" + +# A short state cannot be refused for its length alone: a single-entry sketch is sixteen bytes long, +# below the twenty-four the parser reads for other headers before it validates anything. The length of +# each state is printed next to the value it reads back as - it counts the one-byte length prefix, so +# the sketch itself is one byte smaller. +${CLICKHOUSE_CLIENT} --query " + SELECT + intDiv(length(hex(s)), 2) AS serialized_size, + finalizeAggregation(CAST(unhex(hex(s)), 'AggregateFunction(uniqTheta, UInt64)')) + FROM + ( + SELECT rows, uniqThetaStateIf(number, number < rows) AS s + FROM numbers(2) AS n, (SELECT arrayJoin([0, 1, 2]) AS rows) AS r + GROUP BY rows + ORDER BY rows + )" diff --git a/tests/queries/0_stateless/05217_custom_disk_local_path_alter.reference b/tests/queries/0_stateless/05217_custom_disk_local_path_alter.reference new file mode 100644 index 0000000000000..1f63a15bb0843 --- /dev/null +++ b/tests/queries/0_stateless/05217_custom_disk_local_path_alter.reference @@ -0,0 +1,7 @@ +local +1 +no settings clause +1 +constraints +1 +unrelated change allowed diff --git a/tests/queries/0_stateless/05217_custom_disk_local_path_alter.sh b/tests/queries/0_stateless/05217_custom_disk_local_path_alter.sh new file mode 100755 index 0000000000000..a0cae4db32a75 --- /dev/null +++ b/tests/queries/0_stateless/05217_custom_disk_local_path_alter.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-replicated-database +# Tag no-fasttest: custom disks are not configured in fasttest +# Tag no-replicated-database: creates a user and a settings profile + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Every location on the local filesystem that a disk defined in SQL names has to be inside +# `custom_local_disks_base_directory`. `CREATE TABLE` checks that; `ALTER TABLE ... MODIFY SETTING disk` +# has to check it the same way. +# In this release the check runs after the disk is created, and does not cover object storage disks over +# the local filesystem, so only the rejection of the statement is checked here. + +OUTSIDE="${USER_FILES_PATH}/05217_outside_${CLICKHOUSE_DATABASE}" +INSIDE="${CLICKHOUSE_DISKS_FILES}/05217_${CLICKHOUSE_DATABASE}" + +# Prints 1 if the statement was rejected by the fence. +function check_rejected() +{ + local query=$1 + $CLICKHOUSE_CLIENT -q "$query" 2>&1 | grep -m1 -c -F "must be inside" +} + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_05217" +$CLICKHOUSE_CLIENT -q " + CREATE TABLE t_05217 (x UInt64) ENGINE = MergeTree ORDER BY x + SETTINGS disk = disk(name = '05217_ok_${CLICKHOUSE_DATABASE}', type = local, path = '${INSIDE}_ok/')" + +# `ALTER TABLE ... MODIFY SETTING disk = disk(...)` used to resolve the definition as if it came from +# stored metadata, which registered the disk unchecked, before the check of the statement ran. +echo "local" +check_rejected " + ALTER TABLE t_05217 MODIFY SETTING disk = disk(name = '05217_local_${CLICKHOUSE_DATABASE}', type = local, path = '${OUTSIDE}_local/')" + +$CLICKHOUSE_CLIENT -q "DROP TABLE t_05217" + +# A table created without a SETTINGS clause goes through the same check. +echo "no settings clause" +$CLICKHOUSE_CLIENT -q "CREATE TABLE t_05217_plain (x UInt64) ENGINE = MergeTree ORDER BY x" +check_rejected " + ALTER TABLE t_05217_plain MODIFY SETTING disk = disk(name = '05217_plain_${CLICKHOUSE_DATABASE}', type = local, path = '${OUTSIDE}_plain/')" +$CLICKHOUSE_CLIENT -q "DROP TABLE t_05217_plain" + +# The settings constraints compare the `disk` the table has after the `ALTER`, not one left unresolved: +# a constant `merge_tree_disk` rejects a change of the disk and nothing else. +echo "constraints" +user="u_05217_${CLICKHOUSE_DATABASE}" +profile="p_05217_${CLICKHOUSE_DATABASE}" +$CLICKHOUSE_CLIENT -q " + DROP USER IF EXISTS ${user}; + DROP SETTINGS PROFILE IF EXISTS ${profile}; + CREATE USER ${user} IDENTIFIED WITH no_password; + GRANT ALTER ON ${CLICKHOUSE_DATABASE}.* TO ${user}; + CREATE SETTINGS PROFILE ${profile} SETTINGS merge_tree_disk CONST TO ${user}; + CREATE TABLE t_05217_default (x UInt64) ENGINE = MergeTree ORDER BY x; + CREATE TABLE t_05217_disk (x UInt64) ENGINE = MergeTree ORDER BY x SETTINGS disk = 'local_disk';" + +$CLICKHOUSE_CLIENT --user "${user}" -q "ALTER TABLE t_05217_default MODIFY SETTING disk = 'local_disk'" 2>&1 \ + | grep -m1 -c -F "Setting disk should not be changed" +$CLICKHOUSE_CLIENT --user "${user}" -q "ALTER TABLE t_05217_disk MODIFY SETTING min_bytes_for_wide_part = 0" && echo "unrelated change allowed" + +$CLICKHOUSE_CLIENT -q " + DROP TABLE t_05217_default; + DROP TABLE t_05217_disk; + DROP SETTINGS PROFILE ${profile}; + DROP USER ${user};" + +# In this release a rejected disk still creates its directory, so remove what the statements above left. +rm -rf "${OUTSIDE}"_* diff --git a/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.reference b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.reference new file mode 100644 index 0000000000000..dbf3e42ddd848 --- /dev/null +++ b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.reference @@ -0,0 +1,84 @@ +-- the analyzer wraps the subcolumn into a cast to String +1 +-- mapValues: the index prunes granules through the cast +Granules: 2/2 +Name: idx +Granules: 1/2 +Granules: 2/2 +Name: idx +Granules: 1/2 +-- mapValues: results match the scan +idx 1 +idx 3 +scan 1 +scan 3 +idx 1 +idx 2 +scan 1 +scan 2 +idx 3 +scan 3 +-- mapValues is only a hint: a value that belongs to another key does not match +idx 0 +-- mapValues on LowCardinality(Nullable) values: the cast to Nullable(String) is looked through +Granules: 2/2 +Name: idx +Granules: 1/2 +idx 3 +scan 3 +-- String column: conversions that only add Nullable or LowCardinality are looked through +CAST Nullable Granules: 2/2 +CAST Nullable Name: idx +CAST Nullable Granules: 1/2 +CAST LowCardinality Granules: 2/2 +CAST LowCardinality Name: idx +CAST LowCardinality Granules: 1/2 +toNullable Granules: 2/2 +toNullable Name: idx +toNullable Granules: 1/2 +toLowCardinality Granules: 2/2 +toLowCardinality Name: idx +toLowCardinality Granules: 1/2 +nested Granules: 2/2 +nested Name: idx +nested Granules: 1/2 +idx 1 +idx 2 +scan 1 +scan 2 +not idx 3 +not idx 4 +not scan 3 +not scan 4 +in idx 1 +in idx 3 +in scan 1 +in scan 3 +-- join_use_nulls pushes the filter down to the outer side as toNullable(column) +1 +3 +-- Nullable column: adding LowCardinality is looked through +Granules: 2/2 +Name: idx +Granules: 1/2 +idx 1 +idx 2 +scan 1 +scan 2 +-- Nullable column: dropping Nullable is looked through, the cast does not throw if the granule with NULL is skipped +Granules: 2/2 +Name: idx +Granules: 1/2 +idx 1 +idx 2 +-- Array column: a cast that only wraps the elements is looked through +Array(Nullable) Granules: 2/2 +Array(Nullable) Name: idx +Array(Nullable) Granules: 1/2 +Array(LowCardinality) Granules: 2/2 +Array(LowCardinality) Name: idx +Array(LowCardinality) Granules: 1/2 +idx 1 +idx 2 +scan 1 +scan 2 diff --git a/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql new file mode 100644 index 0000000000000..9a5833c380f14 --- /dev/null +++ b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql @@ -0,0 +1,157 @@ +-- Tags: no-parallel-replicas +-- Tag no-parallel-replicas -- direct read is not compatible with parallel replicas + +-- `arrayElement` on a `Map(K, LowCardinality(V))` returns `V`, while the subcolumn `m.key_` is `LowCardinality(V)`. +-- Hence `optimize_functions_to_subcolumns` rewrites `m['key'] = 'value'` into `_CAST(m.key_, 'V') = 'value'`, +-- and the text index has to look through the cast: for an index on `mapValues(m)`. +-- The same applies to every conversion that does not change a non-NULL value: adding or dropping `LowCardinality` or +-- `Nullable` (adding also as `toNullable`, which `join_use_nulls` emits in pushed-down filters), at any depth of `Array`. + +SET explain_query_plan_default = 'legacy'; +SET enable_analyzer = 1; +SET optimize_functions_to_subcolumns = 1; +SET use_skip_indexes = 1; +SET query_plan_direct_read_from_text_index = 1; + +DROP TABLE IF EXISTS tab_values; + +CREATE TABLE tab_values +( + id UInt32, + m Map(String, LowCardinality(String)), + INDEX idx mapValues(m) TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; + +INSERT INTO tab_values VALUES (1, {'level':'error','msg':'disk is full'}), (2, {'level':'warn','msg':'disk is slow'}), (3, {'level':'error','msg':'network is down'}), (4, {}); + +SELECT '-- the analyzer wraps the subcolumn into a cast to String'; +SELECT count() FROM (EXPLAIN QUERY TREE run_passes = 1 SELECT id FROM tab_values WHERE m['level'] = 'warn') WHERE explain LIKE '%function_name: _CAST%'; + +SELECT '-- mapValues: the index prunes granules through the cast'; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_values WHERE m['level'] = 'warn') WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_values WHERE hasToken(m['msg'], 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; + +SELECT '-- mapValues: results match the scan'; +SELECT 'idx', id FROM tab_values WHERE m['level'] = 'error' ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_values WHERE m['level'] = 'error' ORDER BY id SETTINGS use_skip_indexes = 0; +SELECT 'idx', id FROM tab_values WHERE hasToken(m['msg'], 'disk') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_values WHERE hasToken(m['msg'], 'disk') ORDER BY id SETTINGS use_skip_indexes = 0; +SELECT 'idx', id FROM tab_values WHERE m['msg'] LIKE '%down%' ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_values WHERE m['msg'] LIKE '%down%' ORDER BY id SETTINGS use_skip_indexes = 0; +SELECT '-- mapValues is only a hint: a value that belongs to another key does not match'; +SELECT 'idx', count() FROM tab_values WHERE m['level'] = 'disk' SETTINGS force_data_skipping_indices = 'idx'; + +DROP TABLE tab_values; + +-- A `mapValues` index on `LowCardinality(Nullable(String))` values: the analyzer casts to `Nullable(String)`. +DROP TABLE IF EXISTS tab_values_n; + +CREATE TABLE tab_values_n +( + id UInt32, + m Map(String, LowCardinality(Nullable(String))), + INDEX idx mapValues(m) TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; + +INSERT INTO tab_values_n VALUES (1, {'level':'error'}), (2, {'level':'error'}), (3, {'level':'warn'}), (4, {'level':NULL}); + +SELECT '-- mapValues on LowCardinality(Nullable) values: the cast to Nullable(String) is looked through'; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_values_n WHERE m['level'] = 'warn') WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'idx', id FROM tab_values_n WHERE m['level'] = 'warn' ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_values_n WHERE m['level'] = 'warn' ORDER BY id SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_values_n; + +-- Plain columns with explicit conversions. +DROP TABLE IF EXISTS tab_s; + +CREATE TABLE tab_s +( + id UInt32, + s String, + INDEX idx s TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; + +INSERT INTO tab_s VALUES (1, 'disk is full'), (2, 'disk is slow'), (3, 'network is down'), (4, ''); + +SELECT '-- String column: conversions that only add Nullable or LowCardinality are looked through'; +SELECT 'CAST Nullable', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_s WHERE hasToken(CAST(s, 'Nullable(String)'), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'CAST LowCardinality', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_s WHERE hasToken(CAST(s, 'LowCardinality(String)'), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'toNullable', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_s WHERE hasToken(toNullable(s), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'toLowCardinality', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_s WHERE hasToken(toLowCardinality(s), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'nested', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_s WHERE hasToken(toNullable(toLowCardinality(s)), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'idx', id FROM tab_s WHERE hasToken(toNullable(s), 'disk') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_s WHERE hasToken(toNullable(s), 'disk') ORDER BY id SETTINGS use_skip_indexes = 0; +SELECT 'not idx', id FROM tab_s WHERE NOT hasToken(toNullable(s), 'disk') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'not scan', id FROM tab_s WHERE NOT hasToken(toNullable(s), 'disk') ORDER BY id SETTINGS use_skip_indexes = 0; +SELECT 'in idx', id FROM tab_s WHERE toNullable(s) IN ('disk is full', 'network is down') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'in scan', id FROM tab_s WHERE toNullable(s) IN ('disk is full', 'network is down') ORDER BY id SETTINGS use_skip_indexes = 0; + +SELECT '-- join_use_nulls pushes the filter down to the outer side as toNullable(column)'; +DROP TABLE IF EXISTS tab_ids; +CREATE TABLE tab_ids (id UInt32) ENGINE = MergeTree ORDER BY id; +INSERT INTO tab_ids SELECT number FROM numbers(1, 4); +-- `query_plan_convert_outer_join_to_inner_join` is pinned because the filter is pushed below the join only +-- when the LEFT JOIN becomes an INNER one; otherwise it stays above the join and no index is consulted. +SELECT count() FROM (EXPLAIN indexes = 1 SELECT l.id FROM tab_ids AS l LEFT JOIN tab_s AS r ON l.id = r.id WHERE hasToken(r.s, 'network') SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1) WHERE explain LIKE '%Name: idx%'; +SELECT l.id FROM tab_ids AS l LEFT JOIN tab_s AS r ON l.id = r.id WHERE hasToken(r.s, 'network') ORDER BY l.id SETTINGS join_use_nulls = 1; +DROP TABLE tab_ids; + +DROP TABLE tab_s; + +DROP TABLE IF EXISTS tab_ns; + +CREATE TABLE tab_ns +( + id UInt32, + s Nullable(String), + INDEX idx s TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; + +INSERT INTO tab_ns VALUES (1, 'disk is full'), (2, 'disk is slow'), (3, 'network is down'), (4, NULL); + +SELECT '-- Nullable column: adding LowCardinality is looked through'; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_ns WHERE hasToken(CAST(s, 'LowCardinality(Nullable(String))'), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'idx', id FROM tab_ns WHERE hasToken(CAST(s, 'LowCardinality(Nullable(String))'), 'disk') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_ns WHERE hasToken(CAST(s, 'LowCardinality(Nullable(String))'), 'disk') ORDER BY id SETTINGS use_skip_indexes = 0; + +SELECT '-- Nullable column: dropping Nullable is looked through, the cast does not throw if the granule with NULL is skipped'; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_ns WHERE hasToken(CAST(s, 'String'), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'idx', id FROM tab_ns WHERE hasToken(CAST(s, 'String'), 'disk') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_ns WHERE hasToken(CAST(s, 'String'), 'disk') ORDER BY id SETTINGS use_skip_indexes = 0; -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } + +DROP TABLE tab_ns; + +DROP TABLE IF EXISTS tab_arr; + +CREATE TABLE tab_arr +( + id UInt32, + arr Array(String), + INDEX idx arr TYPE text(tokenizer = 'array') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; + +INSERT INTO tab_arr VALUES (1, ['disk', 'full']), (2, ['disk', 'slow']), (3, ['network', 'down']), (4, []); + +SELECT '-- Array column: a cast that only wraps the elements is looked through'; +SELECT 'Array(Nullable)', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_arr WHERE has(CAST(arr, 'Array(Nullable(String))'), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'Array(LowCardinality)', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_arr WHERE has(CAST(arr, 'Array(LowCardinality(String))'), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'idx', id FROM tab_arr WHERE has(CAST(arr, 'Array(Nullable(String))'), 'disk') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_arr WHERE has(CAST(arr, 'Array(Nullable(String))'), 'disk') ORDER BY id SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_arr; diff --git a/tests/queries/0_stateless/05218_allow_executable_tables.reference b/tests/queries/0_stateless/05218_allow_executable_tables.reference new file mode 100644 index 0000000000000..604b153530121 --- /dev/null +++ b/tests/queries/0_stateless/05218_allow_executable_tables.reference @@ -0,0 +1 @@ +x UInt32 diff --git a/tests/queries/0_stateless/05218_allow_executable_tables.sql b/tests/queries/0_stateless/05218_allow_executable_tables.sql new file mode 100644 index 0000000000000..f56bfcf22caae --- /dev/null +++ b/tests/queries/0_stateless/05218_allow_executable_tables.sql @@ -0,0 +1,35 @@ +-- `allow_executable_tables` gates reading through the `executable` table function and from +-- `Executable` and `ExecutablePool` tables. + +SELECT * FROM executable('nonexist.sh', 'TSV', 'x UInt32'); -- { serverError UNSUPPORTED_METHOD } + +SET allow_executable_tables = 0; +SELECT * FROM executable('nonexist.sh', 'TSV', 'x UInt32'); -- { serverError SUPPORT_IS_DISABLED } +DESCRIBE executable('nonexist.sh', 'TSV', 'x UInt32'); + +-- A definition is still accepted and can be managed; only reading it is refused. +CREATE TABLE t_exec_gate (x UInt32) ENGINE = Executable('nonexist.sh', 'TSV'); +CREATE VIEW v_exec_gate AS SELECT * FROM t_exec_gate; +CREATE MATERIALIZED VIEW mv_exec_gate ENGINE = MergeTree ORDER BY x AS SELECT * FROM t_exec_gate; +CREATE TABLE t_exec_pool_gate (x UInt32) ENGINE = ExecutablePool('nonexist.sh', 'TSV'); +DETACH TABLE t_exec_gate; +ATTACH TABLE t_exec_gate; + +SELECT * FROM t_exec_gate; -- { serverError SUPPORT_IS_DISABLED } +SELECT * FROM v_exec_gate; -- { serverError SUPPORT_IS_DISABLED } +SELECT * FROM t_exec_pool_gate; -- { serverError SUPPORT_IS_DISABLED } + +-- The gate is evaluated on the node that runs the read, so a distributed wrapper does not lift it. +SELECT * FROM remote('127.0.0.1', executable('nonexist.sh', 'TSV', 'x UInt32')); -- { serverError SUPPORT_IS_DISABLED } + +-- A statement that reads as part of its own execution is refused too. +CREATE TABLE t_exec_gate_copy ENGINE = MergeTree ORDER BY x AS SELECT * FROM executable('nonexist.sh', 'TSV', 'x UInt32'); -- { serverError SUPPORT_IS_DISABLED } +CREATE MATERIALIZED VIEW mv_exec_gate_populate ENGINE = MergeTree ORDER BY x POPULATE AS SELECT * FROM t_exec_gate; -- { serverError SUPPORT_IS_DISABLED } + +SET allow_executable_tables = 1; +SELECT * FROM t_exec_gate; -- { serverError UNSUPPORTED_METHOD } + +SET allow_executable_tables = 0; +DROP VIEW mv_exec_gate; +DROP VIEW v_exec_gate; +DROP TABLE t_exec_gate; diff --git a/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.reference b/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.reference new file mode 100644 index 0000000000000..8b916e9e97123 --- /dev/null +++ b/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.reference @@ -0,0 +1,5 @@ +refused +refused +refused +0 1 +10 diff --git a/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.sh b/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.sh new file mode 100755 index 0000000000000..454cce40ebff8 --- /dev/null +++ b/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Tags: no-encrypted-storage + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +BACKUP_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_backup +ATTACHED_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_attached +OTHER_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_other + +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE IF EXISTS $BACKUP_DATABASE_NAME; +DROP DATABASE IF EXISTS $ATTACHED_DATABASE_NAME; +DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME; +CREATE DATABASE $BACKUP_DATABASE_NAME; + +CREATE TABLE $BACKUP_DATABASE_NAME.test_table (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO $BACKUP_DATABASE_NAME.test_table SELECT number FROM numbers(10); + +BACKUP DATABASE $BACKUP_DATABASE_NAME TO Disk('backups', '$BACKUP_DATABASE_NAME') FORMAT Null; +""" + +# `PARALLEL WITH` runs its statements as internal queries, and the full `ATTACH DATABASE ... ENGINE = ...` +# form runs in `ATTACH` mode - the same pair the server's own replay of stored metadata shows. The quoted +# locator is accepted only on that replay, never in a statement a user writes, however it is wrapped. +$CLICKHOUSE_CLIENT -q """ +CREATE DATABASE $OTHER_DATABASE_NAME +PARALLEL WITH +ATTACH DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', 'Disk(\\'backups\\', \\'$BACKUP_DATABASE_NAME\\')'); +""" 2>&1 | grep -q -F 'Expected function' && echo 'refused' + +# Each block below must start from the same state: whether the sibling `CREATE DATABASE` of a +# `PARALLEL WITH` whose other statement threw is committed or rolled back depends on how the +# statements were scheduled, and with `max_threads = 1` it is committed. Drop it in between, so the +# refusal under test is what the block observes, not a leftover "database already exists". +$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME" + +$CLICKHOUSE_CLIENT -q """ +CREATE DATABASE $OTHER_DATABASE_NAME +PARALLEL WITH +CREATE DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', 'Disk(\\'backups\\', \\'$BACKUP_DATABASE_NAME\\')'); +""" 2>&1 | grep -q -F 'Expected function' && echo 'refused' + +$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME" + +# A quoted locator can carry credentials, and `PARALLEL WITH` formats its statements before the engine +# refuses them, so the formatted text must hide it: neither the logged query text (the same text an +# `ON CLUSTER` statement puts into the distributed DDL payload) nor the refusal message may carry it. +$CLICKHOUSE_CLIENT -q """ +CREATE DATABASE $OTHER_DATABASE_NAME +PARALLEL WITH +ATTACH DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', 'S3(\\'http://localhost:11111/05218\\', \\'ak\\', \\'SEKRIT_05218\\')'); +""" 2>&1 | grep -q -F 'Expected function' && echo 'refused' + +$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME" + +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS query_log" +$CLICKHOUSE_CLIENT -q """ +SELECT countIf(query LIKE '%SEKRIT_05218%' OR exception LIKE '%SEKRIT_05218%'), countIf(query LIKE '%Backup(%[HIDDEN]%') > 0 +FROM system.query_log +WHERE current_database = currentDatabase() + AND type != 'QueryStart' + AND event_date >= yesterday() AND event_time > now() - INTERVAL 5 MINUTE; +""" + +# The function form goes through the same wrapper. +$CLICKHOUSE_CLIENT -q """ +CREATE DATABASE IF NOT EXISTS $OTHER_DATABASE_NAME +PARALLEL WITH +ATTACH DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', Disk('backups', '$BACKUP_DATABASE_NAME')); + +SELECT count() FROM $ATTACHED_DATABASE_NAME.test_table; +""" + +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE IF EXISTS $ATTACHED_DATABASE_NAME; +DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME; +DROP DATABASE $BACKUP_DATABASE_NAME; +""" diff --git a/tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.reference b/tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.reference new file mode 100644 index 0000000000000..99b8764f122fa --- /dev/null +++ b/tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.reference @@ -0,0 +1 @@ +{"a":{"a":{}}} diff --git a/tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.sql b/tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.sql new file mode 100644 index 0000000000000..6178d3e164508 --- /dev/null +++ b/tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.sql @@ -0,0 +1,8 @@ +SELECT finalizeAggregation(CAST(unhex('0130008008200000000101613000800820000000010161300080082000000000') AS AggregateFunction(any, Dynamic))); + +-- Turning a Dynamic constant into a literal recurses once per JSON level, and the depth comes from the +-- value, so it must be reported rather than overflow the stack. +SELECT finalizeAggregation(CAST(unhex(concat('01', repeat('3000800820000000010161', 4000), '300080082000000000')) AS AggregateFunction(any, Dynamic))); -- { serverError TOO_DEEP_RECURSION } + +-- Constant folding reads the value as a Field, which walks the same nesting through the columns. +SELECT if(finalizeAggregation(CAST(unhex(concat('01', repeat('3000800820000000010161', 4000), '300080082000000000')) AS AggregateFunction(any, Dynamic))), 1, 2); -- { serverError TOO_DEEP_RECURSION } diff --git a/tests/queries/0_stateless/05218_explain_hide_secrets_definer_view_join.reference b/tests/queries/0_stateless/05218_explain_hide_secrets_definer_view_join.reference new file mode 100644 index 0000000000000..c3da9be8f327d --- /dev/null +++ b/tests/queries/0_stateless/05218_explain_hide_secrets_definer_view_join.reference @@ -0,0 +1,76 @@ +-- the attacker has no privilege to display secrets +0 +-- protected_v: SHOW CREATE hides the secret arguments +2 +0 +-- protected_v: SELECT works and returns nothing +0 +-- protected_v: EXPLAIN PLAN actions = 1: lines with the secret +0 +-- protected_v: EXPLAIN PLAN actions = 1, pretty = 1: lines with the secret +0 +-- protected_v: EXPLAIN PLAN header = 1: lines with the secret +0 +-- protected_v: EXPLAIN PLAN actions = 1, pretty = 0, compact = 0: lines with the secret +0 +-- protected_v: EXPLAIN PLAN json = 1, actions = 1: lines with the secret +0 +-- protected_v: EXPLAIN QUERY TREE and EXPLAIN SYNTAX: lines with the secret +0 +0 +-- derived_v: SHOW CREATE hides the secret arguments +2 +0 +-- derived_v: SELECT works and returns nothing +0 +-- derived_v: EXPLAIN PLAN actions = 1: lines with the secret +0 +-- derived_v: EXPLAIN PLAN actions = 1, pretty = 1: lines with the secret +0 +-- derived_v: EXPLAIN PLAN header = 1: lines with the secret +0 +-- derived_v: EXPLAIN PLAN actions = 1, pretty = 0, compact = 0: lines with the secret +0 +-- derived_v: EXPLAIN PLAN json = 1, actions = 1: lines with the secret +0 +-- derived_v: EXPLAIN QUERY TREE and EXPLAIN SYNTAX: lines with the secret +0 +0 +-- pos_literal_v: SHOW CREATE hides the secret arguments +1 +0 +-- pos_literal_v: SELECT works and returns nothing +0 +-- pos_literal_v: EXPLAIN PLAN actions = 1: lines with the secret +0 +-- pos_literal_v: EXPLAIN PLAN actions = 1, pretty = 1: lines with the secret +0 +-- pos_literal_v: EXPLAIN PLAN header = 1: lines with the secret +0 +-- pos_literal_v: EXPLAIN PLAN actions = 1, pretty = 0, compact = 0: lines with the secret +0 +-- pos_literal_v: EXPLAIN PLAN json = 1, actions = 1: lines with the secret +0 +-- pos_literal_v: EXPLAIN QUERY TREE and EXPLAIN SYNTAX: lines with the secret +0 +0 +-- pos_alias_v: SHOW CREATE hides the secret arguments +1 +1 +-- pos_alias_v: SELECT works and returns nothing +0 +-- pos_alias_v: EXPLAIN PLAN actions = 1: lines with the secret +0 +-- pos_alias_v: EXPLAIN PLAN actions = 1, pretty = 1: lines with the secret +0 +-- pos_alias_v: EXPLAIN PLAN header = 1: lines with the secret +0 +-- pos_alias_v: EXPLAIN PLAN actions = 1, pretty = 0, compact = 0: lines with the secret +0 +-- pos_alias_v: EXPLAIN PLAN json = 1, actions = 1: lines with the secret +0 +-- pos_alias_v: EXPLAIN QUERY TREE and EXPLAIN SYNTAX: lines with the secret +0 +0 +-- without the grant the plan is not reachable at all +ACCESS_DENIED diff --git a/tests/queries/0_stateless/05218_explain_hide_secrets_definer_view_join.sh b/tests/queries/0_stateless/05218_explain_hide_secrets_definer_view_join.sh new file mode 100755 index 0000000000000..2fbecdf4fd1f2 --- /dev/null +++ b/tests/queries/0_stateless/05218_explain_hide_secrets_definer_view_join.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-replicated-database +# Tag no-fasttest: the encryption functions are not available in the fast test build +# Tag no-replicated-database: SQL SECURITY DEFINER views and users are set up per-test + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Reproduces the reported scenario: a SQL SECURITY DEFINER view derives its HMAC key with decrypt() +# on the opposite side of a JOIN (or in a derived table). A user holding only SELECT on the view, +# without the displaySecretsInShowAndSelect privilege, must not recover the decrypted key through +# any EXPLAIN flavour, while SHOW CREATE keeps hiding the decrypt() arguments and SELECT stays usable. +# The plans are not dumped into the reference (they depend on the configuration); only the absence +# of the plaintext is checked. + +db=${CLICKHOUSE_DATABASE} +owner="owner_${db}_$RANDOM" +attacker="attacker_${db}_$RANDOM" +nogrant="nogrant_${db}_$RANDOM" +key='0123456789abcdef' +iv='abcdef9876543210' +plaintext='JOIN_SIDE_SECRET_PLAINTEXT' + +ciphertext=$(${CLICKHOUSE_CLIENT} --query "SELECT hex(encrypt('aes-128-cbc', '$plaintext', '$key', '$iv'))") + +${CLICKHOUSE_CLIENT} <&1 | grep -o "ACCESS_DENIED" | head -1 + +for view in protected_v derived_v pos_literal_v pos_alias_v; do + ${CLICKHOUSE_CLIENT} --query "DROP VIEW $db.$view" +done +${CLICKHOUSE_CLIENT} --query "DROP USER $owner, $attacker, $nogrant" diff --git a/tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.reference b/tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.reference new file mode 100644 index 0000000000000..4cd3790c6b883 --- /dev/null +++ b/tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.reference @@ -0,0 +1,8 @@ +--- url --- +60 30 +--- s3 --- +60 30 +--- forwarded queries of url --- +3 3 60 +--- forwarded queries of s3 --- +3 3 60 diff --git a/tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.sh b/tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.sh new file mode 100755 index 0000000000000..c956ed1d537a3 --- /dev/null +++ b/tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, zookeeper +# Tag no-fasttest: needs MinIO and a Replicated table + +# https://github.com/ClickHouse/ClickHouse/issues/120485 +# +# `INSERT INTO SELECT * FROM url(...)` (or `s3(...)`) with parallel replicas enabled: +# `parallel_replicas_for_cluster_engines` converts the plain table function into a cluster storage on the +# initiator, so the INSERT takes the distributed `parallel_distributed_insert_select` path and forwards the +# query to every replica of the cluster. The forwarded query text still named the plain function, and a replica +# running it as a secondary query created a plain storage that expanded the globs and read every file on its +# own instead of taking its share of the read tasks from the initiator, so N replicas inserted the data N times. +# The forwarded query must name the `*Cluster` variant, the same way the SELECT path does. +# +# The three "replicas" of `test_cluster_one_shard_three_replicas_localhost` are all this server, so the +# secondary queries are visible in the local query log. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -eu + +S3_DIR="http://localhost:11111/test/${CLICKHOUSE_DATABASE}/05218" + +# Three files of different sizes: 10 + 20 + 30 = 60 rows, so a duplicated file changes the count. +for i in 1 2 3 +do + $CLICKHOUSE_CLIENT -q "INSERT INTO FUNCTION s3('${S3_DIR}/part_${i}.tsv', 'TSV', 'x UInt32') SELECT number FROM numbers(${i} * 10)" +done + +$CLICKHOUSE_CLIENT -q " + DROP TABLE IF EXISTS dst_05218 SYNC; + CREATE TABLE dst_05218 (x UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/dst_05218', 'r1') ORDER BY x; +" + +SETTINGS="enable_parallel_replicas = 1, automatic_parallel_replicas_mode = 0, max_parallel_replicas = 3, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + parallel_replicas_for_cluster_engines = 1, parallel_distributed_insert_select = 2, log_queries = 1" + +# The query ids and the query log lookup below must isolate this run: a re-run on the same server +# must not pick up the secondary queries of a previous run. +QUERY_ID_SUFFIX="${CLICKHOUSE_DATABASE}_$(date +%s%N)_${RANDOM}" +QUERY_ID_URL="05218_url_${QUERY_ID_SUFFIX}" +QUERY_ID_S3="05218_s3_${QUERY_ID_SUFFIX}" + +echo "--- url ---" +$CLICKHOUSE_CLIENT --query_id "${QUERY_ID_URL}" -q " + INSERT INTO dst_05218 SELECT * FROM url('${S3_DIR}/part_{1..3}.tsv', 'TSV', 'x UInt32') SETTINGS ${SETTINGS}" +$CLICKHOUSE_CLIENT -q "SELECT count(), uniqExact(x) FROM dst_05218" + +echo "--- s3 ---" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE dst_05218" +$CLICKHOUSE_CLIENT --query_id "${QUERY_ID_S3}" -q " + INSERT INTO dst_05218 SELECT * FROM s3('${S3_DIR}/part_{1..3}.tsv', 'TSV', 'x UInt32') SETTINGS ${SETTINGS}" +$CLICKHOUSE_CLIENT -q "SELECT count(), uniqExact(x) FROM dst_05218" + +# The INSERT must really have been distributed: every replica ran the forwarded INSERT, the forwarded +# query names the `*Cluster` function, and the replicas together read every file exactly once. +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS query_log" +for pair in "${QUERY_ID_URL} urlCluster" "${QUERY_ID_S3} s3Cluster" +do + query_id="${pair%% *}" + cluster_function="${pair##* }" + echo "--- forwarded queries of ${cluster_function%Cluster} ---" + $CLICKHOUSE_CLIENT -q " + WITH initial AS + ( + SELECT query_id + FROM system.query_log + WHERE current_database = currentDatabase() + AND query_id = '${query_id}' + AND is_initial_query = 1 + AND type = 'QueryFinish' + AND event_date >= yesterday() + AND event_time >= now() - INTERVAL 10 MINUTE + ) + SELECT + count() AS replicas, + countIf(query ILIKE '%${cluster_function}(%') AS cluster_function_queries, + sum(read_rows) AS rows_read_by_replicas + FROM system.query_log + WHERE initial_query_id IN (SELECT query_id FROM initial) + AND is_initial_query = 0 + AND query_kind = 'Insert' + AND type = 'QueryFinish' + AND event_date >= yesterday() + AND event_time >= now() - INTERVAL 10 MINUTE" +done + +$CLICKHOUSE_CLIENT -q "DROP TABLE dst_05218 SYNC" diff --git a/tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.reference b/tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.reference new file mode 100644 index 0000000000000..06f2d273f643e --- /dev/null +++ b/tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.reference @@ -0,0 +1,102 @@ +present key, subcolumns=0 1 b1 +present key, subcolumns=0 2 +present key, subcolumns=0 3 +present key, subcolumns=0 4 FIRST +present key, subcolumns=1 1 b1 +present key, subcolumns=1 2 +present key, subcolumns=1 3 +present key, subcolumns=1 4 FIRST +absent key, subcolumns=0 1 +absent key, subcolumns=0 2 +absent key, subcolumns=0 3 +absent key, subcolumns=0 4 +absent key, subcolumns=1 1 +absent key, subcolumns=1 2 +absent key, subcolumns=1 3 +absent key, subcolumns=1 4 +empty key, subcolumns=0 1 +empty key, subcolumns=0 2 +empty key, subcolumns=0 3 +empty key, subcolumns=0 4 empty +empty key, subcolumns=1 1 +empty key, subcolumns=1 2 +empty key, subcolumns=1 3 +empty key, subcolumns=1 4 empty +direct subcolumn 1 b1 +direct subcolumn 2 +direct subcolumn 3 +direct subcolumn 4 FIRST +compact, subcolumns=0 1 b1 +compact, subcolumns=0 2 +compact, subcolumns=0 3 +compact, subcolumns=0 4 FIRST +compact, subcolumns=1 1 b1 +compact, subcolumns=1 2 +compact, subcolumns=1 3 +compact, subcolumns=1 4 FIRST +fixed string key, subcolumns=0 1 b1 +fixed string key, subcolumns=0 2 +fixed string key, subcolumns=1 1 b1 +fixed string key, subcolumns=1 2 +wide map, subcolumns=0 17633783096591347703 10736917531793588130 +wide map, subcolumns=1 17633783096591347703 10736917531793588130 +high dictionary position, subcolumns=0 100000 100 +high dictionary position, subcolumns=1 100000 100 +random maps, subcolumns=0 9043450906938593318 +random maps, subcolumns=1 9043450906938593318 +UUID key, subcolumns=0 1 hit +UUID key, subcolumns=0 2 +UUID key, subcolumns=1 1 hit +UUID key, subcolumns=1 2 +IPv4 key, subcolumns=0 1 hit +IPv4 key, subcolumns=0 2 +IPv4 key, subcolumns=1 1 hit +IPv4 key, subcolumns=1 2 +IPv6 key, subcolumns=0 1 hit +IPv6 key, subcolumns=0 2 +IPv6 key, subcolumns=1 1 hit +IPv6 key, subcolumns=1 2 +Int128 key, subcolumns=0 1 hit +Int128 key, subcolumns=0 2 +Int128 key, subcolumns=1 1 hit +Int128 key, subcolumns=1 2 +UInt256 key, subcolumns=0 1 hit +UInt256 key, subcolumns=0 2 +UInt256 key, subcolumns=1 1 hit +UInt256 key, subcolumns=1 2 +Decimal64 key subcolumn 1 hit +Decimal64 key subcolumn 2 +DateTime64 key subcolumn 1 hit +DateTime64 key subcolumn 2 +UUID value, subcolumns=0 1 61f0c404-5cb3-11e7-907b-a6006ad3dba0 +UUID value, subcolumns=0 2 00000000-0000-0000-0000-000000000000 +UUID value, subcolumns=1 1 61f0c404-5cb3-11e7-907b-a6006ad3dba0 +UUID value, subcolumns=1 2 00000000-0000-0000-0000-000000000000 +IPv6 value, subcolumns=0 1 ::1 +IPv6 value, subcolumns=0 2 :: +IPv6 value, subcolumns=1 1 ::1 +IPv6 value, subcolumns=1 2 :: +Int128 value, subcolumns=0 1 -123456789012345678901234567890 +Int128 value, subcolumns=0 2 0 +Int128 value, subcolumns=1 1 -123456789012345678901234567890 +Int128 value, subcolumns=1 2 0 +Decimal64 value, subcolumns=0 1 1.25 +Decimal64 value, subcolumns=0 2 0 +Decimal64 value, subcolumns=1 1 1.25 +Decimal64 value, subcolumns=1 2 0 +DateTime64 value, subcolumns=0 1 2020-01-01 00:00:00.500 +DateTime64 value, subcolumns=0 2 1970-01-01 00:00:00.000 +DateTime64 value, subcolumns=1 1 2020-01-01 00:00:00.500 +DateTime64 value, subcolumns=1 2 1970-01-01 00:00:00.000 +Nullable(UUID) value, subcolumns=0 1 61f0c404-5cb3-11e7-907b-a6006ad3dba0 +Nullable(UUID) value, subcolumns=0 2 \N +Nullable(UUID) value, subcolumns=0 3 \N +Nullable(UUID) value, subcolumns=1 1 61f0c404-5cb3-11e7-907b-a6006ad3dba0 +Nullable(UUID) value, subcolumns=1 2 \N +Nullable(UUID) value, subcolumns=1 3 \N +Nullable(Decimal64) value, subcolumns=0 1 1.25 +Nullable(Decimal64) value, subcolumns=0 2 \N +Nullable(Decimal64) value, subcolumns=0 3 \N +Nullable(Decimal64) value, subcolumns=1 1 1.25 +Nullable(Decimal64) value, subcolumns=1 2 \N +Nullable(Decimal64) value, subcolumns=1 3 \N diff --git a/tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.sql b/tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.sql new file mode 100644 index 0000000000000..925533b788e1d --- /dev/null +++ b/tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.sql @@ -0,0 +1,203 @@ +-- m[key] on a Map with LowCardinality keys is resolved through dictionary positions instead of +-- comparing the key values. The result must be the same as with the generic comparison, and the +-- same for both the arrayElement path (optimize_functions_to_subcolumns = 0) and the subcolumn +-- path (optimize_functions_to_subcolumns = 1). + +DROP TABLE IF EXISTS t_map_lc; + +CREATE TABLE t_map_lc (id UInt64, m Map(LowCardinality(String), String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; + +-- Row 2 has no 'b' at all, row 3 is an empty map, row 4 has 'b' twice and '' as a key. +INSERT INTO t_map_lc VALUES (1, map('a', 'a1', 'b', 'b1')), (2, map('a', 'a2')), (3, map()), (4, map('', 'empty', 'b', 'FIRST', 'b', 'SECOND')); + +SELECT 'present key, subcolumns=0', id, m['b'] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'present key, subcolumns=1', id, m['b'] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; + +-- A key that is in no row is absent from the dictionary as well, and is resolved without +-- looking at the rows. +SELECT 'absent key, subcolumns=0', id, m['zz'] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'absent key, subcolumns=1', id, m['zz'] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; + +-- The empty string is kept in the reserved default position of the dictionary, not in its index. +SELECT 'empty key, subcolumns=0', id, m[''] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'empty key, subcolumns=1', id, m[''] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; + +-- Reading the subcolumn directly goes through the same key lookup. +SELECT 'direct subcolumn', id, m.key_b FROM t_map_lc ORDER BY id; + +DROP TABLE t_map_lc; + +-- Compact parts exercise the other reader path. +CREATE TABLE t_map_lc (id UInt64, m Map(LowCardinality(String), String)) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_map_lc VALUES (1, map('a', 'a1', 'b', 'b1')), (2, map('a', 'a2')), (3, map()), (4, map('', 'empty', 'b', 'FIRST', 'b', 'SECOND')); + +SELECT 'compact, subcolumns=0', id, m['b'] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'compact, subcolumns=1', id, m['b'] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; + +DROP TABLE t_map_lc; + +-- LowCardinality(FixedString) keys use the same dictionary lookup. +CREATE TABLE t_map_lc_fixed (id UInt64, m Map(LowCardinality(FixedString(3)), String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_lc_fixed VALUES (1, map('aaa', 'a1', 'bbb', 'b1')), (2, map('aaa', 'a2')); + +SELECT 'fixed string key, subcolumns=0', id, m['bbb'], m['zzz'] FROM t_map_lc_fixed ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'fixed string key, subcolumns=1', id, m['bbb'], m['zzz'] FROM t_map_lc_fixed ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; + +DROP TABLE t_map_lc_fixed; + +-- A wide map, where the looked up key is the last one in every row: the shape the dictionary +-- lookup is meant to make cheap. +DROP TABLE IF EXISTS t_map_lc_wide; + +CREATE TABLE t_map_lc_wide (id UInt64, m Map(LowCardinality(String), String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; + +INSERT INTO t_map_lc_wide +SELECT number, mapFromArrays(arrayMap(x -> concat('k', leftPad(toString(x), 2, '0')), range(20)), arrayMap(x -> concat('v', toString(number), '_', toString(x)), range(20))) +FROM numbers(1000); + +SELECT 'wide map, subcolumns=0', sum(cityHash64(m['k19'])), sum(cityHash64(m['k00'])) FROM t_map_lc_wide SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'wide map, subcolumns=1', sum(cityHash64(m['k19'])), sum(cityHash64(m['k00'])) FROM t_map_lc_wide SETTINGS optimize_functions_to_subcolumns = 1; + +DROP TABLE t_map_lc_wide; + +-- A key at a high dictionary position looked up over rows that do not contain it. The dictionary of +-- a part is shared by all of its blocks, so a dictionary position can be out of the range of the +-- index type of an individual block, and must not be truncated into a false match. +DROP TABLE IF EXISTS t_map_lc_many_keys; + +CREATE TABLE t_map_lc_many_keys (id UInt64, m Map(LowCardinality(String), String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; + +INSERT INTO t_map_lc_many_keys +SELECT number, map(concat('k', leftPad(toString(number % 1000), 4, '0')), concat('v', toString(number))) +FROM numbers(100000); + +SELECT 'high dictionary position, subcolumns=0', count(), countIf(m['k0999'] != '') FROM t_map_lc_many_keys SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'high dictionary position, subcolumns=1', count(), countIf(m['k0999'] != '') FROM t_map_lc_many_keys SETTINGS optimize_functions_to_subcolumns = 1; + +DROP TABLE t_map_lc_many_keys; + +-- Pseudo-random wide maps with duplicate keys: both paths must agree, which is the property that +-- the first-match fix of issue #111203 established. +DROP TABLE IF EXISTS t_map_lc_random; + +CREATE TABLE t_map_lc_random (id UInt64, m Map(LowCardinality(String), String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; + +INSERT INTO t_map_lc_random +SELECT + number, + mapFromArrays( + arrayMap(x -> concat('k', toString(cityHash64(number, x) % 12)), range(1 + (number % 15))), + arrayMap(x -> concat('v', toString(cityHash64(number, x, 'value'))), range(1 + (number % 15)))) +FROM numbers(20000); + +SELECT 'random maps, subcolumns=0', sum(cityHash64(m['k0'], m['k5'], m['k11'], m['k99'])) FROM t_map_lc_random SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'random maps, subcolumns=1', sum(cityHash64(m['k0'], m['k5'], m['k11'], m['k99'])) FROM t_map_lc_random SETTINGS optimize_functions_to_subcolumns = 1; + +DROP TABLE t_map_lc_random; + +-- Key types that used to be compared through virtual compareAt. + +DROP TABLE IF EXISTS t_map_key_types; + +CREATE TABLE t_map_key_types (id UInt64, m Map(UUID, String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types VALUES (1, map('61f0c404-5cb3-11e7-907b-a6006ad3dba0', 'hit')), (2, map('00000000-0000-0000-0000-000000000001', 'other')); +SELECT 'UUID key, subcolumns=0', id, m[toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'UUID key, subcolumns=1', id, m[toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_key_types; + +CREATE TABLE t_map_key_types (id UInt64, m Map(IPv4, String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types VALUES (1, map('1.2.3.4', 'hit')), (2, map('5.6.7.8', 'other')); +SELECT 'IPv4 key, subcolumns=0', id, m[toIPv4('1.2.3.4')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'IPv4 key, subcolumns=1', id, m[toIPv4('1.2.3.4')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_key_types; + +CREATE TABLE t_map_key_types (id UInt64, m Map(IPv6, String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types VALUES (1, map('::1', 'hit')), (2, map('::2', 'other')); +SELECT 'IPv6 key, subcolumns=0', id, m[toIPv6('::1')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'IPv6 key, subcolumns=1', id, m[toIPv6('::1')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_key_types; + +-- The wide integers are built from strings, because a literal that wide goes through Float64 and +-- loses precision. +CREATE TABLE t_map_key_types (id UInt64, m Map(Int128, String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types SELECT 1, map(toInt128('-123456789012345678901234567890'), 'hit'); +INSERT INTO t_map_key_types SELECT 2, map(toInt128(1), 'other'); +SELECT 'Int128 key, subcolumns=0', id, m[toInt128('-123456789012345678901234567890')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'Int128 key, subcolumns=1', id, m[toInt128('-123456789012345678901234567890')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_key_types; + +CREATE TABLE t_map_key_types (id UInt64, m Map(UInt256, String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types SELECT 1, map(toUInt256('12345678901234567890123456789012345678901234567890'), 'hit'); +INSERT INTO t_map_key_types SELECT 2, map(toUInt256(1), 'other'); +SELECT 'UInt256 key, subcolumns=0', id, m[toUInt256('12345678901234567890123456789012345678901234567890')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'UInt256 key, subcolumns=1', id, m[toUInt256('12345678901234567890123456789012345678901234567890')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_key_types; + +-- Decimal and DateTime64 keys are rejected by m[key], but reachable through the subcolumn name. + +CREATE TABLE t_map_key_types (id UInt64, m Map(Decimal64(2), String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types SELECT 1, map(toDecimal64(1.5, 2), 'hit', toDecimal64(2.5, 2), 'other'); +INSERT INTO t_map_key_types SELECT 2, map(toDecimal64(2.5, 2), 'other'); +SELECT 'Decimal64 key subcolumn', id, `m.key_1.50` FROM t_map_key_types ORDER BY id; +DROP TABLE t_map_key_types; + +-- The timezone is pinned, because the key is rendered into the subcolumn name. +CREATE TABLE t_map_key_types (id UInt64, m Map(DateTime64(3, 'UTC'), String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types SELECT 1, map(toDateTime64('2020-01-01 00:00:00.500', 3, 'UTC'), 'hit', toDateTime64('2021-01-01 00:00:00.500', 3, 'UTC'), 'other'); +INSERT INTO t_map_key_types SELECT 2, map(toDateTime64('2021-01-01 00:00:00.500', 3, 'UTC'), 'other'); +SELECT 'DateTime64 key subcolumn', id, `m.key_2020-01-01 00:00:00.500` FROM t_map_key_types ORDER BY id; +DROP TABLE t_map_key_types; + +-- Value types that used to be copied through virtual insertFrom. The rows not holding the key +-- cover the default-insert branch, the Nullable variants the null map. + +DROP TABLE IF EXISTS t_map_value_types; + +CREATE TABLE t_map_value_types (id UInt64, m Map(String, UUID)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types VALUES (1, map('a', '61f0c404-5cb3-11e7-907b-a6006ad3dba0')), (2, map('b', '00000000-0000-0000-0000-000000000001')); +SELECT 'UUID value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'UUID value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; + +CREATE TABLE t_map_value_types (id UInt64, m Map(String, IPv6)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types VALUES (1, map('a', '::1')), (2, map('b', '::2')); +SELECT 'IPv6 value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'IPv6 value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; + +CREATE TABLE t_map_value_types (id UInt64, m Map(String, Int128)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types SELECT 1, map('a', toInt128('-123456789012345678901234567890')); +INSERT INTO t_map_value_types SELECT 2, map('b', toInt128(1)); +SELECT 'Int128 value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'Int128 value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; + +CREATE TABLE t_map_value_types (id UInt64, m Map(String, Decimal64(3))) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types VALUES (1, map('a', 1.25)), (2, map('b', 2.5)); +SELECT 'Decimal64 value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'Decimal64 value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; + +-- The timezone is pinned, because the missing key renders the epoch as the default value. +CREATE TABLE t_map_value_types (id UInt64, m Map(String, DateTime64(3, 'UTC'))) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types VALUES (1, map('a', '2020-01-01 00:00:00.500')), (2, map('b', '2021-01-01 00:00:00.500')); +SELECT 'DateTime64 value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'DateTime64 value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; + +CREATE TABLE t_map_value_types (id UInt64, m Map(String, Nullable(UUID))) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types VALUES (1, map('a', '61f0c404-5cb3-11e7-907b-a6006ad3dba0')), (2, map('a', NULL)), (3, map('b', NULL)); +SELECT 'Nullable(UUID) value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'Nullable(UUID) value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; + +CREATE TABLE t_map_value_types (id UInt64, m Map(String, Nullable(Decimal64(3)))) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types VALUES (1, map('a', 1.25)), (2, map('a', NULL)), (3, map('b', NULL)); +SELECT 'Nullable(Decimal64) value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'Nullable(Decimal64) value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; diff --git a/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.reference b/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.reference new file mode 100644 index 0000000000000..018a9a02fa76d --- /dev/null +++ b/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.reference @@ -0,0 +1,60 @@ +-- user granted only pub +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias +[0,1,2,3,4,5,6,7] +SELECT pub FROM t_prewhere_alias PREWHERE pub = 2 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret = 200 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200 +ACCESS_DENIED +SELECT count() FROM t_prewhere_alias PREWHERE secret_alias = 200 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201 +ACCESS_DENIED +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500) +ACCESS_DENIED +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret_alias >= 500) +ACCESS_DENIED +SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 500) +ACCESS_DENIED +SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(pub < 5) +1 +-- user granted pub and the alias columns, but not their source +SELECT secret_alias FROM t_prewhere_alias PREWHERE pub = 2 +200 +SELECT pub FROM t_prewhere_alias WHERE secret_alias = 200 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200 +2 +SELECT pub FROM t_prewhere_alias WHERE secret_alias_expression = 201 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression + 0 = 201 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret = 200 +ACCESS_DENIED +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500) +ACCESS_DENIED +-- user granted pub and the source column, but not the alias columns +SELECT pub FROM t_prewhere_alias PREWHERE secret = 200 +2 +SELECT pub FROM t_prewhere_alias WHERE secret_alias = 200 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias WHERE secret_alias_expression = 201 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201 +ACCESS_DENIED +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500) +[4,5,6,7] +SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 500) +0 +SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 501) +1 +-- row policy over a column the user is not granted +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias +[0,1,3,4,5,6,7] +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias PREWHERE pub > 1 +[3,4,5,6,7] diff --git a/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.sh b/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.sh new file mode 100755 index 0000000000000..89efe05350649 --- /dev/null +++ b/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Tags: no-old-analyzer, no-parallel-replicas +# no-parallel-replicas: reading an ALIAS column with a grant on only that ALIAS column is denied under +# parallel replicas. That is an unrelated pre-existing bug (reproducible on master without this test's +# subject), so exclude the setting rather than encode the wrong behaviour in the reference. + +# Column-level SELECT grants must be enforced for columns that the planner resolves away before the +# access check runs, so that PREWHERE cannot be used as an oracle over a column the user cannot read: +# +# 1. An ALIAS column referenced in PREWHERE is replaced by its expression, so it used to never reach +# the list of selected columns and was not access checked at all, while the same alias in SELECT or +# WHERE was correctly denied. +# 2. A column used only as an `indexHint` argument is never read, only used for index analysis, but +# which granules survive the analysis is observable in the result. +# +# In both cases the required privilege is the one on the name written in the query, exactly as in +# WHERE: referencing an ALIAS column requires a grant on the ALIAS column itself, and a grant on the +# physical columns its expression reads is neither sufficient nor required. Those source columns are +# still read from disk to compute the ALIAS - the administrator authored the ALIAS expression and so +# chose what it exposes - but their values never reach the user. + +# Most of the queries below are expected to be denied, and each denial would otherwise add an +# log line and a stack trace to the output, because the harness streams server-side logs to the client. +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +USER_PUB="user_pub_${CLICKHOUSE_DATABASE}" +USER_ALIAS="user_alias_${CLICKHOUSE_DATABASE}" +USER_SOURCE="user_source_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --multiquery --query " + DROP USER IF EXISTS ${USER_PUB}, ${USER_ALIAS}, ${USER_SOURCE}; + + CREATE TABLE t_prewhere_alias + ( + secret Int32, + pub Int32, + secret_alias Int32 ALIAS secret, + secret_alias_expression Int32 ALIAS secret_alias + 1 + ) + ENGINE = MergeTree ORDER BY secret SETTINGS index_granularity = 1, min_bytes_for_wide_part = 0; + + INSERT INTO t_prewhere_alias (secret, pub) SELECT number * 100, number FROM numbers(8); + + CREATE USER ${USER_PUB}, ${USER_ALIAS}, ${USER_SOURCE}; + GRANT SELECT(pub) ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias TO ${USER_PUB}; + GRANT SELECT(pub, secret_alias, secret_alias_expression) ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias TO ${USER_ALIAS}; + GRANT SELECT(pub, secret) ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias TO ${USER_SOURCE}; +" + +# Run all the queries of one user in a single session, echoing each query before its own result so +# that every line of the reference is attributable. --ignore-error keeps the session going past the +# ACCESS_DENIED answers that most of these queries are expected to produce; without it the first +# denial would end the session and silently drop every query after it. +# +# An exception is printed by the client as exactly three lines - the "Received exception" banner, the +# "Code: ..." message and the echoed query - of which only the error name is stable: the banner holds +# the server version and the message holds the user name, which embeds ${CLICKHOUSE_DATABASE}. Reduce +# the block to that name, keeping it distinct per error so that a change of error shows up as a diff +# rather than being masked. +run_all() +{ + local user=$1 + shift + + local sql="" + local query + for query in "$@" + do + # The echoed query is derived from the query itself, so the two cannot drift apart. + sql+="SELECT '${query//\'/\'\'}'; +${query}; +" + done + + ${CLICKHOUSE_CLIENT} --user "${user}" --multiquery --ignore-error --query "${sql}" 2>&1 \ + | sed -E '/^Received exception/d; /^\(query:/d; s/^Code: [0-9]+\..*\(([A-Z_]+)\)$/\1/' +} + +echo "-- user granted only pub" +run_all "${USER_PUB}" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias" \ + "SELECT pub FROM t_prewhere_alias PREWHERE pub = 2" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret = 200" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200" \ + "SELECT count() FROM t_prewhere_alias PREWHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500)" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret_alias >= 500)" \ + "SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 500)" \ + "SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(pub < 5)" + +echo "-- user granted pub and the alias columns, but not their source" +run_all "${USER_ALIAS}" \ + "SELECT secret_alias FROM t_prewhere_alias PREWHERE pub = 2" \ + "SELECT pub FROM t_prewhere_alias WHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias WHERE secret_alias_expression = 201" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression + 0 = 201" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret = 200" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500)" + +echo "-- user granted pub and the source column, but not the alias columns" +run_all "${USER_SOURCE}" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret = 200" \ + "SELECT pub FROM t_prewhere_alias WHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias WHERE secret_alias_expression = 201" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500)" \ + "SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 500)" \ + "SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 501)" + +# A row policy is defined by an administrator, so it may reference columns the user cannot read. +echo "-- row policy over a column the user is not granted" +${CLICKHOUSE_CLIENT} --query "CREATE ROW POLICY p_prewhere_alias ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias USING secret_alias_expression != 201 TO ${USER_PUB}" +run_all "${USER_PUB}" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias PREWHERE pub > 1" + +${CLICKHOUSE_CLIENT} --multiquery --query " + DROP ROW POLICY p_prewhere_alias ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias; + DROP TABLE t_prewhere_alias; + DROP USER ${USER_PUB}, ${USER_ALIAS}, ${USER_SOURCE}; +" diff --git a/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.reference b/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.reference new file mode 100644 index 0000000000000..17c209230693f --- /dev/null +++ b/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.reference @@ -0,0 +1 @@ +7 8 diff --git a/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.sh b/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.sh new file mode 100755 index 0000000000000..e4d11a0e11bf4 --- /dev/null +++ b/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# A `SET` statement that changes `profile` installs a new constraint set halfway through itself, and the +# assignments and resets after that point must pass it, as they would in a separate statement. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +CONST_PROFILE="profile_const_$CLICKHOUSE_DATABASE" +READONLY_PROFILE="profile_readonly_$CLICKHOUSE_DATABASE" +MAX_PROFILE="profile_max_$CLICKHOUSE_DATABASE" +RELAXED_PROFILE="profile_relaxed_$CLICKHOUSE_DATABASE" + +$CLICKHOUSE_CLIENT -m -q " +DROP SETTINGS PROFILE IF EXISTS $CONST_PROFILE, $READONLY_PROFILE, $MAX_PROFILE, $RELAXED_PROFILE; +CREATE SETTINGS PROFILE $CONST_PROFILE SETTINGS max_execution_time = 10 CONST, SQL_tenant_id = 1 CONST; +CREATE SETTINGS PROFILE $READONLY_PROFILE SETTINGS readonly = 1; +CREATE SETTINGS PROFILE $MAX_PROFILE SETTINGS max_memory_usage MAX 1000000; +CREATE SETTINGS PROFILE $RELAXED_PROFILE SETTINGS max_memory_usage MAX 1099511627776; +" + +# One session on purpose: a rejected statement must leave it untouched, which the statements after it prove. +$CLICKHOUSE_CLIENT -m -q " +SET profile = '$CONST_PROFILE', max_execution_time = 999; -- { serverError SETTING_CONSTRAINT_VIOLATION } +SET profile = '$CONST_PROFILE', max_execution_time = DEFAULT; -- { serverError SETTING_CONSTRAINT_VIOLATION } +SET profile = '$READONLY_PROFILE', max_memory_usage = 1099511627776; -- { serverError READONLY } +SET profile = '$MAX_PROFILE', max_memory_usage = 1099511627776; -- { serverError SETTING_CONSTRAINT_VIOLATION } +-- a later, looser profile does not lift the constraint in force at the assignment +SET profile = '$MAX_PROFILE', max_memory_usage = 5000000, profile = '$RELAXED_PROFILE'; -- { serverError SETTING_CONSTRAINT_VIOLATION } +-- none of the rejected statements applied its profile, which would have installed this setting +SELECT getSetting('SQL_tenant_id'); -- { serverError UNKNOWN_SETTING } +-- assigning the value the profile installs is a no-op and stays allowed +SET profile = '$CONST_PROFILE', max_execution_time = 10; +-- what is not constrained still applies, before and after the profile change +SET SQL_before = 7, profile = '$CONST_PROFILE', SQL_after = 8; +SELECT getSetting('SQL_before'), getSetting('SQL_after'); +" + +$CLICKHOUSE_CLIENT -q "DROP SETTINGS PROFILE $CONST_PROFILE, $READONLY_PROFILE, $MAX_PROFILE, $RELAXED_PROFILE" diff --git a/tests/queries/0_stateless/05219_explain_ast_hide_secrets.reference b/tests/queries/0_stateless/05219_explain_ast_hide_secrets.reference new file mode 100644 index 0000000000000..405f09ad316dd --- /dev/null +++ b/tests/queries/0_stateless/05219_explain_ast_hide_secrets.reference @@ -0,0 +1,101 @@ +-- the inlined view body hides the key +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Identifier encrypted_secret + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Function hex (alias encrypted_secret) (children 1) + ExpressionList (children 1) + Function encrypt (children 1) + ExpressionList (children 3) + Literal \'aes-128-ecb\' + Identifier secret + Literal \'[HIDDEN]\' + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + TableIdentifier default.private_plaintext +-- the session setting alone does not disclose it + Literal \'aes-128-ecb\' + Literal \'[HIDDEN]\' +-- the graph dump hides it too +Literal \'[HIDDEN]\' +-- a secret typed into the explained query itself is hidden as well +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 1) + ExpressionList (children 1) + Function encrypt (children 1) + ExpressionList (children 4) + Literal \'aes-128-ecb\' + Literal \'[HIDDEN]\' + Literal \'[HIDDEN]\' + Function leftPad (children 1) + ExpressionList (children 3) + Literal \'[HIDDEN]\' + Literal \'[HIDDEN]\' + Literal \'[HIDDEN]\' +-- a positional secret written as a comparison collapses to one node, not to 'Function equals' +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 1) + ExpressionList (children 1) + Function encrypt (children 1) + ExpressionList (children 3) + Literal \'aes-128-ecb\' + Literal \'[HIDDEN]\' + Literal \'[HIDDEN]\' +-- a url the finder cannot read collapses to one node, not to 'Function concat' +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Asterisk + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + Function url (children 1) + ExpressionList (children 1) + Literal \'[HIDDEN]\' +-- a mongodb url the finder cannot read collapses to one node, not to 'Function concat' +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Asterisk + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + Function mongodb (children 1) + ExpressionList (children 2) + Literal \'[HIDDEN]\' + Literal \'c\' +-- the same url as a named override keeps its key, the value collapses +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Asterisk + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + Function url (children 1) + ExpressionList (children 2) + Identifier creds + Function equals (children 1) + ExpressionList (children 2) + Identifier url + Literal \'[HIDDEN]\' +-- a nested map keeps its keys and hides its values + Literal \'http://x/f\' + Literal \'Authorization\' + Literal \'[HIDDEN]\' +-- the view itself stays usable for the restricted user +1 diff --git a/tests/queries/0_stateless/05219_explain_ast_hide_secrets.sh b/tests/queries/0_stateless/05219_explain_ast_hide_secrets.sh new file mode 100755 index 0000000000000..dcc3ca50f8c08 --- /dev/null +++ b/tests/queries/0_stateless/05219_explain_ast_hide_secrets.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-replicated-database +# Tag no-fasttest: the encryption functions are not available in the fast test build +# Tag no-replicated-database: SQL SECURITY DEFINER views and users are set up per-test + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `EXPLAIN AST optimize = 1` inlines the body of a view the user may only SELECT from, and the dump +# prints every literal verbatim. The secret arguments must be hidden as `SHOW CREATE` hides them. +# The stateless test server keeps `display_secrets_in_show_and_select` off, so the gate always hides here. + +user="user_05219_${CLICKHOUSE_DATABASE}_$RANDOM" +db=${CLICKHOUSE_DATABASE} +key='Sixteen byte key' + +${CLICKHOUSE_CLIENT} <