From b778e67dd2a616fc17ee522cd044920a6943dec3 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Mon, 31 Aug 2026 19:02:02 -0400 Subject: [PATCH 1/8] fix PRs table being broken by a PR 404 --- .../actions/create_workflow_report/create_workflow_report.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/actions/create_workflow_report/create_workflow_report.py b/.github/actions/create_workflow_report/create_workflow_report.py index 4cad8779a914..dde3813ea7dd 100755 --- a/.github/actions/create_workflow_report/create_workflow_report.py +++ b/.github/actions/create_workflow_report/create_workflow_report.py @@ -192,6 +192,9 @@ def _enrich_prs_in_release_merge_prs(df: pd.DataFrame, repo: str) -> pd.DataFram f"https://api.github.com/repos/{repo}/pulls/{pr_number}", headers=headers, ) + if response.status_code == 404: + # NOTE (strtgbb): not in this repo — upstream PR merged from a fork + continue if response.status_code != 200: raise Exception( f"Failed to fetch pull request info: {response.status_code} {response.text}" @@ -207,6 +210,8 @@ def _enrich_prs_in_release_merge_prs(df: pd.DataFrame, repo: str) -> pd.DataFram "pr_labels": html.escape(", ".join(sorted(label_names)), quote=True), } ) + if not rows: + return pd.DataFrame(columns=["pr_number", "pr_name", "pr_labels"]) return pd.DataFrame(rows) From 2f53ed2d7061d93485b3d9ec022d60471f39ef8d Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Sun, 20 Sep 2026 20:47:22 -0400 Subject: [PATCH 2/8] Recognize Altinity CI report comments in `fetch_ci_report.js`. The tool filtered for `clickhouse-gh[bot]` and path-style S3 URLs, so PR fetches missed the `github-actions[bot]` comment and never printed `--failed` details from the workflow report. Co-authored-by: Cursor --- .claude/tools/fetch_ci_report.js | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.claude/tools/fetch_ci_report.js b/.claude/tools/fetch_ci_report.js index 09b4cab3d82d..4803a71b7c7b 100755 --- a/.claude/tools/fetch_ci_report.js +++ b/.claude/tools/fetch_ci_report.js @@ -574,14 +574,16 @@ async function getCIReportsFromPR(prUrl) { console.log(`Fetching CI reports for PR #${prNumber}...\n`); - // Fetch PR comments to find CI bot comment. + // Report URLs are the signal — do not filter by bot login. + // Altinity posts via github-actions[bot] with the virtual-hosted S3 URL; + // older comments used clickhouse-gh[bot] and path-style S3. // Drop GH_CONFIG_DIR before spawning gh: some agent/runner checkouts set it to a poisoned // config dir (no/expired auth) that makes `gh api` fail, while the default config is fine. // Other repo tooling (patch-release-check) does the same via `env -u GH_CONFIG_DIR gh`. const ghEnv = { ...process.env }; delete ghEnv.GH_CONFIG_DIR; try { - const commentsJson = execSync(`gh api repos/Altinity/ClickHouse/issues/${prNumber}/comments --paginate --jq '.[] | select(.user.login == "clickhouse-gh[bot]") | {body, created_at}'`, { + const commentsJson = execSync(`gh api repos/Altinity/ClickHouse/issues/${prNumber}/comments --paginate --jq '.[] | {body, created_at}'`, { encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], env: ghEnv @@ -589,15 +591,12 @@ async function getCIReportsFromPR(prUrl) { const comments = commentsJson.trim().split('\n').filter(l => l.trim()).map(l => JSON.parse(l)); comments.sort((a, b) => (b.created_at || '').localeCompare(a.created_at || '')); - if (!comments || comments.length === 0) { - throw new Error('No CI bot comment found'); - } - // Search through all bot comments for CI report URLs (not just the latest). Exclude backtick and + // Search through all comments for CI report URLs (not just the latest). Exclude backtick and // quote chars so a URL quoted in markdown (e.g. inside the AI-review text) is not captured with // trailing junk, strip trailing punctuation, and dedupe -- otherwise the same report is fetched - // twice and the summary is doubled. - const reportUrlPattern = /https:\/\/s3\.amazonaws\.com\/altinity-build-artifacts\/json\.html\?[^\s)`'"]+/g; + // twice and the summary is doubled. Match both path-style and virtual-hosted S3 URLs. + const reportUrlPattern = /https:\/\/(?:s3\.amazonaws\.com\/altinity-build-artifacts|altinity-build-artifacts\.s3\.amazonaws\.com)\/json\.html\?[^\s)`'"]+/g; for (const comment of comments) { if (!comment.body) continue; let urls = comment.body.match(reportUrlPattern); @@ -607,9 +606,9 @@ async function getCIReportsFromPR(prUrl) { } } - throw new Error('No CI report URLs found in bot comments'); + throw new Error('No CI report URLs found in PR comments'); } catch (error) { - if (error.message.includes('No CI bot comment found') || error.message.includes('No CI report URLs found')) { + if (error.message.includes('No CI report URLs found')) { throw error; } throw new Error(`Failed to fetch PR comments: ${error.message}`); From 258c09f5ec29fd793d92b121e882a07177631f3d Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 22 Sep 2026 08:53:14 -0400 Subject: [PATCH 3/8] increase retries for stress tests server startup --- tests/docker_scripts/stress_runner.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/docker_scripts/stress_runner.sh b/tests/docker_scripts/stress_runner.sh index 1fdaee27310e..2ae38f58b957 100755 --- a/tests/docker_scripts/stress_runner.sh +++ b/tests/docker_scripts/stress_runner.sh @@ -97,7 +97,9 @@ if [ "$cache_policy" = "SLRU" ]; then sed -i.tmp "s|LRU|SLRU|" /etc/clickhouse-server/config.d/storage_conf*.xml fi -start_server || { echo "Failed to start server"; exit 1; } +# Preload writes system logs the restart must load before port 9000 opens. +# The default wait (~70s) expires first on some ARM runners. +start_server 10 || { echo "Failed to start server"; exit 1; } clickhouse-client --query "SYSTEM STOP THREAD FUZZER" From b63d9a21d272d2f741a8c26bd1e666f530d803cb Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:39:02 -0400 Subject: [PATCH 4/8] try to resolve build failures due to submodule cache failing --- ci/praktika/native_jobs.py | 8 ++++++-- ci/settings/altinity_overrides.py | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ci/praktika/native_jobs.py b/ci/praktika/native_jobs.py index 804b2805a904..e6c9a673f43c 100644 --- a/ci/praktika/native_jobs.py +++ b/ci/praktika/native_jobs.py @@ -292,6 +292,8 @@ def _prepare_submodule_cache(workflow, workflow_config: RunConfig) -> Result: no_strict=True, ) Shell.check(f"rm -f {archive_path}") + if not created and not S3.head_object(s3_path): + raise RuntimeError(f"failed to upload submodule cache {s3_path}") info = ( f"cache miss, created: {cache_hash}" if created @@ -302,10 +304,12 @@ def _prepare_submodule_cache(workflow, workflow_config: RunConfig) -> Result: workflow_config.dump() status = Result.Status.OK except Exception as e: - print(f"WARNING: Submodule cache failed: {e}") + print(f"ERROR: Submodule cache failed: {e}") traceback.print_exc() info = f"{e}\n{traceback.format_exc()}" - status = Result.Status.OK # non-fatal, jobs fall back to GitHub clone + # Do not continue with an empty submodule_cache_hash. Builds would + # skip the restore and clone the same pins themselves. + status = Result.Status.FAIL return Result.create_from( name="Submodule Cache", diff --git a/ci/settings/altinity_overrides.py b/ci/settings/altinity_overrides.py index 4221ccf17b87..9e965980b8ef 100644 --- a/ci/settings/altinity_overrides.py +++ b/ci/settings/altinity_overrides.py @@ -59,6 +59,10 @@ class RunnerLabels: INSTALL_PYTHON_REQS_FOR_NATIVE_JOBS = "" +# NOTE (strtgbb): anonymous submodule fetches of non-tip pins get refused +# after the burst of ~150 clones; send the ambient gh token instead. +ENABLE_SUBMODULE_CLONE_AUTH = True + DISABLED_WORKFLOWS = [ "backport_branches.py", "custom_build_praktika.py", From 4b69eb061d7b32fdf3f4d48ed6e6deacdb140bd9 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 22 Sep 2026 22:08:40 -0400 Subject: [PATCH 5/8] increase timeouts for integration and stress --- ci/jobs/integration_test_job.py | 2 +- tests/docker_scripts/stress_runner.sh | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/jobs/integration_test_job.py b/ci/jobs/integration_test_job.py index df0a298ed775..4ccac05dc242 100644 --- a/ci/jobs/integration_test_job.py +++ b/ci/jobs/integration_test_job.py @@ -1329,7 +1329,7 @@ def main(): # hard subprocess backstop). Used below to keep an empty flaky/targeted result a # best-effort SKIPPED only when a timeout actually exhausted the budget. timed_out = False - session_timeout_parallel = 3600 * 2 + session_timeout_parallel = 3600 * 2.5 session_timeout_sequential = 3600 if is_llvm_coverage: diff --git a/tests/docker_scripts/stress_runner.sh b/tests/docker_scripts/stress_runner.sh index 2ae38f58b957..0d013452f68a 100755 --- a/tests/docker_scripts/stress_runner.sh +++ b/tests/docker_scripts/stress_runner.sh @@ -304,7 +304,9 @@ fi # hang the server under sanitizers and trip the hung check. cp -av --dereference /repo/ci/jobs/scripts/fuzzer/limit-recursion-settings.xml /etc/clickhouse-server/users.d/ -start_server || { echo "Failed to start server"; exit 1; } +# Same wait as the other restarts: ARM sanitizer + S3 + async_load_databases=false +# can miss the default ~70s window before port 9000 opens. +start_server 10 || { echo "Failed to start server"; exit 1; } # clickhouse-test must know which storage backend the server actually uses, or its storage skip # tags are inert and incompatible tests run on an unsupported backend. Both variables are already From f0e5578016ffc0c4a75de905d765f07c811cc6ed Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Wed, 23 Sep 2026 09:18:52 -0400 Subject: [PATCH 6/8] increase timeout in install_thread_pool_fault_injection --- ci/jobs/scripts/stress/stress.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ci/jobs/scripts/stress/stress.py b/ci/jobs/scripts/stress/stress.py index b4e8b47141fd..793e71d1d8d7 100755 --- a/ci/jobs/scripts/stress/stress.py +++ b/ci/jobs/scripts/stress/stress.py @@ -400,7 +400,13 @@ def install_thread_pool_fault_injection() -> None: logging.info("Installing thread-pool fault-injection config: %s -> %s", src, dst) subprocess.run(["ln", "-sf", src, dst], check=True) - if not call_with_retry(make_query_command("SYSTEM RELOAD CONFIG"), timeout=30, retry_count=5): + # NOTE (strtgbb): ARM debug + ThreadFuzzer config reload is ~18-23s; the + # default 15s receive_timeout loses the race and fails the job. + if not call_with_retry( + make_query_command("SYSTEM RELOAD CONFIG", receive_timeout=60), + timeout=90, + retry_count=5, + ): # Fail-close before the verify query: a stale non-zero probability left # over from an earlier reload would otherwise mask the reload failure. raise RuntimeError( @@ -655,9 +661,9 @@ def execute_bash(full_command, timeout=120): raise -def make_query_command(query: str) -> str: +def make_query_command(query: str, receive_timeout: int = 15) -> str: return ( - f'clickhouse client -q "{query}" --receive_timeout=15 --max_untracked_memory=1Gi ' + f'clickhouse client -q "{query}" --receive_timeout={receive_timeout} --max_untracked_memory=1Gi ' "--memory_profiler_step=1Gi --max_memory_usage_for_user=0 --max_memory_usage_in_client=1000000000 " "--enable-progress-table-toggle=0 " "--ast_fuzzer_runs=0", From a4d442f54e5c68f59e39ac050a57311ddcfefe03 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:20:34 -0400 Subject: [PATCH 7/8] Use Antalya version flavour on antalya-26.8 Match the `26.x.y.20001.altinityantalya` convention used on other Antalya branches instead of the rebase `altinitytest` identity. PR: https://github.com/Altinity/ClickHouse/pull/2405 Co-authored-by: Cursor --- cmake/autogenerated_versions.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 581fe1859c69..8820579f055f 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -7,10 +7,9 @@ SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) SET(VERSION_PATCH 6) SET(VERSION_GITHASH ec5605431dacfc812affc406cc81ca398ca68174) -SET(VERSION_DESCRIBE v26.8.6.10001.altinitytest) -SET(VERSION_STRING 26.8.6.10001.altinitytest) +SET(VERSION_DESCRIBE v26.8.6.20001.altinityantalya) +SET(VERSION_STRING 26.8.6.20001.altinityantalya) # end of autochange -SET(VERSION_TWEAK 10001) -SET(VERSION_FLAVOUR altinitytest) - +SET(VERSION_TWEAK 20001) +SET(VERSION_FLAVOUR altinityantalya) From 287493fe9c42fd2796944230a95de6f005f40d6a Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Fri, 2 Oct 2026 09:29:32 -0400 Subject: [PATCH 8/8] round vector-search L2Distance and lengthen swarm shutdown drain ARM sanitizer builds disable SimSIMD so `02354_vector_search_rescoring_distance_in_select_list` disagrees with the binary-build reference. TSan needs more than 3s after `SYSTEM STOP SWARM MODE` before the node is killed. https: //s3.amazonaws.com/altinity-build-artifacts/PRs/2405/a4d442f54e5c68f59e39ac050a57311ddcfefe03/36891348973/ci_run_report.html https: //github.com/Altinity/ClickHouse/pull/2405 Co-authored-by: Cursor --- tests/integration/test_s3_cluster/test.py | 5 +-- ...escoring_distance_in_select_list.reference | 36 +++++++++---------- ...arch_rescoring_distance_in_select_list.sql | 14 ++++---- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/tests/integration/test_s3_cluster/test.py b/tests/integration/test_s3_cluster/test.py index 00978fb7231c..cb18b7a64307 100644 --- a/tests/integration/test_s3_cluster/test.py +++ b/tests/integration/test_s3_cluster/test.py @@ -1729,8 +1729,9 @@ def query_cycle(): node_to_shutdown.query("SYSTEM STOP SWARM MODE") - # enough time to complete processing of objects, started before "SYSTEM STOP SWARM MODE" - time.sleep(3) + # Drain in-flight objects started before STOP SWARM. Query time is 3-4s + # normally; TSan is several times slower, so 3s is not enough. + time.sleep(15) node_to_shutdown.stop_clickhouse(kill=True) diff --git a/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.reference b/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.reference index e307b0ef6ae9..0bf9c660e01a 100644 --- a/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.reference +++ b/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.reference @@ -2,32 +2,32 @@ Create tables with Array(Float32) and Array(BFloat16) column Column: Array(Float32) -- Search vector: Array(Float64) 5 0 -6 0.09375 -7 0.203125 -8 0.296875 +6 0.1 +7 0.2 +8 0.3 -- Search vector: Array(Float32) 5 0 -6 0.09375 -7 0.203125 -8 0.296875 +6 0.1 +7 0.2 +8 0.3 -- Search vector: Array(BFloat16) 5 0 -6 0.09375 -7 0.203125 -8 0.296875 +6 0.1 +7 0.2 +8 0.3 Column: Array(BFloat16) -- Search vector: Array(Float64) 5 0 -6 0.09375 -7 0.1875 -8 0.296875 +6 0.1 +7 0.2 +8 0.3 -- Search vector: Array(Float32) 5 0 -6 0.09375 -7 0.1875 -8 0.296875 +6 0.1 +7 0.2 +8 0.3 -- Search vector: Array(BFloat16) 5 0 -6 0.09375 -7 0.1875 -8 0.296875 +6 0.1 +7 0.2 +8 0.3 diff --git a/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql b/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql index c4aa1debd865..d7797d811353 100644 --- a/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql +++ b/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql @@ -60,26 +60,28 @@ INSERT INTO tab_bf16 VALUES (0, [1.0, 0.0]), (9, [0.0, 2.4]); -- The nearest neighbours to [0.0, 0.2] are 5,6,7,8 +-- Round distances: ARM sanitizer builds disable SimSIMD and scalar USearch +-- differs in the 3rd decimal (0.203125 vs 0.1875). Neighbor order is unchanged. SELECT 'Column: Array(Float32)'; SELECT '-- Search vector: Array(Float64)'; WITH CAST([0.0, 2.0] AS Array(Float64)) AS reference_vec -SELECT id, L2Distance(vec, reference_vec) +SELECT id, round(L2Distance(vec, reference_vec), 1) FROM tab_f32 ORDER BY L2Distance(vec, reference_vec) LIMIT 4; SELECT '-- Search vector: Array(Float32)'; WITH CAST([0.0, 2.0] AS Array(BFloat16)) AS reference_vec -SELECT id, L2Distance(vec, reference_vec) +SELECT id, round(L2Distance(vec, reference_vec), 1) FROM tab_f32 ORDER BY L2Distance(vec, reference_vec) LIMIT 4; SELECT '-- Search vector: Array(BFloat16)'; WITH CAST([0.0, 2.0] AS Array(BFloat16)) AS reference_vec -SELECT id, L2Distance(vec, reference_vec) +SELECT id, round(L2Distance(vec, reference_vec), 1) FROM tab_f32 ORDER BY L2Distance(vec, reference_vec) LIMIT 4; @@ -88,21 +90,21 @@ SELECT 'Column: Array(BFloat16)'; SELECT '-- Search vector: Array(Float64)'; WITH [0.0, 2.0] AS reference_vec -SELECT id, L2Distance(vec, reference_vec) +SELECT id, round(L2Distance(vec, reference_vec), 1) FROM tab_bf16 ORDER BY L2Distance(vec, reference_vec) LIMIT 4; SELECT '-- Search vector: Array(Float32)'; WITH CAST([0.0, 2.0] AS Array(BFloat16)) AS reference_vec -SELECT id, L2Distance(vec, reference_vec) +SELECT id, round(L2Distance(vec, reference_vec), 1) FROM tab_bf16 ORDER BY L2Distance(vec, reference_vec) LIMIT 4; SELECT '-- Search vector: Array(BFloat16)'; WITH CAST([0.0, 2.0] AS Array(Float32)) AS reference_vec -SELECT id, L2Distance(vec, reference_vec) +SELECT id, round(L2Distance(vec, reference_vec), 1) FROM tab_bf16 ORDER BY L2Distance(vec, reference_vec) LIMIT 4;