Merge pull request #2405 from Altinity/rebase-cicd-v26.8.7.19-lts

Antalya 26.8 - CI/CD reliability fixes

Author: strtgbb

.claude/tools/fetch_ci_report.js

@@ -574,30 +574,29 @@ async function getCIReportsFromPR(prUrl) {
 
   console.log(`Fetching CI reports for PR #${prNumber}...\n`);
 
-  // Fetch PR comments to find CI bot comment.
+  // Report URLs are the signal — do not filter by bot login.
+  // Altinity posts via github-actions[bot] with the virtual-hosted S3 URL;
+  // older comments used clickhouse-gh[bot] and path-style S3.
   // Drop GH_CONFIG_DIR before spawning gh: some agent/runner checkouts set it to a poisoned
   // config dir (no/expired auth) that makes `gh api` fail, while the default config is fine.
   // Other repo tooling (patch-release-check) does the same via `env -u GH_CONFIG_DIR gh`.
   const ghEnv = { ...process.env };
   delete ghEnv.GH_CONFIG_DIR;
   try {
-    const commentsJson = execSync(`gh api repos/Altinity/ClickHouse/issues/${prNumber}/comments --paginate --jq '.[] | select(.user.login == "clickhouse-gh[bot]") | {body, created_at}'`, {
+    const commentsJson = execSync(`gh api repos/Altinity/ClickHouse/issues/${prNumber}/comments --paginate --jq '.[] | {body, created_at}'`, {
       encoding: 'utf8',
       stdio: ['pipe', 'pipe', 'pipe'],
       env: ghEnv
     });
 
     const comments = commentsJson.trim().split('\n').filter(l => l.trim()).map(l => JSON.parse(l));
     comments.sort((a, b) => (b.created_at || '').localeCompare(a.created_at || ''));
-    if (!comments || comments.length === 0) {
-      throw new Error('No CI bot comment found');
-    }
 
-    // Search through all bot comments for CI report URLs (not just the latest). Exclude backtick and
+    // Search through all comments for CI report URLs (not just the latest). Exclude backtick and
     // quote chars so a URL quoted in markdown (e.g. inside the AI-review text) is not captured with
     // trailing junk, strip trailing punctuation, and dedupe -- otherwise the same report is fetched
-    // twice and the summary is doubled.
-    const reportUrlPattern = /https:\/\/s3\.amazonaws\.com\/altinity-build-artifacts\/json\.html\?[^\s)`'"]+/g;
+    // twice and the summary is doubled. Match both path-style and virtual-hosted S3 URLs.
+    const reportUrlPattern = /https:\/\/(?:s3\.amazonaws\.com\/altinity-build-artifacts|altinity-build-artifacts\.s3\.amazonaws\.com)\/json\.html\?[^\s)`'"]+/g;
     for (const comment of comments) {
       if (!comment.body) continue;
       let urls = comment.body.match(reportUrlPattern);
@@ -607,9 +606,9 @@ async function getCIReportsFromPR(prUrl) {
       }
     }
 
-    throw new Error('No CI report URLs found in bot comments');
+    throw new Error('No CI report URLs found in PR comments');
   } catch (error) {
-    if (error.message.includes('No CI bot comment found') || error.message.includes('No CI report URLs found')) {
+    if (error.message.includes('No CI report URLs found')) {
       throw error;
     }
     throw new Error(`Failed to fetch PR comments: ${error.message}`);

.github/actions/create_workflow_report/create_workflow_report.py

@@ -192,6 +192,9 @@ def _enrich_prs_in_release_merge_prs(df: pd.DataFrame, repo: str) -> pd.DataFram
             f"https://api.github.com/repos/{repo}/pulls/{pr_number}",
             headers=headers,
         )
+        if response.status_code == 404:
+            # NOTE (strtgbb): not in this repo — upstream PR merged from a fork
+            continue
         if response.status_code != 200:
             raise Exception(
                 f"Failed to fetch pull request info: {response.status_code} {response.text}"
@@ -207,6 +210,8 @@ def _enrich_prs_in_release_merge_prs(df: pd.DataFrame, repo: str) -> pd.DataFram
                 "pr_labels": html.escape(", ".join(sorted(label_names)), quote=True),
             }
         )
+    if not rows:
+        return pd.DataFrame(columns=["pr_number", "pr_name", "pr_labels"])
     return pd.DataFrame(rows)
 
 

ci/jobs/integration_test_job.py

@@ -1329,7 +1329,7 @@ def main():
     # hard subprocess backstop). Used below to keep an empty flaky/targeted result a
     # best-effort SKIPPED only when a timeout actually exhausted the budget.
     timed_out = False
-    session_timeout_parallel = 3600 * 2
+    session_timeout_parallel = 3600 * 2.5
     session_timeout_sequential = 3600
 
     if is_llvm_coverage:

ci/jobs/scripts/stress/stress.py

@@ -400,7 +400,13 @@ def install_thread_pool_fault_injection() -> None:
 
     logging.info("Installing thread-pool fault-injection config: %s -> %s", src, dst)
     subprocess.run(["ln", "-sf", src, dst], check=True)
-    if not call_with_retry(make_query_command("SYSTEM RELOAD CONFIG"), timeout=30, retry_count=5):
+    # NOTE (strtgbb): ARM debug + ThreadFuzzer config reload is ~18-23s; the
+    # default 15s receive_timeout loses the race and fails the job.
+    if not call_with_retry(
+        make_query_command("SYSTEM RELOAD CONFIG", receive_timeout=60),
+        timeout=90,
+        retry_count=5,
+    ):
         # Fail-close before the verify query: a stale non-zero probability left
         # over from an earlier reload would otherwise mask the reload failure.
         raise RuntimeError(
@@ -655,9 +661,9 @@ def execute_bash(full_command, timeout=120):
         raise
 
 
-def make_query_command(query: str) -> str:
+def make_query_command(query: str, receive_timeout: int = 15) -> str:
     return (
-        f'clickhouse client -q "{query}" --receive_timeout=15 --max_untracked_memory=1Gi '
+        f'clickhouse client -q "{query}" --receive_timeout={receive_timeout} --max_untracked_memory=1Gi '
         "--memory_profiler_step=1Gi --max_memory_usage_for_user=0 --max_memory_usage_in_client=1000000000 "
         "--enable-progress-table-toggle=0 "
         "--ast_fuzzer_runs=0",

ci/praktika/native_jobs.py

@@ -292,6 +292,8 @@ def _prepare_submodule_cache(workflow, workflow_config: RunConfig) -> Result:
                 no_strict=True,
             )
             Shell.check(f"rm -f {archive_path}")
+            if not created and not S3.head_object(s3_path):
+                raise RuntimeError(f"failed to upload submodule cache {s3_path}")
             info = (
                 f"cache miss, created: {cache_hash}"
                 if created
@@ -302,10 +304,12 @@ def _prepare_submodule_cache(workflow, workflow_config: RunConfig) -> Result:
         workflow_config.dump()
         status = Result.Status.OK
     except Exception as e:
-        print(f"WARNING: Submodule cache failed: {e}")
+        print(f"ERROR: Submodule cache failed: {e}")
         traceback.print_exc()
         info = f"{e}\n{traceback.format_exc()}"
-        status = Result.Status.OK  # non-fatal, jobs fall back to GitHub clone
+        # Do not continue with an empty submodule_cache_hash. Builds would
+        # skip the restore and clone the same pins themselves.
+        status = Result.Status.FAIL
 
     return Result.create_from(
         name="Submodule Cache",

ci/settings/altinity_overrides.py

@@ -59,6 +59,10 @@ class RunnerLabels:
 
 INSTALL_PYTHON_REQS_FOR_NATIVE_JOBS = ""
 
+# NOTE (strtgbb): anonymous submodule fetches of non-tip pins get refused
+# after the burst of ~150 clones; send the ambient gh token instead.
+ENABLE_SUBMODULE_CLONE_AUTH = True
+
 DISABLED_WORKFLOWS = [
     "backport_branches.py",
     "custom_build_praktika.py",

cmake/autogenerated_versions.txt

@@ -7,10 +7,9 @@ SET(VERSION_MAJOR 26)
 SET(VERSION_MINOR 8)
 SET(VERSION_PATCH 6)
 SET(VERSION_GITHASH ec5605431dacfc812affc406cc81ca398ca68174)
-SET(VERSION_DESCRIBE v26.8.6.10001.altinitytest)
-SET(VERSION_STRING 26.8.6.10001.altinitytest)
+SET(VERSION_DESCRIBE v26.8.6.20001.altinityantalya)
+SET(VERSION_STRING 26.8.6.20001.altinityantalya)
 # end of autochange
 
-SET(VERSION_TWEAK 10001)
-SET(VERSION_FLAVOUR altinitytest)
-
+SET(VERSION_TWEAK 20001)
+SET(VERSION_FLAVOUR altinityantalya)

tests/docker_scripts/stress_runner.sh

@@ -97,7 +97,9 @@ if [ "$cache_policy" = "SLRU" ]; then
     sed -i.tmp "s|<cache_policy>LRU</cache_policy>|<cache_policy>SLRU</cache_policy>|" /etc/clickhouse-server/config.d/storage_conf*.xml
 fi
 
-start_server || { echo "Failed to start server"; exit 1; }
+# Preload writes system logs the restart must load before port 9000 opens.
+# The default wait (~70s) expires first on some ARM runners.
+start_server 10 || { echo "Failed to start server"; exit 1; }
 
 clickhouse-client --query "SYSTEM STOP THREAD FUZZER"
 
@@ -302,7 +304,9 @@ fi
 # hang the server under sanitizers and trip the hung check.
 cp -av --dereference /repo/ci/jobs/scripts/fuzzer/limit-recursion-settings.xml /etc/clickhouse-server/users.d/
 
-start_server || { echo "Failed to start server"; exit 1; }
+# Same wait as the other restarts: ARM sanitizer + S3 + async_load_databases=false
+# can miss the default ~70s window before port 9000 opens.
+start_server 10 || { echo "Failed to start server"; exit 1; }
 
 # clickhouse-test must know which storage backend the server actually uses, or its storage skip
 # tags are inert and incompatible tests run on an unsupported backend. Both variables are already

tests/integration/test_s3_cluster/test.py

@@ -1729,8 +1729,9 @@ def query_cycle():
 
     node_to_shutdown.query("SYSTEM STOP SWARM MODE")
 
-    # enough time to complete processing of objects, started before "SYSTEM STOP SWARM MODE"
-    time.sleep(3)
+    # Drain in-flight objects started before STOP SWARM. Query time is 3-4s
+    # normally; TSan is several times slower, so 3s is not enough.
+    time.sleep(15)
 
     node_to_shutdown.stop_clickhouse(kill=True)
 

tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.reference

@@ -2,32 +2,32 @@ Create tables with Array(Float32) and Array(BFloat16) column
 Column: Array(Float32)
 -- Search vector: Array(Float64)
 5	0
-6	0.09375
-7	0.203125
-8	0.296875
+6	0.1
+7	0.2
+8	0.3
 -- Search vector: Array(Float32)
 5	0
-6	0.09375
-7	0.203125
-8	0.296875
+6	0.1
+7	0.2
+8	0.3
 -- Search vector: Array(BFloat16)
 5	0
-6	0.09375
-7	0.203125
-8	0.296875
+6	0.1
+7	0.2
+8	0.3
 Column: Array(BFloat16)
 -- Search vector: Array(Float64)
 5	0
-6	0.09375
-7	0.1875
-8	0.296875
+6	0.1
+7	0.2
+8	0.3
 -- Search vector: Array(Float32)
 5	0
-6	0.09375
-7	0.1875
-8	0.296875
+6	0.1
+7	0.2
+8	0.3
 -- Search vector: Array(BFloat16)
 5	0
-6	0.09375
-7	0.1875
-8	0.296875
+6	0.1
+7	0.2
+8	0.3