Fix Calico compatibility scraping by release family

static/compatibilities/calico.yaml

@@ -6,7 +6,7 @@ chart_name: tigera-operator
 eolApiSlug: calico
 versions:
 - version: 3.32.2
-  kube: ['1.30', '1.29', '1.28', '1.27']
+  kube: ['1.36', '1.35', '1.34']
   requirements: []
   incompatibilities: []
   summary:
@@ -18,7 +18,7 @@ versions:
   chart_version: 3.32.2
   images: ['quay.io/tigera/operator:v1.42.6']
 - version: 3.32.0
-  kube: ['1.30', '1.29', '1.28', '1.27']
+  kube: ['1.36', '1.35', '1.34']
   requirements: []
   incompatibilities: []
   summary:
@@ -33,7 +33,7 @@ versions:
   chart_version: 3.32.0
   images: ['quay.io/tigera/operator:v1.42.0']
 - version: 3.31.0
-  kube: ['1.30', '1.29', '1.28', '1.27']
+  kube: ['1.35', '1.34', '1.33', '1.32']
   requirements: []
   incompatibilities: []
   summary:
@@ -48,7 +48,7 @@ versions:
   chart_version: 3.31.0
   images: ['quay.io/tigera/operator:v1.40.0']
 - version: 3.30.0
-  kube: ['1.30', '1.29', '1.28', '1.27']
+  kube: ['1.35', '1.34', '1.33', '1.32', '1.31']
   requirements: []
   incompatibilities: []
   summary:
@@ -64,7 +64,7 @@ versions:
   images: ['quay.io/tigera/operator:v1.38.0']
   eolAt: '2026-04-30'
 - version: 3.29.0
-  kube: ['1.30', '1.29', '1.28', '1.27']
+  kube: ['1.32', '1.31', '1.30', '1.29']
   requirements: []
   incompatibilities: []
   summary:
@@ -92,7 +92,7 @@ versions:
   images: ['quay.io/tigera/operator:v1.34.0']
   eolAt: '2025-05-05'
 - version: 3.27.0
-  kube: ['1.30', '1.29', '1.28', '1.27']
+  kube: ['1.29', '1.28', '1.27']
   requirements: []
   incompatibilities: []
   summary:

utils/compatibility/scrapers/calico.py

@@ -1,39 +1,93 @@
-# scrapers/new-scraper.py
+# Calico compatibility scraper
 
+import re
 from bs4 import BeautifulSoup
 from collections import OrderedDict
 from utils import (
     update_compatibility_info,
     get_chart_versions,
-    expand_kube_versions,
+    fetch_page,
     get_github_releases,
+    print_error,
 )
 
 app_name = "calico"
-compatibility_url = "https://docs.tigera.io/calico/latest/getting-started/kubernetes/requirements#kubernetes-requirements"
+compatibility_url = (
+    "https://docs.tigera.io/calico/{version}/getting-started/kubernetes/requirements"
+)
+
+
+def _release_family(version):
+    """Return the major/minor Calico documentation family for a release."""
+    match = re.fullmatch(r"v?(\d+\.\d+)(?:\.\d+)", version)
+    if not match:
+        raise ValueError(f"Invalid Calico release version: {version}")
+    return match.group(1)
+
+
+def parse_kube_versions(content, calico_family):
+    """Extract Kubernetes versions explicitly tested by a Calico family."""
+    text = BeautifulSoup(content, "html.parser").get_text(" ", strip=True)
+    pattern = (
+        rf"We test Calico v?{re.escape(calico_family)}\s+against\s+"
+        r"the following Kubernetes versions\."
+        r"(.*?)\s+Due to changes in the Kubernetes API"
+    )
+    match = re.search(pattern, text, flags=re.IGNORECASE | re.DOTALL)
+    if not match:
+        raise ValueError(
+            f"Could not find the supported Kubernetes versions for Calico {calico_family}"
+        )
+
+    versions = re.findall(r"\bv?(\d+\.\d+)\b", match.group(1))
+    versions = sorted(
+        set(versions),
+        key=lambda version: tuple(int(part) for part in version.split(".")),
+        reverse=True,
+    )
+    if not versions:
+        raise ValueError(
+            f"Calico {calico_family} documentation contained no Kubernetes versions"
+        )
+    return versions
 
 
 def do_scrape(app_name):
-    # https://docs.tigera.io/calico/latest/getting-started/kubernetes/requirements#supported-versions
-    # We test Calico v3.28 against the following Kubernetes versions. Other versions may work,
-    # but we are not actively testing them.
-    # v1.27, v1.28, v1.29, v1.30
-    # Due to changes in the Kubernetes API, Calico v3.28 will not work on Kubernetes v1.20 or below. v1.21 may work,
-    # but is no longer tested. Newer versions may also work, but we recommend upgrading to
-    # a version of Calico that is tested against the newer Kubernetes version.
     versions = []
-    kube_versions = expand_kube_versions("1.27", "1.30")
     releases = get_github_releases("projectcalico", "calico")
     chart_versions = get_chart_versions(app_name, "tigera-operator")
+
+    release_versions = []
     for release in releases:
         ver = release.lstrip("v")
         chart_version = chart_versions.get(ver)
-        if not chart_version:
+        if chart_version:
+            release_versions.append((ver, chart_version))
+
+    if not release_versions:
+        print_error("No released Calico versions have matching operator charts")
+        return
+
+    kube_versions_by_family = {}
+    for ver, _ in release_versions:
+        family = _release_family(ver)
+        if family in kube_versions_by_family:
             continue
+        content = fetch_page(compatibility_url.format(version=family))
+        if not content:
+            print_error(f"Could not fetch Calico {family} requirements")
+            return
+        try:
+            kube_versions_by_family[family] = parse_kube_versions(content, family)
+        except ValueError as error:
+            print_error(str(error))
+            return
+
+    for ver, chart_version in release_versions:
         version_info = OrderedDict(
             [
                 ("version", ver),
-                ("kube", kube_versions),
+                ("kube", kube_versions_by_family[_release_family(ver)]),
                 ("chart_version", chart_version),
                 ("images", []),
                 ("requirements", []),
@@ -42,10 +96,8 @@ def do_scrape(app_name):
         )
         versions.append(version_info)
 
-    update_compatibility_info(
-        f"../../static/compatibilities/{app_name}.yaml", versions
-    )
+    update_compatibility_info(f"../../static/compatibilities/{app_name}.yaml", versions)
 
 
 def scrape():
-    do_scrape("calico")
+    do_scrape("calico")

utils/compatibility/tests/test_calico.py

@@ -0,0 +1,95 @@
+"""Offline coverage for Calico's version-specific Kubernetes requirements."""
+
+import importlib
+from pathlib import Path
+import sys
+import unittest
+from unittest.mock import patch
+
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+scraper = importlib.import_module("scrapers.calico")
+
+
+def requirements_page(family, versions):
+    values = " ".join(f"v{version}" for version in versions)
+    return (
+        "<html><body><h4>Supported versions</h4>"
+        f"<p>We test Calico v{family} against the following Kubernetes versions. "
+        f"{values} Due to changes in the Kubernetes API, Calico v{family} will not work "
+        "on Kubernetes v1.20 or below.</p></body></html>"
+    ).encode()
+
+
+class CalicoTests(unittest.TestCase):
+    def test_parser_uses_the_release_family_and_sorts_versions(self):
+        page = requirements_page("3.32", ["1.34", "1.36", "1.35", "1.35"])
+        self.assertEqual(
+            scraper.parse_kube_versions(page, "3.32"),
+            ["1.36", "1.35", "1.34"],
+        )
+
+    def test_parser_rejects_missing_or_malformed_source(self):
+        for page in (
+            b"<p>Supported versions: 1.34, 1.35</p>",
+            requirements_page("3.31", ["1.34"]).replace(
+                b"Due to changes in the Kubernetes API", b"Source changed"
+            ),
+            requirements_page("3.32", []),
+        ):
+            with self.subTest(page=page):
+                with self.assertRaises(ValueError):
+                    scraper.parse_kube_versions(page, "3.32")
+
+    def test_scrape_fetches_each_family_once_and_builds_rows(self):
+        releases = ["v3.32.2", "v3.31.7", "v3.32.1"]
+        charts = {version.lstrip("v"): version.lstrip("v") for version in releases}
+        pages = {
+            scraper.compatibility_url.format(version="3.32"): requirements_page(
+                "3.32", ["1.34", "1.35", "1.36"]
+            ),
+            scraper.compatibility_url.format(version="3.31"): requirements_page(
+                "3.31", ["1.32", "1.33", "1.34", "1.35"]
+            ),
+        }
+
+        with patch.object(
+            scraper, "get_github_releases", return_value=releases
+        ), patch.object(
+            scraper, "get_chart_versions", return_value=charts
+        ), patch.object(
+            scraper, "fetch_page", side_effect=pages.get
+        ) as fetch, patch.object(
+            scraper, "update_compatibility_info"
+        ) as update:
+            scraper.do_scrape("calico")
+
+        self.assertEqual(fetch.call_count, 2)
+        self.assertEqual(
+            [call.args[0] for call in fetch.call_args_list],
+            list(pages),
+        )
+        rows = update.call_args.args[1]
+        self.assertEqual(
+            [row["version"] for row in rows], ["3.32.2", "3.31.7", "3.32.1"]
+        )
+        self.assertEqual(rows[0]["kube"], ["1.36", "1.35", "1.34"])
+        self.assertEqual(rows[1]["kube"], ["1.35", "1.34", "1.33", "1.32"])
+
+    def test_source_failure_does_not_write_partial_compatibility_data(self):
+        with patch.object(
+            scraper, "get_github_releases", return_value=["v3.32.2"]
+        ), patch.object(
+            scraper, "get_chart_versions", return_value={"3.32.2": "3.32.2"}
+        ), patch.object(
+            scraper, "fetch_page", return_value=None
+        ), patch.object(
+            scraper, "update_compatibility_info"
+        ) as update:
+            scraper.do_scrape("calico")
+
+        update.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()