ci: sync shared check-upstream.sh and link sibling release notes

build/check-upstream.sh

@@ -158,18 +158,37 @@ discover_github_tag() { # <owner/repo> [tag-strip-expr]
       | awk -F/ '{ print $NF }' | sed -E "s/^v//; ${2:-s/^//}" | clean_versions | newest
 }
 
-# nuget.org's flat container: the package's own version index, and the .nupkg beside it. Used for
-# the platform binding packages an umbrella repository pins - this author's own releases, published
-# from the sibling repositories - which nothing else was watching.
+# nuget.org's registration index, filtered to *listed* versions.
 #
-# The ID is lowercased because the flat container is case-sensitive and serves only lowercase.
-# clean_versions drops the -beta.N.M prereleases every pull request publishes, which is what makes
-# this usable at all: the newest *stable* release is the only thing an umbrella should re-pin to.
+# Deliberately not the flat container (v3-flatcontainer/<id>/index.json), which is the obvious
+# choice and the wrong one: it enumerates every version ever pushed, including delisted ones, and
+# a delisted package is still downloadable - so neither the version list nor the stage-2 download
+# check can tell the difference. FFmpegKit.Net.Full.Android is the case in point: 8.1.7.1 through
+# 8.1.7.3 are all delisted, the flat container still lists them, and a watcher built on it reports
+# "re-pin 8.1.2.5 to 8.1.7.3" - pointing at packages the author deliberately withdrew.
+#
+# The registration index carries catalogEntry.listed per version. Delisted entries also carry the
+# published sentinel 1900-01-01, which is checked too: listed is absent on some older entries.
+#
+# Paginated for packages with enough versions, and the pages are gzip whether or not they are
+# asked for, hence --compressed on the outer fetch and the magic-byte check on the inner ones.
 discover_nuget() { # <PackageId>
     local id
     id="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]')"
-    fetch "https://api.nuget.org/v3-flatcontainer/${id}/index.json" \
-      | jq_lines 'import sys,json;[print(v) for v in json.load(sys.stdin)["versions"]]' \
+    fetch --compressed "https://api.nuget.org/v3/registration5-gz-semver2/${id}/index.json" \
+      | jq_lines 'import sys,json,gzip,urllib.request
+def load(u):
+    r=urllib.request.Request(u,headers={"Accept-Encoding":"gzip"})
+    b=urllib.request.urlopen(r,timeout=60).read()
+    if b[:2]==b"\x1f\x8b": b=gzip.decompress(b)
+    return json.loads(b)
+for page in json.load(sys.stdin)["items"]:
+    items=page.get("items")
+    if items is None: items=load(page["@id"])["items"]
+    for it in items:
+        ce=it["catalogEntry"]
+        if ce.get("listed",True) and not str(ce.get("published","")).startswith("1900"):
+            print(ce["version"])' \
       | clean_versions | newest
 }
 
@@ -186,6 +205,23 @@ strip_expr() { case "$1" in */*:*) printf '%s' "${1#*:}" ;; *) printf 's/^//' ;;
 # `pod install` finds the bytes, and is exactly what this repository's fetch-*.sh scripts already
 # resolve - so a version this script confirms is one the existing tooling can fetch.
 
+# For a component published from one of this author's own repositories - the nuget rows, whose
+# optional seventh manifest column names the owner/repo - a drift finding can carry more than the
+# version: the changelog for it already exists, because in these families merging
+# docs/release-notes/<version>.md IS the release. So the finding links straight to that note, and
+# the issue asking for a re-pin says what the re-pin brings. HEAD rather than a branch name,
+# because the repositories disagree on master versus main and a blob URL under HEAD follows the
+# default branch either way. The note file is confirmed before it is linked, in the same spirit as
+# every other pointer here; when it is missing (a release cut before the convention, say) the
+# releases page is the honest fallback.
+sibling_links() { # sibling_links <owner/repo> <version>
+    if fetch -r 0-0 -o /dev/null "https://raw.githubusercontent.com/$1/HEAD/docs/release-notes/$2.md" 2>/dev/null; then
+        printf ' · [release notes](https://github.com/%s/blob/HEAD/docs/release-notes/%s.md) · [releases](https://github.com/%s/releases)' "$1" "$2" "$1"
+    else
+        printf ' · [releases](https://github.com/%s/releases)' "$1"
+    fi
+}
+
 # CocoaPods' CDN shards a pod's spec directory by the first three hex digits of the *pod name's*
 # MD5 (not the version's), so every version of a pod lives under the same shard.
 podspec_url() { # <PodName> <version>
@@ -234,7 +270,9 @@ download_url() { # download_url <kind> <locator> <confirm-template> <version>
 # --- the pass ------------------------------------------------------------------------------
 
 checked=0
-while IFS=$'\t' read -r group label kind pin locator confirm; do
+# src is the optional seventh column and most manifests stop at six; read leaves it empty there,
+# which is every row except the sibling-package nuget ones.
+while IFS=$'\t' read -r group label kind pin locator confirm src; do
     case "${group}" in ''|'#'*) continue ;; esac
     checked=$((checked + 1))
 
@@ -308,7 +346,8 @@ while IFS=$'\t' read -r group label kind pin locator confirm; do
         # as a pin ahead of upstream and say nothing.
         case "${current}" in
             *-*)
-                note "${group}" "**${label}**: pinned the prerelease \`${current}\` while \`${latest}\` is published on nuget.org — a released umbrella must not depend on a beta. Re-pin \`${pin}\` in \`Directory.Build.props\`."
+                links=""; [ -z "${src}" ] || links="$(sibling_links "${src}" "${latest}" || true)"
+                note "${group}" "**${label}**: pinned the prerelease \`${current}\` while \`${latest}\` is published on nuget.org — a released umbrella must not depend on a beta. Re-pin \`${pin}\` in \`Directory.Build.props\`.${links}"
                 continue
                 ;;
         esac
@@ -319,7 +358,8 @@ while IFS=$'\t' read -r group label kind pin locator confirm; do
         else
             url="$(download_url nuget "${locator}" "${confirm}" "${latest}" || true)"
             if [ -n "${url}" ] && downloadable "${url}"; then
-                note "${group}" "**${label}**: pinned \`${current}\`, but \`${latest}\` is published on nuget.org — re-pin \`${pin}\` in \`Directory.Build.props\`. [nupkg](${url})"
+                links=""; [ -z "${src}" ] || links="$(sibling_links "${src}" "${latest}" || true)"
+                note "${group}" "**${label}**: pinned \`${current}\`, but \`${latest}\` is published on nuget.org — re-pin \`${pin}\` in \`Directory.Build.props\`. [nupkg](${url})${links}"
             else
                 echo "    -   ${label}: ${latest} is indexed but its .nupkg does not download yet (pinned ${current})"
             fi

build/upstream.tsv

@@ -19,7 +19,11 @@
 #            NuGet package id
 #   confirm  download URL template with {version}, or "-" to derive it from the locator
 #
-# group	label	kind	pin	locator	confirm
-Platform packages	OpenTok.Net.Android	nuget	OpenTokAndroidPackageVersion	OpenTok.Net.Android	-
-Platform packages	OpenTok.Net.iOS	nuget	OpenTokIosPackageVersion	OpenTok.Net.iOS	-
-Platform packages	OpenTok.Net.Win	nuget	OpenTokWinPackageVersion	OpenTok.Net.Win	-
+#   source   owner/repo publishing the component - optional, and only meaningful on nuget
+#            rows, where the drift finding links that repository's release notes for the
+#            version it reports
+#
+# group	label	kind	pin	locator	confirm	source
+Platform packages	OpenTok.Net.Android	nuget	OpenTokAndroidPackageVersion	OpenTok.Net.Android	-	sbokatuk/OpenTok.Net.Android
+Platform packages	OpenTok.Net.iOS	nuget	OpenTokIosPackageVersion	OpenTok.Net.iOS	-	sbokatuk/OpenTok.Net.iOS
+Platform packages	OpenTok.Net.Win	nuget	OpenTokWinPackageVersion	OpenTok.Net.Win	-	sbokatuk/OpenTok.Net.Win