From 9dea75bc4fa459605bb2b58738486c3d82e867ac Mon Sep 17 00:00:00 2001 From: MajorMoth <61519600+MajorMoth@users.noreply.github.com> Date: Mon, 20 Jul 2026 22:18:36 +0200 Subject: [PATCH 1/2] changelog workflow update --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/workflows/publish.yml | 4 +- Tools/actions_changelogs_since_last_run.py | 228 ------------------ .../{ => changelogs}/actions_changelog_rss.py | 0 .../actions_changelogs_since_last_run.py | 62 +++++ Tools/changelogs/changelog_helperfunctions.py | 208 ++++++++++++++++ 6 files changed, 273 insertions(+), 231 deletions(-) delete mode 100755 Tools/actions_changelogs_since_last_run.py rename Tools/{ => changelogs}/actions_changelog_rss.py (100%) mode change 100755 => 100644 create mode 100644 Tools/changelogs/actions_changelogs_since_last_run.py create mode 100644 Tools/changelogs/changelog_helperfunctions.py diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 0fbb0b8ae6b..912b5383b02 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -15,7 +15,7 @@ Describe how you tested the pull request, and how someone reviewing this PR can --> ## Media - + ## Requirements diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1910a630ad7..f290002065c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -54,13 +54,13 @@ jobs: - name: Publish changelog (Discord) continue-on-error: true - run: Tools/actions_changelogs_since_last_run.py + run: Tools/changelogs/actions_changelogs_since_last_run.py env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DISCORD_WEBHOOK_URL: ${{ secrets.CHANGELOG_DISCORD_WEBHOOK }} - name: Publish changelog (RSS) continue-on-error: true - run: Tools/actions_changelog_rss.py + run: Tools/changelogs/actions_changelog_rss.py env: CHANGELOG_RSS_KEY: ${{ secrets.CHANGELOG_RSS_KEY }} diff --git a/Tools/actions_changelogs_since_last_run.py b/Tools/actions_changelogs_since_last_run.py deleted file mode 100755 index a50dc73a4bc..00000000000 --- a/Tools/actions_changelogs_since_last_run.py +++ /dev/null @@ -1,228 +0,0 @@ -#!/usr/bin/env python3 - -""" -Sends updates to a Discord webhook for new changelog entries since the last GitHub Actions publish run. - -Automatically figures out the last run and changelog contents with the GitHub API. -""" - -import itertools -import os -from pathlib import Path -from typing import Any, Iterable - -import requests -import yaml -import time - -DEBUG = False -DEBUG_CHANGELOG_FILE_OLD = Path("Resources/Changelog/Old.yml") -GITHUB_API_URL = os.environ.get("GITHUB_API_URL", "https://api.github.com") - -# https://discord.com/developers/docs/resources/webhook -DISCORD_SPLIT_LIMIT = 2000 -DISCORD_WEBHOOK_URL = os.environ.get("DISCORD_WEBHOOK_URL") - -CHANGELOG_FILE = "Resources/Changelog/Den.yml" - -TYPES_TO_EMOJI = {"Fix": "🐛", "Add": "🆕", "Remove": "❌", "Tweak": "⚒️"} - -ChangelogEntry = dict[str, Any] - - -def main(): - if not DISCORD_WEBHOOK_URL: - print("No discord webhook URL found, skipping discord send") - return - - if DEBUG: - # to debug this script locally, you can use - # a separate local file as the old changelog - last_changelog_stream = DEBUG_CHANGELOG_FILE_OLD.read_text() - else: - # when running this normally in a GitHub actions workflow, - # it will get the old changelog from the GitHub API - last_changelog_stream = get_last_changelog() - - last_changelog = yaml.safe_load(last_changelog_stream) - with open(CHANGELOG_FILE, "r") as f: - cur_changelog = yaml.safe_load(f) - - diff = diff_changelog(last_changelog, cur_changelog) - message_lines = changelog_entries_to_message_lines(diff) - send_message_lines(message_lines) - - -def get_most_recent_workflow( - sess: requests.Session, github_repository: str, github_run: str -) -> Any: - workflow_run = get_current_run(sess, github_repository, github_run) - past_runs = get_past_runs(sess, workflow_run) - for run in past_runs["workflow_runs"]: - # First past successful run that isn't our current run. - if run["id"] == workflow_run["id"]: - continue - - return run - - -def get_current_run( - sess: requests.Session, github_repository: str, github_run: str -) -> Any: - resp = sess.get( - f"{GITHUB_API_URL}/repos/{github_repository}/actions/runs/{github_run}" - ) - resp.raise_for_status() - return resp.json() - - -def get_past_runs(sess: requests.Session, current_run: Any) -> Any: - """ - Get all successful workflow runs before our current one. - """ - params = {"status": "success", "created": f"<={current_run['created_at']}"} - resp = sess.get(f"{current_run['workflow_url']}/runs", params=params) - resp.raise_for_status() - return resp.json() - - -def get_last_changelog() -> str: - github_repository = os.environ["GITHUB_REPOSITORY"] - github_run = os.environ["GITHUB_RUN_ID"] - github_token = os.environ["GITHUB_TOKEN"] - - session = requests.Session() - session.headers["Authorization"] = f"Bearer {github_token}" - session.headers["Accept"] = "Accept: application/vnd.github+json" - session.headers["X-GitHub-Api-Version"] = "2022-11-28" - - most_recent = get_most_recent_workflow(session, github_repository, github_run) - last_sha = most_recent["head_commit"]["id"] - print(f"Last successful publish job was {most_recent['id']}: {last_sha}") - last_changelog_stream = get_last_changelog_by_sha( - session, last_sha, github_repository - ) - - return last_changelog_stream - - -def get_last_changelog_by_sha( - sess: requests.Session, sha: str, github_repository: str -) -> str: - """ - Use GitHub API to get the previous version of the changelog YAML (Actions builds are fetched with a shallow clone) - """ - params = { - "ref": sha, - } - headers = {"Accept": "application/vnd.github.raw"} - - resp = sess.get( - f"{GITHUB_API_URL}/repos/{github_repository}/contents/{CHANGELOG_FILE}", - headers=headers, - params=params, - ) - resp.raise_for_status() - return resp.text - - -def diff_changelog( - old: dict[str, Any], cur: dict[str, Any] -) -> Iterable[ChangelogEntry]: - """ - Find all new entries not present in the previous publish. - """ - old_entry_ids = {e["id"] for e in old["Entries"]} - return (e for e in cur["Entries"] if e["id"] not in old_entry_ids) - - -def get_discord_body(content: str): - return { - "content": content, - # Do not allow any mentions. - "allowed_mentions": {"parse": []}, - # SUPPRESS_EMBEDS - "flags": 1 << 2, - } - - -def send_discord_webhook(lines: list[str]): - content = "".join(lines) - body = get_discord_body(content) - retry_attempt = 0 - - try: - response = requests.post(DISCORD_WEBHOOK_URL, json=body, timeout=10) - while response.status_code == 429: - retry_attempt += 1 - if retry_attempt > 20: - print("Too many retries on a single request despite following retry_after header... giving up") - exit(1) - retry_after = response.json().get("retry_after", 5) - print(f"Rate limited, retrying after {retry_after} seconds") - time.sleep(retry_after) - response = requests.post(DISCORD_WEBHOOK_URL, json=body, timeout=10) - response.raise_for_status() - except requests.exceptions.RequestException as e: - print(f"Failed to send message: {e}") - exit(1) - - -def changelog_entries_to_message_lines(entries: Iterable[ChangelogEntry]) -> list[str]: - """Process structured changelog entries into a list of lines making up a formatted message.""" - message_lines = [] - - for contributor_name, group in itertools.groupby(entries, lambda x: x["author"]): - message_lines.append("\n") - message_lines.append(f"**{contributor_name}** updated:\n") - - for entry in group: - url = entry.get("url") - if url and not url.strip(): - url = None - - for change in entry["changes"]: - emoji = TYPES_TO_EMOJI.get(change["type"], "❓") - message = change["message"] - - # if a single line is longer than the limit, it needs to be truncated - if len(message) > DISCORD_SPLIT_LIMIT: - message = message[: DISCORD_SPLIT_LIMIT - 100].rstrip() + " [...]" - - if url is not None: - pr_number = url.split("/")[-1] - line = f"{emoji} - {message} ([#{pr_number}]({url}))\n" - else: - line = f"{emoji} - {message}\n" - - message_lines.append(line) - - return message_lines - - -def send_message_lines(message_lines: list[str]): - """Join a list of message lines into chunks that are each below Discord's message length limit, and send them.""" - chunk_lines = [] - chunk_length = 0 - - for line in message_lines: - line_length = len(line) - new_chunk_length = chunk_length + line_length - - if new_chunk_length > DISCORD_SPLIT_LIMIT: - print("Split changelog and sending to discord") - send_discord_webhook(chunk_lines) - - new_chunk_length = line_length - chunk_lines.clear() - - chunk_lines.append(line) - chunk_length = new_chunk_length - - if chunk_lines: - print("Sending final changelog to discord") - send_discord_webhook(chunk_lines) - - -if __name__ == "__main__": - main() diff --git a/Tools/actions_changelog_rss.py b/Tools/changelogs/actions_changelog_rss.py old mode 100755 new mode 100644 similarity index 100% rename from Tools/actions_changelog_rss.py rename to Tools/changelogs/actions_changelog_rss.py diff --git a/Tools/changelogs/actions_changelogs_since_last_run.py b/Tools/changelogs/actions_changelogs_since_last_run.py new file mode 100644 index 00000000000..9f7be1f2e7d --- /dev/null +++ b/Tools/changelogs/actions_changelogs_since_last_run.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +""" +Sends updates to a Discord webhook for new changelog entries since the last GitHub Actions publish run. + +Automatically figures out the last run and changelog contents with the GitHub API. +""" + +import requests # REMOVE + +from changelog_helperfunctions import log, get_pr_json, grab_image_urls_from_pr_body, validate_environment, create_session, get_changes, send_discord_webhook, TYPES_TO_EMOJI + + +def main(): + if not validate_environment(): + exit(1) + + sess: requests.Session = create_session() + changes = get_changes(sess) + + data = {"embeds": []} + embed_count: int = 0 + + for author, changes, id, time, url in (entry.values() for entry in changes): + if embed_count >= 10: # discord allows up to 10 embeds per message + if not send_discord_webhook(data): + exit(1) + embed_count = 0 + data = {"embeds": []} + + pr = get_pr_json(sess, url) + + if not pr: + log.error(f"Could not find the pull request from: {url}") + + embed = { + "title": f"{author}", + "thumbnail": {"url": pr["user"]["avatar_url"]}, + "description": f"### {pr["title"]} [[PR]]({url})", + "color": int("#81BABA"[1:], base=16), # embeds want an integer color representation for some reason, this allows you to put in a hex value in code and have a preview of the color in your ide + # change this to whatever color fits your repository + "fields": [] + } + + images = grab_image_urls_from_pr_body(pr["body"]) + if images: + embed.update({"image": { "url": images[0] }}) # maybe discord adds nice multi-image embed support later down the line, for now, only the first image. + else: + log.info(f"Could not find any images in pull request at: {url}") + + for message, type in ((change["message"], change["type"]) for change in changes): + emoji = TYPES_TO_EMOJI.get(type, "❓") + embed["fields"].append({"name":f"", "value":f"{emoji} - {message}"}) + + data["embeds"].append(embed) + embed_count += 1 + + return + + +if __name__ == "__main__": + main() diff --git a/Tools/changelogs/changelog_helperfunctions.py b/Tools/changelogs/changelog_helperfunctions.py new file mode 100644 index 00000000000..dceca0f9e81 --- /dev/null +++ b/Tools/changelogs/changelog_helperfunctions.py @@ -0,0 +1,208 @@ +from pathlib import Path +from typing import Any, Iterable + +import re +import os +import yaml +import time +import logging +import requests + +FORMAT = "%(message)s" +logging.basicConfig( + level="NOTSET", format=FORMAT, datefmt="[%X]" +) +log = logging.getLogger("changelog") + +ChangelogEntry = dict[str, Any] + +DEBUG = False +DEBUG_CHANGELOG_FILE_OLD = Path("Resources/Changelog/Old.yml") +CHANGELOG_FILE = "Resources/Changelog/Den.yml" + +TYPES_TO_EMOJI = {"Fix": "🐛", "Add": "🆕", "Remove": "❌", "Tweak": "⚒️"} + +# https://discord.com/developers/docs/resources/webhook +DISCORD_SPLIT_LIMIT = 2000 + +DISCORD_WEBHOOK_URL = os.environ.get("DISCORD_WEBHOOK_URL") +GITHUB_API_URL = os.environ.get("GITHUB_API_URL", "https://api.github.com") +GITHUB_REPOSITORY = os.environ.get("GITHUB_REPOSITORY") +GITHUB_RUN = os.environ.get("GITHUB_RUN_ID") +GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN") + +if DEBUG: + GITHUB_REPOSITORY = "" + GITHUB_RUN = 1 + GITHUB_TOKEN = "" # replace with your personal access token or the user token from your repository's secrets for debugging DO NOT COMMIT IT WITH THE TOKEN + DISCORD_WEBHOOK_URL = "" # similar deal with this + + +def validate_environment() -> bool: + """Validates whether the current environment has all the required environment variables. Returns true/false based on success/failure.""" + + if not DISCORD_WEBHOOK_URL: + log.error("No Discord webhook URL found.") + return False + + if not os.environ.get("GITHUB_API_URL"): + log.warning("No Github API URL found - if the fallback API URL is deprecated, this script will stop working.") + + if not GITHUB_REPOSITORY: + log.error("No Github repository found.") + return False + + if not GITHUB_RUN: + log.error("No Github run identifier found.") + return False + + if not GITHUB_TOKEN: + log.error("No Github user token found.") + return False + + return True + +def create_session() -> requests.Session: + """Creates a session using the requests module to be used for interacting with REST APIs.""" + sess = requests.Session() + sess.headers["Authorization"] = f"Bearer {GITHUB_TOKEN}" + sess.headers["Accept"] = "Accept: application/vnd.github+json" + sess.headers["X-GitHub-Api-Version"] = "2026-03-10" # upgrade + return sess + +def get_changes( + sess: requests.Session +) -> str: + if DEBUG: + log.info("Debug mode active.") + # to debug this script locally, you can use + # a separate local file as the old changelog + # with a couple of entries removed + with open(DEBUG_CHANGELOG_FILE_OLD, "r", encoding="utf-8-sig") as file: + last_changelog_stream = file.read() + else: + # when running this normally in a GitHub actions workflow, + # it will get the old changelog from the GitHub API + last_changelog_stream = get_last_changelog(sess) + + last_changelog = yaml.safe_load(last_changelog_stream) + with open(CHANGELOG_FILE, "r", encoding="utf-8-sig") as file: + cur_changelog = yaml.safe_load(file) + + return diff_changelog(last_changelog, cur_changelog) # diff_changelog expects a clean string with no byte order mark otherwise it crashes. ask me how I found out. + +def get_most_recent_workflow( + sess: requests.Session, github_repository: str, github_run: str +) -> Any: + workflow_run = get_current_run(sess, github_repository, github_run) + past_runs = get_past_runs(sess, workflow_run) + for run in past_runs["workflow_runs"]: + # First past successful run that isn't our current run. + if run["id"] == workflow_run["id"]: + continue + + return run + + +def get_current_run( + sess: requests.Session, github_repository: str, github_run: str +) -> Any: + resp = sess.get( + f"{GITHUB_API_URL}/repos/{github_repository}/actions/runs/{github_run}" + ) + resp.raise_for_status() + return resp.json() + + +def get_past_runs(sess: requests.Session, current_run: Any) -> Any: + """ + Get all successful workflow runs before our current one. + """ + params = {"status": "success", "created": f"<={current_run['created_at']}"} + resp = sess.get(f"{current_run['workflow_url']}/runs", params=params) + resp.raise_for_status() + return resp.json() + + +def get_last_changelog( + sess: requests.Session +) -> str: + most_recent = get_most_recent_workflow(sess, GITHUB_REPOSITORY, GITHUB_RUN) + last_sha = most_recent["head_commit"]["id"] + print(f"Last successful publish job was {most_recent['id']}: {last_sha}") + last_changelog_stream = get_last_changelog_by_sha( + sess, last_sha, GITHUB_REPOSITORY + ) + + return last_changelog_stream + + +def get_last_changelog_by_sha( + sess: requests.Session, sha: str, github_repository: str +) -> str: + """ + Use GitHub API to get the previous version of the changelog YAML (Actions builds are fetched with a shallow clone) + """ + params = { + "ref": sha, + } + headers = {"Accept": "application/vnd.github.raw"} + + resp = sess.get( + f"{GITHUB_API_URL}/repos/{github_repository}/contents/{CHANGELOG_FILE}", + params=params, + ) + resp.raise_for_status() + return resp.text + + +def diff_changelog( + old: dict[str, Any], cur: dict[str, Any] +) -> Iterable[ChangelogEntry]: + """ + Find all new entries not present in the previous publish. + """ + old_entry_ids = {e["id"] for e in old["Entries"]} + return (e for e in cur["Entries"] if e["id"] not in old_entry_ids) + +def get_pr_json( + sess: requests.Session, pr_url: str +) -> Any: + """Gets the JSON body of the PR using Github's API. The function expects urls in the standard format, not ones already pointing to the API.""" + match = re.match(r"https://github\.com/([^/]+)/([^/]+)/pull/(\d+)", pr_url) + owner, repo, number = match.groups() + + resp = sess.get( + f"{GITHUB_API_URL}/repos/{owner}/{repo}/pulls/{number}" + ) + resp.raise_for_status() + return resp.json() + +def grab_image_urls_from_pr_body( + body: str +) -> list[str]: + """Returns a list of all images contained in HTML tags that appear in the body of the PR. This function expects the body as a string.""" + return re.findall(r']+src="([^"]+)"', body) + +def send_discord_webhook( + json: list[str] +) -> bool: + """Handles actually sending the webhook, and deals with rate limiting/exceptions. Returns true/false based on success/failure.""" + retry_attempt = 0 + + try: + response = requests.post(DISCORD_WEBHOOK_URL, json=json, timeout=10) + while response.status_code == 429: + retry_attempt += 1 + if retry_attempt > 20: + log.error("Too many retries on a single request despite following retry_after header... giving up.") + return False + retry_after = response.json().get("retry_after", 5) + log.info(f"Rate limited, retrying after {retry_after} seconds.") + time.sleep(retry_after) + response = requests.post(DISCORD_WEBHOOK_URL, json=json, timeout=10) + response.raise_for_status() + except requests.exceptions.RequestException as e: + log.exception(f"Failed to send message: {e}") + return False + return True \ No newline at end of file From 64d21a46b5caf9139dc89facba737c5082697dbd Mon Sep 17 00:00:00 2001 From: MajorMoth <61519600+MajorMoth@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:41:39 +0200 Subject: [PATCH 2/2] shurrup pylance --- Tools/changelogs/changelog_helperfunctions.py | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/Tools/changelogs/changelog_helperfunctions.py b/Tools/changelogs/changelog_helperfunctions.py index dceca0f9e81..a2ff9ac5c3d 100644 --- a/Tools/changelogs/changelog_helperfunctions.py +++ b/Tools/changelogs/changelog_helperfunctions.py @@ -25,15 +25,15 @@ # https://discord.com/developers/docs/resources/webhook DISCORD_SPLIT_LIMIT = 2000 -DISCORD_WEBHOOK_URL = os.environ.get("DISCORD_WEBHOOK_URL") +DISCORD_WEBHOOK_URL = os.environ.get("DISCORD_WEBHOOK_URL", "") GITHUB_API_URL = os.environ.get("GITHUB_API_URL", "https://api.github.com") -GITHUB_REPOSITORY = os.environ.get("GITHUB_REPOSITORY") -GITHUB_RUN = os.environ.get("GITHUB_RUN_ID") -GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN") +GITHUB_REPOSITORY = os.environ.get("GITHUB_REPOSITORY", "") +GITHUB_RUN = os.environ.get("GITHUB_RUN_ID", "1") +GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN", "") if DEBUG: GITHUB_REPOSITORY = "" - GITHUB_RUN = 1 + GITHUB_RUN = "1" GITHUB_TOKEN = "" # replace with your personal access token or the user token from your repository's secrets for debugging DO NOT COMMIT IT WITH THE TOKEN DISCORD_WEBHOOK_URL = "" # similar deal with this @@ -72,7 +72,7 @@ def create_session() -> requests.Session: def get_changes( sess: requests.Session -) -> str: +) -> Iterable[ChangelogEntry]: if DEBUG: log.info("Debug mode active.") # to debug this script locally, you can use @@ -170,13 +170,15 @@ def get_pr_json( ) -> Any: """Gets the JSON body of the PR using Github's API. The function expects urls in the standard format, not ones already pointing to the API.""" match = re.match(r"https://github\.com/([^/]+)/([^/]+)/pull/(\d+)", pr_url) - owner, repo, number = match.groups() - - resp = sess.get( - f"{GITHUB_API_URL}/repos/{owner}/{repo}/pulls/{number}" - ) - resp.raise_for_status() - return resp.json() + if match: + owner, repo, number = match.groups() + resp = sess.get( + f"{GITHUB_API_URL}/repos/{owner}/{repo}/pulls/{number}" + ) + resp.raise_for_status() + return resp.json() + else: + return None def grab_image_urls_from_pr_body( body: str @@ -185,7 +187,7 @@ def grab_image_urls_from_pr_body( return re.findall(r']+src="([^"]+)"', body) def send_discord_webhook( - json: list[str] + json: dict[str, list] ) -> bool: """Handles actually sending the webhook, and deals with rate limiting/exceptions. Returns true/false based on success/failure.""" retry_attempt = 0