diff --git a/.github/workflows/star-history.yaml b/.github/workflows/star-history.yaml new file mode 100644 index 000000000..a07e4995e --- /dev/null +++ b/.github/workflows/star-history.yaml @@ -0,0 +1,120 @@ +name: Star History + +on: + push: + branches: [main] + paths: + - .github/workflows/star-history.yaml + - scripts/star_history.py + - scripts/star_history_lib/** + + schedule: + - cron: '23 4 * * *' + + watch: + types: [started] + + workflow_dispatch: + +concurrency: + group: star-history + cancel-in-progress: false + +permissions: + contents: write + +env: + RELEASE_TAG: star-history/latest + OUTPUT_DIR: dist/star-history + SCALE: '2' + +jobs: + publish: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Mirror upstream charts + env: + STAR_HISTORY_TOKEN: ${{ secrets.STAR_HISTORY_TOKEN }} + run: | + set -euo pipefail + + python3 scripts/star_history.py mirror \ + --repository "$GITHUB_REPOSITORY" \ + --output-dir "$OUTPUT_DIR" \ + > manifest.json + + python3 -m json.tool manifest.json + + - name: Resolve a Chrome binary + id: chrome + run: | + set -euo pipefail + + for candidate in google-chrome google-chrome-stable chromium chromium-browser; do + if command -v "$candidate" >/dev/null 2>&1; then + echo "bin=$candidate" >> "$GITHUB_OUTPUT" + "$candidate" --version + exit 0 + fi + done + + echo "::error::No Chrome or Chromium binary found on the runner" + exit 1 + + - name: Rasterize PNG variants + env: + CHROME: ${{ steps.chrome.outputs.bin }} + run: | + set -euo pipefail + + python3 scripts/star_history.py rasterize \ + --manifest manifest.json \ + --chrome "$CHROME" \ + --scale "$SCALE" + + - name: Publish release assets + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + + UPDATED_AT=$(date -u +'%Y-%m-%d %H:%M UTC') + ASSET_LIST=$(python3 scripts/star_history.py assets --manifest manifest.json) + mapfile -t ASSETS <<< "$ASSET_LIST" + + printf '%s\n' \ + "Star history charts for \`$GITHUB_REPOSITORY\`, sanitized and mirrored from star-history.com." \ + "" \ + "- **Updated:** $UPDATED_AT" \ + "" \ + "The \`.png\` assets are the ones embedded in the README. GitHub serves" \ + "release assets as \`application/octet-stream\`, which its image proxy" \ + "renders for raster images but blocks for SVG. The \`.svg\` assets are the" \ + "same charts at any resolution, for slides and docs." \ + > release-notes.md + + if gh release view "$RELEASE_TAG" >/dev/null 2>&1; then + gh release edit "$RELEASE_TAG" --notes-file release-notes.md + else + gh release create "$RELEASE_TAG" \ + --title "Star History" \ + --notes-file release-notes.md \ + --target "$GITHUB_SHA" \ + --prerelease \ + --latest=false + fi + + gh release upload "$RELEASE_TAG" "${ASSETS[@]}" --clobber + + printf '%s\n' \ + "### Star history updated" \ + "" \ + "- **Updated:** $UPDATED_AT" \ + "- **Release:** [\`$RELEASE_TAG\`](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/releases/tag/star-history%2Flatest)" \ + >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index 8ec53bdf4..a63d3277a 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ config.local.* **/build /install/ core/**/bin/ +**/__pycache__/ # Ignore all hidden files and directories .* diff --git a/scripts/star_history.py b/scripts/star_history.py new file mode 100644 index 000000000..dbd977dc3 --- /dev/null +++ b/scripts/star_history.py @@ -0,0 +1,5 @@ +from star_history_lib.cli import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/star_history_lib/__init__.py b/scripts/star_history_lib/__init__.py new file mode 100644 index 000000000..c6b10f8f3 --- /dev/null +++ b/scripts/star_history_lib/__init__.py @@ -0,0 +1,2 @@ +class ChartError(RuntimeError): + pass diff --git a/scripts/star_history_lib/cli.py b/scripts/star_history_lib/cli.py new file mode 100644 index 000000000..f97c7a4fc --- /dev/null +++ b/scripts/star_history_lib/cli.py @@ -0,0 +1,113 @@ +import argparse +import json +import os +import sys + +from . import ChartError +from .client import CHART_ENDPOINT, chart_url, fetch_svg +from .raster import asset_paths, rasterize, write_raster_page +from .svg import background_of, sanitize_svg, viewport_of + + +THEMES = ("light", "dark") + + +def add_mirror_parser(subparsers) -> None: + parser = subparsers.add_parser("mirror") + parser.add_argument("--repository", required=True) + parser.add_argument("--output-dir", required=True) + parser.add_argument("--basename", default="star-history") + parser.add_argument("--token", default=os.environ.get("STAR_HISTORY_TOKEN", "")) + parser.add_argument("--chart-type", default="date") + parser.add_argument("--legend", default="top-left") + parser.set_defaults(handler=mirror) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Mirror star-history.com charts") + subparsers = parser.add_subparsers(required=True) + add_mirror_parser(subparsers) + + raster_parser = subparsers.add_parser("rasterize") + raster_parser.add_argument("--manifest", required=True) + raster_parser.add_argument("--chrome", required=True) + raster_parser.add_argument("--scale", default="2") + raster_parser.set_defaults(handler=run_rasterize) + + assets_parser = subparsers.add_parser("assets") + assets_parser.add_argument("--manifest", required=True) + assets_parser.set_defaults(handler=print_assets) + return parser.parse_args() + + +def fetch_sanitized_svg(url: str) -> str: + return sanitize_svg(fetch_svg(url)) + + +def build_manifest(args: argparse.Namespace) -> dict: + if args.repository.count("/") != 1: + raise ChartError(f"--repository must be owner/name, got '{args.repository}'") + + os.makedirs(args.output_dir, exist_ok=True) + charts = [] + assets = [] + + for theme in THEMES: + url = chart_url( + args.repository, theme, args.token, args.chart_type, args.legend + ) + svg = fetch_sanitized_svg(url) + svg_path = os.path.join(args.output_dir, f"{args.basename}-{theme}.svg") + png_path = os.path.join(args.output_dir, f"{args.basename}-{theme}.png") + with open(svg_path, "w", encoding="utf-8") as handle: + handle.write(svg) + + width, height = viewport_of(svg, theme) + background = background_of(svg, theme) + charts.append( + { + "theme": theme, + "svg": svg_path, + "png": png_path, + "raster_page": write_raster_page( + svg_path, background, width, height + ), + "background": background, + "width": width, + "height": height, + "bytes": len(svg.encode("utf-8")), + } + ) + assets.extend((svg_path, png_path)) + + return { + "repository": args.repository, + "source": CHART_ENDPOINT, + "authenticated": bool(args.token), + "assets": assets, + "charts": charts, + } + + +def mirror(args: argparse.Namespace) -> int: + print(json.dumps(build_manifest(args))) + return 0 + + +def run_rasterize(args: argparse.Namespace) -> int: + rasterize(args.manifest, args.chrome, args.scale) + return 0 + + +def print_assets(args: argparse.Namespace) -> int: + print("\n".join(asset_paths(args.manifest))) + return 0 + + +def main() -> int: + args = parse_args() + try: + return args.handler(args) + except ChartError as error: + print(f"::error::{error}", file=sys.stderr) + return 1 diff --git a/scripts/star_history_lib/client.py b/scripts/star_history_lib/client.py new file mode 100644 index 000000000..384ff87fa --- /dev/null +++ b/scripts/star_history_lib/client.py @@ -0,0 +1,67 @@ +import re +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +from . import ChartError + + +CHART_ENDPOINT = "https://api.star-history.com/chart" +USER_AGENT = "opentaint-star-history" +REQUEST_TIMEOUT = 60 +RETRIES = 3 +RETRY_BACKOFF_SECONDS = 5 +MIN_CHART_BYTES = 2048 + + +def chart_url( + repository: str, theme: str, token: str, chart_type: str, legend: str +) -> str: + query = [("repos", repository), ("type", chart_type), ("legend", legend)] + if theme != "light": + query.append(("theme", theme)) + if token: + query.append(("sealed_token", token)) + return f"{CHART_ENDPOINT}?{urllib.parse.urlencode(query)}" + + +def redact(url: str) -> str: + return re.sub(r"(sealed_token=)[^&]*", r"\1***", url) + + +def fetch_svg(url: str) -> str: + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + last_error = "" + + for attempt in range(1, RETRIES + 1): + try: + with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT) as response: + content_type = response.headers.get("Content-Type", "") + body = response.read().decode("utf-8") + except urllib.error.HTTPError as error: + last_error = f"HTTP {error.code}" + except (urllib.error.URLError, TimeoutError) as error: + last_error = str(error) + except UnicodeDecodeError as error: + raise ChartError( + f"{redact(url)} returned non-text data: {error}" + ) from error + else: + if "svg" not in content_type: + last_error = f"unexpected Content-Type '{content_type}'" + elif len(body) < MIN_CHART_BYTES: + last_error = f"chart is suspiciously small ({len(body)} bytes)" + else: + return body + + if attempt < RETRIES: + print( + f"::warning::Attempt {attempt}/{RETRIES} for {redact(url)} failed: " + f"{last_error}; retrying", + file=sys.stderr, + ) + time.sleep(RETRY_BACKOFF_SECONDS * attempt) + + raise ChartError(f"{redact(url)} failed after {RETRIES} attempts: {last_error}") diff --git a/scripts/star_history_lib/raster.py b/scripts/star_history_lib/raster.py new file mode 100644 index 000000000..01900fa9b --- /dev/null +++ b/scripts/star_history_lib/raster.py @@ -0,0 +1,77 @@ +import html +import json +import os +import subprocess +import tempfile +from pathlib import Path + +from . import ChartError + + +CSP = ( + "default-src 'none'; img-src 'self' data:; " + "style-src 'unsafe-inline'; font-src data:" +) + + +def write_raster_page( + svg_path: str, background: str, width: int, height: int +) -> str: + page_path = f"{svg_path}.html" + image_name = html.escape(os.path.basename(svg_path), quote=True) + safe_background = html.escape(background, quote=True) + document = f""" + + + + + + + +""" + with open(page_path, "w", encoding="utf-8") as handle: + handle.write(document) + return page_path + + +def load_manifest(path: str) -> dict: + with open(path, encoding="utf-8") as handle: + return json.load(handle) + + +def asset_paths(path: str) -> list[str]: + return load_manifest(path)["assets"] + + +def rasterize(path: str, chrome: str, scale: str) -> None: + for chart in load_manifest(path)["charts"]: + png = Path(chart["png"]).resolve() + page = Path(chart["raster_page"]).resolve() + with tempfile.TemporaryDirectory() as profile: + subprocess.run( + [ + chrome, + "--headless", + "--disable-gpu", + "--disable-dev-shm-usage", + "--hide-scrollbars", + f"--user-data-dir={profile}", + f"--force-device-scale-factor={scale}", + "--virtual-time-budget=10000", + f"--window-size={chart['width']},{chart['height']}", + f"--screenshot={png}", + page.as_uri(), + ], + check=True, + ) + if not png.is_file() or png.stat().st_size == 0: + raise ChartError(f"Chrome produced no screenshot for {chart['theme']}") diff --git a/scripts/star_history_lib/svg.py b/scripts/star_history_lib/svg.py new file mode 100644 index 000000000..1cde575df --- /dev/null +++ b/scripts/star_history_lib/svg.py @@ -0,0 +1,137 @@ +import math +import re +import sys +import xml.etree.ElementTree as ET + +from . import ChartError + + +SVG_NAMESPACE = "http://www.w3.org/2000/svg" +DEFAULT_BACKGROUNDS = {"light": "#ffffff", "dark": "#0d1117"} +DEFAULT_SIZE = (800, 534) + +BACKGROUND_RE = re.compile(r"background:\s*(#[0-9a-fA-F]{3,8}|[a-zA-Z]+)") +WIDTH_RE = re.compile(r'\bwidth="([0-9.]+)"') +HEIGHT_RE = re.compile(r'\bheight="([0-9.]+)"') +CSS_URL_RE = re.compile(r"url\(\s*(['\"]?)(.*?)\1\s*\)", re.IGNORECASE | re.DOTALL) +CSS_URL_START_RE = re.compile(r"url\s*\(", re.IGNORECASE) +UNSAFE_CSS_RE = re.compile( + r"@import|expression\s*\(|behavior\s*:|" + r"(?:file|ftp|https?|javascript|vbscript)\s*:", + re.IGNORECASE, +) +SAFE_RASTER_DATA_RE = re.compile( + r"^data:image/(?:gif|jpeg|png|webp)(?:[;,])", re.IGNORECASE +) +SAFE_FONT_DATA_RE = re.compile( + r"^data:(?:application/font-woff|font/woff2?)(?:[;,])", re.IGNORECASE +) +DANGEROUS_ELEMENTS = { + "a", + "animate", + "animatemotion", + "animatetransform", + "discard", + "embed", + "foreignobject", + "handler", + "iframe", + "listener", + "object", + "script", + "set", +} + + +def split_xml_name(name: str) -> tuple[str, str]: + if name.startswith("{"): + namespace, local_name = name[1:].split("}", 1) + return namespace, local_name + return "", name + + +def validate_resource_reference(reference: str, *, allow_font: bool) -> None: + reference = reference.strip() + if reference.startswith("#") or SAFE_RASTER_DATA_RE.match(reference): + return + if allow_font and SAFE_FONT_DATA_RE.match(reference): + return + raise ChartError(f"unsafe SVG resource reference: {reference[:80]!r}") + + +def validate_css(css: str) -> None: + if "\\" in css: + raise ChartError("SVG CSS escapes are not allowed") + if UNSAFE_CSS_RE.search(css): + raise ChartError("SVG CSS contains an active or external construct") + + matches = list(CSS_URL_RE.finditer(css)) + if len(matches) != len(CSS_URL_START_RE.findall(css)): + raise ChartError("SVG CSS contains a malformed url() reference") + for match in matches: + validate_resource_reference(match.group(2), allow_font=True) + + +def sanitize_svg(svg: str) -> str: + lowered = svg.lower() + if " is not allowed" + ) + if element_name.lower() in DANGEROUS_ELEMENTS: + raise ChartError(f"active SVG element <{element_name}> is not allowed") + + for attribute_name, value in element.attrib.items(): + _, local_name = split_xml_name(attribute_name) + lowered_name = local_name.lower() + if lowered_name.startswith("on"): + raise ChartError(f"SVG event attribute {local_name!r} is not allowed") + if lowered_name in {"href", "src"}: + validate_resource_reference(value, allow_font=False) + if lowered_name == "style" or CSS_URL_START_RE.search(value): + validate_css(value) + + if element_name.lower() == "style": + validate_css(element.text or "") + + ET.register_namespace("", SVG_NAMESPACE) + return ET.tostring(root, encoding="unicode") + + +def background_of(svg: str, theme: str) -> str: + match = BACKGROUND_RE.search(svg[:1024]) + if match: + return match.group(1) + print( + f"::warning::No background declared in the {theme} chart; " + f"falling back to {DEFAULT_BACKGROUNDS[theme]}", + file=sys.stderr, + ) + return DEFAULT_BACKGROUNDS[theme] + + +def viewport_of(svg: str, theme: str) -> tuple[int, int]: + root = svg[:1024] + width, height = WIDTH_RE.search(root), HEIGHT_RE.search(root) + if width and height: + return math.ceil(float(width.group(1))), math.ceil(float(height.group(1))) + print( + f"::warning::No intrinsic size on the {theme} chart; " + f"falling back to {DEFAULT_SIZE[0]}x{DEFAULT_SIZE[1]}", + file=sys.stderr, + ) + return DEFAULT_SIZE