Frontend regression testing

.github/workflows/tests.yml

@@ -131,7 +131,7 @@ jobs:
           python manage.py scss
 
       - name: Run tests (shuffled)
-        run: coverage run manage.py test --shuffle --tag selenium
+        run: coverage run manage.py test --shuffle --tag selenium --exclude-tag vrt
       - name: Upload coverage
         uses: codecov/codecov-action@v5
         with:

.github/workflows/vrt.yml

@@ -0,0 +1,58 @@
+name: Visual Test Suite
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+jobs:
+  visual-regression-tests:
+    runs-on: ubuntu-22.04
+    name: Visual Regression Tests
+    env:
+      VRT_APIURL: ${{ vars.VRT_APIURL }}
+      VRT_APIKEY: ${{ secrets.VRT_APIKEY }}
+      VRT_PROJECT: ${{ secrets.VRT_PROJECT }}
+      VRT_BRANCHNAME: ${{ github.event.pull_request.head.label }}
+    steps:
+      - name: Check if user is collaborator
+        uses: actions/github-script@v8
+        env:
+          TRIGGERING_USERNAME: ${{ github.triggering_actor }}
+        with:
+          script: |
+            const username = process.env.TRIGGERING_USERNAME;
+
+            const isCollaborator = await github.rest.repos.checkCollaborator({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                username: username
+            })
+                .then(({status}) => status === 204)
+                .catch((error) => {
+                    if (error.status === 404) return false;
+                    throw error;
+                });
+
+            if(!isCollaborator) {
+              console.error(`Insufficient privileges, ${username} is not a collaborator`);
+              process.exit(1)
+            }
+
+            console.log(`User ${username} is a project collaborator, continuing...`)
+
+      - name: Checkout PR
+        uses: actions/checkout@v6
+        with:
+          submodules: true
+          repository: '${{ github.event.pull_request.head.repo.full_name }}'
+          ref: '${{ github.event.pull_request.head.ref }}'
+      - uses: ./.github/setup_evap
+        with:
+          shell: .#evap-frontend-dev
+          npm-ci: true
+          start-db: true
+      - name: Load Self-Signed Certificate
+        run: |
+          echo "${{ secrets.VRT_SSL_CERTIFICATE }}" > evap-vrt.crt
+          sudo cp evap-vrt.crt /usr/local/share/ca-certificates/
+          sudo update-ca-certificates
+      - name: Run visual regression tests
+        run: python manage.py vrt_test

evap/evaluation/management/commands/vrt_test.py

@@ -0,0 +1,43 @@
+import os.path
+import sys
+from subprocess import run
+
+from django.core.management import call_command
+from django.core.management.base import BaseCommand
+
+from evap.evaluation.management.commands.tools import subprocess_run_or_exit
+
+
+class Command(BaseCommand):
+    help = "Run the visual regression testing suite"
+
+    def handle(self, *args, **options):
+        assert os.environ.get("VRT_APIURL"), "env var VRT_APIURL must be set"
+        assert os.environ.get("VRT_APIKEY"), "env var VRT_APIKEY must be set"
+        assert os.environ.get("VRT_PROJECT"), "env var VRT_PROJECT must be set"
+
+        commit_hash = os.environ.get("VRT_CIBUILDID")
+        if not commit_hash:
+            commit_hash = run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, check=False)
+            if commit_hash.returncode != 0:
+                self.stderr.write(self.style.ERROR("Could not get commit hash: " + str(commit_hash.stderr)))
+                sys.exit(1)
+            commit_hash = commit_hash.stdout.decode().strip()
+            self.stdout.write(self.style.NOTICE(f"using VRT_CIBUILDID = '{commit_hash}'"))
+            os.environ["VRT_CIBUILDID"] = commit_hash
+
+        branch_name = os.environ.get("VRT_BRANCHNAME")
+        if not branch_name:
+            branch_name = run(["git", "rev-parse", "--abbrev-ref", "HEAD"], capture_output=True, check=False)
+            if branch_name.returncode != 0:
+                self.stderr.write(self.style.ERROR("Could not get branch name: " + branch_name.stderr))
+                sys.exit(1)
+            branch_name = branch_name.stdout.decode().strip()
+            self.stdout.write(self.style.NOTICE(f"using VRT_BRANCHNAME = '{branch_name}'"))
+            os.environ["VRT_BRANCHNAME"] = branch_name
+
+        call_command("ts", "compile")
+        call_command("scss")
+
+        # subprocess call so our sys.argv check in settings.py works
+        subprocess_run_or_exit(["./manage.py", "test", "--tag", "vrt"], self.stdout)

evap/evaluation/tests/tools.py

@@ -1,3 +1,5 @@
+import json
+import os
 import time
 from collections.abc import Iterator, Sequence
 from contextlib import contextmanager
@@ -7,18 +9,21 @@
 
 import django.test
 import django_webtest
+import requests
 import webtest
 from django.conf import settings
 from django.contrib.auth import login
 from django.contrib.auth.models import Group
 from django.contrib.staticfiles.handlers import StaticFilesHandler
 from django.db import DEFAULT_DB_ALIAS, connections
 from django.http.request import HttpRequest, QueryDict
+from django.test import override_settings, tag
 from django.test.runner import DiscoverRunner
 from django.test.selenium import SeleniumTestCase
 from django.test.utils import CaptureQueriesContext
 from django.urls import reverse
 from django.utils import timezone, translation
+from freezegun import freeze_time
 from model_bakery import baker
 from selenium.webdriver.common.by import By
 from selenium.webdriver.firefox.webdriver import WebDriver
@@ -41,15 +46,15 @@
 
 
 class EvapTestRunner(DiscoverRunner):
-    """Skips selenium tests by default, if no other tags are specified."""
+    """Skips selenium and vrt tests by default, if no other tags are specified."""
 
     def __init__(self, *args: Any, headed=False, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
 
         self.__headed = headed
 
         if not self.tags and not self.exclude_tags:
-            self.exclude_tags = {"selenium"}
+            self.exclude_tags = {"selenium", "vrt"}
 
     @classmethod
     def add_arguments(cls, parser):
@@ -305,6 +310,7 @@ class LiveServerTest(SeleniumTestCase):
 
     def setUp(self) -> None:
         super().setUp()
+
         self.request = self.make_request()
         self.manager = make_manager()
         self.selenium.get(self.live_server_url)
@@ -360,6 +366,111 @@ def setUpClass(cls) -> None:
         cls.selenium.set_window_size(*cls.window_size)
 
 
+@tag("vrt")
+@override_settings(SLOGANS_EN=["Einigermaßen verlässlich aussehende Pixeltestung"])
+class VisualRegressionTestCase(LiveServerTest):
+    window_size = (1920, 1080)
+    _http_timeout_seconds = 3
+    _freezer: Any
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.api_url = os.environ.get("VRT_APIURL")
+
+        self.headers = {
+            "apiKey": os.environ.get("VRT_APIKEY"),
+            "Content-Type": "application/json",
+        }
+
+        project_id = os.environ.get("VRT_PROJECT")
+        self.data = {
+            "project": project_id,
+            "projectId": project_id,  # this is not a typo, depending on the request either project/projectId is used
+            "branchName": os.environ.get("VRT_BRANCHNAME"),
+            "ciBuildId": os.environ.get("VRT_CIBUILDID"),
+        }
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        super().setUpClass()
+
+        cls._freezer = freeze_time("2025-10-27")
+        cls._freezer.start()
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        super().tearDownClass()
+        cls._freezer.stop()
+
+    @property
+    def viewport(self):
+        return f"{self.window_size[0]}x{self.window_size[1]}"
+
+    def setUp(self) -> None:
+        super().setUp()
+        self.build_id = self._start_vrt_session()
+
+    def tearDown(self) -> None:
+        super().tearDown()
+        self._stop_vrt_session()
+
+    def _start_vrt_session(self) -> str:
+        registration_response = requests.post(
+            f"{self.api_url}/builds",
+            data=json.dumps(self.data),
+            headers=self.headers,
+            timeout=self._http_timeout_seconds,
+        )
+
+        registration_response.raise_for_status()
+        return registration_response.json().get("id")
+
+    def _stop_vrt_session(self):
+        # marks the session of the current as done
+        response = requests.patch(
+            f"{self.api_url}/builds/{self.build_id}",
+            data=json.dumps({"isRunning": False}),
+            headers=self.headers,
+            timeout=self._http_timeout_seconds,
+        )
+        response.raise_for_status()
+
+    def _post_screenshot(self, name) -> tuple[str, str]:
+        test_data = self.data | {
+            "name": name,
+            "imageBase64": self.selenium.get_screenshot_as_base64(),
+            "viewport": self.viewport,
+            "buildId": self.build_id,
+        }
+
+        test_response = requests.post(
+            f"{self.api_url}/test-runs",
+            data=json.dumps(test_data),
+            headers=self.headers,
+            timeout=self._http_timeout_seconds,
+        )
+
+        test_response.raise_for_status()
+        payload = test_response.json()
+        return payload.get("status"), payload.get("url", "<url-not-found>")
+
+    def trigger_screenshot(self, name: str):
+        full_name = self.__class__.__name__ + "_" + name
+
+        status, review_url = self._post_screenshot(full_name)
+
+        switcher = {
+            "new": f"No Baseline! Review manually: {review_url}",
+            "unresolved": f"Difference found: {review_url}",
+        }
+
+        error_message = switcher.get(status)
+
+        if error_message:
+            self.fail(error_message)
+
+
 def classes_of_element(element: WebElement) -> list[str]:
     classes = element.get_attribute("class")
     if classes is None:

evap/grades/tests/test_live.py

@@ -0,0 +1,28 @@
+from model_bakery import baker
+
+from evap.evaluation.models import Course, Evaluation, Group, Semester
+from evap.evaluation.tests.tools import VisualRegressionTestCase
+
+
+class GradesViewTest(VisualRegressionTestCase):
+    def test_grades_semester_view(self):
+        baker.seed(31902)
+
+        semester = baker.make(Semester)
+
+        self.manager.groups.add(Group.objects.get(name="Grade publisher"))
+        with self.enter_staff_mode():
+            self.selenium.get(self.reverse("grades:semester_view", args=[semester.id]))
+            self.trigger_screenshot("grades:semester - no courses")
+
+            courses = baker.make(Course, semester=semester, _quantity=30)
+            baker.make(
+                Evaluation,
+                course=iter(courses),
+                wait_for_grade_upload_before_publishing=True,
+                state=Evaluation.State.IN_EVALUATION,
+                _quantity=len(courses),
+            )
+            self.selenium.get(self.reverse("grades:semester_view", args=[semester.id]))
+
+            self.trigger_screenshot("grades:semester - 30 courses")