diff --git a/.github/setup_evap/action.yml b/.github/setup_evap/action.yml index a5e91d3d26..10f2f3f8b6 100644 --- a/.github/setup_evap/action.yml +++ b/.github/setup_evap/action.yml @@ -23,8 +23,10 @@ runs: with: arguments: "${{ inputs.shell }}" - - name: Add localsettings - run: cp evap/settings_test.py evap/localsettings.py + - run: | + echo "DJANGO_SETTINGS_MODULE=localsettings" >> "$GITHUB_ENV" + ln -s evap/development/cisettings.py localsettings.py + ln -s evap/settings/schema.pyi localsettings.pyi shell: bash - name: Create data/ directory diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 55da9b7b6d..88372396ff 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -69,6 +69,7 @@ jobs: with: submodules: true - uses: DeterminateSystems/nix-installer-action@main + - run: echo "DJANGO_SETTINGS_MODULE=evap.development.cisettings" >> "$GITHUB_ENV" - run: nix run .#build-dist - run: tar tvf dist/*.tar.gz - run: unzip -l dist/*.whl @@ -92,6 +93,8 @@ jobs: - run: pip install *.whl - name: Check that "evaluation" section appears in help string run: python -m evap --help | grep --fixed-strings "[evaluation]" + env: + DJANGO_SETTINGS_MODULE: evap.development.cisettings test_frontend: name: Test Frontend @@ -225,14 +228,13 @@ jobs: WHEEL=$(echo *.whl) pip install $WHEEL[psycopg-binary] + - run: echo "DJANGO_SETTINGS_MODULE=evap.development.cisettings" >> "$GITHUB_ENV" - name: Load test data working-directory: deployment run: | cat <(echo 'from evap.settings import *') ../main/evap/settings_test.py | tee deployment_settings.py python -m evap migrate python -m evap loaddata test_data - env: - DJANGO_SETTINGS_MODULE: deployment_settings - name: Backup database working-directory: deployment run: ./update_production.sh backup.json @@ -240,13 +242,11 @@ jobs: EVAP_OVERRIDE_BACKUP_FILENAME: true EVAP_SKIP_UPDATE: true EVAP_SKIP_APACHE_STEPS: true - DJANGO_SETTINGS_MODULE: deployment_settings - name: Reload backup working-directory: deployment run: echo "yy" | ./load_production_backup.sh backup.json env: EVAP_SKIP_APACHE_STEPS: true - DJANGO_SETTINGS_MODULE: deployment_settings macos-nix-build: runs-on: macos-14 diff --git a/.gitignore b/.gitignore index 78ecec9641..f14eb3d615 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,8 @@ dist/ .DS_Store thumbs.db +localsettings.py +localsettings.pyi evap/localsettings.py evap/static/css/evap.css evap/static/css/evap.css.map diff --git a/evap/__main__.py b/evap/__main__.py index b09b03d085..1083eb6c7f 100755 --- a/evap/__main__.py +++ b/evap/__main__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import os +import logging import sys from django.conf import settings @@ -8,9 +8,14 @@ def main(): - assert not settings.configured - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "evap.settings") settings.DATADIR.mkdir(exist_ok=True) + + if settings.TESTING: + from typeguard import install_import_hook # noqa: PLC0415 + + install_import_hook(("evap", "tools")) + logging.disable() + execute_from_command_line(sys.argv) diff --git a/evap/development/cisettings.py b/evap/development/cisettings.py new file mode 100644 index 0000000000..c24fabb12b --- /dev/null +++ b/evap/development/cisettings.py @@ -0,0 +1,10 @@ +from evap.settings import default, dev, local_services, open_id, test # noqa: TID251 +from evap.settings_resolver import resolve_settings + + +class CiSettings: + DEBUG = True + SECRET_KEY = "evap-github-actions-secret-key" # nosec + + +globals().update(resolve_settings([default, open_id, dev, local_services, test, CiSettings])) diff --git a/evap/development/localsettings.template.py b/evap/development/localsettings.template.py index 0258d5c4f2..0610d2bdb1 100644 --- a/evap/development/localsettings.template.py +++ b/evap/development/localsettings.template.py @@ -1,59 +1,12 @@ # noqa: N999 -from fractions import Fraction -from pathlib import Path +from evap.settings import debug_toolbar, default, dev, local_services, open_id, test # noqa: TID251 +from evap.settings_resolver import resolve_settings -from django.utils.safestring import mark_safe -DATABASES = { - "default": { - "ENGINE": "django.db.backends.postgresql", - "NAME": "evap", - "USER": "evap", - "PASSWORD": "evap", - # Absolute path to use unix domain socket - "HOST": Path("./data/").resolve(), - "CONN_MAX_AGE": 600, - } -} +class LocalSettings: + DEBUG = True + SECRET_KEY = "$SECRET_KEY" # nosec -REDIS_URL = f"unix://{Path('./data/redis.socket').resolve()}" -CACHES = { - "default": { - "BACKEND": "django.core.cache.backends.redis.RedisCache", - "LOCATION": f"{REDIS_URL}?db=0", - }, - "results": { - "BACKEND": "django.core.cache.backends.redis.RedisCache", - "LOCATION": f"{REDIS_URL}?db=1", - "TIMEOUT": None, # is always invalidated manually - }, - "sessions": { - "BACKEND": "django.core.cache.backends.redis.RedisCache", - "LOCATION": f"{REDIS_URL}?db=2", - }, -} - -# Make this unique, and don't share it with anybody. -SECRET_KEY = "$SECRET_KEY" # nosec - -# Make apache work when DEBUG == False -ALLOWED_HOSTS = ["localhost", "127.0.0.1"] - -### Evaluation progress rewards -GLOBAL_EVALUATION_PROGRESS_REWARDS: list[tuple[Fraction, str]] = [ - (Fraction("0"), "0€"), - (Fraction("0.25"), "1.000€"), - (Fraction("0.6"), "3.000€"), - (Fraction("0.7"), "7.000€"), - (Fraction("0.9"), "10.000€"), -] -GLOBAL_EVALUATION_PROGRESS_EXCLUDED_COURSE_TYPE_IDS: list[int] = [] -GLOBAL_EVALUATION_PROGRESS_EXCLUDED_EVALUATION_IDS: list[int] = [] -GLOBAL_EVALUATION_PROGRESS_INFO_TEXT = { - "de": mark_safe("Deine Teilnahme am Evaluationsprojekt wird helfen. Evaluiere also jetzt!"), - "en": mark_safe("Your participation in the evaluation helps, so evaluate now!"), -} -# Questionnaires automatically added to exam evaluations -EXAM_QUESTIONNAIRE_IDS = [111] +globals().update(resolve_settings([default, local_services, open_id, dev, test, debug_toolbar, LocalSettings])) diff --git a/evap/evaluation/tests/test_tools.py b/evap/evaluation/tests/test_tools.py index 7545a62f3e..69e059e88a 100644 --- a/evap/evaluation/tests/test_tools.py +++ b/evap/evaluation/tests/test_tools.py @@ -1,3 +1,5 @@ +import graphlib +from argparse import Namespace from unittest.mock import patch from uuid import UUID @@ -18,6 +20,7 @@ inside_transaction, is_prefetched, ) +from evap.settings_resolver import derived, not_set, required, resolve_settings class TestLanguageMiddleware(WebTest): @@ -242,3 +245,83 @@ def test_inside_transaction(self): with transaction.atomic(): self.assertTrue(inside_transaction()) + + +class TestResolveSettings(TestCase): + def test_not_set(self): + # We don't use the "remove a previous setting" functionality anywhere, but this behavior is the natural composition of layers and not_set + self.assertEqual(resolve_settings([Namespace(FOO=42), Namespace(FOO=not_set())]), {}) + self.assertEqual(resolve_settings([Namespace(FOO=not_set()), Namespace(FOO=42)]), {"FOO": 42}) + + def test_key_names(self): + # See SettingResolver.iter_settings + self.assertEqual(resolve_settings([Namespace(foo=1, _FOO=2, FoO=3)]), {}) + + def test_derived(self): + self.assertEqual( + resolve_settings( + [ + Namespace( + FOO=1, + BAR=derived(final={"FOO"})(lambda prev, final: final.FOO + 10), + ), + Namespace( + FOO=derived(prev={"FOO"})(lambda prev, final: prev.FOO + 100), + BAR=derived(prev={"BAR"})(lambda prev, final: prev.BAR + 1000), + ), + ] + ), + { + "FOO": 101, + "BAR": 1111, + }, + ) + + def test_cycle(self): + with self.assertRaises(graphlib.CycleError): + resolve_settings( + [ + Namespace( + FOO=derived(final={"BAR"})(lambda prev, final: final.BAR), + BAR=derived(final={"FOO"})(lambda prev, final: final.FOO), + ), + Namespace( + FOO=derived(prev={"FOO"})(lambda prev, final: prev.FOO), + BAR=derived(prev={"BAR"})(lambda prev, final: prev.BAR), + ), + ] + ) + + def test_required(self): + with self.assertRaises(ValueError): + resolve_settings([Namespace(FOO=required())]) + with self.assertRaises(ValueError): + resolve_settings( + [Namespace(FOO=required()), Namespace(FOO=42, BAR=derived(prev={"FOO"})(lambda prev, final: prev.FOO))] + ) + self.assertEqual( + resolve_settings( + [ + Namespace(FOO=required()), + Namespace(FOO=42, BAR=derived(final={"FOO"})(lambda prev, final: final.FOO)), + ] + ), + {"FOO": 42, "BAR": 42}, + ) + + def test_derived_final_dependencies(self): + # Ensure that derived final dependencies are ordered before in the topological order + self.assertEqual( + resolve_settings( + [ + Namespace(FOO=required()), + Namespace( + FOO=42, + BAR=derived(final={"BAZ"})(lambda prev, final: final.BAZ), + BAZ=derived(final={"FOO"})(lambda prev, final: final.FOO), + ), + Namespace(), + ] + ), + {"FOO": 42, "BAR": 42, "BAZ": 42}, + ) diff --git a/evap/settings/__init__.py b/evap/settings/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/evap/settings/debug_toolbar.py b/evap/settings/debug_toolbar.py new file mode 100644 index 0000000000..cc0618a66a --- /dev/null +++ b/evap/settings/debug_toolbar.py @@ -0,0 +1,40 @@ +# pylint: disable=unused-argument,invalid-name + +from evap.settings_resolver import derived + +# Very helpful but eats a lot of performance on sql-heavy pages. +# Works only with DEBUG = True and Django's development server (so no apache). +ENABLE_DEBUG_TOOLBAR = False + + +@derived(final={"DEBUG", "ENABLE_DEBUG_TOOLBAR", "TESTING"}) +def REALLY_ENABLE_DEBUG_TOOLBAR(prev, final): + return final.ENABLE_DEBUG_TOOLBAR and final.DEBUG and not final.TESTING + + +@derived(prev={"INSTALLED_APPS"}, final={"REALLY_ENABLE_DEBUG_TOOLBAR"}) +def INSTALLED_APPS(prev, final): + if final.REALLY_ENABLE_DEBUG_TOOLBAR: + return prev.INSTALLED_APPS + ["debug_toolbar"] + return prev.INSTALLED_APPS + + +@derived(prev={"MIDDLEWARE"}, final={"REALLY_ENABLE_DEBUG_TOOLBAR"}) +def MIDDLEWARE(prev, final): + if final.REALLY_ENABLE_DEBUG_TOOLBAR: + return ["debug_toolbar.middleware.DebugToolbarMiddleware"] + prev.MIDDLEWARE + return prev.MIDDLEWARE + + +def show_toolbar(request): + return True + + +@derived(prev={"DEBUG_TOOLBAR_CONFIG"}, final={"REALLY_ENABLE_DEBUG_TOOLBAR"}) +def DEBUG_TOOLBAR_CONFIG(prev, final): + if final.REALLY_ENABLE_DEBUG_TOOLBAR: + return { + "SHOW_TOOLBAR_CALLBACK": "evap.settings.debug_toolbar.show_toolbar", + "JQUERY_URL": "", + } + return prev.DEBUG_TOOLBAR_CONFIG diff --git a/evap/settings.py b/evap/settings/default.py similarity index 66% rename from evap/settings.py rename to evap/settings/default.py index a2aad495f7..d7b3ca6031 100644 --- a/evap/settings.py +++ b/evap/settings/default.py @@ -1,15 +1,6 @@ -""" -Django settings for EvaP project. +# ruff: noqa: E731, N803 +# pylint: disable=unused-argument,invalid-name -For more information on this file, see -https://docs.djangoproject.com/en/2.0/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/2.0/ref/settings/ -""" - -import logging -import sys from datetime import timedelta from fractions import Fraction from pathlib import Path @@ -17,18 +8,27 @@ from django.contrib.staticfiles.storage import ManifestStaticFilesStorage +import evap +from evap.settings_resolver import derived, required from evap.tools import MonthAndDay -MODULE = Path(__file__).parent.resolve() -CWD = Path(".").resolve() -DATADIR = CWD / "data" -### Debugging +class ManifestStaticFilesStorageWithJsReplacement(ManifestStaticFilesStorage): + support_js_module_import_aggregation = True + + +MODULE = Path(evap.__file__).parent +CWD = Path.cwd().resolve() + + +@derived(final={"CWD"}) +def DATADIR(prev, final): + return final.CWD / "data" -DEBUG = True -# Very helpful but eats a lot of performance on sql-heavy pages. -# Works only with DEBUG = True and Django's development server (so no apache). +### Debugging + +DEBUG = required() ENABLE_DEBUG_TOOLBAR = False ### EvaP logic @@ -127,7 +127,10 @@ ] # The page URL that is used in email templates. -PAGE_URL = "localhost:8000" +PAGE_URL = required() + +# Key used for Django's signing module +SECRET_KEY = required() DATABASES = { "default": { @@ -156,76 +159,84 @@ }, } - -class ManifestStaticFilesStorageWithJsReplacement(ManifestStaticFilesStorage): - support_js_module_import_aggregation = True - - STORAGES = { "default": { "BACKEND": "django.core.files.storage.FileSystemStorage", }, "staticfiles": { - "BACKEND": "evap.settings.ManifestStaticFilesStorageWithJsReplacement", + "BACKEND": "evap.settings.default.ManifestStaticFilesStorageWithJsReplacement", }, } -CONTACT_EMAIL = "webmaster@localhost" +CONTACT_EMAIL = required() ALLOW_ANONYMOUS_FEEDBACK_MESSAGES = True -LEGAL_NOTICE_TEXT = "Objection! (this is a default setting that the administrators should change, please contact them)" +LEGAL_NOTICE_TEXT = required() # Config for mail system -DEFAULT_FROM_EMAIL = "webmaster@localhost" -REPLY_TO_EMAIL = DEFAULT_FROM_EMAIL -SEND_ALL_EMAILS_TO_ADMINS_IN_BCC = False -if DEBUG: - EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" - - -LOGGING = { - "version": 1, - "disable_existing_loggers": False, - "formatters": { - "default": { - "format": "[%(asctime)s] %(levelname)s: %(message)s", - }, - }, - "handlers": { - "file": { - "level": "DEBUG", - "class": "logging.handlers.RotatingFileHandler", - "filename": DATADIR / "evap.log", - "maxBytes": 1024 * 1024 * 10, - "backupCount": 5, - "formatter": "default", - }, - "mail_admins": { - "level": "ERROR", - "class": "django.utils.log.AdminEmailHandler", - }, - "console": { - "class": "logging.StreamHandler", - "formatter": "default", - }, - }, - "loggers": { - "django": { - "handlers": ["console", "file", "mail_admins"], - "level": "INFO", - "propagate": True, +DEFAULT_FROM_EMAIL = required() + + +@derived(final={"DEFAULT_FROM_EMAIL"}) +def REPLY_TO_EMAIL(prev, final): + return final.DEFAULT_FROM_EMAIL + + +SEND_ALL_EMAILS_TO_ADMINS_IN_BCC = required() + + +@derived(prev={"EMAIL_BACKEND"}, final={"DEBUG"}) +def EMAIL_BACKEND(prev, final): + if final.DEBUG: + return "django.core.mail.backends.console.EmailBackend" + return prev.EMAIL_BACKEND + + +@derived(final={"DATADIR"}) +def LOGGING(prev, final): + return { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "default": { + "format": "[%(asctime)s] %(levelname)s: %(message)s", + }, }, - "evap": { - "handlers": ["console", "file", "mail_admins"], - "level": "DEBUG", - "propagate": True, + "handlers": { + "file": { + "level": "DEBUG", + "class": "logging.handlers.RotatingFileHandler", + "filename": final.DATADIR / "evap.log", + "maxBytes": 1024 * 1024 * 10, + "backupCount": 5, + "formatter": "default", + }, + "mail_admins": { + "level": "ERROR", + "class": "django.utils.log.AdminEmailHandler", + }, + "console": { + "class": "logging.StreamHandler", + "formatter": "default", + }, }, - "mozilla_django_oidc": { - "handlers": ["console", "file", "mail_admins"], - "level": "DEBUG", - "propagate": True, + "loggers": { + "django": { + "handlers": ["console", "file", "mail_admins"], + "level": "INFO", + "propagate": True, + }, + "evap": { + "handlers": ["console", "file", "mail_admins"], + "level": "DEBUG", + "propagate": True, + }, + "mozilla_django_oidc": { + "handlers": ["console", "file", "mail_admins"], + "level": "DEBUG", + "propagate": True, + }, }, - }, -} + } ### Application definition @@ -283,7 +294,6 @@ class ManifestStaticFilesStorageWithJsReplacement(ManifestStaticFilesStorage): "builtins": ["django.templatetags.i18n"], } - TEMPLATES: Any = [ { "BACKEND": "django.template.backends.django.DjangoTemplates", @@ -333,7 +343,6 @@ class ManifestStaticFilesStorageWithJsReplacement(ManifestStaticFilesStorage): # see https://docs.djangoproject.com/en/5.0/ref/settings/#data-upload-max-number-fields DATA_UPLOAD_MAX_NUMBER_FIELDS = None - ### Internationalization LANGUAGE_CODE = "en" @@ -344,7 +353,11 @@ class ManifestStaticFilesStorageWithJsReplacement(ManifestStaticFilesStorage): USE_TZ = False -LOCALE_PATHS = [MODULE / "locale"] + +@derived(final={"MODULE"}) +def LOCALE_PATHS(prev, final): + return [final.MODULE / "locale"] + FORMAT_MODULE_PATH = ["evap.locale"] @@ -353,30 +366,37 @@ class ManifestStaticFilesStorageWithJsReplacement(ManifestStaticFilesStorage): ("de", "Deutsch"), ] - ### Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/2.0/howto/static-files/ STATIC_URL = "/static/" + # Additional locations of static files -STATICFILES_DIRS = [ - MODULE / "static", -] +@derived(final={"MODULE"}) +def STATICFILES_DIRS(prev, final): + return [final.MODULE / "static"] + # Absolute path to the directory static files should be collected to. -STATIC_ROOT = DATADIR / "static_collected" +@derived(final={"DATADIR"}) +def STATIC_ROOT(prev, final): + return final.DATADIR / "static_collected" ### User-uploaded files + # Absolute filesystem path to the directory that will hold user-uploaded files. -MEDIA_ROOT = DATADIR / "upload" +@derived(final={"DATADIR"}) +def MEDIA_ROOT(prev, final): + return final.DATADIR / "upload" + ### Evaluation progress rewards -GLOBAL_EVALUATION_PROGRESS_REWARDS: list[tuple[Fraction, str]] = ( - [] -) # (required_voter_ratio between 0 and 1, reward_text) + +# (required_voter_ratio between 0 and 1, reward_text) +GLOBAL_EVALUATION_PROGRESS_REWARDS: list[tuple[Fraction, str]] = [] GLOBAL_EVALUATION_PROGRESS_EXCLUDED_COURSE_TYPE_IDS: list[int] = [] GLOBAL_EVALUATION_PROGRESS_EXCLUDED_EVALUATION_IDS: list[int] = [] GLOBAL_EVALUATION_PROGRESS_INFO_TEXT: dict[str, str] = {"de": "", "en": ""} @@ -417,94 +437,3 @@ def CHARACTER_ALLOWED_IN_NAME(character): # pylint: disable=invalid-name ord(character) in range(256, 384), # Latin Extended-A ) ) - - -### OpenID Login -# replace 'example.com', OIDC_RP_CLIENT_ID and OIDC_RP_CLIENT_SECRET with real values in localsettings when activating -ACTIVATE_OPEN_ID_LOGIN = False -OIDC_RENEW_ID_TOKEN_EXPIRY_SECONDS = 60 * 60 * 24 * 7 # one week -OIDC_RP_SIGN_ALGO = "RS256" -OIDC_USERNAME_ALGO = "" -OIDC_RP_SCOPES = "openid email profile" - -OIDC_RP_CLIENT_ID = "evap" -OIDC_RP_CLIENT_SECRET = "evap-secret" # nosec - -OIDC_OP_AUTHORIZATION_ENDPOINT = "https://example.com/auth" -OIDC_OP_TOKEN_ENDPOINT = "https://example.com/token" # nosec -OIDC_OP_USER_ENDPOINT = "https://example.com/me" -OIDC_OP_JWKS_ENDPOINT = "https://example.com/certs" - -# Mapping of email domain transition which users may undergo during the lifetime of their account. -# Given an item (k, v) of the mapping, if an account with domain k would be created through OpenID, -# but an account with the same name at domain v exists, the existing account is migrated to the -# domain k instead. -OIDC_EMAIL_TRANSITIONS: dict[str, str] = { - "institution.example.com": "student.institution.example.com", -} - - -### Other - -# Create a localsettings.py if you want to locally override settings -# and don't want the changes to appear in 'git status'. -try: - # localsettings file may or may not exist (for example in CI) - - # the import can overwrite locals with a slightly different type (e.g. DATABASES), which is fine. - from evap.localsettings import * # type: ignore # noqa: F403,PGH003 -except ImportError: - pass - -TEST_RUNNER = "evap.evaluation.tests.tools.EvapTestRunner" -TESTING = "test" in sys.argv or "pytest" in sys.modules - -# speed up tests and activate typeguard introspection -if TESTING: - from typeguard import install_import_hook - - install_import_hook(("evap", "tools")) - - # do not use ManifestStaticFilesStorage as it requires running collectstatic beforehand - STORAGES["staticfiles"]["BACKEND"] = "django.contrib.staticfiles.storage.StaticFilesStorage" - - logging.disable(logging.CRITICAL) # disable logging, primarily to prevent console spam - - # use the database for caching. it's properly reset between tests in constrast to redis, - # and does not change behaviour in contrast to disabling the cache entirely. - CACHES = { - "default": { - "BACKEND": "django.core.cache.backends.db.DatabaseCache", - "LOCATION": "testing_cache_default", - }, - "results": { - "BACKEND": "django.core.cache.backends.db.DatabaseCache", - "LOCATION": "testing_cache_results", - }, - "sessions": { - "BACKEND": "django.core.cache.backends.db.DatabaseCache", - "LOCATION": "testing_cache_sessions", - }, - } - from model_bakery import random_gen - - # give random char field values a reasonable length - BAKER_CUSTOM_FIELDS_GEN = {"django.db.models.CharField": lambda: random_gen.gen_string(20)} - - -# Development helpers -if DEBUG: - INSTALLED_APPS += ["evap.development"] - - # Django debug toolbar settings - if not TESTING and ENABLE_DEBUG_TOOLBAR: - INSTALLED_APPS += ["debug_toolbar"] - MIDDLEWARE = ["debug_toolbar.middleware.DebugToolbarMiddleware"] + MIDDLEWARE - - def show_toolbar(request): - return True - - DEBUG_TOOLBAR_CONFIG = { - "SHOW_TOOLBAR_CALLBACK": "evap.settings.show_toolbar", - "JQUERY_URL": "", - } diff --git a/evap/settings/dev.py b/evap/settings/dev.py new file mode 100644 index 0000000000..c57f4e75c3 --- /dev/null +++ b/evap/settings/dev.py @@ -0,0 +1,45 @@ +# pylint: disable=invalid-name + +from fractions import Fraction + +from django.utils.safestring import mark_safe + +from evap.settings_resolver import derived + +PAGE_URL = "localhost:8000" +ACTIVATE_OPEN_ID_LOGIN = False + +# Make apache work when DEBUG == False +ALLOWED_HOSTS = ["localhost", "127.0.0.1"] + + +@derived(prev={"INSTALLED_APPS"}, final={"DEBUG"}) +def INSTALLED_APPS(prev, final): + if final.DEBUG: + return prev.INSTALLED_APPS + ["evap.development"] + return prev.INSTALLED_APPS + + +CONTACT_EMAIL = "webmaster@localhost" +DEFAULT_FROM_EMAIL = "webmaster@localhost" +REPLY_TO_EMAIL = DEFAULT_FROM_EMAIL +SEND_ALL_EMAILS_TO_ADMINS_IN_BCC = False + +LEGAL_NOTICE_TEXT = "Objection! (this is a default setting that the administrators should change, please contact them)" + +### Evaluation progress rewards +GLOBAL_EVALUATION_PROGRESS_REWARDS: list[tuple[Fraction, str]] = [ + (Fraction("0"), "0€"), + (Fraction("0.25"), "1.000€"), + (Fraction("0.6"), "3.000€"), + (Fraction("0.7"), "7.000€"), + (Fraction("0.9"), "10.000€"), +] +GLOBAL_EVALUATION_PROGRESS_EXCLUDED_COURSE_TYPE_IDS: list[int] = [] +GLOBAL_EVALUATION_PROGRESS_EXCLUDED_EVALUATION_IDS: list[int] = [] +GLOBAL_EVALUATION_PROGRESS_INFO_TEXT = { + "de": mark_safe("Deine Teilnahme am Evaluationsprojekt wird helfen. Evaluiere also jetzt!"), + "en": mark_safe("Your participation in the evaluation helps, so evaluate now!"), +} +# Questionnaires automatically added to exam evaluations +EXAM_QUESTIONNAIRE_IDS = [111] diff --git a/evap/settings/local_services.py b/evap/settings/local_services.py new file mode 100644 index 0000000000..4cf5b189bf --- /dev/null +++ b/evap/settings/local_services.py @@ -0,0 +1,42 @@ +# pylint: disable=unused-argument,invalid-name + +from evap.settings_resolver import derived + + +@derived(final={"DATADIR"}) +def DATABASES(prev, final): + return { + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": "evap", + "USER": "evap", + "PASSWORD": "evap", + # Absolute path to use unix domain socket + "HOST": final.DATADIR.resolve(), + "CONN_MAX_AGE": 600, + } + } + + +@derived(final={"DATADIR"}) +def REDIS_URL(prev, final): + return f"unix://{(final.DATADIR / 'redis.socket').resolve()}" + + +@derived(final={"REDIS_URL"}) +def CACHES(prev, final): + return { + "default": { + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": f"{final.REDIS_URL}?db=0", + }, + "results": { + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": f"{final.REDIS_URL}?db=1", + "TIMEOUT": None, # is always invalidated manually + }, + "sessions": { + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": f"{final.REDIS_URL}?db=2", + }, + } diff --git a/evap/settings/open_id.py b/evap/settings/open_id.py new file mode 100644 index 0000000000..b6a67a35a8 --- /dev/null +++ b/evap/settings/open_id.py @@ -0,0 +1,25 @@ +from evap.settings_resolver import required + +ACTIVATE_OPEN_ID_LOGIN = required() + +# replace 'example.com', OIDC_RP_CLIENT_ID and OIDC_RP_CLIENT_SECRET with real values in localsettings when activating +OIDC_RENEW_ID_TOKEN_EXPIRY_SECONDS = 60 * 60 * 24 * 7 # one week +OIDC_RP_SIGN_ALGO = "RS256" +OIDC_USERNAME_ALGO = "" +OIDC_RP_SCOPES = "openid email profile" + +OIDC_RP_CLIENT_ID = "evap" +OIDC_RP_CLIENT_SECRET = "evap-secret" # nosec + +OIDC_OP_AUTHORIZATION_ENDPOINT = "https://example.com/auth" +OIDC_OP_TOKEN_ENDPOINT = "https://example.com/token" # nosec +OIDC_OP_USER_ENDPOINT = "https://example.com/me" +OIDC_OP_JWKS_ENDPOINT = "https://example.com/certs" + +# Mapping of email domain transition which users may undergo during the lifetime of their account. +# Given an item (k, v) of the mapping, if an account with domain k would be created through OpenID, +# but an account with the same name at domain v exists, the existing account is migrated to the +# domain k instead. +OIDC_EMAIL_TRANSITIONS: dict[str, str] = { + "institution.example.com": "student.institution.example.com", +} diff --git a/evap/settings/schema.pyi b/evap/settings/schema.pyi new file mode 100644 index 0000000000..f90bee94f8 --- /dev/null +++ b/evap/settings/schema.pyi @@ -0,0 +1,37 @@ +from datetime import timedelta +from fractions import Fraction +from pathlib import Path +from typing import Any + +ACTIVATE_OPEN_ID_LOGIN: bool +ADMINS: list[tuple[str, str]] +CONTACT_EMAIL: str +DATADIR: Path +ENABLE_DEBUG_TOOLBAR: bool +EVALUATION_END_OFFSET_HOURS: int +EXAM_QUESTIONNAIRE_IDS: list[int] +GLOBAL_EVALUATION_PROGRESS_EXCLUDED_COURSE_TYPE_IDS: list[int] +GLOBAL_EVALUATION_PROGRESS_EXCLUDED_EVALUATION_IDS: list[int] +GLOBAL_EVALUATION_PROGRESS_INFO_TEXT: dict[str, str] +GLOBAL_EVALUATION_PROGRESS_REWARDS: list[tuple[Fraction, str]] +GRADE_REMINDER_EMAIL_RECIPIENTS: list[str] +IGNORE_PROGRAMS: set[str] +IMPORTER_COURSE_NAME_SIMILARITY_WARNING_THRESHOLD: float +IMPORTER_GRADED_NO: list[str] +IMPORTER_GRADED_YES: list[str] +IMPORTER_MAX_ENROLLMENTS: int +INSTITUTION_EMAIL_DOMAINS: list[str] +INSTITUTION_EMAIL_REPLACEMENTS: list[tuple[str, str]] +LEGAL_NOTICE_TEXT: str +MEDIA_ROOT: Path +MODULE: Path +NON_RESPONSIBLE_USERS: set[str] +OIDC_EMAIL_TRANSITIONS: dict[str, str] +PAGE_URL: str +PARTICIPATION_DELETION_AFTER_INACTIVE_TIME: timedelta +REPLY_TO_EMAIL: str +REWARD_POINTS: list[tuple[float, int]] +SEND_ALL_EMAILS_TO_ADMINS_IN_BCC: bool +SMALL_COURSE_SIZE: int +TEMPLATES: Any +VOTER_COUNT_NEEDED_FOR_PUBLISHING_RATING_RESULTS: int diff --git a/evap/settings/test.py b/evap/settings/test.py new file mode 100644 index 0000000000..cf9657baff --- /dev/null +++ b/evap/settings/test.py @@ -0,0 +1,51 @@ +# pylint: disable=invalid-name + +import sys +from copy import deepcopy + +from evap.settings_resolver import derived + +TEST_RUNNER = "evap.evaluation.tests.tools.EvapTestRunner" +TESTING = "test" in sys.argv or "pytest" in sys.modules + + +@derived(prev={"STORAGES"}, final={"TESTING"}) +def STORAGES(prev, final): + if final.TESTING: + storages = deepcopy(prev.STORAGES) + # do not use ManifestStaticFilesStorage as it requires running collectstatic beforehand + storages["staticfiles"]["BACKEND"] = "django.contrib.staticfiles.storage.StaticFilesStorage" + return storages + return prev.STORAGES + + +@derived(prev={"CACHES"}, final={"TESTING"}) +def CACHES(prev, final): + if final.TESTING: + # use the database for caching. it's properly reset between tests in constrast to redis, + # and does not change behaviour in contrast to disabling the cache entirely. + return { + "default": { + "BACKEND": "django.core.cache.backends.db.DatabaseCache", + "LOCATION": "testing_cache_default", + }, + "results": { + "BACKEND": "django.core.cache.backends.db.DatabaseCache", + "LOCATION": "testing_cache_results", + }, + "sessions": { + "BACKEND": "django.core.cache.backends.db.DatabaseCache", + "LOCATION": "testing_cache_sessions", + }, + } + return prev.CACHES + + +@derived(prev={"BAKER_CUSTOM_FIELDS_GEN"}, final={"TESTING"}) +def BAKER_CUSTOM_FIELDS_GEN(prev, final): + if final.TESTING: + from model_bakery import random_gen # noqa: PLC0415 + + # give random char field values a reasonable length + return {"django.db.models.CharField": lambda: random_gen.gen_string(20)} + return prev.BAKER_CUSTOM_FIELDS_GEN diff --git a/evap/settings_resolver.py b/evap/settings_resolver.py new file mode 100644 index 0000000000..22eef776cb --- /dev/null +++ b/evap/settings_resolver.py @@ -0,0 +1,155 @@ +from argparse import Namespace +from collections import defaultdict +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from enum import Enum, auto +from functools import partial +from graphlib import TopologicalSorter +from typing import Any, Generic, TypeVar + +T = TypeVar("T") + + +@dataclass +class Derived: + """A setting that is derived from other settings.""" + + fn: Callable + prev: set[str] + final: set[str] + + +def derived(*, prev: set[str] | None = None, final: set[str] | None = None) -> Callable[[Callable], Derived]: + """Decorator to create derived settings.""" + return partial(Derived, prev=prev or set(), final=final or set()) + + +class Required(Enum): + """Type for marking settings as required. + + The resolver throws an error if a setting is required in the final mapping of settings. Thus, a setting declared as + required must be overwritten in a later layer.""" + REQUIRED = auto() + + +def required() -> Required: + """Shorthand function to create a required setting.""" + return Required.REQUIRED + + +class NotSet(Enum): + """Type for marking settings as not set, that is, for keeping Django's default value.""" + NOT_SET = auto() + + +def not_set() -> NotSet: + """Shorthand function to create a not-set setting.""" + return NotSet.NOT_SET + + +class SettingResolver(Generic[T]): + """Main class for setting resolution. + + By default, Django settings are configured in a module like + ```python + DEBUG = True + DATADIR = Path("./data") + STATIC_ROOT = DATADIR / "static_collected" + ``` + + This interface works for simple setups, but lacks some features that we desire. Most importantly, we want to declare + a set of default settings that do not necessarily need to be changed when deploying EvaP. Some of these settings + depend on other settings though, for example the STATIC_ROOT setting above depends on the DATADIR setting. If users + would change the value of DATADIR, they would also have to reassign STATIC_ROOT accordingly. + + To resolve this issue, we provide the SettingResolver which takes setting modules and aggregates them into a final + mapping of settings to pass to Django. Notably, the input modules to the resolver can explicitly declare + dependencies between settings using the `@derived` decorator. Concretely, the settings are organized into different + layers. Later layers overwrite previous settings. A derived setting can then declare a dependency on another setting + value from either the previous layer (for example to append an entry to a list) or the final layer (for example to + use a path overwritten by the user). + + To compute the final set of setting values, we form a dependency graph on the settings and layers and compute the + according values in topological order. + + Note that the SettingResolver class is generic over the type of the setting values T, however we only use it below + with T = Any. The type T is only used to track at what times values can still be derived. + """ + + @staticmethod + def iter_settings(namespace: Any) -> Iterable[str]: + """Filter for attributes with SCREAMING_SNAKE_CASE names.""" + for name in dir(namespace): + if name == name.upper() and not name.startswith("_"): + yield name + + def __init__(self) -> None: + self.layers: list[dict[str, T | Derived | Required | NotSet]] = [] + self.layers.append(defaultdict(not_set)) # base level: nothing is set + + def add_layer(self, namespace: Any) -> None: + self.layers.append({name: getattr(namespace, name) for name in self.iter_settings(namespace)}) + + @property + def final_layer(self) -> int: + return len(self.layers) - 1 + + def all_setting_names(self) -> Iterable[str]: + return {name for layer in self.layers for name in layer} + + def get_setting_at_layer(self, name, layer_index) -> tuple[int, T | Derived | Required | NotSet]: + for i in range(layer_index, 0, -1): + if name in self.layers[i]: + return i, self.layers[i][name] + return 0, NotSet.NOT_SET + + def compute_derived_values(self) -> None: + sorter = TopologicalSorter[tuple[int, str]]() + + for i, layer in enumerate(self.layers): + for name, value in layer.items(): + deps: list[tuple[int, str]] = [] + if isinstance(value, Derived): + deps.extend((i - 1, dep_name) for dep_name in value.prev) + deps.extend((self.final_layer, dep_name) for dep_name in value.final) + sorter.add((i, name), *deps) + + for index, name in sorter.static_order(): + match self.get_setting_at_layer(name, index): + case actual_index, Derived(fn, prev, final): + prev_ns = Namespace(**{name: self.get_setting_at_layer(name, actual_index - 1)[1] for name in prev}) + final_ns = Namespace( + **{name: self.get_setting_at_layer(name, self.final_layer)[1] for name in final} + ) + assert all(not isinstance(val, Derived) for val in vars(prev_ns).values()) + assert all(not isinstance(val, Derived) for val in vars(final_ns).values()) + self.layers[actual_index][name] = fn(prev=prev_ns, final=final_ns) + + def get_final_values(self, keys: Iterable[str]) -> dict[str, T]: + resolved = {} + missing_settings = set() + for name in keys: + _, value = self.get_setting_at_layer(name, self.final_layer) + match value: + case Derived(): + raise AssertionError("derived values should have been resolved by now") + case Required.REQUIRED: + missing_settings.add(name) + case NotSet.NOT_SET: + pass + case _: + resolved[name] = value + if missing_settings: + raise ValueError(f"The following settings must be set: {', '.join(missing_settings)}") + return resolved + + def resolve(self) -> dict[str, T]: + self.compute_derived_values() + return self.get_final_values(self.all_setting_names()) + + +def resolve_settings(namespaces: list[Any]) -> dict[str, Any]: + resolver = SettingResolver[Any]() + for ns in namespaces: + resolver.add_layer(ns) + return resolver.resolve() diff --git a/evap/settings_test.py b/evap/settings_test.py deleted file mode 100644 index 447b6cbc05..0000000000 --- a/evap/settings_test.py +++ /dev/null @@ -1,30 +0,0 @@ -from pathlib import Path - -SECRET_KEY = "evap-github-actions-secret-key" # nosec -DATABASES = { - "default": { - "ENGINE": "django.db.backends.postgresql", - "NAME": "evap", - "USER": "evap", - "PASSWORD": "evap", - "HOST": Path("./data/").resolve(), - } -} - -REDIS_URL = f"unix://{Path('./data/redis.socket').resolve()}" - -CACHES = { - "default": { - "BACKEND": "django.core.cache.backends.redis.RedisCache", - "LOCATION": f"{REDIS_URL}?db=0", - }, - "results": { - "BACKEND": "django.core.cache.backends.redis.RedisCache", - "LOCATION": f"{REDIS_URL}?db=1", - "TIMEOUT": None, # is always invalidated manually - }, - "sessions": { - "BACKEND": "django.core.cache.backends.redis.RedisCache", - "LOCATION": f"{REDIS_URL}?db=2", - }, -} diff --git a/flake.nix b/flake.nix index 5bd7fd2dbb..4127793eb8 100644 --- a/flake.nix +++ b/flake.nix @@ -75,7 +75,7 @@ pkgs = pkgsFor.${system}; pc-modules = import ./nix/services.nix { inherit pkgs; - inherit (self.devShells.${system}.evap.passthru) venv; + inherit (self.devShells.${system}.evap-dev.passthru) venv; }; make-process-compose = with-devenv-setup: (import inputs.process-compose-flake.lib { inherit pkgs; }).makeProcessCompose { modules = [ @@ -130,9 +130,9 @@ name = "clean-setup"; runtimeInputs = with pkgs; [ git ]; text = '' - read -r -p "Delete node_modules/, data/, generated CSS and JS files in evap/static/, and evap/localsettings.py? [y/N] " + read -r -p "Delete node_modules/, data/, generated CSS and JS files in evap/static/, and localsettings? [y/N] " [[ "$REPLY" =~ ^[Yy]$ ]] || exit 1 - git clean -f -X evap/static/ node_modules/ data/ evap/localsettings.py + git clean -f -X evap/static/ node_modules/ data/ localsettings.py localsettings.pyi ''; }; }); diff --git a/nix/services.nix b/nix/services.nix index 83df7f698b..f1fa4eeb56 100644 --- a/nix/services.nix +++ b/nix/services.nix @@ -58,14 +58,15 @@ runtimeInputs = with pkgs; [ venv git gnused gettext coreutils ]; text = '' set -e - if [[ -f evap/localsettings.py ]]; then - echo "Found evap/localsettings.py, exiting." + if [[ -f localsettings.py ]]; then + echo "Found localsettings.py, exiting." echo "If you want to install a fresh environment, run clean-setup in a nix develop shell." exit 0 fi set -x - cp evap/development/localsettings.template.py evap/localsettings.py - sed -i -e "s/\$SECRET_KEY/$(head /dev/urandom | LC_ALL=C tr -dc A-Za-z0-9 | head -c 32)/" evap/localsettings.py + cp evap/development/localsettings.template.py localsettings.py + ln -s evap/settings/schema.pyi localsettings.pyi + sed -i -e "s/\$SECRET_KEY/$(head /dev/urandom | LC_ALL=C tr -dc A-Za-z0-9 | head -c 32)/" localsettings.py git submodule update --init ./manage.py compilemessages --locale de ./manage.py reload_testdata --noinput diff --git a/nix/shell.nix b/nix/shell.nix index b832d9140b..d361a86c2e 100644 --- a/nix/shell.nix +++ b/nix/shell.nix @@ -58,6 +58,7 @@ pkgs.mkShell { passthru = { inherit venv; }; env = { + DJANGO_SETTINGS_MODULE = "localsettings"; # this file is set up by services-full UV_NO_SYNC = "1"; UV_PYTHON = python3.interpreter; UV_PYTHON_DOWNLOADS = "never"; diff --git a/pyproject.toml b/pyproject.toml index 9eb115c72d..91ad527597 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,7 +129,6 @@ exclude = ["**/urls.py"] jobs = 0 load-plugins = ["pylint_django"] -django-settings-module = "evap.settings" [tool.pylint.basic] # For most code: snake_case, or PascalCaseFormset, because django does it that way. @@ -199,7 +198,6 @@ packages = ["evap", "tools"] plugins = ["mypy_django_plugin.main"] [tool.django-stubs] -django_settings_module = "evap.settings" [[tool.mypy.overrides]] module = [ @@ -222,7 +220,6 @@ ignore_missing_imports = true [tool.pytest.ini_options] # We don't officially use pytest, but last time we wanted to, this worked for us with pytest-django # pytest-xdist worked for parallelizing tests. -DJANGO_SETTINGS_MODULE = "evap.settings" python_files = ["tests.py", "test_*.py", "*_tests.py"] testpaths = ["evap", "tools"] norecursedirs=["locale", "logs", "static", "static_collected", "upload"]