diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78ce82e722..3ae56a81ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,6 +84,7 @@ jobs: python src/manage.py compilemessages python src/manage.py collectstatic --noinput --link + mkdir private_media coverage run \ --concurrency=multiprocessing \ --parallel-mode \ @@ -113,6 +114,20 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} + - name: Check that private-media tests are properly isolated + run: | + num_private_media_files=$(find private_media -type f | wc -l) + if [[ "$num_private_media_files" == "0" ]]; then + exit 0 + else + echo "# Writes to private_media detected!" >> $GITHUB_STEP_SUMMARY + echo "$num_private_media_files were written into the private media storage, while tests should not leave any files." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Files:" >> $GITHUB_STEP_SUMMARY + find private_media -type f >> $GITHUB_STEP_SUMMARY + exit 1 + fi + tests-reverse: name: Run the Django test suite in reverse runs-on: ubuntu-latest @@ -148,6 +163,7 @@ jobs: run: | python src/manage.py compilemessages python src/manage.py collectstatic --noinput --link + mkdir private_media src/manage.py test src \ --force-color \ --parallel 4 \ diff --git a/docs/developers/backend/file-uploads.rst b/docs/developers/backend/file-uploads.rst index cc24f6d125..5da0593fce 100644 --- a/docs/developers/backend/file-uploads.rst +++ b/docs/developers/backend/file-uploads.rst @@ -13,43 +13,62 @@ The following events happen during that process: - The user adds a file to the component: - - A ``POST`` request is made to ``/api/v1/formio/fileupload`` with the content of the file. - - If configured, the file is scanned for viruses (more details :ref:`here`). In case - a virus is found, the file is not saved and the user receives an error alerting them that a virus was found in the file. - - An instance of the :class:`openforms.submissions.models.TemporaryFileUpload` model is created. - - The endpoint returns the url of the file ``/api/v1/submissions/files/``, the file name and size. This information is added - to the Formio submission step data. - - The UUID of the :class:`openforms.submissions.models.TemporaryFileUpload` is added to the user session. - - The content of the file is saved to the disk. The file is placed in the private media directory (configured through - the ``PRIVATE_MEDIA_ROOT`` setting), within the ``temporary-uploads`` folder. + - A ``POST`` request is made to ``/api/v1/formio/fileupload`` with the content of the + file. + - If configured, the file is scanned for viruses (more details + :ref:`here`). In case a virus is found, the file + is not saved and the user receives an error alerting them that a virus was found in + the file. + - An instance of the :class:`openforms.submissions.models.TemporaryFileUpload` model + is created. It is related to the submission currently in the session. + - The endpoint returns the url of the file ``/api/v1/submissions/files/``, the + file name and size. This information is added to the Formio submission step data. + - The content of the file is saved to the disk. The file is placed in the private + media directory (configured through the ``STORAGES`` setting), within the + ``temporary-uploads`` folder. - The user saves the form step: - - An instance of :class:`openforms.submissions.models.SubmissionFileAttachment` is created (with a relation to the - :class:`openforms.submissions.models.TemporaryFileUpload`). - - The file gets copied to the ``submission-uploads`` folder (which is also in the private media directory). + - An instance of :class:`openforms.submissions.models.SubmissionFileAttachment` is + created (with a relation to the :class:`openforms.submissions.models.TemporaryFileUpload`). + - The file gets copied to the ``submission-uploads`` folder (which is also in the + private media directory). - The user completes the submission: - - The UUID of the :class:`openforms.submissions.models.TemporaryFileUpload` is removed from the session. - - The task ``cleanup_temporary_files_for`` deletes all :class:`openforms.submissions.models.TemporaryFileUpload` - associated with the submission that has been completed. + - The task ``cleanup_temporary_files_for`` deletes all + :class:`openforms.submissions.models.TemporaryFileUpload` associated with the + submission that has been completed. .. note:: When instances of :class:`openforms.submissions.models.TemporaryFileUpload` and - :class:`openforms.submissions.models.SubmissionFileAttachment` are deleted, the associated - files are removed from the file system (thanks to the :class:`openforms.utils.files.DeleteFileFieldFilesMixin` mixin). + :class:`openforms.submissions.models.SubmissionFileAttachment` are deleted, the + associated files are removed from the file system (thanks to the + :class:`openforms.utils.files.DeleteFileFieldFilesMixin` mixin). +Access control +-------------- + +Users must have an active submission in the session before they can upload any +(temporary) files. Additionally, the temporary file is related to the submission UUID +in the session, and users can only download or delete files that belong to a submission +present in their session data. Periodical clean up =================== There are Celery beat tasks that periodically clean up files: -- The task ``cleanup_unclaimed_temporary_files`` cleans up any :class:`openforms.submissions.models.TemporaryFileUpload` which is not related to a - :class:`openforms.submissions.models.SubmissionFileAttachment`. This task runs once a day. -- The task ``delete_submissions`` deletes any successful/incomplete/errored submission that are older than a - configured amount of time. This deletes the associated :class:`openforms.submissions.models.SubmissionFileAttachment`. This task runs once a day. -- The task ``make_sensitive_data_anonymous`` clears any sensitive data from a submission. It also deletes any - :class:`openforms.submissions.models.SubmissionFileAttachment` related to the submission being cleaned. This task runs once a day. +- The task ``cleanup_unclaimed_temporary_files`` cleans up any + :class:`openforms.submissions.models.TemporaryFileUpload` which is not related to a + :class:`openforms.submissions.models.SubmissionFileAttachment`. This task runs once a + day. +- The task ``delete_submissions`` deletes any successful/incomplete/errored submission + that are older than a configured amount of time. This deletes the associated + :class:`openforms.submissions.models.SubmissionFileAttachment`. This task runs once a + day. +- The task ``make_sensitive_data_anonymous`` clears any sensitive data from a + submission. It also deletes any + :class:`openforms.submissions.models.SubmissionFileAttachment` related to the + submission being cleaned. This task runs once a day. diff --git a/requirements/base.txt b/requirements/base.txt index 7d84e827d7..9707b726b8 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -168,7 +168,7 @@ django-csp==4.0 # via -r requirements/base.in django-csp-reports==1.10.0 # via -r requirements/base.in -django-digid-eherkenning==0.24.0 +django-digid-eherkenning==0.25.1 # via -r requirements/base.in django-filter==25.2 # via -r requirements/base.in @@ -206,10 +206,11 @@ django-otp==1.7.0 # maykin-2fa django-phonenumber-field==8.4.0 # via django-two-factor-auth -django-privates==3.1.1 +django-privates==4.0.2 # via # -r requirements/base.in # django-simple-certmanager + # zgw-consumers django-redis==6.0.0 # via -r requirements/base.in django-relativedelta==2.0.0 @@ -222,7 +223,7 @@ django-setup-configuration==0.11.0 # via # mozilla-django-oidc-db # zgw-consumers -django-simple-certmanager==2.5.0 +django-simple-certmanager==4.0.0 # via # -r requirements/base.in # django-digid-eherkenning @@ -709,7 +710,7 @@ xmltodict==1.0.3 # via -r requirements/base.in zeep==4.3.2 # via -r requirements/base.in -zgw-consumers==1.2.0 +zgw-consumers==2.0.2 # via -r requirements/base.in zipp==3.23.0 # via importlib-metadata diff --git a/requirements/ci.txt b/requirements/ci.txt index 64fab1c59e..c85a634258 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -259,7 +259,7 @@ django-csp-reports==1.10.0 # via # -c requirements/base.txt # -r requirements/base.txt -django-digid-eherkenning==0.24.0 +django-digid-eherkenning==0.25.1 # via # -c requirements/base.txt # -r requirements/base.txt @@ -326,11 +326,12 @@ django-phonenumber-field==8.4.0 # -c requirements/base.txt # -r requirements/base.txt # django-two-factor-auth -django-privates==3.1.1 +django-privates==4.0.2 # via # -c requirements/base.txt # -r requirements/base.txt # django-simple-certmanager + # zgw-consumers django-redis==6.0.0 # via # -c requirements/base.txt @@ -354,7 +355,7 @@ django-setup-configuration==0.11.0 # via # -c requirements/base.txt # -r requirements/base.txt -django-simple-certmanager==2.5.0 +django-simple-certmanager==4.0.0 # via # -c requirements/base.txt # -r requirements/base.txt @@ -947,7 +948,6 @@ pyyaml==6.0.3 # pydantic-settings # sphinxcontrib-mermaid # vcrpy - # zgw-consumers qrcode==8.2 # via # -c requirements/base.txt @@ -992,9 +992,7 @@ requests-file==3.0.1 # -r requirements/base.txt # zeep requests-mock==1.12.1 - # via - # -r requirements/test-tools.in - # zgw-consumers + # via -r requirements/test-tools.in requests-oauthlib==2.0.0 # via # -c requirements/base.txt @@ -1273,7 +1271,7 @@ zeep==4.3.2 # via # -c requirements/base.txt # -r requirements/base.txt -zgw-consumers==1.2.0 +zgw-consumers==2.0.2 # via # -c requirements/base.txt # -r requirements/base.txt diff --git a/requirements/dev.txt b/requirements/dev.txt index 8d6ebafefb..58e8770f40 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -284,7 +284,7 @@ django-csp-reports==1.10.0 # -r requirements/ci.txt django-debug-toolbar==6.2.0 # via -r requirements/dev.in -django-digid-eherkenning==0.24.0 +django-digid-eherkenning==0.25.1 # via # -c requirements/ci.txt # -r requirements/ci.txt @@ -353,11 +353,12 @@ django-phonenumber-field==8.4.0 # -c requirements/ci.txt # -r requirements/ci.txt # django-two-factor-auth -django-privates==3.1.1 +django-privates==4.0.2 # via # -c requirements/ci.txt # -r requirements/ci.txt # django-simple-certmanager + # zgw-consumers django-redis==6.0.0 # via # -c requirements/ci.txt @@ -385,7 +386,7 @@ django-setup-configuration==0.11.0 # -r requirements/ci.txt django-silk==5.4.3 # via -r requirements/dev.in -django-simple-certmanager==2.5.0 +django-simple-certmanager==4.0.0 # via # -c requirements/ci.txt # -r requirements/ci.txt @@ -1419,7 +1420,7 @@ zeep==4.3.2 # via # -c requirements/ci.txt # -r requirements/ci.txt -zgw-consumers==1.2.0 +zgw-consumers==2.0.2 # via # -c requirements/ci.txt # -r requirements/ci.txt diff --git a/requirements/extensions.txt b/requirements/extensions.txt index 68eafb6f9b..7819bb0b62 100644 --- a/requirements/extensions.txt +++ b/requirements/extensions.txt @@ -250,7 +250,7 @@ django-csp-reports==1.10.0 # via # -c requirements/base.txt # -r requirements/base.txt -django-digid-eherkenning==0.24.0 +django-digid-eherkenning==0.25.1 # via # -c requirements/base.txt # -r requirements/base.txt @@ -317,11 +317,12 @@ django-phonenumber-field==8.4.0 # -c requirements/base.txt # -r requirements/base.txt # django-two-factor-auth -django-privates==3.1.1 +django-privates==4.0.2 # via # -c requirements/base.txt # -r requirements/base.txt # django-simple-certmanager + # zgw-consumers django-redis==6.0.0 # via # -c requirements/base.txt @@ -345,7 +346,7 @@ django-setup-configuration==0.11.0 # via # -c requirements/base.txt # -r requirements/base.txt -django-simple-certmanager==2.5.0 +django-simple-certmanager==4.0.0 # via # -c requirements/base.txt # -r requirements/base.txt @@ -1156,7 +1157,7 @@ zeep==4.3.2 # via # -c requirements/base.txt # -r requirements/base.txt -zgw-consumers==1.2.0 +zgw-consumers==2.0.2 # via # -c requirements/base.txt # -r requirements/base.txt diff --git a/requirements/type-checking.txt b/requirements/type-checking.txt index 5a895aecd3..9ba804e280 100644 --- a/requirements/type-checking.txt +++ b/requirements/type-checking.txt @@ -276,7 +276,7 @@ django-csp-reports==1.10.0 # via # -c requirements/ci.txt # -r requirements/ci.txt -django-digid-eherkenning==0.24.0 +django-digid-eherkenning==0.25.1 # via # -c requirements/ci.txt # -r requirements/ci.txt @@ -343,11 +343,12 @@ django-phonenumber-field==8.4.0 # -c requirements/ci.txt # -r requirements/ci.txt # django-two-factor-auth -django-privates==3.1.1 +django-privates==4.0.2 # via # -c requirements/ci.txt # -r requirements/ci.txt # django-simple-certmanager + # zgw-consumers django-redis==6.0.0 # via # -c requirements/ci.txt @@ -371,7 +372,7 @@ django-setup-configuration==0.11.0 # via # -c requirements/ci.txt # -r requirements/ci.txt -django-simple-certmanager==2.5.0 +django-simple-certmanager==4.0.0 # via # -c requirements/ci.txt # -r requirements/ci.txt @@ -1405,7 +1406,7 @@ zeep==4.3.2 # via # -c requirements/ci.txt # -r requirements/ci.txt -zgw-consumers==1.2.0 +zgw-consumers==2.0.2 # via # -c requirements/ci.txt # -r requirements/ci.txt diff --git a/src/openapi.yaml b/src/openapi.yaml index 8ed4e859e7..cb5941e62d 100644 --- a/src/openapi.yaml +++ b/src/openapi.yaml @@ -7030,9 +7030,17 @@ components: - ztc - drc - brc + - rc + - kic + - oc + - ic + - pc + - ptc + - vrc + - tc + - bc - cmc - kc - - vrc - orc type: string description: |- @@ -7042,9 +7050,17 @@ components: * `ztc` - ZTC (Zaaktypen) * `drc` - DRC (Informatieobjecten) * `brc` - BRC (Besluiten) - * `cmc` - Contactmomenten API - * `kc` - Klanten API + * `rc` - Referentielijsten API + * `kic` - Klantinteracties API + * `oc` - Organisatie API + * `ic` - Identiteit API + * `pc` - Producten API + * `ptc` - Producttypen API * `vrc` - Verzoeken API + * `tc` - Taken API + * `bc` - Berichten API + * `cmc` - Contactmomenten API - (Deprecated) + * `kc` - Klanten API - (Deprecated) * `orc` - ORC (Overige) AppearanceEnum: enum: diff --git a/src/openforms/appointments/tests/test_pdf.py b/src/openforms/appointments/tests/test_pdf.py index a3043ce7b5..2e0529aa77 100644 --- a/src/openforms/appointments/tests/test_pdf.py +++ b/src/openforms/appointments/tests/test_pdf.py @@ -3,6 +3,8 @@ from django.test import RequestFactory, TestCase, override_settings, tag from django.utils.html import escape +from privates.test import temp_private_root + from openforms.accounts.tests.factories import SuperUserFactory from openforms.submissions.dev_views import SubmissionPDFTestView from openforms.submissions.tests.factories import SubmissionFactory @@ -11,6 +13,7 @@ from .factories import AppointmentFactory, AppointmentProductFactory +@temp_private_root() class PDFGenerationTests(TestCase): @classmethod def setUpTestData(cls): diff --git a/src/openforms/authentication/contrib/digid/migrations/0001_convert_digid_setting_to_db.py b/src/openforms/authentication/contrib/digid/migrations/0001_convert_digid_setting_to_db.py index d8b4f80b14..8dff250254 100644 --- a/src/openforms/authentication/contrib/digid/migrations/0001_convert_digid_setting_to_db.py +++ b/src/openforms/authentication/contrib/digid/migrations/0001_convert_digid_setting_to_db.py @@ -7,7 +7,7 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ("digid_eherkenning", "0001_initial"), + ("digid_eherkenning", "0001_initial_to_v019"), ] # Data migration removed - this was part of the 2.1 release cycle. Our upgrade checks diff --git a/src/openforms/authentication/contrib/digid/tests/test_auth_procedure.py b/src/openforms/authentication/contrib/digid/tests/test_auth_procedure.py index db92fafa67..e7109adad9 100644 --- a/src/openforms/authentication/contrib/digid/tests/test_auth_procedure.py +++ b/src/openforms/authentication/contrib/digid/tests/test_auth_procedure.py @@ -82,7 +82,7 @@ def setUp(self): self.addCleanup(clear_caches) -@temp_private_root() +@temp_private_root(reset_storage=False) @override_settings(CORS_ALLOW_ALL_ORIGINS=True, IS_HTTPS=True) class AuthenticationStep2Tests(DigiDConfigMixin, TestCase): def test_redirect_to_digid(self): @@ -277,7 +277,7 @@ def test_authn_request_uses_default_loa_if_not_overriden(self, mock_id): self.assertEqual(auth_context_class_ref.text, DigiDAssuranceLevels.middle.value) -@temp_private_root() +@temp_private_root(reset_storage=False) @override_settings(CORS_ALLOW_ALL_ORIGINS=True) @requests_mock.Mocker() class AuthenticationStep5Tests(DigiDConfigMixin, TestCase): @@ -477,7 +477,7 @@ def test_cancel_login( ) -@temp_private_root() +@temp_private_root(reset_storage=False) @override_settings(CORS_ALLOW_ALL_ORIGINS=True) @requests_mock.Mocker() class CoSignLoginAuthenticationTests(SubmissionsMixin, DigiDConfigMixin, TestCase): diff --git a/src/openforms/authentication/contrib/digid/tests/test_signicat_integration.py b/src/openforms/authentication/contrib/digid/tests/test_signicat_integration.py index 7506cc1971..4688c897e1 100644 --- a/src/openforms/authentication/contrib/digid/tests/test_signicat_integration.py +++ b/src/openforms/authentication/contrib/digid/tests/test_signicat_integration.py @@ -39,7 +39,7 @@ "onelogin.saml2.authn_request.OneLogin_Saml2_Utils.generate_unique_id", lambda *_, **__: "ONELOGIN_123456", ) -@temp_private_root() +@temp_private_root(reset_storage=False) @override_settings(COOKIE_CONSENT_ENABLED=False) class SignicatDigiDIntegrationTests(OFVCRMixin, TestCase): """Test using Signicat broker. diff --git a/src/openforms/authentication/contrib/eherkenning/migrations/0001_convert_eherkenning_setting_to_db.py b/src/openforms/authentication/contrib/eherkenning/migrations/0001_convert_eherkenning_setting_to_db.py index fc6058368e..0a338fb48e 100644 --- a/src/openforms/authentication/contrib/eherkenning/migrations/0001_convert_eherkenning_setting_to_db.py +++ b/src/openforms/authentication/contrib/eherkenning/migrations/0001_convert_eherkenning_setting_to_db.py @@ -8,7 +8,7 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ("digid_eherkenning", "0001_initial"), + ("digid_eherkenning", "0001_initial_to_v019"), ] # Data migration removed - this was part of the 2.1 release cycle. Our upgrade checks diff --git a/src/openforms/authentication/contrib/eherkenning/tests/test_eherkenning_auth.py b/src/openforms/authentication/contrib/eherkenning/tests/test_eherkenning_auth.py index 8b68d40d25..081bffd954 100644 --- a/src/openforms/authentication/contrib/eherkenning/tests/test_eherkenning_auth.py +++ b/src/openforms/authentication/contrib/eherkenning/tests/test_eherkenning_auth.py @@ -102,7 +102,7 @@ def _get_encrypted_attribute(kvk: str): @override_settings(CORS_ALLOW_ALL_ORIGINS=True, IS_HTTPS=True) -@temp_private_root() +@temp_private_root(reset_storage=False) class AuthenticationStep2Tests(EHerkenningConfigMixin, TestCase): def test_redirect_to_eherkenning_login(self): form = FormFactory.create( @@ -201,7 +201,7 @@ def test_authn_request(self, mock_id): @override_settings(CORS_ALLOW_ALL_ORIGINS=True) -@temp_private_root() +@temp_private_root(reset_storage=False) @requests_mock.Mocker() class AuthenticationStep5Tests(EHerkenningConfigMixin, TestCase): @classmethod @@ -368,7 +368,7 @@ def test_attribute_extraction_failure(self, m): @override_settings(CORS_ALLOW_ALL_ORIGINS=True) -@temp_private_root() +@temp_private_root(reset_storage=False) @requests_mock.Mocker() class CoSignLoginAuthenticationTests( SubmissionsMixin, EHerkenningConfigMixin, TestCase diff --git a/src/openforms/authentication/contrib/eherkenning/tests/test_eidas_auth.py b/src/openforms/authentication/contrib/eherkenning/tests/test_eidas_auth.py index 8ce2df82ed..fcd51a5d7e 100644 --- a/src/openforms/authentication/contrib/eherkenning/tests/test_eidas_auth.py +++ b/src/openforms/authentication/contrib/eherkenning/tests/test_eidas_auth.py @@ -104,7 +104,7 @@ def _get_encrypted_attribute(pseudo_id: str): @override_settings(CORS_ALLOW_ALL_ORIGINS=True, IS_HTTPS=True) -@temp_private_root() +@temp_private_root(reset_storage=False) class AuthenticationStep2Tests(EIDASConfigMixin, TestCase): def test_redirect_to_eIDAS_login(self): form = FormFactory.create( @@ -204,7 +204,7 @@ def test_authn_request(self, mock_id): @override_settings(CORS_ALLOW_ALL_ORIGINS=True) -@temp_private_root() +@temp_private_root(reset_storage=False) @requests_mock.Mocker() class AuthenticationStep5Tests(EIDASConfigMixin, TestCase): @mock_saml2_return_flow(mock_saml_art_verification=True) @@ -292,7 +292,7 @@ def test_cancel_login(self, m): @override_settings(CORS_ALLOW_ALL_ORIGINS=True) -@temp_private_root() +@temp_private_root(reset_storage=False) @requests_mock.Mocker() class CoSignLoginAuthenticationTests(SubmissionsMixin, EIDASConfigMixin, TestCase): @patch( diff --git a/src/openforms/authentication/contrib/eherkenning/tests/test_signicat_integration.py b/src/openforms/authentication/contrib/eherkenning/tests/test_signicat_integration.py index 29d5f0e263..0865be6e56 100644 --- a/src/openforms/authentication/contrib/eherkenning/tests/test_signicat_integration.py +++ b/src/openforms/authentication/contrib/eherkenning/tests/test_signicat_integration.py @@ -43,7 +43,7 @@ "onelogin.saml2.authn_request.OneLogin_Saml2_Utils.generate_unique_id", lambda *_, **__: "ONELOGIN_123456", ) -@temp_private_root() +@temp_private_root(reset_storage=False) @override_settings(COOKIE_CONSENT_ENABLED=False) class SignicatEHerkenningIntegrationTests(OFVCRMixin, TestCase): """Test using Signicat broker. diff --git a/src/openforms/authentication/tests/test_tasks.py b/src/openforms/authentication/tests/test_tasks.py index 81272ccbca..cc3934eeba 100644 --- a/src/openforms/authentication/tests/test_tasks.py +++ b/src/openforms/authentication/tests/test_tasks.py @@ -4,6 +4,7 @@ from django.test import TestCase from digid_eherkenning.models import DigidConfiguration +from privates.test import temp_private_root from ..tasks import update_saml_metadata @@ -14,6 +15,7 @@ EHERKENNING_TEST_METADATA_FILE = BASE_DIR / "files" / "eherkenning" / "metadata" +@temp_private_root() class UpdateSamlTaskTests(TestCase): @patch( "onelogin.saml2.idp_metadata_parser.OneLogin_Saml2_IdPMetadataParser.get_metadata" diff --git a/src/openforms/conf/base.py b/src/openforms/conf/base.py index 1f09c44729..ec29119f3e 100644 --- a/src/openforms/conf/base.py +++ b/src/openforms/conf/base.py @@ -378,21 +378,24 @@ f"{config('STATICFILES_STORAGE_CLASS', default='StaticFilesStorage')}" ), }, + "privates": { + "BACKEND": "django.core.files.storage.FileSystemStorage", + "OPTIONS": { + "location": BASE_DIR / "private_media", + "base_url": "/private-media/", + }, + }, } MEDIA_ROOT = BASE_DIR / "media" MEDIA_URL = "/media/" -PRIVATE_MEDIA_ROOT = BASE_DIR / "private_media" - -PRIVATE_MEDIA_URL = "/private-media/" - FILE_UPLOAD_PERMISSIONS = 0o644 SENDFILE_BACKEND = config("SENDFILE_BACKEND", default="django_sendfile.backends.nginx") -SENDFILE_ROOT = PRIVATE_MEDIA_ROOT -SENDFILE_URL = PRIVATE_MEDIA_URL +SENDFILE_ROOT = STORAGES["privates"]["OPTIONS"]["location"] +SENDFILE_URL = STORAGES["privates"]["OPTIONS"]["base_url"] # # Sending EMAIL diff --git a/src/openforms/config/migrations/0001_initial_to_v250.py b/src/openforms/config/migrations/0001_initial_to_v250.py index 4653b722a8..9754562b72 100644 --- a/src/openforms/config/migrations/0001_initial_to_v250.py +++ b/src/openforms/config/migrations/0001_initial_to_v250.py @@ -90,7 +90,7 @@ class Migration(migrations.Migration): dependencies = [ ("contenttypes", "0002_remove_content_type_name"), - ("digid_eherkenning", "0006_digidconfiguration_metadata_file_source_and_more"), + ("digid_eherkenning", "0001_initial_to_v019"), ("payments_ogone", "0002_auto_20210902_2120"), ("cookie_consent", "0002_auto__add_logitem"), ] diff --git a/src/openforms/emails/digest.py b/src/openforms/emails/digest.py index caf836ba09..9aa00652d0 100644 --- a/src/openforms/emails/digest.py +++ b/src/openforms/emails/digest.py @@ -360,7 +360,7 @@ def collect_invalid_certificates() -> list[InvalidCertificate]: for cert in configured_certificates: error_message = "" is_valid_pair = cert.is_valid_key_pair() - time_until_expiry = cert.expiry_date - today + time_until_expiry = cert.not_valid_after - today is_expired = time_until_expiry <= timedelta(days=0) no_longer_valid_in_two_weeks = time_until_expiry <= timedelta(days=14) @@ -389,7 +389,7 @@ def collect_invalid_certificates() -> list[InvalidCertificate]: label=str(cert), error_message=str(error_message), is_valid_pair=is_valid_pair or is_valid_pair is None, - expiry_date=cert.expiry_date, + expiry_date=cert.not_valid_after, ) ) diff --git a/src/openforms/emails/tests/test_digest_functions.py b/src/openforms/emails/tests/test_digest_functions.py index b4211b2509..b3c2ddb6e2 100644 --- a/src/openforms/emails/tests/test_digest_functions.py +++ b/src/openforms/emails/tests/test_digest_functions.py @@ -10,6 +10,7 @@ import requests_mock from django_yubin.models import Message from freezegun import freeze_time +from privates.test import temp_private_root from rest_framework import serializers from simple_certmanager.test.factories import CertificateFactory from zgw_consumers.constants import AuthTypes @@ -1250,6 +1251,7 @@ def test_partners_with_hc_and_no_immutable_variable_not_collected(self): @override_settings(LANGUAGE_CODE="en") +@temp_private_root() class InvalidCertificatesTests(TestCase): def test_expiring_certificates_not_used_by_a_service_are_not_collected(self): # the certificate (test.certificate) expires on Mar 26 10:15:40 2027 GMT diff --git a/src/openforms/emails/tests/test_tasks_integration.py b/src/openforms/emails/tests/test_tasks_integration.py index a6f4aa39ad..a9c72ddd9d 100644 --- a/src/openforms/emails/tests/test_tasks_integration.py +++ b/src/openforms/emails/tests/test_tasks_integration.py @@ -11,6 +11,7 @@ from django_yubin.models import Message from freezegun import freeze_time from furl import furl +from privates.test import temp_private_root from requests import RequestException from simple_certmanager.test.factories import CertificateFactory from zgw_consumers.constants import AuthTypes @@ -153,6 +154,7 @@ def test_no_email_sent_if_no_recipients(self): return_value=BRKConfig(service=INVALID_BRK_SERVICE), ) @freeze_time("2023-01-03T01:00:00+01:00") + @temp_private_root() @override_settings(BASE_URL="http://testserver") @requests_mock.Mocker() def test_email_sent_when_there_are_failures(self, brk_config, m): diff --git a/src/openforms/formio/formatters/tests/test_kitchensink.py b/src/openforms/formio/formatters/tests/test_kitchensink.py index c593e3ef23..065157ed5c 100644 --- a/src/openforms/formio/formatters/tests/test_kitchensink.py +++ b/src/openforms/formio/formatters/tests/test_kitchensink.py @@ -3,6 +3,8 @@ from django.utils.translation import gettext as _ +from privates.test import temp_private_root + from openforms.submissions.models import Submission from openforms.submissions.tests.factories import ( SubmissionFactory, @@ -35,6 +37,7 @@ def _get_printable_data(submission: Submission) -> list[tuple[str, Any]]: return printable_data +@temp_private_root() class KitchensinkFormatterTestCase(BaseFormatterTestCase): def test_kitchensink_formio(self): self.run_kitchensink_test("kitchensink_data", "kitchensink_printable_text") diff --git a/src/openforms/formio/rendering/tests/test_vanilla_formio_components.py b/src/openforms/formio/rendering/tests/test_vanilla_formio_components.py index 48572bb9ec..c73712510d 100644 --- a/src/openforms/formio/rendering/tests/test_vanilla_formio_components.py +++ b/src/openforms/formio/rendering/tests/test_vanilla_formio_components.py @@ -7,6 +7,8 @@ from django.test import TestCase, override_settings +from privates.test import temp_private_root + from openforms.forms.tests.factories import FormFactory, FormStepFactory from openforms.submissions.rendering import Renderer, RenderModes from openforms.submissions.tests.factories import ( @@ -20,6 +22,7 @@ from ..nodes import ComponentNode +@temp_private_root() @override_settings(LANGUAGE_CODE="en") class FormNodeTests(TestCase): @classmethod diff --git a/src/openforms/formio/tests/test_api_fileupload.py b/src/openforms/formio/tests/test_api_fileupload.py index 515a6839e5..701d84ef18 100644 --- a/src/openforms/formio/tests/test_api_fileupload.py +++ b/src/openforms/formio/tests/test_api_fileupload.py @@ -1,5 +1,4 @@ -import os -import tempfile +import itertools from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from unittest.mock import patch @@ -10,6 +9,7 @@ from django.utils.translation import gettext as _ import clamd +from privates.storages import private_media_storage from privates.test import temp_private_root from rest_framework import status from rest_framework.reverse import reverse @@ -17,6 +17,7 @@ from openforms.config.models import GlobalConfiguration from openforms.submissions.attachments import temporary_upload_from_url +from openforms.submissions.models import TemporaryFileUpload from openforms.submissions.tests.factories import SubmissionFactory from openforms.submissions.tests.mixins import SubmissionsMixin @@ -28,13 +29,16 @@ class FormIOTemporaryFileUploadTest(SubmissionsMixin, APITestCase): @classmethod def setUpTestData(cls): + super().setUpTestData() + cls.submission = SubmissionFactory.create() cls.submission_url = reverse( "api:submission-detail", kwargs={"uuid": cls.submission.uuid} ) - def tearDown(self): - self._clear_session() + def setUp(self): + super().setUp() + self.addCleanup(self._clear_session) def test_upload_view_requires_active_submission(self): url = reverse("api:formio:temporary-file-upload") @@ -76,7 +80,7 @@ def test_upload_view(self): # check if we can retrieve the file from returned url response = self.client.get(body["url"]) - self.assertEqual(b"".join(response.streaming_content), b"my content") + self.assertEqual(response.content, b"my content") self.assertIn("Content-Disposition", response) self.assertIn("attachment", response["Content-Disposition"]) self.assertIn("my-file.txt", response["Content-Disposition"]) @@ -273,18 +277,16 @@ def test_file_contains_virus(self, m_config): "my-file.bin", clamd.EICAR, content_type="application/octet-stream" ) - tmpdir = tempfile.mkdtemp() - with override_settings(PRIVATE_MEDIA_ROOT=tmpdir, SENDFILE_ROOT=tmpdir): - with patch.object( - clamd.ClamdNetworkSocket, - "instream", - return_value={"stream": ("FOUND", "Win.Test.EICAR_HDB-1")}, - ): - response_virus = self.client.post( - url, - {"file": file_with_virus, "submission": self.submission_url}, - format="multipart", - ) + with patch.object( + clamd.ClamdNetworkSocket, + "instream", + return_value={"stream": ("FOUND", "Win.Test.EICAR_HDB-1")}, + ): + response_virus = self.client.post( + url, + {"file": file_with_virus, "submission": self.submission_url}, + format="multipart", + ) self.assertEqual(status.HTTP_400_BAD_REQUEST, response_virus.status_code) self.assertEqual( @@ -292,9 +294,10 @@ def test_file_contains_virus(self, m_config): "File did not pass the virus scan. It was found to contain 'Win.Test.EICAR_HDB-1'.", ) - tmpdir_contents = os.listdir(tmpdir) - - self.assertEqual(0, len(tmpdir_contents)) + storage_contents = [ + *itertools.chain.from_iterable(private_media_storage.listdir("")) + ] + self.assertEqual(0, len(storage_contents)) @patch("openforms.formio.api.validators.GlobalConfiguration.get_solo") def test_file_does_not_contains_virus(self, m_config): @@ -307,31 +310,32 @@ def test_file_does_not_contains_virus(self, m_config): "my-file.bin", b"I am a nice file", content_type="application/octet-stream" ) - tmpdir = tempfile.mkdtemp() - with override_settings(PRIVATE_MEDIA_ROOT=tmpdir, SENDFILE_ROOT=tmpdir): - with patch( + with ( + patch( "openforms.submissions.models.submission_files.fmt_upload_to", return_value="my-file.bin", - ): - with patch.object( - clamd.ClamdNetworkSocket, - "instream", - return_value={"stream": ("OK", None)}, - ): - response_no_virus = self.client.post( - url, - { - "file": file_without_virus, - "submission": self.submission_url, - }, - format="multipart", - ) + ), + patch.object( + clamd.ClamdNetworkSocket, + "instream", + return_value={"stream": ("OK", None)}, + ), + ): + response_no_virus = self.client.post( + url, + { + "file": file_without_virus, + "submission": self.submission_url, + }, + format="multipart", + ) self.assertEqual(status.HTTP_200_OK, response_no_virus.status_code) - tmpdir_contents = os.listdir(tmpdir) - - self.assertEqual(["my-file.bin"], tmpdir_contents) + storage_contents = [ + *itertools.chain.from_iterable(private_media_storage.listdir("")) + ] + self.assertEqual(["my-file.bin"], storage_contents) @patch("openforms.formio.api.validators.GlobalConfiguration.get_solo") def test_file_scan_returns_error(self, m_config): @@ -344,18 +348,16 @@ def test_file_scan_returns_error(self, m_config): "my-file.bin", b"I am a nice file", content_type="application/octet-stream" ) - tmpdir = tempfile.mkdtemp() - with override_settings(PRIVATE_MEDIA_ROOT=tmpdir, SENDFILE_ROOT=tmpdir): - with patch.object( - clamd.ClamdNetworkSocket, - "instream", - return_value={"stream": ("ERROR", "I am an error")}, - ): - response_virus = self.client.post( - url, - {"file": file_with_virus, "submission": self.submission_url}, - format="multipart", - ) + with patch.object( + clamd.ClamdNetworkSocket, + "instream", + return_value={"stream": ("ERROR", "I am an error")}, + ): + response_virus = self.client.post( + url, + {"file": file_with_virus, "submission": self.submission_url}, + format="multipart", + ) self.assertEqual(status.HTTP_400_BAD_REQUEST, response_virus.status_code) self.assertEqual( @@ -363,9 +365,11 @@ def test_file_scan_returns_error(self, m_config): "The virus scan on this file returned an error.", ) - tmpdir_contents = os.listdir(tmpdir) + storage_contents = [ + *itertools.chain.from_iterable(private_media_storage.listdir("")) + ] - self.assertEqual(0, len(tmpdir_contents)) + self.assertEqual(0, len(storage_contents)) @patch("openforms.formio.api.validators.GlobalConfiguration.get_solo") def test_file_scan_returns_unexpected_status(self, m_config): @@ -378,18 +382,16 @@ def test_file_scan_returns_unexpected_status(self, m_config): "my-file.bin", b"I am a nice file", content_type="application/octet-stream" ) - tmpdir = tempfile.mkdtemp() - with override_settings(PRIVATE_MEDIA_ROOT=tmpdir, SENDFILE_ROOT=tmpdir): - with patch.object( - clamd.ClamdNetworkSocket, - "instream", - return_value={"stream": ("UNEXPECTED", "I am message")}, - ): - response_virus = self.client.post( - url, - {"file": file_with_virus, "submission": self.submission_url}, - format="multipart", - ) + with patch.object( + clamd.ClamdNetworkSocket, + "instream", + return_value={"stream": ("UNEXPECTED", "I am message")}, + ): + response_virus = self.client.post( + url, + {"file": file_with_virus, "submission": self.submission_url}, + format="multipart", + ) self.assertEqual(status.HTTP_400_BAD_REQUEST, response_virus.status_code) self.assertEqual( @@ -397,9 +399,11 @@ def test_file_scan_returns_unexpected_status(self, m_config): "The virus scan returned an unexpected status.", ) - tmpdir_contents = os.listdir(tmpdir) + storage_contents = [ + *itertools.chain.from_iterable(private_media_storage.listdir("")) + ] - self.assertEqual(0, len(tmpdir_contents)) + self.assertEqual(0, len(storage_contents)) @patch("openforms.formio.api.validators.GlobalConfiguration.get_solo") def test_cannot_connect_to_clamdav(self, m_config): @@ -412,18 +416,16 @@ def test_cannot_connect_to_clamdav(self, m_config): "my-file.bin", b"I am a nice file", content_type="application/octet-stream" ) - tmpdir = tempfile.mkdtemp() - with override_settings(PRIVATE_MEDIA_ROOT=tmpdir, SENDFILE_ROOT=tmpdir): - with patch.object( - clamd.ClamdNetworkSocket, - "instream", - side_effect=clamd.ConnectionError("Cannot connect!"), - ): - response_virus = self.client.post( - url, - {"file": file_with_virus, "submission": self.submission_url}, - format="multipart", - ) + with patch.object( + clamd.ClamdNetworkSocket, + "instream", + side_effect=clamd.ConnectionError("Cannot connect!"), + ): + response_virus = self.client.post( + url, + {"file": file_with_virus, "submission": self.submission_url}, + format="multipart", + ) self.assertEqual(status.HTTP_400_BAD_REQUEST, response_virus.status_code) self.assertEqual( @@ -431,9 +433,11 @@ def test_cannot_connect_to_clamdav(self, m_config): "The virus scan could not be performed at this time. Please retry later.", ) - tmpdir_contents = os.listdir(tmpdir) + storage_contents = [ + *itertools.chain.from_iterable(private_media_storage.listdir("")) + ] - self.assertEqual(0, len(tmpdir_contents)) + self.assertEqual(0, len(storage_contents)) def test_filename_with_spaces(self): self._add_submission_to_session(self.submission) @@ -500,6 +504,12 @@ def test_filename_with_spaces(self): class ConcurrentUploadTests(SubmissionsMixin, APITransactionTestCase): @tag("gh-3858") def test_concurrent_file_uploads(self): + def _cleanup_storage_files(): + for upload in TemporaryFileUpload.objects.all(): + upload.content.delete(save=False) + + self.addCleanup(_cleanup_storage_files) + submission = SubmissionFactory.from_components( [ { diff --git a/src/openforms/formio/tests/validation/test_file.py b/src/openforms/formio/tests/validation/test_file.py index cc03c76a5f..1ac07bce19 100644 --- a/src/openforms/formio/tests/validation/test_file.py +++ b/src/openforms/formio/tests/validation/test_file.py @@ -5,6 +5,7 @@ from django.test import TestCase, tag from django.utils.translation import gettext_lazy as _ +from privates.test import temp_private_root from rest_framework.settings import api_settings from openforms.config.models import GlobalConfiguration @@ -33,6 +34,7 @@ } +@temp_private_root() class FileValidationMaxFilesAndRequiredTests(TestCase): """Tests related to ``validate.required`` and ``maxNumberOfFiles``.""" @@ -231,6 +233,7 @@ def test_file_nested_in_editgrid(self): self.assertTrue(is_valid) +@temp_private_root() class FileValidationTests(TestCase): def test_different_data(self): """Test consistency between ``url/size`` and ``data.url/data.size``.""" @@ -357,6 +360,7 @@ def test_passes_validation(self): self.assertTrue(is_valid) +@temp_private_root() class FileValidationMimeTypeTests(TestCase): @tag("GHSA-h85r-xv4w-cg8g") def test_attach_upload_validates_file_content_types_malicious_content(self): diff --git a/src/openforms/forms/admin/tasks.py b/src/openforms/forms/admin/tasks.py index 3a2b4939f3..c6d25a2906 100644 --- a/src/openforms/forms/admin/tasks.py +++ b/src/openforms/forms/admin/tasks.py @@ -34,7 +34,7 @@ def process_forms_export(forms_uuids: list, user_id: int) -> None: user = User.objects.get(id=user_id) # This deletes the temp dir once the context manager is exited - with tempfile.TemporaryDirectory(dir=settings.PRIVATE_MEDIA_ROOT) as temp_dir: + with tempfile.TemporaryDirectory(dir=private_media_storage.location) as temp_dir: output_files = [] for form in forms: output_files.append( diff --git a/src/openforms/forms/admin/views.py b/src/openforms/forms/admin/views.py index 7d84ae77b7..38b7afbe2a 100644 --- a/src/openforms/forms/admin/views.py +++ b/src/openforms/forms/admin/views.py @@ -73,7 +73,7 @@ def get(self, request, *args, **kwargs): user=request.user.username, export_id=forms_export.pk, ) - return FileResponse(open(forms_export.export_content.path, "rb")) + return FileResponse(forms_export.export_content.open("rb")) class ImportFormsView(ExportImportPermissionMixin, SuccessMessageMixin, FormView): diff --git a/src/openforms/forms/migrations/0130_alter_formsexport_export_content.py b/src/openforms/forms/migrations/0130_alter_formsexport_export_content.py new file mode 100644 index 0000000000..0999b40b84 --- /dev/null +++ b/src/openforms/forms/migrations/0130_alter_formsexport_export_content.py @@ -0,0 +1,23 @@ +# Generated by Django 5.2.14 on 2026-05-15 20:47 + +from django.db import migrations + +import privates.fields + + +class Migration(migrations.Migration): + dependencies = [ + ("forms", "0129_remove_form_new_renderer_enabled"), + ] + + operations = [ + migrations.AlterField( + model_name="formsexport", + name="export_content", + field=privates.fields.PrivateMediaFileField( + help_text="Zip file containing all the exported forms.", + upload_to="exports/%Y/%m/%d", + verbose_name="export content", + ), + ), + ] diff --git a/src/openforms/forms/tests/admin/test_tasks.py b/src/openforms/forms/tests/admin/test_tasks.py index 38b93f33b1..556dc47fa9 100644 --- a/src/openforms/forms/tests/admin/test_tasks.py +++ b/src/openforms/forms/tests/admin/test_tasks.py @@ -1,5 +1,4 @@ import zipfile -from pathlib import Path from unittest.mock import patch from django.core import mail @@ -45,7 +44,10 @@ def test_zip_file_contains_data(self): self.assertEqual(user, forms_export.user) # Test that the zip file contains the right forms - with zipfile.ZipFile(forms_export.export_content.path, "r") as file: + with ( + forms_export.export_content.open("rb") as content, + zipfile.ZipFile(content, "r") as file, + ): names_list = file.namelist() self.assertEqual(2, len(names_list)) @@ -67,10 +69,12 @@ def test_zip_file_contains_data(self): self.assertIn("test@email.nl", sent_mail.to) -@temp_private_root() +@temp_private_root(reset_storage=False) class ImportFormsTaskTests(TestCase): @classmethod def setUpTestData(cls): + super().setUpTestData() + form1, form2 = FormFactory.create_batch(2) user = SuperUserFactory.create(email="test@email.nl") @@ -93,13 +97,13 @@ def _copy_file_to_imports_tempdir(self): def test_import_forms(self): imported_file_path = self._copy_file_to_imports_tempdir() + assert private_media_storage.exists(imported_file_path) process_forms_import(str(imported_file_path), self.user.id) self.assertEqual(4, Form.objects.count()) - # Check that no files are left over - dir_path = Path(private_media_storage.path(imported_file_path)).parent - self.assertEqual(0, len(list(dir_path.iterdir()))) + # Check that the import file is cleaned up + self.assertFalse(private_media_storage.exists(imported_file_path)) @patch( "openforms.forms.admin.tasks.import_form", diff --git a/src/openforms/prefill/contrib/suwinet/tests/test_plugin.py b/src/openforms/prefill/contrib/suwinet/tests/test_plugin.py index c6f95d9f67..19d6ab676f 100644 --- a/src/openforms/prefill/contrib/suwinet/tests/test_plugin.py +++ b/src/openforms/prefill/contrib/suwinet/tests/test_plugin.py @@ -1,6 +1,8 @@ from pathlib import Path from unittest.mock import patch +from privates.test import temp_private_root + from openforms.submissions.tests.factories import SubmissionFactory from suwinet.tests.factories import SuwinetConfigFactory from suwinet.tests.test_client import SuwinetTestCase @@ -11,6 +13,7 @@ DATA_DIR = Path(__file__).parent / "data" +@temp_private_root(reset_storage=False) class SuwinetPrefillTests(SuwinetTestCase): VCR_TEST_FILES = DATA_DIR diff --git a/src/openforms/registrations/contrib/email/tests/test_backend.py b/src/openforms/registrations/contrib/email/tests/test_backend.py index d37fb4837a..b5d65c86ee 100644 --- a/src/openforms/registrations/contrib/email/tests/test_backend.py +++ b/src/openforms/registrations/contrib/email/tests/test_backend.py @@ -10,6 +10,7 @@ import tablib from furl import furl +from privates.test import temp_private_root from openforms.authentication.service import AuthAttribute from openforms.config.models import GlobalConfiguration @@ -74,6 +75,7 @@ def _get_sent_email(index: int = 0) -> tuple[mail.EmailMultiAlternatives, str, s return message, str(text_body), html_body +@temp_private_root() @override_settings( DEFAULT_FROM_EMAIL="info@open-forms.nl", BASE_URL="https://example.com", @@ -522,6 +524,7 @@ def test_register_and_update_paid_product(self): form__payment_backend="demo", registration_success=True, public_registration_reference="XYZ", + with_report=True, ) payment = SubmissionPaymentFactory.for_submission( submission=submission, status=PaymentStatus.completed @@ -853,6 +856,7 @@ def test_submission_with_email_backend_export_pdf(self): ], submitted_data={"someField": "value0", "someList": ["value1", "value2"]}, completed=True, + with_report=True, ) submission_step = ( submission.submissionstep_set.get() # pyright: ignore[reportAttributeAccessIssue] diff --git a/src/openforms/registrations/contrib/generic_json/tests/test_backend.py b/src/openforms/registrations/contrib/generic_json/tests/test_backend.py index 2edb7d7d8e..775e3fcaf3 100644 --- a/src/openforms/registrations/contrib/generic_json/tests/test_backend.py +++ b/src/openforms/registrations/contrib/generic_json/tests/test_backend.py @@ -8,6 +8,7 @@ import requests_mock from freezegun import freeze_time +from privates.test import temp_private_root from requests import RequestException from zgw_consumers.constants import AuthTypes from zgw_consumers.test.factories import ServiceFactory @@ -28,6 +29,7 @@ from ..typing import GenericJSONOptions +@temp_private_root() class GenericJSONBackendTests(OFVCRMixin, TestCase): maxDiff = None diff --git a/src/openforms/registrations/contrib/microsoft_graph/tests/test_backend.py b/src/openforms/registrations/contrib/microsoft_graph/tests/test_backend.py index 7e11053cc0..081dca9856 100644 --- a/src/openforms/registrations/contrib/microsoft_graph/tests/test_backend.py +++ b/src/openforms/registrations/contrib/microsoft_graph/tests/test_backend.py @@ -18,7 +18,6 @@ from openforms.submissions.tests.factories import ( SubmissionFactory, SubmissionFileAttachmentFactory, - SubmissionReportFactory, SubmissionStepFactory, ) from openforms.utils.tests.cache import clear_caches @@ -387,16 +386,17 @@ def test_folder_path_with_date(self, upload_mock): self.assertEqual(call.args[1], path) +@temp_private_root() class MSGraphRegistrationBackendFailureTests(TestCase): def test_no_service_configured_raises_registration_error(self): submission = SubmissionFactory.create( form__registration_backend="microsoft-graph", + with_report=True, ) SubmissionStepFactory.create( submission=submission, data={"foo": "bar", "some_list": ["value1", "value2"]}, ) - SubmissionReportFactory.create(submission=submission) options: MicrosoftGraphOptions = { "folder_path": "/open-forms/", diff --git a/src/openforms/registrations/contrib/objects_api/tests/test_backend.py b/src/openforms/registrations/contrib/objects_api/tests/test_backend.py index b678a12c0f..c4faec7708 100644 --- a/src/openforms/registrations/contrib/objects_api/tests/test_backend.py +++ b/src/openforms/registrations/contrib/objects_api/tests/test_backend.py @@ -8,6 +8,7 @@ from django.test import TestCase +from privates.test import temp_private_root from requests import RequestException from vcr.request import Request as VCRRequest @@ -46,6 +47,7 @@ def __call__(self, request: VCRRequest) -> VCRRequest | None: return request +@temp_private_root() class ObjectsAPIBackendVCRTests(OFVCRMixin, TestCase): _vcr_before_record_request: BeforeRecordRequestWrapper = ( BeforeRecordRequestWrapper() @@ -115,6 +117,7 @@ def fail_csv_request(request: VCRRequest): completed=True, # the version of the document types are valid on this timestamp completed_on=datetime(2024, 7, 1, 12, 0, 0).replace(tzinfo=UTC), + with_report=True, ) with self.assertRaises(RegistrationFailed): @@ -479,6 +482,7 @@ def test_prefer_dynamic_resolution_over_fixed_url(self): completed=True, # Use a stable timestamp to get stable request params completed_on=datetime(2024, 7, 1, 12, 0, 0).replace(tzinfo=UTC), + with_report=True, ) attachment = SubmissionFileAttachmentFactory.create( submission_step=submission.steps[0], diff --git a/src/openforms/registrations/contrib/objects_api/tests/test_backend_v1.py b/src/openforms/registrations/contrib/objects_api/tests/test_backend_v1.py index 324cde7c8e..68c299d254 100644 --- a/src/openforms/registrations/contrib/objects_api/tests/test_backend_v1.py +++ b/src/openforms/registrations/contrib/objects_api/tests/test_backend_v1.py @@ -6,6 +6,8 @@ from django.test import TestCase, override_settings, tag from django.utils import timezone +from privates.test import temp_private_root + from openforms.authentication.service import AuthAttribute from openforms.contrib.objects_api.clients import get_documents_client from openforms.contrib.objects_api.tests.factories import ObjectsAPIGroupConfigFactory @@ -32,6 +34,7 @@ FIXED_SUBMISSION_UUID = UUID(hex="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") +@temp_private_root() class ObjectsAPIBackendV1Tests(OFVCRMixin, TestCase): maxDiff = None @@ -137,6 +140,7 @@ def test_submission_with_objects_api_backend_override_defaults(self): }, language_code="en", uuid=FIXED_SUBMISSION_UUID, + with_report=True, ) submission_step = submission.steps[0] assert submission_step.form_step @@ -237,6 +241,7 @@ def test_submission_with_objects_api_backend_override_defaults_upload_csv_defaul }, ], submitted_data={"voornaam": "Foo"}, + with_report=True, ) objects_form_options = { "version": 1, @@ -291,6 +296,7 @@ def test_submission_with_objects_api_backend_override_defaults_do_not_upload_csv }, ], submitted_data={"voornaam": "Foo"}, + with_report=True, ) plugin = ObjectsAPIRegistration(PLUGIN_IDENTIFIER) @@ -371,6 +377,7 @@ def test_submission_with_objects_api_backend_override_content_json(self): submitted_data={"voornaam": "Foo"}, language_code="en", uuid=FIXED_SUBMISSION_UUID, + with_report=True, ) submission_step = submission.steps[0] assert submission_step.form_step @@ -431,6 +438,7 @@ def test_submission_with_objects_api_backend_use_config_defaults(self): submitted_data={"voornaam": "Foo"}, language_code="en", uuid=FIXED_SUBMISSION_UUID, + with_report=True, ) submission_step = submission.steps[0] assert submission_step.form_step @@ -506,6 +514,7 @@ def test_submission_with_objects_api_backend_attachments(self): submitted_data={}, language_code="en", completed=True, + with_report=True, ) submission_step = submission.steps[0] # Set up two attachments to upload to the documents API @@ -613,6 +622,7 @@ def test_submission_with_objects_api_backend_attachments_specific_iotypen(self): ], language_code="en", completed=True, + with_report=True, ) submission_step = submission.steps[0] file_attachment_1 = SubmissionFileAttachmentFactory.create( @@ -716,6 +726,7 @@ def test_submission_with_objects_api_backend_attachments_component_overwrites(se }, language_code="en", completed=True, + with_report=True, ) submission_step = submission.steps[0] SubmissionFileAttachmentFactory.create( @@ -800,6 +811,7 @@ def test_submission_with_objects_api_backend_attachments_component_inside_fields }, language_code="en", completed=True, + with_report=True, ) submission_step = submission.steps[0] SubmissionFileAttachmentFactory.create( @@ -860,6 +872,7 @@ def test_submission_with_objects_api_escapes_html(self): ], submitted_data={"voornaam": ""}, language_code="en", + with_report=True, ) submission_step = submission.steps[0] @@ -909,6 +922,7 @@ def test_submission_with_payment(self): language_code="en", form__payment_backend="demo", form__product__price=10, + with_report=True, ) SubmissionPaymentFactory.for_submission( submission=submission, @@ -1048,6 +1062,7 @@ def test_object_ownership_not_validated_if_new_object(self): }, submitted_data={"textfield": "test"}, initial_data_reference="some ref", + with_report=True, ) try: @@ -1089,6 +1104,7 @@ def test_date_related_objects_as_separate_variables_in_template(self): }, language_code="en", uuid=FIXED_SUBMISSION_UUID, + with_report=True, ) objects_form_options: RegistrationOptionsV1 = { diff --git a/src/openforms/registrations/contrib/objects_api/tests/test_backend_v2.py b/src/openforms/registrations/contrib/objects_api/tests/test_backend_v2.py index d93983af22..64cfd7fbc7 100644 --- a/src/openforms/registrations/contrib/objects_api/tests/test_backend_v2.py +++ b/src/openforms/registrations/contrib/objects_api/tests/test_backend_v2.py @@ -7,6 +7,7 @@ from django.utils import timezone from freezegun import freeze_time +from privates.test import temp_private_root from zgw_consumers.constants import AuthTypes from zgw_consumers.test.factories import ServiceFactory @@ -47,6 +48,7 @@ @freeze_time("2024-03-19T13:40:34.222258+00:00") +@temp_private_root() class ObjectsAPIBackendV2Tests(OFVCRMixin, TestCase): """This test case requires the Objects & Objecttypes API and Open Zaak to be running. @@ -93,6 +95,7 @@ def test_submission_with_objects_api_v2(self): "coordinates": [4.893164274470299, 52.36673378967122], }, }, + with_report=True, ) v2_options: RegistrationOptionsV2 = { diff --git a/src/openforms/registrations/contrib/objects_api/tests/test_template.py b/src/openforms/registrations/contrib/objects_api/tests/test_template.py index 38fe88a5d3..440a285371 100644 --- a/src/openforms/registrations/contrib/objects_api/tests/test_template.py +++ b/src/openforms/registrations/contrib/objects_api/tests/test_template.py @@ -6,6 +6,7 @@ from django.test import TestCase, override_settings, tag from freezegun import freeze_time +from privates.test import temp_private_root from openforms.contrib.objects_api.tests.factories import ObjectsAPIGroupConfigFactory from openforms.formio.constants import DataSrcOptions @@ -21,6 +22,7 @@ from ..typing import RegistrationOptionsV1 +@temp_private_root() class JSONTemplatingTests(OFVCRMixin, TestCase): maxDiff = None @@ -359,6 +361,7 @@ def test_submission_with_objects_api_content_json_not_valid_json(self): plugin.register_submission(submission, options) +@temp_private_root() class JSONTemplatingRegressionTests(OFVCRMixin, SubmissionsMixin, TestCase): @classmethod def setUpTestData(cls): diff --git a/src/openforms/registrations/contrib/stuf_zds/tests/test_backend.py b/src/openforms/registrations/contrib/stuf_zds/tests/test_backend.py index 2a168a2449..40559a8549 100644 --- a/src/openforms/registrations/contrib/stuf_zds/tests/test_backend.py +++ b/src/openforms/registrations/contrib/stuf_zds/tests/test_backend.py @@ -191,14 +191,15 @@ class StufZDSPluginTests(StUFZDSTestBase): test the plugin function """ - def setUp(self): - super().setUp() + @classmethod + def setUpTestData(cls): + super().setUpTestData() - self.service = StufServiceFactory.create() + cls.service = StufServiceFactory.create() config = StufZDSConfig.get_solo() - config.service = self.service + config.service = cls.service config.save() - self.addCleanup(StufZDSConfig.clear_cache) + cls.addClassCleanup(StufZDSConfig.clear_cache) @patch("celery.app.task.Task.request") def test_plugin(self, m, mock_task): @@ -265,6 +266,7 @@ def test_plugin(self, m, mock_task): language_code="en", cosigned=True, co_sign_data__value="123456782", + with_report=True, ) attachment = SubmissionFileAttachmentFactory.create( @@ -543,6 +545,7 @@ def test_plugin_natuurlijk_persoon_initiator(self, m, mock_task): form__name="my-form", form__product__price=Decimal("0"), form__payment_backend="demo", + with_report=True, ) attachment = SubmissionFileAttachmentFactory.create( @@ -783,6 +786,7 @@ def test_plugin_natuurlijk_persoon_without_auth(self, m, mock_task): "voorletters": "J.W.", "geslachtsaanduiding": "mannelijk", }, + with_report=True, ) attachment = SubmissionFileAttachmentFactory.create( @@ -905,6 +909,7 @@ def test_plugin_nietNatuurlijkPersoon_without_auth(self, m, mock_task): "postcode": "2022XY", "straatnaam": "foo bar", }, + with_report=True, ) attachment = SubmissionFileAttachmentFactory.create( @@ -1043,6 +1048,7 @@ def test_plugin_vestiging_initiator(self, m, mock_task): form__name="my-form", form__product__price=Decimal("0"), form__payment_backend="demo", + with_report=True, ) attachment = SubmissionFileAttachmentFactory.create( @@ -1229,6 +1235,7 @@ def test_plugin_vestiging_initiator_kvk_only(self, m, mock_task): form__name="my-form", form__product__price=Decimal("0"), form__payment_backend="demo", + with_report=True, ) attachment = SubmissionFileAttachmentFactory.create( @@ -1419,6 +1426,7 @@ def test_plugin_vestiging_initiator_kvk_and_vestigingsnummer(self, m, mock_task) form__name="my-form", form__product__price=Decimal("0"), form__payment_backend="demo", + with_report=True, ) attachment = SubmissionFileAttachmentFactory.create( @@ -1580,6 +1588,7 @@ def test_plugin_vestiging_initiator_kvk_and_vestigingsnummer_through_auth(self, form__name="my-form", form__product__price=Decimal("0"), form__payment_backend="demo", + with_report=True, ) SubmissionFileAttachmentFactory.create( @@ -1720,6 +1729,7 @@ def test_plugin_medewerker(self, m, mock_task): form__name="my-form", form__product__price=Decimal("0"), form__payment_backend="demo", + with_report=True, ) RegistratorInfoFactory.create(submission=submission, value="123456782") config = StufZDSConfig.get_solo() @@ -1829,6 +1839,7 @@ def test_plugin_medewerker_without_auth(self, m, mock_task): submitted_data={ "medewerker_nummer": "007", }, + with_report=True, ) attachment = SubmissionFileAttachmentFactory.create( @@ -1936,6 +1947,7 @@ def test_plugin_payment(self, m, mock_task): "voornaam": "Foo", "achternaam": "Bar", }, + with_report=True, ) self.assertTrue(submission.payment_required) @@ -2056,6 +2068,7 @@ def test_retried_registration_with_internal_reference(self, m, mock_task): pre_registration_completed=False, registration_result={"temporary_internal_reference": "OF-1234"}, components_list=[{"key": "dummy"}], + with_report=True, ) m.post( @@ -2168,6 +2181,7 @@ def test_plugin_optional_fields(self, m, mock_task): }, "extra": "BuzzBazz", }, + with_report=True, ) SubmissionFileAttachmentFactory.create( submission_step=submission.steps[0], @@ -2315,6 +2329,7 @@ def test_plugin_map_with_pointer(self, m, mock_task): "coordinates": [4.893164274470299, 52.36673378967122], }, }, + with_report=True, ) SubmissionFileAttachmentFactory.create( submission_step=submission.steps[0], @@ -2398,6 +2413,7 @@ def test_plugin_map_with_polygon(self, m, mock_task): ], }, }, + with_report=True, ) SubmissionFileAttachmentFactory.create( submission_step=submission.steps[0], @@ -2482,6 +2498,7 @@ def test_plugin_map_with_line_string(self, m, mock_task): ], }, }, + with_report=True, ) SubmissionFileAttachmentFactory.create( submission_step=submission.steps[0], @@ -2595,6 +2612,7 @@ def test_plugin_optional_fields_missing_status_description(self, m, mock_task): }, "extra": "BuzzBazz", }, + with_report=True, ) SubmissionFileAttachmentFactory.create( submission_step=submission.steps[0], @@ -2780,6 +2798,7 @@ def test_plugin_optional_fields_missing_status_code(self, m, mock_task): }, "extra": "BuzzBazz", }, + with_report=True, ) SubmissionFileAttachmentFactory.create( submission_step=submission.steps[0], @@ -3049,6 +3068,7 @@ def test_user_defined_variables(self, m, mock_task): }, language_code="en", completed=True, + with_report=True, ) SubmissionValueVariableFactory.create( @@ -3155,6 +3175,7 @@ def test_user_defined_and_static_variables_have_stuf_name_in_extra_elementen( }, language_code="en", completed=True, + with_report=True, ) # can't pass this as part of `SubmissionFactory.from_components` submission.price = Decimal("40.00") @@ -3239,6 +3260,7 @@ def test_plugin_with_extra_unmapped_number_data(self, m, mock_task): registration_result={"intermediate": {"zaaknummer": "foo-zaak"}}, submitted_data={"extra_number": 2023}, language_code="en", + with_report=True, ) m.post( @@ -3295,6 +3317,7 @@ def test_plugin_with_extra_unmapped_number_data(self, m, mock_task): ) +@temp_private_root() class XMLSanitizerVCRTests(OFVCRMixin, StUFAssertionsMixin, HypothesisTestCase): VCR_TEST_FILES = TESTS_DIR / "files" @@ -3347,6 +3370,7 @@ def test_xml_generation_with_various_texts(self, text): "achternaam": text, }, language_code="en", + with_report=True, ) plugin = StufZDSRegistration("stuf") @@ -3358,6 +3382,7 @@ def test_xml_generation_with_various_texts(self, text): self.assertSoapXMLCommon(xml_doc) +@temp_private_root() class StufZDSPluginVCRTests(OFVCRMixin, StUFZDSTestBase): VCR_TEST_FILES = TESTS_DIR / "files" @@ -3410,6 +3435,7 @@ def test_date_related_values_in_extra_elementen(self): "datetime": "2025-11-18T14:21:00+01:00", }, language_code="en", + with_report=True, ) plugin = StufZDSRegistration("stuf") @@ -3465,6 +3491,7 @@ def test_key_with_period_used_as_registration_attribute(self): "extra": "Extra tekst", }, language_code="en", + with_report=True, ) plugin = StufZDSRegistration("stuf") @@ -3511,6 +3538,7 @@ def test_illegal_characters_in_xml_are_removed(self): "achternaam": "bad" + chr(1) + "value", }, language_code="en", + with_report=True, ) plugin = StufZDSRegistration("stuf") @@ -3555,6 +3583,7 @@ def test_extra_mapped_extra_elementen_are_picked_up(self): language_code="en", completed=True, completed_on=datetime(2026, 4, 30, 12, 0, 0, tzinfo=UTC), + with_report=True, ) plugin = StufZDSRegistration("stuf") @@ -3775,7 +3804,7 @@ def test_payment_status_is_correct_when_no_payment_required(self, m): ) -@temp_private_root() +@temp_private_root(reset_storage=False) class StufZDSPluginPaymentVCRTests(OFVCRMixin, StUFZDSTestBase): VCR_TEST_FILES = TESTS_DIR / "files" @@ -3814,6 +3843,7 @@ def setUpTestData(cls): language_code="en", public_registration_reference="abc123", registration_result={"zaak": "1234"}, + with_report=True, ) # can't pass this as part of `SubmissionFactory.from_components` cls.submission.price = Decimal("40.00") @@ -3980,6 +4010,7 @@ def test_emit_order_ids_as_csv_list_instead_of_using_suffixes(self): ) +@temp_private_root(reset_storage=False) class StufZDSPluginPartnersComponentVCRTests(OFVCRMixin, StUFZDSTestBase): VCR_TEST_FILES = TESTS_DIR / "files" @@ -4044,6 +4075,7 @@ def test_create_zaak_with_partners_as_betrokkene(self): language_code="en", public_registration_reference="abc123", registration_result={"zaak": "1234"}, + with_report=True, ) FormVariableFactory.create( key="partners_immutable", @@ -4129,6 +4161,7 @@ def test_create_zaak_with_partners_as_extraElementen(self): language_code="en", public_registration_reference="abc890", registration_result={"zaak": "890"}, + with_report=True, ) FormVariableFactory.create( key="partners_immutable", @@ -4189,6 +4222,7 @@ def test_create_zaak_with_no_partners_retrieved(self): language_code="en", public_registration_reference="abc999", registration_result={"zaak": "9990"}, + with_report=True, ) FormVariableFactory.create( key="partners_immutable", @@ -4236,6 +4270,7 @@ def test_create_zaak_with_hidden_partners(self): language_code="en", public_registration_reference="abc999", registration_result={"zaak": "9990"}, + with_report=True, ) FormVariableFactory.create( key="partners_immutable", @@ -4281,6 +4316,7 @@ def test_only_relevant_variables_are_taken_into_account(self): language_code="en", public_registration_reference="abc890", registration_result={"zaak": "890"}, + with_report=True, ) FormVariableFactory.create( key="irrelevant_variable", @@ -4361,6 +4397,7 @@ def test_partners_registration_with_date_of_birth_as_str(self): ] }, completed=True, + with_report=True, ) SubmissionValueVariableFactory.create( @@ -4424,6 +4461,7 @@ def test_registration_as_extra_elementen_when_partners_component_is_hidden(self) submitted_data={"textfield": "foo"}, language_code="nl", completed=True, + with_report=True, ) SubmissionValueVariableFactory.create( @@ -4466,6 +4504,7 @@ def test_registration_as_extra_elementen_when_partners_component_is_hidden(self) ) +@temp_private_root(reset_storage=False) class StufZDSPluginChildrenComponentVCRTests(OFVCRMixin, StUFZDSTestBase): VCR_TEST_FILES = TESTS_DIR / "files" @@ -4540,6 +4579,7 @@ def test_create_zaak_with_one_form_step_and_children_as_betrokkene(self): language_code="en", public_registration_reference="abc123", registration_result={"zaak": "1234"}, + with_report=True, ) FormVariableFactory.create( key="children_immutable", @@ -4640,6 +4680,7 @@ def test_create_zaak_with_one_form_step_and_enabled_children_selection( language_code="en", public_registration_reference="abc123", registration_result={"zaak": "1234"}, + with_report=True, ) FormVariableFactory.create( key="children_immutable", @@ -4776,6 +4817,7 @@ def test_create_zaak_with_one_form_step_enabled_selection_and_manually_added_chi }, ] }, + with_report=True, ) FormVariableFactory.create( key="children_immutable", @@ -4888,6 +4930,7 @@ def test_create_zaak_with_one_form_step_disabled_selection_and_manually_added_ch }, ] }, + with_report=True, ) FormVariableFactory.create( key="children_immutable", @@ -4976,6 +5019,7 @@ def test_create_zaak_with_one_form_step_and_children_as_extraelementen(self): language_code="en", public_registration_reference="abc123", registration_result={"zaak": "1234"}, + with_report=True, ) FormVariableFactory.create( key="children_immutable", @@ -5772,6 +5816,7 @@ def test_registration_as_extra_elementen_when_children_component_is_hidden(self) public_registration_reference="abc123", registration_result={"zaak": "1234"}, submitted_data={"textfield": "foo"}, + with_report=True, ) SubmissionValueVariableFactory.create( @@ -5820,6 +5865,7 @@ def test_registration_as_extra_elementen_when_children_component_is_hidden(self) ) +@temp_private_root(reset_storage=False) class StufZDSConfirmationEmailVCRTests(OFVCRMixin, StUFZDSTestBase): VCR_TEST_FILES = TESTS_DIR / "files" @@ -5854,6 +5900,7 @@ def setUpTestData(cls): public_registration_reference="abc123", registration_result={"zaak": "bar"}, confirmation_email_sent=True, + with_report=True, ) @patch( diff --git a/src/openforms/registrations/contrib/stuf_zds/tests/test_backend_extra_elements.py b/src/openforms/registrations/contrib/stuf_zds/tests/test_backend_extra_elements.py index d1cf14fe55..3df661a887 100644 --- a/src/openforms/registrations/contrib/stuf_zds/tests/test_backend_extra_elements.py +++ b/src/openforms/registrations/contrib/stuf_zds/tests/test_backend_extra_elements.py @@ -81,6 +81,7 @@ def test_register_with_variables_mapping_initiator(self): language_code="en", public_registration_reference="OF-1234", registration_result={"zaak": "1234"}, + with_report=True, ) # can't pass this as part of `SubmissionFactory.from_components` submission.price = Decimal("40.00") diff --git a/src/openforms/registrations/contrib/stuf_zds/tests/test_failure_modes.py b/src/openforms/registrations/contrib/stuf_zds/tests/test_failure_modes.py index 01e4fa1bee..2542122b9e 100644 --- a/src/openforms/registrations/contrib/stuf_zds/tests/test_failure_modes.py +++ b/src/openforms/registrations/contrib/stuf_zds/tests/test_failure_modes.py @@ -22,7 +22,7 @@ @tag("gh-1183") @freeze_time("2020-12-22") -@temp_private_root() +@temp_private_root(reset_storage=False) class PartialRegistrationFailureTests(StUFZDSTestBase): """ Test that partial results are stored and don't cause excessive registration calls. @@ -71,6 +71,7 @@ def setUpTestData(cls): "voornaam": "Foo", "achternaam": "Bar", }, + with_report=True, ) FormRegistrationBackendFactory.create( form=cls.submission.form, diff --git a/src/openforms/registrations/contrib/zgw_apis/tests/test_backend.py b/src/openforms/registrations/contrib/zgw_apis/tests/test_backend.py index f90bc9f705..066180e9a7 100644 --- a/src/openforms/registrations/contrib/zgw_apis/tests/test_backend.py +++ b/src/openforms/registrations/contrib/zgw_apis/tests/test_backend.py @@ -232,6 +232,7 @@ def test_submission_with_registrator(self): completed=True, # Pin to a known case & document type version completed_on=datetime(2024, 6, 9, 15, 30, 0).replace(tzinfo=UTC), + with_report=True, ) RegistratorInfoFactory.create(submission=submission, value="123456782") options: RegistrationOptions = { @@ -353,6 +354,7 @@ def test_create_zaak_with_natuurlijk_persoon_initiator_and_legacy_config(self): form__payment_backend="demo", language_code="en", completed=True, + with_report=True, ) SubmissionFileAttachmentFactory.create( submission_step=submission.steps[0], @@ -547,6 +549,7 @@ def test_create_zaak_with_vestiging_and_kvk_initiator_and_legacy_config(self): }, kvk="12345678", completed=True, + with_report=True, ) catalogi_root = self.zgw_group.ztc_service.api_root options: RegistrationOptions = { @@ -630,6 +633,7 @@ def test_create_zaak_with_vestiging_and_kvk_initiator_and_legacy_config_through_ kvk="12345678", branch_number="000038509490", completed=True, + with_report=True, ) catalogi_root = self.zgw_group.ztc_service.api_root options: RegistrationOptions = { @@ -735,6 +739,7 @@ def test_create_zaak_with_kvk_initiator_only_and_legacy_config(self): }, kvk="12345678", completed=True, + with_report=True, ) catalogi_root = self.zgw_group.ztc_service.api_root @@ -806,6 +811,7 @@ def test_create_zaak_with_case_identification_reference(self): completed=True, # Pin to a known case type version completed_on=datetime(2024, 9, 9, 15, 30, 0).replace(tzinfo=UTC), + with_report=True, ) RegistratorInfoFactory.create(submission=submission, value="employee-123") options: RegistrationOptions = { @@ -897,6 +903,7 @@ def test_create_zaak_with_zaaktype_where_initiator_roltype_is_missing(self): completed=True, # Pin to a known case & document type version completed_on=datetime(2025, 6, 9, 15, 30, 0).replace(tzinfo=UTC), + with_report=True, ) options: RegistrationOptions = { "zgw_api_group": self.zgw_group, @@ -940,6 +947,7 @@ def test_create_zaak_with_case_identification_reference_and_product(self): completed=True, # Pin to a known case type version completed_on=datetime(2024, 11, 1, 15, 30, 0).replace(tzinfo=UTC), + with_report=True, ) options: RegistrationOptions = { "zgw_api_group": self.zgw_group, @@ -1002,6 +1010,7 @@ def test_allow_registration_with_unpublished_case_types(self): completed=True, # Pin to a known case type version completed_on=datetime(2024, 9, 9, 15, 30, 0).replace(tzinfo=UTC), + with_report=True, ) options: RegistrationOptions = { "zgw_api_group": zgw_group, @@ -1051,6 +1060,7 @@ def test_create_document_with_document_type_description_reference(self): completed=True, # Pin to a known case & document type version completed_on=datetime(2024, 6, 9, 15, 30, 0).replace(tzinfo=UTC), + with_report=True, ) SubmissionFileAttachmentFactory.create(submission_step=submission.steps[0]) options: RegistrationOptions = { @@ -1169,6 +1179,7 @@ def test_submission_with_multiple_eigenschappen_creation(self): completed=True, # Pin to a known case type version (2024-10-31) completed_on=datetime(2024, 11, 9, 15, 30, 0, tzinfo=UTC), + with_report=True, ) options: RegistrationOptions = { "zgw_api_group": self.zgw_group, @@ -1259,6 +1270,7 @@ def test_submission_with_nested_component_columns_and_eigenschap(self): completed=True, # Pin to a known case type version (2024-10-31) completed_on=datetime(2024, 11, 9, 15, 30, 0, tzinfo=UTC), + with_report=True, ) options: RegistrationOptions = { "zgw_api_group": self.zgw_group, @@ -1319,6 +1331,7 @@ def test_register_and_update_paid_product(self): completed=True, # Pin to a known case & document type version completed_on=datetime(2024, 6, 9, 15, 30, 0).replace(tzinfo=UTC), + with_report=True, ) assert submission.payment_required options: RegistrationOptions = { @@ -1407,6 +1420,7 @@ def test_file_attachments_respect_field_specific_overrides(self): completed=True, # Pin to a known case & document type version completed_on=datetime(2024, 11, 9, 15, 30, 0).replace(tzinfo=UTC), + with_report=True, ) submission_step = SubmissionStep.objects.get() options: RegistrationOptions = { @@ -1577,6 +1591,7 @@ def test_submission_with_zgw_and_objects_api_backends(self): completed=True, # Pin to a known case & document type version completed_on=datetime(2024, 11, 9, 15, 30, 0).replace(tzinfo=UTC), + with_report=True, ) options: RegistrationOptions = { "zgw_api_group": self.zgw_group, @@ -1768,6 +1783,7 @@ def test_confirmation_emails_are_attached_when_updating_registration( completed=True, # Pin to a known case & document type version completed_on=datetime(2024, 6, 9, 15, 30, 0).replace(tzinfo=UTC), + with_report=True, confirmation_email_sent=True, ) options: RegistrationOptions = { @@ -1843,6 +1859,7 @@ def test_confirmation_email_is_only_attached_once(self, mock_get_last_email): completed=True, # Pin to a known case & document type version completed_on=datetime(2024, 6, 9, 15, 30, 0).replace(tzinfo=UTC), + with_report=True, confirmation_email_sent=True, ) options: RegistrationOptions = { @@ -1902,6 +1919,7 @@ def test_updating_registration_skips_when_confirmation_email_was_not_sent(self): completed=True, # Pin to a known case & document type version completed_on=datetime(2024, 6, 9, 15, 30, 0).replace(tzinfo=UTC), + with_report=True, confirmation_email_sent=False, ) options: RegistrationOptions = { @@ -1947,6 +1965,7 @@ def test_updating_registration_raises_when_confirmation_email_was_not_found( completed=True, # Pin to a known case & document type version completed_on=datetime(2024, 6, 9, 15, 30, 0).replace(tzinfo=UTC), + with_report=True, confirmation_email_sent=True, ) options: RegistrationOptions = { @@ -2008,6 +2027,7 @@ def test_submission_with_partners_component(self, m, n): auth_info__value="000009921", auth_info__attribute=AuthAttribute.bsn, completed_on=datetime(2024, 11, 9, 15, 30, 0, tzinfo=UTC), + with_report=True, ) FormVariableFactory.create( key="partners_immutable", @@ -2130,6 +2150,7 @@ def test_submission_with_partners_component_and_manually_added_data(self): }, ] }, + with_report=True, ) options: RegistrationOptions = { @@ -2231,6 +2252,7 @@ def test_submission_with_children_component_and_selection_disabled(self, m, n): auth_info__value="999970094", auth_info__attribute=AuthAttribute.bsn, completed_on=datetime(2024, 11, 9, 15, 30, 0, tzinfo=UTC), + with_report=True, ) FormVariableFactory.create( key="children_immutable", @@ -2376,6 +2398,7 @@ def test_submission_with_children_component_and_selection_enabled(self, m, n): auth_info__value="999970094", auth_info__attribute=AuthAttribute.bsn, completed_on=datetime(2024, 11, 9, 15, 30, 0, tzinfo=UTC), + with_report=True, ) FormVariableFactory.create( key="children_immutable", @@ -2523,6 +2546,7 @@ def test_submission_with_manually_added_children_and_selection_enabled(self): }, ] }, + with_report=True, ) FormVariableFactory.create( key="children_immutable", @@ -2645,6 +2669,7 @@ def test_submission_with_manually_added_children_and_selection_disabled(self): }, ] }, + with_report=True, ) FormVariableFactory.create( key="children_immutable", @@ -2772,6 +2797,7 @@ def test_submission_with_children_component_and_manually_added_data(self): }, ] }, + with_report=True, ) catalogi_root = self.zgw_group.ztc_service.api_root @@ -2849,6 +2875,7 @@ def test_documents_use_public_form_name(self): completed=True, # Pin to a known case & document type version completed_on=datetime(2024, 11, 9, 15, 30, 0).replace(tzinfo=UTC), + with_report=True, ) options: RegistrationOptions = { "zgw_api_group": self.zgw_group, @@ -2900,6 +2927,7 @@ def test_create_zaak_with_templated_description_and_explanation(self): completed=True, # Pin to a known case type version completed_on=datetime(2024, 11, 1, 15, 30, 0).replace(tzinfo=UTC), + with_report=True, ) options: RegistrationOptions = { "zgw_api_group": self.zgw_group, @@ -2955,6 +2983,7 @@ def test_create_zaak_with_empty_description_and_explanation(self): completed=True, # Pin to a known case type version completed_on=datetime(2024, 11, 1, 15, 30, 0).replace(tzinfo=UTC), + with_report=True, ) options: RegistrationOptions = { "zgw_api_group": self.zgw_group, @@ -3016,6 +3045,7 @@ def test_can_upload_attachments_with_indirect_document_type_reference(self): completed=True, # Pin to a known case & document type version completed_on=datetime(2024, 11, 9, 15, 30, 0).replace(tzinfo=UTC), + with_report=True, ) attachment = SubmissionFileAttachmentFactory.create( submission_step=submission.steps[0], diff --git a/src/openforms/registrations/contrib/zgw_apis/tests/test_backend_partial_failure.py b/src/openforms/registrations/contrib/zgw_apis/tests/test_backend_partial_failure.py index 2a607e456d..8727c11138 100644 --- a/src/openforms/registrations/contrib/zgw_apis/tests/test_backend_partial_failure.py +++ b/src/openforms/registrations/contrib/zgw_apis/tests/test_backend_partial_failure.py @@ -33,7 +33,7 @@ def __call__(self, request: VCRRequest) -> VCRRequest | None: @tag("gh-1183") -@temp_private_root() +@temp_private_root(reset_storage=False) class PartialRegistrationFailureTests(OFVCRMixin, TestCase): """ Test that partial results are stored and don't cause excessive registration calls. @@ -102,6 +102,7 @@ def _reset_vcr_hook(): bsn="111222333", # Pin to a known case & document type version completed_on=datetime(2024, 11, 9, 15, 30, 0).replace(tzinfo=UTC), + with_report=True, ) def _get_vcr_kwargs(self, **kwargs): diff --git a/src/openforms/submissions/migrations/0012_alter_submissionfileattachment_content_and_more.py b/src/openforms/submissions/migrations/0012_alter_submissionfileattachment_content_and_more.py new file mode 100644 index 0000000000..1732b2353b --- /dev/null +++ b/src/openforms/submissions/migrations/0012_alter_submissionfileattachment_content_and_more.py @@ -0,0 +1,43 @@ +# Generated by Django 5.2.14 on 2026-05-15 20:47 + +from django.db import migrations + +import privates.fields + +import openforms.submissions.models.submission_files + + +class Migration(migrations.Migration): + dependencies = [ + ("submissions", "0011_merge_20260304_1610"), + ] + + operations = [ + migrations.AlterField( + model_name="submissionfileattachment", + name="content", + field=privates.fields.PrivateMediaFileField( + help_text="Content of the submission file attachment.", + upload_to=openforms.submissions.models.submission_files.submission_file_upload_to, + verbose_name="content", + ), + ), + migrations.AlterField( + model_name="submissionreport", + name="content", + field=privates.fields.PrivateMediaFileField( + help_text="Content of the submission report", + upload_to="submission-reports/%Y/%m/%d", + verbose_name="content", + ), + ), + migrations.AlterField( + model_name="temporaryfileupload", + name="content", + field=privates.fields.PrivateMediaFileField( + help_text="content of the file attachment.", + upload_to=openforms.submissions.models.submission_files.temporary_file_upload_to, + verbose_name="content", + ), + ), + ] diff --git a/src/openforms/submissions/tests/factories.py b/src/openforms/submissions/tests/factories.py index c3576a36c9..617f644346 100644 --- a/src/openforms/submissions/tests/factories.py +++ b/src/openforms/submissions/tests/factories.py @@ -220,7 +220,7 @@ def from_components( remember to generate from privates.test import temp_private_root """ - kwargs.setdefault("with_report", True) + kwargs.setdefault("with_report", False) bsn = kwargs.pop("bsn", None) kvk = kwargs.pop("kvk", None) diff --git a/src/openforms/submissions/tests/test_attachment_download_view.py b/src/openforms/submissions/tests/test_attachment_download_view.py index 29ba7a5aca..dc3887b6f6 100644 --- a/src/openforms/submissions/tests/test_attachment_download_view.py +++ b/src/openforms/submissions/tests/test_attachment_download_view.py @@ -15,7 +15,6 @@ from .factories import SubmissionFileAttachmentFactory -@override_settings(SENDFILE_BACKEND="django_sendfile.backends.nginx") @temp_private_root() class SubmissionAttachmentDownloadTest(WebTest): @classmethod @@ -97,6 +96,7 @@ def test_valid_preconditions_invalid_hash_403(self): self.assertEqual(response.status_code, 403) + @override_settings(SENDFILE_BACKEND="django_sendfile.backends.nginx") def test_nginx_sendfile_response(self): submission_file_attachment = SubmissionFileAttachmentFactory.create( submission_step__submission__completed=True, diff --git a/src/openforms/submissions/tests/test_models.py b/src/openforms/submissions/tests/test_models.py index be6f61f174..61bfb22502 100644 --- a/src/openforms/submissions/tests/test_models.py +++ b/src/openforms/submissions/tests/test_models.py @@ -1,4 +1,3 @@ -import os from unittest.mock import patch from django.core.exceptions import ValidationError @@ -193,7 +192,7 @@ def test_submission_delete_file_uploads_cascade_file_already_gone(self): attachment = SubmissionFileAttachmentFactory.create( submission_step__submission=submission ) - os.remove(attachment.content.path) + attachment.content.storage.delete(attachment.content.name) with self.subTest("test setup validation"): self.assertFalse(attachment.content.storage.exists(attachment.content.path)) @@ -202,7 +201,7 @@ def test_submission_delete_file_uploads_cascade_file_already_gone(self): with ( patch( - "django.core.files.storage.FileSystemStorage.delete", side_effect=exc + "django.core.files.storage.InMemoryStorage.delete", side_effect=exc ) as mock_delete, self.captureOnCommitCallbacks(execute=True), ): diff --git a/src/openforms/submissions/tests/test_on_cosign.py b/src/openforms/submissions/tests/test_on_cosign.py index 78e0483f1f..7a6f423cde 100644 --- a/src/openforms/submissions/tests/test_on_cosign.py +++ b/src/openforms/submissions/tests/test_on_cosign.py @@ -3,6 +3,8 @@ from django.core import mail from django.test import TestCase, override_settings +from privates.test import temp_private_root + from openforms.emails.tests.factories import ConfirmationEmailTemplateFactory from openforms.registrations.contrib.email.models import EmailConfig @@ -12,6 +14,7 @@ @override_settings(CELERY_TASK_ALWAYS_EAGER=True) +@temp_private_root() class OnCosignTests(TestCase): def test_submission_on_cosign(self): submission = SubmissionFactory.from_components( diff --git a/src/openforms/submissions/tests/test_submission_co_sign.py b/src/openforms/submissions/tests/test_submission_co_sign.py index 5eb2fb5fbc..38cd6a8286 100644 --- a/src/openforms/submissions/tests/test_submission_co_sign.py +++ b/src/openforms/submissions/tests/test_submission_co_sign.py @@ -4,6 +4,7 @@ from django.urls import resolve from furl import furl +from privates.test import temp_private_root from rest_framework import status from rest_framework.reverse import reverse from rest_framework.test import APITestCase @@ -82,6 +83,7 @@ def test_submission_co_sign_status_no_co_sign(self): ) +@temp_private_root() class SubmissionCosignEndpointTests(SubmissionsMixin, APITestCase): def test_submission_must_be_in_session(self): submission = SubmissionFactory.from_components( @@ -156,6 +158,7 @@ def test_cosign_happy_flow_calls_on_cosign_task(self): "cosign": "test@example.com", }, registration_success=True, + with_report=True, ) session = self.client.session diff --git a/src/openforms/submissions/tests/test_submission_completion.py b/src/openforms/submissions/tests/test_submission_completion.py index 3316cf1509..15dcc45292 100644 --- a/src/openforms/submissions/tests/test_submission_completion.py +++ b/src/openforms/submissions/tests/test_submission_completion.py @@ -1082,6 +1082,7 @@ def test_price_rules_specified_but_no_match(self): @override_settings(CELERY_TASK_ALWAYS_EAGER=True) +@temp_private_root() class SetRegistrationBackendTests(SubmissionsMixin, APITestCase): "Registration backend can be set with a form action" diff --git a/src/openforms/submissions/tests/test_submission_report.py b/src/openforms/submissions/tests/test_submission_report.py index 0819208787..a4fb3f5f3f 100644 --- a/src/openforms/submissions/tests/test_submission_report.py +++ b/src/openforms/submissions/tests/test_submission_report.py @@ -1,5 +1,4 @@ import json -import os from datetime import timedelta from unittest.mock import patch from uuid import UUID @@ -692,24 +691,26 @@ class DeleteReportTests(TestCase): def test_file_deletion(self): submission_report = SubmissionReportFactory.create() - file_path = submission_report.content.path + storage = submission_report.content.storage + name = submission_report.content.name - self.assertTrue(os.path.exists(file_path)) + self.assertTrue(storage.exists(name)) submission_report.delete() - self.assertFalse(os.path.exists(file_path)) + self.assertFalse(storage.exists(name)) def test_file_deleted_on_submission_deletion(self): submission_report = SubmissionReportFactory.create() - file_path = submission_report.content.path + storage = submission_report.content.storage + name = submission_report.content.name - self.assertTrue(os.path.exists(file_path)) + self.assertTrue(storage.exists(name)) submission_report.submission.delete() - self.assertFalse(os.path.exists(file_path)) + self.assertFalse(storage.exists(name)) SIGNATURE = ( diff --git a/src/openforms/submissions/tests/test_submission_status.py b/src/openforms/submissions/tests/test_submission_status.py index 668ce91b04..c2775f88d5 100644 --- a/src/openforms/submissions/tests/test_submission_status.py +++ b/src/openforms/submissions/tests/test_submission_status.py @@ -532,6 +532,7 @@ def test_succesful_processing_submission_with_cosign(self): public_registration_reference="OF-ABCDE", metadata__tasks_ids=["some-id"], metadata__trigger_event=PostSubmissionEvents.on_completion, + with_report=True, ) token = submission_status_token_generator.make_token(submission) check_status_url = reverse( diff --git a/src/openforms/submissions/tests/test_tasks_pdf.py b/src/openforms/submissions/tests/test_tasks_pdf.py index 96e5c539fe..82a4d64297 100644 --- a/src/openforms/submissions/tests/test_tasks_pdf.py +++ b/src/openforms/submissions/tests/test_tasks_pdf.py @@ -317,6 +317,7 @@ def test_timestamp_included(self): }, ], public_registration_reference="OF-12345", + with_report=True, ) html = submission.report.generate_submission_report_pdf() diff --git a/src/openforms/submissions/tests/test_temporary_uploads.py b/src/openforms/submissions/tests/test_temporary_uploads.py index d3f833936b..987cc835a6 100644 --- a/src/openforms/submissions/tests/test_temporary_uploads.py +++ b/src/openforms/submissions/tests/test_temporary_uploads.py @@ -1,4 +1,3 @@ -import os import uuid from datetime import timedelta @@ -119,8 +118,8 @@ def test_delete_view_requires_registered_uploads(self): def test_delete_view(self): upload = TemporaryFileUploadFactory.create() - path = upload.content.path - self.assertTrue(os.path.exists(path)) + name = upload.content.name + self.assertTrue(upload.content.storage.exists(name)) self._add_submission_to_session(upload.submission) url = reverse("api:submissions:temporary-file", kwargs={"uuid": upload.uuid}) @@ -135,7 +134,7 @@ def test_delete_view(self): self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) # expect the file and instance to be deleted - self.assertFalse(os.path.exists(path)) + self.assertFalse(upload.content.storage.exists(name)) with self.assertRaisesRegex(FileNotFoundError, r"No such file or directory:"): upload.content.read() @@ -150,14 +149,14 @@ def test_delete_view(self): def test_delete_instance_method(self): upload = TemporaryFileUploadFactory.create() - path = upload.content.path - self.assertTrue(os.path.exists(path)) + name = upload.content.path + self.assertTrue(upload.content.storage.exists(name)) with self.captureOnCommitCallbacks(execute=True): upload.delete() # expect the file and instance to be deleted - self.assertFalse(os.path.exists(path)) + self.assertFalse(upload.content.storage.exists(name)) with self.assertRaisesMessage( ValueError, "The 'content' attribute has no file associated with it." @@ -169,15 +168,14 @@ def test_delete_instance_method(self): def test_delete_queryset_method(self): uploads = TemporaryFileUploadFactory.create_batch(3) - paths = [u.content.path for u in uploads] - for path in paths: - self.assertTrue(os.path.exists(path)) + for upload in uploads: + self.assertTrue(upload.content.storage.exists(upload.content.name)) with self.captureOnCommitCallbacks(execute=True): TemporaryFileUpload.objects.all().delete() - for path in paths: - self.assertFalse(os.path.exists(path)) + for upload in uploads: + self.assertFalse(upload.content.storage.exists(upload.content.name)) def test_delete_temporary_file_attachement_deletes_the_saved_one_as_well(self): upload = TemporaryFileUploadFactory.create() diff --git a/src/openforms/translations/migrations/0003_alter_translationsmetadata_compiled_asset_and_more.py b/src/openforms/translations/migrations/0003_alter_translationsmetadata_compiled_asset_and_more.py new file mode 100644 index 0000000000..0b0c558483 --- /dev/null +++ b/src/openforms/translations/migrations/0003_alter_translationsmetadata_compiled_asset_and_more.py @@ -0,0 +1,34 @@ +# Generated by Django 5.2.14 on 2026-05-15 20:47 + +from django.db import migrations + +import privates.fields + + +class Migration(migrations.Migration): + dependencies = [ + ("of_translations", "0002_alter_translationsmetadata_language_code"), + ] + + operations = [ + migrations.AlterField( + model_name="translationsmetadata", + name="compiled_asset", + field=privates.fields.PrivateMediaFileField( + editable=False, + help_text="JSON file containing user's custom translations after it has been successfully compiled.", + upload_to="messages/compiled/%Y/%m/%d", + verbose_name="compiled translations JSON file", + ), + ), + migrations.AlterField( + model_name="translationsmetadata", + name="messages_file", + field=privates.fields.PrivateMediaFileField( + blank=True, + help_text="JSON file containing user's custom translations.", + upload_to="messages/uploaded/%Y/%m/%d", + verbose_name="messages JSON file", + ), + ), + ] diff --git a/src/openforms/translations/subprocesses.py b/src/openforms/translations/subprocesses.py index 7d1f3de403..030b0c3f63 100644 --- a/src/openforms/translations/subprocesses.py +++ b/src/openforms/translations/subprocesses.py @@ -1,8 +1,12 @@ +import contextlib import subprocess import tempfile +from collections.abc import Generator from pathlib import Path from django.conf import settings +from django.core.files.storage import FileSystemStorage +from django.db.models.fields.files import FieldFile # make sure the right path is used for the subporcess (different paths are used based on # the environment) @@ -10,43 +14,64 @@ FORMATJS_BIN = PROJECT_ROOT / "node_modules" / ".bin" / "formatjs" -def compile_messages_file(input_path: str) -> tuple[bool, str]: +@contextlib.contextmanager +def ensure_input_file_exists_on_disk(input_file: FieldFile) -> Generator[str]: + # ensure that the input file exists on disk, in case non-filesystem storage backends + # are used + if isinstance(input_file.storage, FileSystemStorage): + yield input_file.path + else: + with tempfile.NamedTemporaryFile(mode="wb", suffix=".json") as tmp_input_file: + # copy the contents to the temp file + input_file.seek(0) + for chunk in input_file.chunks(): + tmp_input_file.write(chunk) + tmp_input_file.flush() + yield tmp_input_file.name + + +def compile_messages_file(input_file: FieldFile) -> tuple[bool, str]: """ Compiles a messages JSON file using formatjs as a subprocess. + :param input_file: The model field containing the input file. + Returns: (True, compiled_json) on success (False, error_message) on failure """ - with tempfile.NamedTemporaryFile( - mode="r", - suffix=".json", - encoding="utf-8", - delete=False, - ) as output_tmp: - output_path = output_tmp.name - - try: - subprocess.run( - [ - str(FORMATJS_BIN), - "compile", - input_path, - "--ast", - "--out-file", - output_path, - ], - stderr=subprocess.PIPE, - check=True, - ) - with open(output_path, encoding="utf-8") as f: - compiled_json: str = f.read() - - return True, compiled_json - except subprocess.CalledProcessError as exc: - error_msg: str = exc.stderr.decode("utf-8").strip() - return False, error_msg - - finally: - # Clean up the temporary file - Path(output_path).unlink(missing_ok=True) + with ensure_input_file_exists_on_disk(input_file) as input_path: + with ( + tempfile.NamedTemporaryFile( + mode="r", + suffix=".json", + encoding="utf-8", + delete=False, + ) as output_tmp, + ): + output_path = output_tmp.name + + try: + subprocess.run( + [ + str(FORMATJS_BIN), + "compile", + input_path, + "--ast", + "--out-file", + output_path, + ], + stderr=subprocess.PIPE, + check=True, + ) + with open(output_path, encoding="utf-8") as f: + compiled_json: str = f.read() + + return True, compiled_json + except subprocess.CalledProcessError as exc: + error_msg: str = exc.stderr.decode("utf-8").strip() + return False, error_msg + + finally: + # Clean up the temporary file + Path(output_path).unlink(missing_ok=True) diff --git a/src/openforms/translations/tasks.py b/src/openforms/translations/tasks.py index 77f4d8c37d..9e916eb39d 100644 --- a/src/openforms/translations/tasks.py +++ b/src/openforms/translations/tasks.py @@ -59,7 +59,7 @@ def process_custom_translation_assets(translations_metadata_pk: str) -> None: # the file is a valid JSON file, begin the subprocess log.info("custom_translations_file_compiling_started") - success, result = compile_messages_file(instance.messages_file.path) + success, result = compile_messages_file(instance.messages_file) if success: log.info("custom_translations_file_compiling_succeeded") diff --git a/src/openforms/translations/tests/test_admin.py b/src/openforms/translations/tests/test_admin.py index 7a44c9605a..951fcf0bea 100644 --- a/src/openforms/translations/tests/test_admin.py +++ b/src/openforms/translations/tests/test_admin.py @@ -6,6 +6,7 @@ from django_webtest import TransactionWebTest, WebTest from furl import furl from maykin_2fa.test import disable_admin_mfa +from privates.test import temp_private_root from openforms.accounts.tests.factories import ( StaffUserFactory, @@ -53,6 +54,7 @@ def test_user_preference_overrides_browser_prefs(self): @disable_admin_mfa() +@temp_private_root() class AdminTranslationMetaDataTests(WebTest): def test_changelist_page_access(self): user = UserFactory.create(is_staff=False, is_superuser=False) @@ -213,6 +215,7 @@ def test_compiled_asset_download_link_with_no_asset(self): @disable_admin_mfa() +@temp_private_root() class AdminTranslationMetaDataTransactionTests(TransactionWebTest): def test_saving_model(self): super_user = SuperUserFactory.create() @@ -250,11 +253,11 @@ def test_saving_model(self): obj = TranslationsMetaData.objects.get() with ( - open(obj.messages_file.path) as messages_file_path, - open(obj.compiled_asset.path) as compiled_asset_path, + obj.messages_file.open("rb") as messages_file, + obj.compiled_asset.open("rb") as compiled_asset, ): - messages_file_data = json.load(messages_file_path) - compiled_asset_data = json.load(compiled_asset_path) + messages_file_data = json.load(messages_file) + compiled_asset_data = json.load(compiled_asset) self.assertEqual(messages_file_data, expected_messages_file_data) self.assertEqual(compiled_asset_data, expected_compiled_asset_data) diff --git a/src/openforms/translations/tests/test_subprocesses.py b/src/openforms/translations/tests/test_subprocesses.py index b1b0159efc..501e468cc6 100644 --- a/src/openforms/translations/tests/test_subprocesses.py +++ b/src/openforms/translations/tests/test_subprocesses.py @@ -1,99 +1,100 @@ import json -import tempfile -from pathlib import Path from django.core.files.base import ContentFile -from django.test import TestCase, override_settings +from django.test import TestCase + +from privates.test import temp_private_root from ..subprocesses import compile_messages_file from .factories import TranslationsMetaDataFactory +@temp_private_root() class CompileCustomTranslationFileTests(TestCase): + # Note that we can't really test for cleanup of temp files, because we don't know + # the location of the tempdir that will be created by `compile_messages_file` + def test_uploaded_messages_are_successfully_compiled(self): - tmpdir = tempfile.mkdtemp() - with override_settings(PRIVATE_MEDIA_ROOT=tmpdir, SENDFILE_ROOT=tmpdir): - translation_metadata = TranslationsMetaDataFactory.create() + translation_metadata = TranslationsMetaDataFactory.create() - result, compiled_json = compile_messages_file( - translation_metadata.messages_file.path - ) + result, compiled_json = compile_messages_file( + translation_metadata.messages_file + ) - assert compiled_json is not None + assert compiled_json is not None - self.assertTrue(result) - self.assertEqual( - json.loads(compiled_json), - { - "abc123": [ - { - "offset": 0, - "options": { - "one": {"value": [{"type": 0, "value": "1 item"}]}, - "other": { - "value": [ - {"type": 1, "value": "count"}, - {"type": 0, "value": " items"}, - ] - }, - }, - "pluralType": "cardinal", - "type": 6, - "value": "count", - } - ], - "skjd8uh": [ - { - "type": 0, - "value": "A modified translated text", - } - ], - }, - ) - - def test_invalid_uploaded_messages_fail_and_return_errors(self): - tmpdir = tempfile.mkdtemp() - with override_settings(PRIVATE_MEDIA_ROOT=tmpdir, SENDFILE_ROOT=tmpdir): - messages = { - "skjd8uh": [{"type": 0, "value": "A modified translated text"}], + self.assertTrue(result) + self.assertEqual( + json.loads(compiled_json), + { "abc123": [ { - "type": 6, + "offset": 0, "options": { - "one": [{"type": 0, "value": "1 item"}], - "other": [{"type": 0, "value": "{count} items"}], + "one": {"value": [{"type": 0, "value": "1 item"}]}, + "other": { + "value": [ + {"type": 1, "value": "count"}, + {"type": 0, "value": " items"}, + ] + }, }, + "pluralType": "cardinal", + "type": 6, + "value": "count", } ], - } - json_bytes = json.dumps(messages, ensure_ascii=False).encode("utf-8") - - translation_metadata = TranslationsMetaDataFactory.create( - messages_file=ContentFile(json_bytes, name="messages_test_en.json") - ) - - result, error_msg = compile_messages_file( - translation_metadata.messages_file.path - ) - - self.assertFalse(result) - self.assertIsInstance(error_msg, str) - - def test_temp_output_file_is_removed_after_subprocess_finished(self): - tmpdir = tempfile.mkdtemp() - with override_settings(PRIVATE_MEDIA_ROOT=tmpdir, SENDFILE_ROOT=tmpdir): - translation_metadata = TranslationsMetaDataFactory.create() - tmp_path = Path(translation_metadata.messages_file.path) - - self.assertTrue(tmp_path.exists()) - - result, compiled_json = compile_messages_file( - translation_metadata.messages_file.path - ) - remaining_files = list(Path(tmpdir).rglob("*")) - remaining_files = [f for f in remaining_files if f.is_file()] - - self.assertTrue(result) - # only the initial messages file should be present in the directory, the temp - # output file (compiled asset) should have been removed by now - self.assertEqual(remaining_files, [tmp_path]) + "skjd8uh": [ + { + "type": 0, + "value": "A modified translated text", + } + ], + }, + ) + + def test_invalid_uploaded_messages_fail_and_return_errors(self): + messages = { + "skjd8uh": [{"type": 0, "value": "A modified translated text"}], + "abc123": [ + { + "type": 6, + "options": { + "one": [{"type": 0, "value": "1 item"}], + "other": [{"type": 0, "value": "{count} items"}], + }, + } + ], + } + json_bytes = json.dumps(messages, ensure_ascii=False).encode("utf-8") + + translation_metadata = TranslationsMetaDataFactory.create( + messages_file=ContentFile(json_bytes, name="messages_test_en.json") + ) + + result, error_msg = compile_messages_file(translation_metadata.messages_file) + + self.assertFalse(result) + self.assertIsInstance(error_msg, str) + + +class RealFileSystemStorageTranslationCompilationTests(TestCase): + """ + Run tests that rely on the real filesystem storage being used. + """ + + def test_uploaded_messages_are_successfully_compiled(self): + translation_metadata = TranslationsMetaDataFactory.create() + + def delete_files(): + translation_metadata.messages_file.delete(save=False) + + self.addCleanup(delete_files) + + result, compiled_json = compile_messages_file( + translation_metadata.messages_file + ) + + assert compiled_json is not None + + self.assertTrue(result) diff --git a/src/openforms/translations/tests/test_tasks.py b/src/openforms/translations/tests/test_tasks.py index c7afa7f373..0df47090d0 100644 --- a/src/openforms/translations/tests/test_tasks.py +++ b/src/openforms/translations/tests/test_tasks.py @@ -5,6 +5,7 @@ from django.test import TestCase, override_settings from freezegun import freeze_time +from privates.test import temp_private_root from ..constants import StatusChoices from ..tasks import process_custom_translation_assets @@ -12,6 +13,7 @@ @override_settings(LANGUAGE_CODE="en") +@temp_private_root() class ProcessingCustomTranslationAssetTests(TestCase): @freeze_time("2026-01-27T18:00:00+01:00") def test_input_messages_file_successfully_processed(self): diff --git a/src/openforms/translations/tests/test_views.py b/src/openforms/translations/tests/test_views.py index 0adf6989e6..4da78d80c9 100644 --- a/src/openforms/translations/tests/test_views.py +++ b/src/openforms/translations/tests/test_views.py @@ -1,16 +1,17 @@ -from django.conf import settings from django.http import HttpResponse from django.test import override_settings from django.urls import reverse +from privates.test import temp_private_root from rest_framework import status from rest_framework.test import APITestCase from .factories import TranslationsMetaDataFactory -@override_settings(SENDFILE_BACKEND="django_sendfile.backends.nginx") +@temp_private_root() class CustomizedCompiledTranslationsTests(APITestCase): + @override_settings(SENDFILE_BACKEND="django_sendfile.backends.nginx") def test_view_returns_compiled_file_json_data(self): TranslationsMetaDataFactory.create(language_code="en", with_compiled_asset=True) @@ -23,10 +24,11 @@ def test_view_returns_compiled_file_json_data(self): # make sure nginx serves the file directly via the private media directory self.assertTrue( - response.headers["X-Accel-Redirect"].startswith(settings.PRIVATE_MEDIA_URL) + response.headers["X-Accel-Redirect"].startswith("/private-media/") ) self.assertIn("compiled_test_en", response.headers["X-Accel-Redirect"]) + @override_settings(SENDFILE_BACKEND="django_sendfile.backends.nginx") def test_returns_empty_object_when_no_compiled_asset_found(self): endpoint = reverse( "api:i18n:customized-translations", kwargs={"language_code": "nl"} diff --git a/src/soap/models.py b/src/soap/models.py index 11017faf99..a37a769dd3 100644 --- a/src/soap/models.py +++ b/src/soap/models.py @@ -1,3 +1,5 @@ +from functools import partial + from django.db import models from django.utils.translation import gettext_lazy as _ @@ -6,6 +8,7 @@ from zeep.wsse.username import UsernameToken from .constants import EndpointSecurity, SOAPVersion +from .utils import ensure_file_exists_on_disk class _Signature(Signature): @@ -90,18 +93,23 @@ def get_cert(self) -> None | str | tuple[str, str]: if not certificate: return None + get_pub_cert_path = partial( + ensure_file_exists_on_disk, certificate.public_certificate + ) + if certificate.public_certificate and certificate.private_key: - return (certificate.public_certificate.path, certificate.private_key.path) + privkey_path = ensure_file_exists_on_disk(certificate.private_key) + return (get_pub_cert_path(), privkey_path) if certificate.public_certificate: - return certificate.public_certificate.path + return get_pub_cert_path() return None def get_verify(self) -> bool | str: certificate = self.server_certificate if certificate: - return certificate.public_certificate.path + return ensure_file_exists_on_disk(certificate.public_certificate) return True def get_auth(self) -> tuple[str, str] | None: @@ -118,8 +126,8 @@ def get_wsse( self, ) -> Signature | UsernameToken | tuple[UsernameToken, Signature] | None: sig = lambda: _Signature( # noqa: E731 - self.client_certificate.private_key.path, - self.client_certificate.public_certificate.path, + ensure_file_exists_on_disk(self.client_certificate.private_key), + ensure_file_exists_on_disk(self.client_certificate.public_certificate), ) basic = lambda: UsernameToken(self.user, self.password) # noqa: E731 diff --git a/src/soap/tests/test_client.py b/src/soap/tests/test_client.py index da6ceaebeb..8b78692228 100644 --- a/src/soap/tests/test_client.py +++ b/src/soap/tests/test_client.py @@ -8,6 +8,7 @@ import requests_mock from ape_pie import InvalidURLError +from privates.test import temp_private_root from requests.exceptions import RequestException from simple_certmanager.test.factories import CertificateFactory from zeep.exceptions import XMLSyntaxError @@ -25,6 +26,7 @@ WSDL_URI = str(WSDL) +@temp_private_root(reset_storage=False) class ClientTransportTests(OFVCRMixin, TestCase): VCR_TEST_FILES = DATA_DIR @@ -52,6 +54,16 @@ def setUpTestData(cls): with_private_key=True, ) + def _reset_file_pointers(self): + super().setUp() + + self.client_cert_only.public_certificate.seek(0) + self.client_cert_and_privkey.public_certificate.seek(0) + self.client_cert_and_privkey.private_key.seek(0) + self.server_cert.public_certificate.seek(0) + self.server_cert_and_privkey.public_certificate.seek(0) + self.server_cert_and_privkey.private_key.seek(0) + def test_no_server_cert_specified(self): service = SoapServiceFactory.build(url=WSDL_URI) @@ -68,8 +80,11 @@ def test_server_cert_specified(self): client = build_client(service) + self._reset_file_pointers() + cert_path = client.transport.session.verify self.assertEqual( - client.transport.session.verify, self.server_cert.public_certificate.path + Path(cert_path).read_bytes(), + self.server_cert.public_certificate.read(), ) def test_no_client_cert_specified(self): @@ -86,8 +101,11 @@ def test_client_cert_only_public_cert_specified(self): client = build_client(service) + self._reset_file_pointers() + cert_path = client.transport.session.cert self.assertEqual( - client.transport.session.cert, self.client_cert_only.public_certificate.path + Path(cert_path).read_bytes(), + self.client_cert_only.public_certificate.read(), ) def test_client_cert_public_cert_and_privkey_specified(self): @@ -97,11 +115,16 @@ def test_client_cert_public_cert_and_privkey_specified(self): client = build_client(service) + self._reset_file_pointers() + cert_path, key_path = client.transport.session.cert self.assertEqual( - client.transport.session.cert, ( - self.client_cert_and_privkey.public_certificate.path, - self.client_cert_and_privkey.private_key.path, + Path(cert_path).read_bytes(), + Path(key_path).read_bytes(), + ), + ( + self.client_cert_and_privkey.public_certificate.read(), + self.client_cert_and_privkey.private_key.read(), ), ) diff --git a/src/soap/utils.py b/src/soap/utils.py new file mode 100644 index 0000000000..f96c9771b0 --- /dev/null +++ b/src/soap/utils.py @@ -0,0 +1,23 @@ +import tempfile + +from django.core.files.storage import FileSystemStorage, InMemoryStorage +from django.db.models.fields.files import FieldFile + + +def ensure_file_exists_on_disk(field: FieldFile) -> str: + # Copied from django-digid-eherkenning + match field.storage: + case FileSystemStorage(): # pragma: no cover + return field.path + case InMemoryStorage(): + # TODO: figure out a solution to get these files/directories cleaned up once + # tests complete. Maybe setting up a signal dispatch? + tmp_input_file = tempfile.NamedTemporaryFile(mode="wb", delete=False) + field.open("rb") + field.seek(0) + for chunk in field.chunks(): + tmp_input_file.write(chunk) + tmp_input_file.flush() + return tmp_input_file.name + case _: # pragma: no cover + raise NotImplementedError() diff --git a/src/stuf/stuf_zds/tests/test_client.py b/src/stuf/stuf_zds/tests/test_client.py index 99666ac430..215f24e816 100644 --- a/src/stuf/stuf_zds/tests/test_client.py +++ b/src/stuf/stuf_zds/tests/test_client.py @@ -1,9 +1,11 @@ +from pathlib import Path from unittest.mock import patch from django.template.loader import render_to_string from django.test import SimpleTestCase, TestCase, tag import requests_mock +from privates.test import temp_private_root from simple_certmanager.constants import CertificateTypes from simple_certmanager.test.factories import CertificateFactory @@ -18,6 +20,7 @@ @requests_mock.Mocker() +@temp_private_root(reset_storage=False) class StufZdsClientTest(TestCase): @classmethod def setUpTestData(cls): @@ -68,16 +71,21 @@ def test_mutual_tls(self, m): request_with_tls = m.last_request + self.server_certificate.public_certificate.seek(0) + self.client_certificate.public_certificate.seek(0) + self.client_certificate.private_key.seek(0) + + server_cert = self.server_certificate.public_certificate.read() + client_cert = self.client_certificate.public_certificate.read() + client_key = self.client_certificate.private_key.read() + self.assertEqual( - self.server_certificate.public_certificate.path, request_with_tls.verify - ) - self.assertEqual( - ( - self.client_certificate.public_certificate.path, - self.client_certificate.private_key.path, - ), - request_with_tls.cert, + Path(request_with_tls.verify).read_bytes(), + server_cert, ) + mtls_cert, mtls_key = request_with_tls.cert + self.assertEqual(Path(mtls_cert).read_bytes(), client_cert) + self.assertEqual(Path(mtls_key).read_bytes(), client_key) def test_mutual_tls_no_private_key(self, m): stuf_service = StufServiceFactory.create( @@ -99,13 +107,17 @@ def test_mutual_tls_no_private_key(self, m): history = m.request_history request_with_tls = history[-1] + self.server_certificate.public_certificate.seek(0) + self.client_certificate_only.public_certificate.seek(0) + + server_cert = self.server_certificate.public_certificate.read() + client_cert = self.client_certificate_only.public_certificate.read() + self.assertEqual( - self.server_certificate.public_certificate.path, request_with_tls.verify - ) - self.assertEqual( - self.client_certificate_only.public_certificate.path, - request_with_tls.cert, + Path(request_with_tls.verify).read_bytes(), + server_cert, ) + self.assertEqual(Path(request_with_tls.cert).read_bytes(), client_cert) def test_no_mutual_tls(self, m): stuf_service = StufServiceFactory.create() diff --git a/src/suwinet/tests/test_client.py b/src/suwinet/tests/test_client.py index 55b94a7187..19290d5b06 100644 --- a/src/suwinet/tests/test_client.py +++ b/src/suwinet/tests/test_client.py @@ -22,6 +22,7 @@ DATA_DIR = Path(__file__).parent / "data" +@temp_private_root() class SuwinetConfigTests(TestCase): def test_client_requires_a_service(self): config = SuwinetConfigFactory.build(service=None) @@ -73,7 +74,7 @@ def test_iterating_client_yields_configured_service_names(self): self.assertEqual(list(client), ["KadasterDossierGSD"]) -@temp_private_root() +@temp_private_root(reset_storage=False) class SuwinetTestCase(OFVCRMixin, TestCase): VCR_TEST_FILES = DATA_DIR