diff --git a/.github/workflows/data/conda/geolibs-pg17.yml b/.github/workflows/data/conda/geolibs-pg17.yml new file mode 100644 index 000000000000..468301b639fe --- /dev/null +++ b/.github/workflows/data/conda/geolibs-pg17.yml @@ -0,0 +1,11 @@ +name: geodjango +channels: + - conda-forge +dependencies: + - python=3.14 + - pip=26.* + - postgresql=17.* + - postgis=3.5.* + - libgdal=3.11.* + - geos=3.14.* + - proj=9.6.* diff --git a/.github/workflows/data/conda/geolibs-pg18.yml b/.github/workflows/data/conda/geolibs-pg18.yml new file mode 100644 index 000000000000..d0f65e3cde36 --- /dev/null +++ b/.github/workflows/data/conda/geolibs-pg18.yml @@ -0,0 +1,11 @@ +name: geodjango +channels: + - conda-forge +dependencies: + - python=3.14 + - pip=26.* + - postgresql=18.* + - postgis=3.6.* + - libgdal=3.13.* + - geos=3.14.* + - proj=9.8.* diff --git a/.github/workflows/postgis.yml b/.github/workflows/postgis.yml index ae5559616801..13e2af8552b2 100644 --- a/.github/workflows/postgis.yml +++ b/.github/workflows/postgis.yml @@ -17,54 +17,48 @@ jobs: postgis: if: contains(github.event.pull_request.labels.*.name, 'geodjango') runs-on: ubuntu-latest + defaults: + run: + # Use Conda shell in all steps. See https://github.com/marketplace/actions/setup-miniconda#use-a-default-shell + shell: bash -el {0} strategy: fail-fast: false matrix: - postgis-version: ["latest", "18-3.6-alpine", "17-master"] - name: PostGIS ${{ matrix.postgis-version }} - services: - postgres: - image: postgis/postgis:${{ matrix.postgis-version }} - env: - POSTGRES_DB: geodjango - POSTGRES_USER: user - POSTGRES_PASSWORD: postgres - ports: - - 5432:5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - timeout-minutes: 60 + include: + - conda-environment-file: ".github/workflows/data/conda/geolibs-pg17.yml" + pg_major: "17" + - conda-environment-file: ".github/workflows/data/conda/geolibs-pg18.yml" + pg_major: "18" + name: PostgreSQL ${{ matrix.pg_major }} steps: - name: Checkout uses: actions/checkout@v6 with: persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v6 + - name: Setup Miniforge + # Pinned to v4.0.1 + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 with: - python-version: '3.14' - cache: 'pip' - cache-dependency-path: 'tests/requirements/py3.txt' - - name: Update apt repo - run: sudo apt update + activate-environment: geodjango + environment-file: ${{ matrix.conda-environment-file }} + channel-priority: strict - name: Install libmemcached-dev for pylibmc run: sudo apt install -y libmemcached-dev - - name: Install geospatial dependencies - run: sudo apt install -y binutils libproj-dev gdal-bin - - name: Print PostGIS versions - env: - POSTGRES_DB: geodjango - POSTGRES_USER: user - POSTGRES_PASSWORD: postgres - run: | - PGPASSWORD=$POSTGRES_PASSWORD psql -U $POSTGRES_USER -d $POSTGRES_DB -h localhost -c "SELECT PostGIS_full_version();" - name: Install and upgrade packaging tools run: python -m pip install --upgrade pip wheel - run: python -m pip install -r tests/requirements/py3.txt -r tests/requirements/postgres.txt -e . - name: Create PostgreSQL settings file run: mv ./.github/workflows/data/test_postgis.py.tpl ./tests/test_postgis.py + - name: Initialize and start local PostgreSQL + env: + PGPASSWORD: postgres + run: | + initdb -D "$GITHUB_WORKSPACE/.tmp/pgdata" --username="user" --auth=scram-sha-256 --pwfile=<(echo "$PGPASSWORD") + pg_ctl -D "$GITHUB_WORKSPACE/.tmp/pgdata" -w start + psql -U user -d postgres -c "CREATE EXTENSION IF NOT EXISTS postgis;" + psql -U user -d postgres -c "SELECT PostGIS_full_version();" + - name: Print geospatial versions + run: | + python -c "from django.contrib.gis import gdal, geos; print(f'GDAL: {gdal.gdal_version()}'); print(f'GEOS: {geos.geos_version()}')" - name: Run PostGIS tests run: python -Wall tests/runtests.py -v2 --settings=test_postgis diff --git a/django/middleware/cache.py b/django/middleware/cache.py index 60c219064a1f..f4feab96eb19 100644 --- a/django/middleware/cache.py +++ b/django/middleware/cache.py @@ -56,10 +56,9 @@ learn_cache_key, patch_response_headers, patch_vary_headers, - split_header_value, ) from django.utils.deprecation import MiddlewareMixin -from django.utils.http import parse_http_date_safe +from django.utils.http import parse_http_date_safe, split_directive_names class UpdateCacheMiddleware(MiddlewareMixin): @@ -105,17 +104,12 @@ def process_response(self, request, response): return response # Don't cache responses when the Cache-Control header is set to - # private, no-cache, or no-store. - cache_control = response.get("Cache-Control", "").lower() - cache_control_parts = list(split_header_value(cache_control)) - if cache_control and any( - directive in cache_control_parts - for directive in ( - "private", - "no-cache", - "no-store", - ) - ): + # private, no-cache, or no-store. Qualified forms like + # `private="Set-Cookie"` reduce to their directive name and match too. + cache_control_parts = set( + split_directive_names(response.get("Cache-Control", "")) + ) + if cache_control_parts.intersection({"private", "no-cache", "no-store"}): return response # Don't cache responses when the Vary header contains '*'. diff --git a/django/middleware/http.py b/django/middleware/http.py index ec9fecdcfdb3..b5d2b8a08200 100644 --- a/django/middleware/http.py +++ b/django/middleware/http.py @@ -1,6 +1,6 @@ from django.utils.cache import get_conditional_response, set_response_etag from django.utils.deprecation import MiddlewareMixin -from django.utils.http import parse_http_date_safe, split_header_value +from django.utils.http import parse_http_date_safe, split_directive_names class ConditionalGetMiddleware(MiddlewareMixin): @@ -37,5 +37,5 @@ def process_response(self, request, response): def needs_etag(self, response): """Return True if an ETag header should be added to response.""" - cache_control_headers = split_header_value(response.get("Cache-Control", "")) - return all(header.lower() != "no-store" for header in cache_control_headers) + directives = split_directive_names(response.get("Cache-Control", "")) + return "no-store" not in directives diff --git a/django/utils/cache.py b/django/utils/cache.py index 4bdae65b7bc4..e797e9ece5c9 100644 --- a/django/utils/cache.py +++ b/django/utils/cache.py @@ -50,7 +50,7 @@ def patch_cache_control(response, **kwargs): def dictitem(s): t = s.split("=", 1) if len(t) > 1: - return (t[0].lower(), t[1]) + return (t[0].strip().lower(), t[1]) else: return (t[0].lower(), True) diff --git a/django/utils/http.py b/django/utils/http.py index 040b2841f375..3d5b7b6be6aa 100644 --- a/django/utils/http.py +++ b/django/utils/http.py @@ -209,6 +209,20 @@ def split_header_value(value, sep=","): yield stripped +def split_directive_names(value): + """Yield the lowercased directive names from an HTTP header value. + + Any qualified value is discarded, so that qualified forms permitted by + RFC 9111 (e.g. `private="Set-Cookie"`) reduce to their directive name + ("private"). + + Use to check for the presence of a directive when its value is not needed; + use `split_header_value()` when the value matters (e.g. `max-age`). + """ + for part in split_header_value(value): + yield part.split("=", 1)[0].strip().lower() + + def parse_etags(etag_str): """ Parse a string of ETags given in an If-None-Match or If-Match header as diff --git a/docs/faq/models.txt b/docs/faq/models.txt index 92d2c69cf8db..f6d5eb621041 100644 --- a/docs/faq/models.txt +++ b/docs/faq/models.txt @@ -60,14 +60,7 @@ immediately after :djadmin:`migrate` was executed. Do Django models support multiple-column primary keys? ====================================================== -No. Only single-column primary keys are supported. - -But this isn't an issue in practice, because there's nothing stopping you from -adding other constraints (using the ``unique_together`` model option or -creating the constraint directly in your database), and enforcing the -uniqueness at that level. Single-column primary keys are needed for things such -as the admin interface to work; e.g., you need a single value to specify -an object to edit or delete. +Partially. See :doc:`/topics/composite-primary-key`. Does Django support NoSQL databases? ==================================== diff --git a/docs/topics/async.txt b/docs/topics/async.txt index 6d4e22de3717..d14482274db7 100644 --- a/docs/topics/async.txt +++ b/docs/topics/async.txt @@ -5,16 +5,16 @@ Asynchronous support .. currentmodule:: asgiref.sync Django has support for writing asynchronous ("async") views, along with an -entirely async-enabled request stack if you are running under -:doc:`ASGI `. Async views will still work under -WSGI, but with performance penalties, and without the ability to have efficient -long-running requests. +entirely async-enabled request stack if you are running under :doc:`ASGI +`. Async views will still work under WSGI, but +with a small per-request adaptation cost (see :ref:`async_performance`), and +without the ability to have efficient long-running requests. -We're still working on async support for the ORM and other parts of Django. -You can expect to see this in future releases. For now, you can use the -:func:`sync_to_async` adapter to interact with the sync parts of Django. -There is also a whole range of async-native Python libraries that you can -integrate with. +Many parts of Django provide asynchronous APIs, including :ref:`the ORM +`, the cache framework, authentication, sessions, and signals. +For other code, the :func:`sync_to_async` adapter is a low-cost bridge (see +:ref:`async_performance`). A wide range of async-native Python libraries can +also be integrated. Async views =========== @@ -44,18 +44,18 @@ other exciting response types. If you want to use these, you will need to deploy Django using :doc:`ASGI ` instead. -.. warning:: +.. note:: - You will only get the benefits of a fully-asynchronous request stack if you - have *no synchronous middleware* loaded into your site. If there is a piece - of synchronous middleware, then Django must use a thread per request to - safely emulate a synchronous environment for it. + A fully asynchronous request stack requires async middleware end-to-end. + Where a piece of synchronous middleware sits between an ASGI server and an + async view, Django adapts it by running it in its own thread; see + :ref:`async_performance` for the cost trade-off. - Middleware can be built to support :ref:`both sync and async - ` contexts. Some of Django's middleware is built like - this, but not all. To see what middleware Django has to adapt for, you can - turn on debug logging for the ``django.request`` logger and look for log - messages about *"Asynchronous handler adapted for middleware ..."*. + Django's bundled middleware supports both :ref:`sync and async + `. Third-party middleware may not. To see which + middleware Django adapts, turn on debug logging for the ``django.request`` + logger and look for log messages about *"Asynchronous handler adapted for + middleware ..."*. In both ASGI and WSGI mode, you can still safely use asynchronous support to run code concurrently rather than serially. This is especially handy when @@ -140,7 +140,7 @@ function. Queries & the ORM ----------------- -With some exceptions, Django can run ORM queries asynchronously as well:: +With some exceptions, Django can run ORM queries asynchronously:: async for author in Author.objects.filter(name__startswith="A"): book = await author.books.afirst() @@ -153,7 +153,7 @@ Detailed notes can be found in :ref:`async-queries`, but in short: * ``async for`` is supported on all QuerySets (including the output of ``values()`` and ``values_list()``.) -Django also supports some asynchronous model methods that use the database:: +Asynchronous model methods that use the database are also supported:: async def make_book(*args, **kwargs): book = Book(...) @@ -171,7 +171,9 @@ synchronous function and call it using :func:`sync_to_async`. :ref:`Persistent database connections `, set via the :setting:`CONN_MAX_AGE` setting, should also be disabled in async mode. Instead, use your database backend's built-in connection pooling if available, -or investigate a third-party connection pooling option if required. +or investigate a third-party connection pooling option if required. As in +synchronous Django, concurrent requests in a single process share that pool, so +size it to the target in-flight query concurrency. .. _async_performance: @@ -180,20 +182,31 @@ Performance When running in a mode that does not match the view (e.g. an async view under WSGI, or a traditional sync view under ASGI), Django must emulate the other -call style to allow your code to run. This context-switch causes a small -performance penalty of around a millisecond. - -This is also true of middleware. Django will attempt to minimize the number of -context-switches between sync and async. If you have an ASGI server, but all -your middleware and views are synchronous, it will switch just once, before it -enters the middleware stack. +call style to allow your code to run. The per-call cost of this adaptation is +small: tens of microseconds in the in-request ASGI path, where the running +event loop is reused, and a few hundred microseconds in the cold-start path +used by management commands, background tasks, and scripts. Against typical +request times measured in milliseconds, this is rarely visible in itself, but +can become so under GIL contention as the number of active threads grows. + +If you find yourself wrapping individual rows or operations in a tight loop, +restructure your code so the loop runs inside a single :func:`sync_to_async` +(or :func:`async_to_sync`) crossing. The per-call cost of the context switch is +then spread across the whole loop and effectively disappears. + +The same per-call adaptation cost applies to middleware. Django will attempt to +minimize the number of context-switches between sync and async. If you have an +ASGI server, but all your middleware and views are synchronous, it will switch +just once, before it enters the middleware stack. However, if you put synchronous middleware between an ASGI server and an asynchronous view, it will have to switch into sync mode for the middleware and then back to async mode for the view. Django will also hold the sync thread -open for middleware exception propagation. This may not be noticeable at first, -but adding this penalty of one thread per request can remove any async -performance advantage. +open for middleware exception propagation. For request/response views that hit +the ORM and return, this is not usually a meaningful penalty. It matters most +when you are using ASGI for high in-process concurrency over non-ORM I/O (for +example upstream HTTP fan-out, server-sent events, or other long-lived +requests), where the extra thread per request caps that concurrency. You should do your own performance testing to see what effect ASGI versus WSGI has on your code. In some cases, there may be a performance increase even for @@ -232,8 +245,8 @@ Async safety Certain key parts of Django are not able to operate safely in an async environment, as they have global state that is not coroutine-aware. These parts of Django are classified as "async-unsafe", and are protected from execution in -an async environment. The ORM is the main example, but there are other parts -that are also protected in this way. +an async environment. The synchronous API of the ORM is the main example, but +there are other parts that are also protected in this way. If you try to run any of these parts from a thread where there is a *running event loop*, you will get a @@ -334,9 +347,12 @@ Threadlocals and contextvars values are preserved across the boundary in both directions. :func:`async_to_sync` is essentially a more powerful version of the -:func:`asyncio.run` function in Python's standard library. As well -as ensuring threadlocals work, it also enables the ``thread_sensitive`` mode of -:func:`sync_to_async` when that wrapper is used below it. +:func:`asyncio.run` function in Python's standard library. As well as ensuring +threadlocals work, it also enables the ``thread_sensitive`` mode of +:func:`sync_to_async` when that wrapper is used below it. In the cold path +(no running event loop) it pays the cost of starting a fresh event loop, like +:func:`asyncio.run`; when an event loop is already running (the in-request ASGI +case), the running loop is reused and the cost drops accordingly. ``sync_to_async()`` ------------------- @@ -369,14 +385,6 @@ thread, so :func:`sync_to_async` has two threading modes: * ``thread_sensitive=False``: the sync function will run in a brand new thread which is then closed once the invocation completes. -.. warning:: - - ``asgiref`` version 3.3.0 changed the default value of the - ``thread_sensitive`` parameter to ``True``. This is a safer default, and in - many cases interacting with Django the correct value, but be sure to - evaluate uses of ``sync_to_async()`` if updating ``asgiref`` from a prior - version. - Thread-sensitive mode is quite special, and does a lot of work to run all functions in the same thread. Note, though, that it *relies on usage of* :func:`async_to_sync` *above it in the stack* to correctly run things on the @@ -396,6 +404,14 @@ always be in a *different* thread to any async code that is calling it, so you should avoid passing raw database handles or other thread-sensitive references around. +Within a single request, multiple ``thread_sensitive`` calls serialize on that +request's worker thread, but each request gets its own per-context worker, so +concurrent requests do *not* serialize against each other. This mirrors +Django's connection-per-thread model, and the same constraint applies in other +async database libraries, where concurrent queries on a single connection +serialize on a lock. To support more concurrent requests, increase the +connection pool size accordingly rather than disabling ``thread_sensitive``. + In practice this restriction means that you should not pass features of the database ``connection`` object when calling ``sync_to_async()``. Doing so will trigger the thread safety checks: diff --git a/tests/cache/tests.py b/tests/cache/tests.py index 6ccc8f2a3eca..f65603002c4b 100644 --- a/tests/cache/tests.py +++ b/tests/cache/tests.py @@ -2466,6 +2466,17 @@ def test_patch_cache_control(self): parts = {cc for cc in response.headers["Cache-Control"].split(", ")} self.assertEqual(parts, expected_cc) + def test_patch_cache_control_whitespace_around_equals(self): + # Whitespace around "=" must not be retained in the directive name; + # otherwise no_cache=True fails to collapse the qualified field-list + # form (i.e. dictitem() lacks a strip()). + for initial_cc in ('no-cache ="Set-Cookie"', 'no-cache = "Set-Cookie"'): + with self.subTest(initial_cc=initial_cc): + response = HttpResponse(headers={"Cache-Control": initial_cc}) + patch_cache_control(response, no_cache=True) + parts = {cc for cc in response.headers["Cache-Control"].split(", ")} + self.assertEqual(parts, {"no-cache"}) + def test_has_vary_header(self): tests = [ ("*", "*", True), @@ -3061,6 +3072,40 @@ def view(request, value): response = view(request, "2") self.assertEqual(response.content, b"Hello World 1") + def test_qualified_cache_control_value_not_cached(self): + for cc in ( + 'private="Set-Cookie"', + 'no-cache="Set-Cookie"', + 'no-store="Set-Cookie"', + # Malformed whitespace around "=" still fails safe. + 'private ="Set-Cookie"', + 'no-cache = "Set-Cookie"', + ): + with self.subTest(cache_control=cc): + + @cache_page(3) + def view(request, value): + return HttpResponse( + f"Hello World {value}", headers={"Cache-Control": cc} + ) + + request = self.factory.get("/view/") + response = view(request, "1") + self.assertEqual(response.content, b"Hello World 1") + response = view(request, "2") + self.assertEqual(response.content, b"Hello World 2") + + def test_authorization_header_exception_qualified_public_directive(self): + @cache_page(3) + def view(request, value): + return HttpResponse( + f"Hello World {value}", headers={"Cache-Control": 'public="abc"'} + ) + + request = self.factory.get("/view/", headers={"Authorization": "token"}) + response = view(request, "1") + self.assertIs(has_vary_header(response, "Authorization"), False) + def test_vary_asterisk_not_cached(self): views_with_cache = ( cache_page(3)(hello_world_view_patch_vary_headers_asterisk), diff --git a/tests/middleware/tests.py b/tests/middleware/tests.py index 374dc7399a58..e8c30b744ace 100644 --- a/tests/middleware/tests.py +++ b/tests/middleware/tests.py @@ -614,6 +614,15 @@ def test_etag_extended_cache_control(self): ConditionalGetMiddleware(self.get_response)(self.req).has_header("ETag") ) + def test_no_etag_no_store_qualified(self): + # A qualified no-store directive (including whitespace around "=") + # reduces to its name and must still prevent ETag generation. + for cc in ('no-store="x"', 'no-store ="x"', "no-store = x"): + with self.subTest(cache_control=cc): + self.resp_headers["Cache-Control"] = cc + response = ConditionalGetMiddleware(self.get_response)(self.req) + self.assertIs(response.has_header("ETag"), False) + def test_if_none_match_and_no_etag(self): self.req.META["HTTP_IF_NONE_MATCH"] = "spam" resp = ConditionalGetMiddleware(self.get_response)(self.req) diff --git a/tests/utils_tests/test_http.py b/tests/utils_tests/test_http.py index 6bbc7eb92005..f7525885c2d2 100644 --- a/tests/utils_tests/test_http.py +++ b/tests/utils_tests/test_http.py @@ -18,6 +18,7 @@ parse_header_parameters, parse_http_date, quote_etag, + split_directive_names, split_header_value, url_has_allowed_host_and_scheme, urlencode, @@ -357,6 +358,29 @@ def test_custom_sep(self): self.assertEqual(list(split_header_value(value, sep=";")), expected) +class SplitDirectiveNamesTests(unittest.TestCase): + def test_basic(self): + tests = [ + ("", []), + ("no-store", ["no-store"]), + # Names are lowercased. + ("No-Store, PRIVATE", ["no-store", "private"]), + # Qualified values are dropped, leaving the directive name. + ('private="Set-Cookie"', ["private"]), + ('no-cache="Set-Cookie", max-age=0', ["no-cache", "max-age"]), + # Whitespace around the "=" is stripped from the name. + ('private ="Set-Cookie"', ["private"]), + ("no-cache = foo", ["no-cache"]), + # Superstrings are preserved (not confused for shorter names). + ("myprivate", ["myprivate"]), + # A nameless directive yields an empty name. + ('="Set-Cookie"', [""]), + ] + for value, expected in tests: + with self.subTest(value=value): + self.assertEqual(list(split_directive_names(value)), expected) + + class ETagProcessingTests(unittest.TestCase): def test_parsing(self): self.assertEqual(