From dda1c2169525d826e7b7a633ac567d907a1c758d Mon Sep 17 00:00:00 2001 From: nilsmechtel Date: Thu, 16 Jul 2026 10:40:22 +0200 Subject: [PATCH 1/3] feat(proxy): reject at capacity instead of queueing unboundedly When an app's ProxyDeployment semaphore is full, the deployment function now raises RuntimeError("...has reached its maximum of N concurrent requests and is currently very busy. Please try again in a moment.") instead of awaiting a free slot. A blocked caller holds its already-decoded payload (e.g. an inference image) in the proxy actor's memory while it waits, so an open-ended queue grows proxy memory without bound. Failing fast returns capacity pressure to the caller and frees the payload immediately. The cap is the existing per-app max_ongoing_requests knob; behaviour changes from "queue when full" to "reject when full" for every app. .locked() is race-free here: because we reject rather than await, the semaphore never accumulates waiters, and there is no await between the check and the acquire. Co-Authored-By: Claude Opus 4.8 (1M context) --- bioengine/apps/proxy_deployment.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/bioengine/apps/proxy_deployment.py b/bioengine/apps/proxy_deployment.py index e97b659..79481a7 100644 --- a/bioengine/apps/proxy_deployment.py +++ b/bioengine/apps/proxy_deployment.py @@ -428,6 +428,26 @@ async def deployment_function(*args, context: Dict[str, Any], **kwargs) -> Any: # Semaphore bounds concurrency for both WebSocket and WebRTC entry paths — # they share this single wrapper, so a long-running call from either # transport consumes one slot until it returns. + # + # Fail fast instead of queueing unboundedly when at capacity: a + # blocked caller still holds its decoded payload (e.g. an inference + # image) in this actor's memory while it waits, so an open-ended + # backlog grows proxy memory without limit. Because we reject rather + # than await, the semaphore never accumulates waiters — so + # ``.locked()`` means "no free slot", and there is no ``await`` + # between this check and the acquire below, making the decision + # race-free. + if self.service_semaphore.locked(): + logger.warning( + f"⚠️ '{self.application_id}' at capacity " + f"({self.max_ongoing_requests} concurrent requests); " + f"rejecting '{method_name}'." + ) + raise RuntimeError( + f"Application '{self.application_id}' has reached its maximum " + f"of {self.max_ongoing_requests} concurrent requests and is " + f"currently very busy. Please try again in a moment." + ) async with self.service_semaphore: try: await self._check_permissions(context, method_name=method_name) From 05d2f548d8861e38ecafd407f08636977ab205cf Mon Sep 17 00:00:00 2001 From: nilsmechtel Date: Thu, 16 Jul 2026 11:02:38 +0200 Subject: [PATCH 2/3] refactor(proxy): drop redundant "very busy" from capacity error The "maximum of N concurrent requests" wording already conveys that the app is at capacity; the extra "currently very busy" was redundant. Co-Authored-By: Claude Opus 4.8 (1M context) --- bioengine/apps/proxy_deployment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bioengine/apps/proxy_deployment.py b/bioengine/apps/proxy_deployment.py index 79481a7..705719b 100644 --- a/bioengine/apps/proxy_deployment.py +++ b/bioengine/apps/proxy_deployment.py @@ -445,8 +445,8 @@ async def deployment_function(*args, context: Dict[str, Any], **kwargs) -> Any: ) raise RuntimeError( f"Application '{self.application_id}' has reached its maximum " - f"of {self.max_ongoing_requests} concurrent requests and is " - f"currently very busy. Please try again in a moment." + f"of {self.max_ongoing_requests} concurrent requests. Please " + f"try again in a moment." ) async with self.service_semaphore: try: From c724f1720c9041403a3133c6726b04e9a9fe9d4a Mon Sep 17 00:00:00 2001 From: nilsmechtel Date: Thu, 16 Jul 2026 11:02:56 +0200 Subject: [PATCH 3/3] chore(release): bump version to 0.11.26 Co-Authored-By: Claude Opus 4.8 (1M context) --- bioengine/_version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bioengine/_version.py b/bioengine/_version.py index 83f80f7..af6c015 100644 --- a/bioengine/_version.py +++ b/bioengine/_version.py @@ -13,4 +13,4 @@ Must stay in lock-step with ``pyproject.toml``'s ``version`` field. The ``version-check.yml`` CI workflow enforces the match. """ -__version__ = "0.11.25" +__version__ = "0.11.26" diff --git a/pyproject.toml b/pyproject.toml index 12eb65a..80fcbbd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "bioengine" -version = "0.11.25" +version = "0.11.26" description = "BioEngine — CLI and SDK for deploying and calling AI model services on BioEngine workers" requires-python = ">=3.11" authors = [