diff --git a/bioengine/_app/bootstrap.py b/bioengine/_app/bootstrap.py index 36fc746..1d092da 100644 --- a/bioengine/_app/bootstrap.py +++ b/bioengine/_app/bootstrap.py @@ -113,9 +113,11 @@ def introspect_app_in_ray_task( from bioengine._app.replica_init import _ensure_source # Apply env_vars from runtime_env onto the process so _ensure_source - # reads the same keys as the eventual replicas. + # reads the same keys as the eventual replicas. Overwrite (not + # setdefault): this build's version/URI must win even on a worker whose + # process env somehow already carries a prior build's values. for key, value in env_vars.items(): - os.environ.setdefault(key, value) + os.environ[key] = value app_dir = Path(env_vars["BIOENGINE_APP_DIR"]) version = env_vars.get("BIOENGINE_ARTIFACT_VERSION", "") @@ -510,11 +512,12 @@ def build_and_run_application( # Re-apply env_vars from the task's runtime_env so the source loader # below sees the same BIOENGINE_* keys as the eventual replicas. + # Overwrite (not setdefault) so this build's values win. for key, value in replica_env_vars.items(): - os.environ.setdefault(key, value) + os.environ[key] = value # The Ray-GCS source URI lives on the task env explicitly; replicas # get it via per-deployment env_vars injected in `_with_pkg`. - os.environ.setdefault("BIOENGINE_APP_SOURCE_URI", app_source_uri) + os.environ["BIOENGINE_APP_SOURCE_URI"] = app_source_uri head_app_dir = Path(replica_env_vars["BIOENGINE_APP_DIR"]) head_version = replica_env_vars.get("BIOENGINE_ARTIFACT_VERSION") or spec.get( diff --git a/bioengine/_version.py b/bioengine/_version.py index e720791..3de1b06 100644 --- a/bioengine/_version.py +++ b/bioengine/_version.py @@ -13,4 +13,4 @@ Must stay in lock-step with ``pyproject.toml``'s ``version`` field. The ``version-check.yml`` CI workflow enforces the match. """ -__version__ = "0.11.27" +__version__ = "0.11.28" diff --git a/bioengine/apps/builder.py b/bioengine/apps/builder.py index d6cbaab..ef2293a 100644 --- a/bioengine/apps/builder.py +++ b/bioengine/apps/builder.py @@ -301,7 +301,14 @@ async def _introspect_via_ray_task( try: return await asyncio.to_thread( ray.get, - ray.remote(num_cpus=0, runtime_env=submit_runtime_env)( + # max_calls=1 forces a fresh worker process per introspect: + # Ray pools task workers, and _load_app_class imports the user + # modules into that process's sys.modules. A reused worker + # would return a stale cached module (Python resolves imports + # from sys.modules before sys.path), so the spec — and the + # by-value-pickled deployment class — would capture old code + # even after the on-disk source is refreshed. + ray.remote(num_cpus=0, max_calls=1, runtime_env=submit_runtime_env)( introspect_app_in_ray_task ).remote(entry_id, task_env_vars), ) @@ -734,7 +741,12 @@ async def submit(self, built_app: "BuiltApp", application_id: str) -> None: try: await asyncio.to_thread( ray.get, - ray.remote(num_cpus=0, runtime_env=built_app.runtime_env)( + # max_calls=1: same reason as the introspect task — a fresh + # process guarantees the deployment class is imported from the + # refreshed source, not a stale sys.modules entry cached on a + # reused worker (the class is cloudpickled by value to the + # replicas, so the build worker's import is what they run). + ray.remote(num_cpus=0, max_calls=1, runtime_env=built_app.runtime_env)( build_and_run_application ).remote( _ensure_jsonable(built_app.spec), diff --git a/pyproject.toml b/pyproject.toml index 08fe7af..051b1da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "bioengine" -version = "0.11.27" +version = "0.11.28" description = "BioEngine — CLI and SDK for deploying and calling AI model services on BioEngine workers" requires-python = ">=3.11" authors = [