From 9da5bba0c003880cdbc213e6c135c65967f5d9bf Mon Sep 17 00:00:00 2001 From: nilsmechtel Date: Wed, 15 Jul 2026 18:54:09 +0200 Subject: [PATCH 1/3] feat(worker): expand ${VAR} env references in startup-application config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit read_startup_applications now recursively runs os.path.expandvars over each parsed startup-app config, so a startup app can reference the worker's own environment — including secrets mounted via valueFrom.secretKeyRef — as ${VAR}. This lets a startup app carry a secret in its env (e.g. HF_READ_TOKEN for model-runner's gated HuggingFace weight downloads) without the secret value living in git-tracked helm values: "application_env_vars": {"*": {"_HF_READ_TOKEN": "${HF_READ_TOKEN}"}} Expansion runs only on operator-supplied --startup-applications CLI config, never on tenant deploy_app calls, so sourcing from the worker env is safe. An unset ${VAR} is left verbatim. Co-Authored-By: Claude Opus 4.8 (1M context) --- bioengine/worker/__main__.py | 32 +++++++++- .../worker/test_startup_app_env_expansion.py | 59 +++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 tests/worker/test_startup_app_env_expansion.py diff --git a/bioengine/worker/__main__.py b/bioengine/worker/__main__.py index 54ec5f8a..1c2b8e73 100644 --- a/bioengine/worker/__main__.py +++ b/bioengine/worker/__main__.py @@ -48,10 +48,11 @@ import argparse import asyncio import json +import os import shlex import signal import sys -from typing import Dict +from typing import Any, Dict from bioengine.worker import BioEngineWorker @@ -487,6 +488,30 @@ def get_args_by_group(parser: argparse.ArgumentParser) -> Dict[str, Dict[str, an return group_configs +def _expand_env_in_config(value: Any) -> Any: + """Recursively expand ``${VAR}`` / ``$VAR`` from the worker's own + environment in every string within a startup-application config. + + This runs ONLY on operator-supplied ``--startup-applications`` config + (never on a tenant's ``deploy_app`` call), so it is safe to source + values from the worker's environment — including secrets mounted via + ``valueFrom.secretKeyRef`` (the same way ``HYPHA_TOKEN`` reaches the + worker). It lets a startup app pull a k8s secret into an app's env + without writing the secret value into git-tracked helm values, e.g.:: + + "application_env_vars": {"*": {"_HF_READ_TOKEN": "${HF_READ_TOKEN}"}} + + An unset ``${VAR}`` is left verbatim (``os.path.expandvars`` semantics). + """ + if isinstance(value, str): + return os.path.expandvars(value) + if isinstance(value, dict): + return {k: _expand_env_in_config(v) for k, v in value.items()} + if isinstance(value, list): + return [_expand_env_in_config(v) for v in value] + return value + + def read_startup_applications( group_configs: Dict[str, Dict[str, any]], ) -> Dict[str, Dict[str, any]]: @@ -527,6 +552,11 @@ def read_startup_applications( if not isinstance(application_config, dict): raise ValueError("Application configuration must be a JSON object") + # Resolve ${VAR} references (e.g. secrets injected into the + # worker env via valueFrom.secretKeyRef) so a startup app can + # carry a secret in its env without the value living in git. + application_config = _expand_env_in_config(application_config) + # Validate required fields if "artifact_id" not in application_config: raise ValueError( diff --git a/tests/worker/test_startup_app_env_expansion.py b/tests/worker/test_startup_app_env_expansion.py new file mode 100644 index 00000000..5f3b67bb --- /dev/null +++ b/tests/worker/test_startup_app_env_expansion.py @@ -0,0 +1,59 @@ +"""Env-var expansion in worker startup-application config. + +Startup apps may reference the worker's own environment (e.g. a secret +mounted via ``valueFrom.secretKeyRef``) as ``${VAR}``; the worker expands +these before deploying, so the secret value never has to live in +git-tracked helm values. +""" + +from bioengine.worker.__main__ import ( + _expand_env_in_config, + read_startup_applications, +) + + +def test_expand_env_recurses_and_preserves_types(monkeypatch): + monkeypatch.setenv("SECRET_TOKEN", "s3kret") + config = { + "artifact_id": "bioimage-io/model-runner", + "application_env_vars": {"*": {"_HF_READ_TOKEN": "${SECRET_TOKEN}"}}, + "nested": [{"k": "$SECRET_TOKEN"}], + "disable_gpu": False, + "num_gpus": 1, + } + + expanded = _expand_env_in_config(config) + + assert expanded["application_env_vars"]["*"]["_HF_READ_TOKEN"] == "s3kret" + assert expanded["nested"][0]["k"] == "s3kret" + # Non-string values pass through untouched. + assert expanded["disable_gpu"] is False + assert expanded["num_gpus"] == 1 + + +def test_expand_env_leaves_unset_vars_verbatim(monkeypatch): + monkeypatch.delenv("DEFINITELY_UNSET_VAR", raising=False) + assert _expand_env_in_config("${DEFINITELY_UNSET_VAR}") == "${DEFINITELY_UNSET_VAR}" + + +def test_read_startup_applications_expands_env(monkeypatch): + monkeypatch.setenv("HF_READ_TOKEN", "hf_fromsecret") + group_configs = { + "Core Options": { + "startup_applications": [ + '{"artifact_id": "bioimage-io/model-runner", ' + '"application_env_vars": {"*": {"_HF_READ_TOKEN": "${HF_READ_TOKEN}"}}}' + ] + } + } + + result = read_startup_applications(group_configs) + + apps = result["Core Options"]["startup_applications"] + assert isinstance(apps, list) and len(apps) == 1 + assert apps[0]["application_env_vars"]["*"]["_HF_READ_TOKEN"] == "hf_fromsecret" + + +def test_read_startup_applications_noop_without_apps(): + group_configs = {"Core Options": {}} + assert read_startup_applications(group_configs) == group_configs From 3ecaa5f3776a92e6a3456f7101d99242127c0c0e Mon Sep 17 00:00:00 2001 From: nilsmechtel Date: Wed, 15 Jul 2026 19:35:44 +0200 Subject: [PATCH 2/3] chore(release): bump version to 0.11.25 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 245cbdc2..12eb65af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "bioengine" -version = "0.11.24" +version = "0.11.25" description = "BioEngine — CLI and SDK for deploying and calling AI model services on BioEngine workers" requires-python = ">=3.11" authors = [ From 3493ecddc1e320c0b6673a579fa4f843f1ae253c Mon Sep 17 00:00:00 2001 From: nilsmechtel Date: Wed, 15 Jul 2026 19:37:30 +0200 Subject: [PATCH 3/3] chore(release): sync bioengine/_version.py to 0.11.25 --- bioengine/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bioengine/_version.py b/bioengine/_version.py index 4a4fd578..83f80f70 100644 --- a/bioengine/_version.py +++ b/bioengine/_version.py @@ -13,4 +13,4 @@ Must stay in lock-step with ``pyproject.toml``'s ``version`` field. The ``version-check.yml`` CI workflow enforces the match. """ -__version__ = "0.11.24" +__version__ = "0.11.25"