From a81c61b18d6fd8bb58f3eaa2e18f1ffbc7281704 Mon Sep 17 00:00:00 2001 From: SearchSavior Date: Wed, 16 Sep 2026 22:53:43 -0400 Subject: [PATCH 1/5] new around here? *prepare yourself, its yaml-time* --- config.yaml | 258 +++++++++------ src/cli/groups/add.py | 61 +++- src/cli/groups/serve.py | 6 +- src/cli/modules/server_config.py | 293 ++++++++++++++---- src/cli/utils.py | 4 +- src/server/main.py | 67 ++-- src/server/model_registry.py | 37 ++- src/server/routes/openai.py | 108 +++++-- src/server/schemas/modeling/config_blocks.py | 152 +++++++++ src/server/schemas/registration.py | 51 ++- src/server/utils/merge.py | 135 ++++++++ tests/unit/test_cli_add_unit.py | 73 ++++- tests/unit/test_config_merge_unit.py | 229 ++++++++++++++ .../test_openai_audio_transcriptions_unit.py | 6 +- tests/unit/test_tool_call_parser_unit.py | 16 +- tests/unit/test_yaml_config_unit.py | 221 +++++++++++++ 16 files changed, 1454 insertions(+), 263 deletions(-) create mode 100644 src/server/schemas/modeling/config_blocks.py create mode 100644 src/server/utils/merge.py create mode 100644 tests/unit/test_config_merge_unit.py create mode 100644 tests/unit/test_yaml_config_unit.py diff --git a/config.yaml b/config.yaml index 88c2286..e479751 100644 --- a/config.yaml +++ b/config.yaml @@ -1,6 +1,5 @@ server: -port: - + port: # comma seperated list of model_name to load on startup @@ -9,16 +8,21 @@ startup_models : ${OPENRARC_AUTOLOAD_MODELS} models: + # Each model entry nests its load-time fields under `load_config`. Model-level + # request defaults (sampler_config and the per-model_type *_config blocks) are + # siblings of load_config and are overridden per request: + # request-time > config.yaml > engine default qwen35-08b: - engine: ovgenai - model_type: vlm - model_path: /mnt/Ironwolf-4TB/Models/OpenVINO/Qwen3.5/Qwen3.5-0.8B-int8_asym-ov/ - device: CPU - tool_call_parser: qwen35 + load_config: + engine: ovgenai + model_type: vlm + model_path: /mnt/Ironwolf-4TB/Models/OpenVINO/Qwen3.5/Qwen3.5-0.8B-int8_asym-ov/ + device: CPU + tool_call_parser: qwen35 runtime_config: PERFORMANCE_HINT: LATENCY scheduler_config: - max_num_batched_tokens: + max_num_batched_tokens: num_kv_blocks: cache_size: num_linear_attention_blocks: @@ -47,7 +51,7 @@ models: runtime_config: PERFORMANCE_HINT: LATENCY scheduler_config: - max_num_batched_tokens: + max_num_batched_tokens: num_kv_blocks: cache_size: num_linear_attention_blocks: @@ -62,99 +66,159 @@ models: top_p: 0.95 repetition_penalty: 1.05 max_tokens: 1024 +####################################################################################################### +####################################################################################################### +####################################################################################################### + orion-v1: + load_config: + engine: ovgenai + model_type: vlm + model_path: /mnt/Ironwolf-4TB/Models/OpenVINO/Orion-26B-A4B-v1-int4-ov/ + device: GPU.1 + tool_call_parser: gemma4 + runtime_config: + ATTENTION_BACKEND: PA + CACHE_DIR: /mnt/Ironwolf-4TB/Models/OpenVINO/Orion-26B-A4B-v1-int4-ov/model_cache +####################################################################################################### +####################################################################################################### +####################################################################################################### + ornith-9b: + load_config: + engine: ovgenai + model_type: vlm + model_path: /mnt/Ironwolf-4TB/Models/OpenVINO/Qwen/Ornith-1.5-9B/Ornith-1.5-9B-int4_asym-awq-se256-ov + device: GPU.0 + runtime_config: + ATTENTION_BACKEND: PA + CACHE_DIR: /mnt/Ironwolf-4TB/Models/OpenVINO/Qwen/Ornith-1.5-9B/Ornith-1.5-9B-int4_asym-awq-se256-ov/model_cache +####################################################################################################### +####################################################################################################### +####################################################################################################### + qwen38-27b-davidau: + load_config: + engine: ovgenai + model_type: vlm + model_path: /mnt/Ironwolf-4TB/Models/OpenVINO/Huihui-Qwen3.8-27B-abliterated-int4-awq + device: GPU.0 + tool_call_parser: qwen35 + runtime_config: + ATTENTION_BACKEND: PA + CACHE_DIR: /mnt/Ironwolf-4TB/Models/OpenVINO/Huihui-Qwen3.8-27B-abliterated-int4-awq/model_cache + scheduler_config: + cache_interval_multiplier: 128 + max_num_batched_tokens: 2048 + enable_prefix_caching: true + use_sparse_attention: true + max_num_seqs: 1 + num_linear_attention_blocks: 0 +# =========================== +# LLMs +# =========================== + qwen38-27b: + load_config: + engine: ovgenai + model_type: vlm + model_path: /mnt/Ironwolf-4TB/Models/OpenVINO/Qwen3.5/Qwen3.8-27B-int4-ov/ + device: GPU.0 + tool_call_parser: qwen35 + runtime_config: + ATTENTION_BACKEND: PA + CACHE_DIR: /mnt/Ironwolf-4TB/Models/OpenVINO/Qwen3.5/Qwen3.8-27B-int4-ov/model_cache + scheduler_config: + cache_interval_multiplier: 128 + max_num_batched_tokens: 2048 + enable_prefix_caching: true + use_sparse_attention: true + max_num_seqs: 1 + num_linear_attention_blocks: 0 kokoro: - engine: openvino - model_type: kokoro - model_path: - device: CPU + load_config: + engine: openvino + model_type: kokoro + model_path: /mnt/Ironwolf-4TB/Models/OpenVINO/Kokoro-82M-FP16-OpenVINO + device: CPU kokoro_config: - voice: af_sarah - voice_blend: af_heart:0.7,af_nicole:0.3 - lang_code: - speed: 1.0 - chracter_count_chunk: 100 + voice_blend: af_heart:0.7,af_nicole:0.3 - qwen3asr: - engine: openvino - model_type: qwen3_asr - model_path: - device: CPU - qwen3_asr_config: - language: - max_tokens: - max_chunk_sec: - search_expand_sec: - min_window_ms: + whisper: + load_config: + engine: ovgenai + model_type: whisper + model_path: /mnt/Ironwolf-4TB/Models/OpenVINO/Whisper/distil-whisper-large-v3-int8-ov + device: GPU.0 - - qwen3_tts_elmo: - model_type: qwen3_tts_voice_clone - model_path: - engine: openvino - device: GPU.0 - qwen3_tts_custom_voice_config: - language: - speaker: - instruct: - qwen3_tts_config: - max_new_tokens: - do_sample: - top_k: - top_p: - temperature: - repetition_penalty: - subtalker_do_sample: - subtalker_top_k: - subtalker_top_p: - subtalker_temperature: - stream: - stream_chunk_frames: - stream_left_context: + # --- Incomplete examples: fill in model_path to enable these. --- + # kokoro: + # load_config: + # engine: openvino + # model_type: kokoro + # model_path: + # device: CPU + # kokoro_config: + # voice: af_sarah + # voice_blend: af_heart:0.7,af_nicole:0.3 + # lang_code: + # speed: 1.0 + # character_count_chunk: 100 + # qwen3asr: + # load_config: + # engine: openvino + # model_type: qwen3_asr + # model_path: + # device: CPU + # qwen3_asr_config: + # language: + # max_tokens: + # max_chunk_sec: + # search_expand_sec: + # min_window_ms: - qwen3_tts_oscar: - model_type: qwen3_tts_voice_design - model_path: - engine: openvino - device: GPU.0 - qwen3_tts_voice_design_config: - voice_description: - qwen3_tts_config: - max_new_tokens: - do_sample: - top_k: - top_p: - temperature: - repetition_penalty: - subtalker_do_sample: - subtalker_top_k: - subtalker_top_p: - subtalker_temperature: - stream: - stream_chunk_frames: - stream_left_context: + # qwen3_tts_oscar: + # load_config: + # model_type: qwen3_tts_voice_design + # model_path: + # engine: openvino + # device: GPU.0 + # qwen3_tts_voice_design_config: + # voice_description: + # qwen3_tts_config: + # max_new_tokens: + # do_sample: + # top_k: + # top_p: + # temperature: + # repetition_penalty: + # subtalker_do_sample: + # subtalker_top_k: + # subtalker_top_p: + # subtalker_temperature: + # stream: + # stream_chunk_frames: + # stream_left_context: - qwen3_tts_oscar: - model_type: qwen3_tts_voice_clone - model_path: - engine: openvino - device: GPU.0 - qwen3_tts_voice_clone_config: - ref_text: - x_vector_only: - instruct: - qwen3_tts_config: - max_new_tokens: - do_sample: - top_k: - top_p: - temperature: - repetition_penalty: - subtalker_do_sample: - subtalker_top_k: - subtalker_top_p: - subtalker_temperature: - stream: - stream_chunk_frames: - stream_left_context: \ No newline at end of file + # qwen3_tts_elmo: + # load_config: + # model_type: qwen3_tts_voice_clone + # model_path: + # engine: openvino + # device: GPU.0 + # qwen3_tts_voice_clone_config: + # ref_text: + # x_vector_only: + # instruct: + # qwen3_tts_config: + # max_new_tokens: + # do_sample: + # top_k: + # top_p: + # temperature: + # repetition_penalty: + # subtalker_do_sample: + # subtalker_top_k: + # subtalker_top_p: + # subtalker_temperature: + # stream: + # stream_chunk_frames: + # stream_left_context: diff --git a/src/cli/groups/add.py b/src/cli/groups/add.py index 6516834..42b9b55 100644 --- a/src/cli/groups/add.py +++ b/src/cli/groups/add.py @@ -6,6 +6,7 @@ import click from pydantic import ValidationError +from src.server.schemas.modeling.config_blocks import validate_block from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import SchedulerConfigSchema from ..main import cli, console @@ -40,6 +41,9 @@ @click.option("--scheduler-config", "-sc", default=None, help='OpenVINO runtime scheduler configuration as JSON string (e.g., \'{"use_sparse_attention": true}\').') +@click.option("--sampler-config", "--smc", + default=None, + help='Default sampler settings for llm/vlm models as JSON string (e.g., \'{"temperature": 0.7, "top_k": 40}\'). Overridden per request.') @click.option('--cache-dir', '--cd', required=False, default=None, @@ -68,7 +72,7 @@ default=None, help='Tool-call output format for this model (qwen35 XML, hermes JSON, gemma4 call syntax, or museglimmer Harmony atem). llm/vlm only; required for tool calling.') @click.pass_context -def add(ctx, model_path, model_name, engine, model_type, device, runtime_config, scheduler_config, cache_dir, draft_model_path, draft_device, num_assistant_tokens, assistant_confidence_threshold, tool_call_parser): +def add(ctx, model_path, model_name, engine, model_type, device, runtime_config, scheduler_config, sampler_config, cache_dir, draft_model_path, draft_device, num_assistant_tokens, assistant_confidence_threshold, tool_call_parser): """- Add a model configuration to the config file.""" # Validate model path @@ -106,36 +110,59 @@ def add(ctx, model_path, model_name, engine, model_type, device, runtime_config, console.print(e) ctx.exit(1) - # Legacy configs may still contain vlm_type, but new configs resolve VLM tokens from config.json. - load_config = { - "model_name": model_name, - "model_path": model_path, - "model_type": model_type, - "engine": engine, - "device": device, - "runtime_config": parsed_runtime_config, - "scheduler_config": parsed_scheduler_config, + # Validate against the contract that backs the block, so an invalid default + # is caught here rather than at model load time. + parsed_sampler_config = {} + if sampler_config: + try: + parsed_sampler_config = json.loads(sampler_config) + if not isinstance(parsed_sampler_config, dict): + console.print(f"[red]Error: sampler_config must be a JSON object (dictionary), got {type(parsed_sampler_config).__name__}[/red]") + console.print('[yellow]Example format: \'{"temperature": 0.7, "top_k": 40}\'[/yellow]') + ctx.exit(1) + validate_block("sampler_config", parsed_sampler_config, model_type) + except (json.JSONDecodeError, ValidationError) as e: + console.print("[red]Error: Failed validating sampler_config:[/red]") + console.print('[yellow]Example format: \'{"temperature": 0.7, "top_k": 40}\'[/yellow]') + console.print('') + console.print('[yellow]Error:[/yellow]') + console.print(e) + ctx.exit(1) + + # Model-level request defaults are stored as siblings of load_config. + entry = { + "load_config": { + "model_name": model_name, + "model_path": model_path, + "model_type": model_type, + "engine": engine, + "device": device, + "runtime_config": parsed_runtime_config, + "scheduler_config": parsed_scheduler_config, + } } + if parsed_sampler_config: + entry["sampler_config"] = parsed_sampler_config # Store the cache directory (resolved relative to the config file at load time) if cache_dir: - load_config["cache_dir"] = cache_dir + entry["load_config"]["cache_dir"] = cache_dir # Add speculative decoding options if provided if draft_model_path: if not validate_model_path(draft_model_path): console.print(f"[red]Model file check failed! {draft_model_path} does not contain openvino model files OR your chosen path is malformed. Verify chosen path is correct and acquired model files match source on the hub, or the destination of converted model.[/red]") ctx.exit(1) - load_config["draft_model_path"] = draft_model_path + entry["load_config"]["draft_model_path"] = draft_model_path if draft_device: - load_config["draft_device"] = draft_device + entry["load_config"]["draft_device"] = draft_device if num_assistant_tokens is not None: - load_config["num_assistant_tokens"] = num_assistant_tokens + entry["load_config"]["num_assistant_tokens"] = num_assistant_tokens if assistant_confidence_threshold is not None: - load_config["assistant_confidence_threshold"] = assistant_confidence_threshold + entry["load_config"]["assistant_confidence_threshold"] = assistant_confidence_threshold if tool_call_parser: - load_config["tool_call_parser"] = tool_call_parser + entry["load_config"]["tool_call_parser"] = tool_call_parser - ctx.obj.server_config.save_model_config(model_name, load_config) + ctx.obj.server_config.save_model_entry(model_name, entry) console.print(f"[green]Model configuration saved:[/green] {model_name}") console.print(f"[dim]Use 'openarc load {model_name}' to load this model.[/dim]") diff --git a/src/cli/groups/serve.py b/src/cli/groups/serve.py index 2913663..30e62ef 100644 --- a/src/cli/groups/serve.py +++ b/src/cli/groups/serve.py @@ -48,9 +48,9 @@ def start(ctx, host, port, load_models, use_api_key, verbose, startup_models): """ from ..modules.launch_server import start_server - # Save server configuration for other CLI commands to use - config_path = ctx.obj.server_config.save_server_config(host, port) - console.print(f"[dim]Configuration saved to: {config_path}[/dim]") + # config.yaml is never rewritten here: it is hand-authored and may carry + # comments and ${VAR} references that a YAML round-trip would destroy. + console.print(f"[dim]Using configuration: {ctx.obj.server_config.config_file}[/dim]") # Handle startup models models_to_load = [] diff --git a/src/cli/modules/server_config.py b/src/cli/modules/server_config.py index 006ef93..17468ed 100644 --- a/src/cli/modules/server_config.py +++ b/src/cli/modules/server_config.py @@ -3,12 +3,21 @@ This module handles all configuration file operations without any CLI/presentation logic. """ -import json +import logging import os +import re from pathlib import Path from typing import Optional, Dict, Any, List, cast + +import yaml + from ..utils import get_config_file_path +logger = logging.getLogger(__name__) + +# Matches ${VAR} and ${VAR:-default} inside a string scalar. +_ENV_PATTERN = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}") + class ServerConfig: """Manages OpenArc server and model configurations.""" @@ -16,74 +25,116 @@ class ServerConfig: def __init__(self, config_file: Optional[Path] = None): """ Initialize ServerConfig with a config file path. - + Args: config_file: Path to the config file. If None, OPENARC_CONFIG_FILE env var when set, - otherwise defaults to openarc_config.json in project root. + otherwise defaults to config.yaml in project root. """ self.config_file = get_config_file_path() if config_file is None else config_file - + + @staticmethod + def _interpolate_env(value: Any, path: str = "") -> Any: + """Recursively expand ${VAR} / ${VAR:-default} in string scalars. + + Interpolation is applied to values only, never to mapping keys. An + unset variable with no default expands to an empty string and logs a + warning rather than raising, so a config referencing an unset + OPENARC_AUTOLOAD_MODELS still loads. + """ + if isinstance(value, str): + def _replace(match: re.Match) -> str: + name, default = match.group(1), match.group(2) + if name in os.environ: + return os.environ[name] + if default is not None: + return default + logger.warning( + f"Config: ${{{name}}} at '{path or ''}' is unset; using empty string" + ) + return "" + + return _ENV_PATTERN.sub(_replace, value) + if isinstance(value, dict): + return { + key: ServerConfig._interpolate_env( + val, f"{path}.{key}" if path else str(key) + ) + for key, val in value.items() + } + if isinstance(value, list): + return [ + ServerConfig._interpolate_env(item, f"{path}[{idx}]") + for idx, item in enumerate(value) + ] + return value + def load_config(self) -> Dict[str, Any]: """ - Load full configuration from JSON config file. - + Load full configuration from the YAML config file. + + ${VAR} references in string scalars are resolved from the environment + after parsing. + Returns: Configuration dictionary, or empty dict if file doesn't exist or is invalid. """ if not self.config_file.exists(): return {} - + try: - with open(self.config_file, "r") as f: - config = json.load(f) - return config if config else {} - except (json.JSONDecodeError, FileNotFoundError): + with open(self.config_file, "r", encoding="utf-8") as f: + config = yaml.safe_load(f) + except (yaml.YAMLError, FileNotFoundError, OSError) as exc: + logger.error(f"Config: failed to read {self.config_file}: {exc}") return {} - + + if not config: + return {} + if not isinstance(config, dict): + logger.error( + f"Config: {self.config_file} must contain a mapping at the top level, " + f"got {type(config).__name__}" + ) + return {} + + return cast(Dict[str, Any], self._interpolate_env(config)) + def save_config(self, config: Dict[str, Any]) -> None: """ - Save configuration to JSON config file. Does nothing if an on-disk - config file exists and its contents are identical to the provided + Save configuration to the YAML config file. Does nothing if an on-disk + config file exists and its parsed contents are identical to the provided config, allowing the config to live on a read-only filesystem. - + + Comparison is done on the parsed structure rather than raw text, since + YAML round-tripping legitimately reformats the document. + Args: config: Configuration dictionary to save. """ - + if self.config_file.exists(): - try: - with open(self.config_file, "r") as f: - existing_config = json.load(f) - if existing_config == config: - return # No changes, skip writing - except: - pass # Overwrite if the existing file is invalid or missing - - with open(self.config_file, "w") as f: - json.dump(config, f, indent=2) + if self.load_config() == self._interpolate_env(config): + return # No changes, skip writing + + with open(self.config_file, "w", encoding="utf-8") as f: + yaml.safe_dump(config, f, sort_keys=False, default_flow_style=False) def save_server_config(self, host: str, port: int) -> Path: """ - Save server configuration. - + Record the server bind address. + + Deliberately a no-op on disk. config.yaml is hand-authored and may + contain comments and ${VAR} references that a YAML round-trip cannot + preserve, so `openarc serve start` must not rewrite it. host/port are + supplied on the command line and read back from there. + Args: host: Server host address. port: Server port number. - + Returns: - Path to the saved config file. + Path to the config file. """ - config = self.load_config() - config.update({ - "server": { - "host": host, - "port": port - }, - "created_by": "openarc-cli", - "version": "1.0" - }) - - self.save_config(config) return self.config_file def load_server_config(self) -> Dict[str, Any]: @@ -95,54 +146,120 @@ def load_server_config(self) -> Dict[str, Any]: Returns defaults if not configured. """ config = self.load_config() - if config and "server" in config: - return config["server"] - + server = config.get("server") if config else None + if isinstance(server, dict): + # Blank values (e.g. a bare `port:` key) parse as None; fall back to + # the default rather than handing a None back to callers. + host = server.get("host") or "localhost" + port = server.get("port") or 8000 + return {"host": host, "port": port} + return {"host": "localhost", "port": 8000} def save_model_config(self, model_name: str, model_config: Dict[str, Any]) -> None: """ - Save model configuration. - + Save model configuration under models..load_config. + + Writing into load_config keeps every model entry in the canonical + nested shape, whether it was authored by this command or by hand. + Args: model_name: Name of the model. - model_config: Model configuration dictionary. + model_config: Model load configuration dictionary. """ config = self.load_config() - + if "models" not in config: config["models"] = {} - - config["models"][model_name] = model_config + + config["models"][model_name] = {"load_config": model_config} self.save_config(config) - + + def save_model_entry(self, model_name: str, entry: Dict[str, Any]) -> None: + """Save a full models. entry, preserving the nested shape. + + Unlike save_model_config, this stores sibling blocks (sampler_config, + *_config) alongside load_config rather than only load fields. + """ + config = self.load_config() + if "models" not in config: + config["models"] = {} + config["models"][model_name] = entry + self.save_config(config) + + @staticmethod + def _split_model_entry( + model_config: Dict[str, Any], + ) -> tuple[Dict[str, Any], Dict[str, Any]]: + """Split a models. entry into (load fields, config blocks). + + The canonical shape nests load fields under `load_config` and keeps the + request-default blocks (sampler_config, runtime_config, + scheduler_config, *_config) as siblings. Returns a tuple so callers can + route each half to the right place. + """ + from src.server.schemas.modeling.config_blocks import BLOCK_CONTRACTS + + if not isinstance(model_config, dict): + return {}, {} + + nested = model_config.get("load_config") + if not isinstance(nested, dict): + # Flat-form entry: the whole thing is load fields. + return dict(model_config), {} + + load_fields = dict(nested) + blocks: Dict[str, Any] = {} + for key, value in model_config.items(): + if key == "load_config": + continue + if key in BLOCK_CONTRACTS: + blocks[key] = value + else: + # runtime_config / scheduler_config and other load-time siblings. + load_fields[key] = value + return load_fields, blocks + + @staticmethod + def _unwrap_model_entry(model_config: Dict[str, Any]) -> Dict[str, Any]: + """Flatten a nested models. entry into a single load-config dict.""" + load_fields, _ = ServerConfig._split_model_entry(model_config) + return load_fields + def get_model_config(self, model_name: str) -> Optional[Dict[str, Any]]: """ Get model configuration by name. - + Args: model_name: Name of the model. - + Returns: - Model configuration dict, or None if not found. Relative paths are resolved - against the config file's directory, allowing for configs to be packaged with models. + Flattened model configuration dict, or None if not found. Relative paths are + resolved against the config file's directory, allowing for configs to be + packaged with models. """ config = self.load_config() models = cast(Dict[str, Dict[str, Any]], config.get("models", {})) model = models.get(model_name) - return self._resolve_model_paths(model) if model else None - + if not model: + return None + return self._resolve_model_paths(self._unwrap_model_entry(model)) + def get_all_models(self) -> Dict[str, Dict[str, Any]]: """ Get all model configurations. - + Returns: - Dictionary mapping model names to their configurations. Relative paths are resolved - against the config file's directory, allowing for configs to be packaged with models. + Dictionary mapping model names to their flattened configurations. Relative + paths are resolved against the config file's directory, allowing for configs + to be packaged with models. """ config = self.load_config() models = cast(Dict[str, Dict[str, Any]], config.get("models", {})) - return {name: self._resolve_model_paths(cfg) for name, cfg in models.items()} + return { + name: self._resolve_model_paths(self._unwrap_model_entry(cfg)) + for name, cfg in models.items() + } def _resolve_model_paths(self, model_config: Dict[str, Any]) -> Dict[str, Any]: """Return a copy of model_config with relative model_path, draft_model_path, @@ -162,32 +279,72 @@ def _resolve_model_paths(self, model_config: Dict[str, Any]) -> Dict[str, Any]: resolved["cache_dir"] = str((self.config_file.parent / cache_dir).resolve()) return resolved - + + def get_model_load_config(self, model_name: str) -> Optional["ModelLoadConfig"]: + """Build a validated ModelLoadConfig for a named model in config.yaml. + + model_name is injected from the mapping key, overriding anything the + entry itself declares, so the key is authoritative. + + Returns: + A validated ModelLoadConfig, or None if the model is not configured. + + Raises: + ValueError: If the entry is malformed or carries config blocks that + do not match its model_type. + """ + from pydantic import ValidationError + + from src.server.schemas.registration import ModelLoadConfig + + config = self.load_config() + models = cast(Dict[str, Dict[str, Any]], config.get("models", {})) + entry = models.get(model_name) + if not entry: + return None + + flattened, blocks = self._split_model_entry(entry) + flattened = self._resolve_model_paths(flattened) + flattened["model_name"] = model_name + if blocks: + flattened["model_config_blocks"] = blocks + + try: + load_config = ModelLoadConfig(**flattened) + except ValidationError as exc: + raise ValueError( + f"Invalid configuration for model '{model_name}': {exc}" + ) from exc + + load_config.validate_config_blocks() + return load_config + def remove_model_config(self, model_name: str) -> bool: """ Remove model configuration by name. - + Args: model_name: Name of the model to remove. - + Returns: True if model was removed, False if model was not found. """ config = self.load_config() models = config.get("models", {}) - + if model_name not in models: return False - + del models[model_name] config["models"] = models self.save_config(config) - + return True - + def model_exists(self, model_name: str) -> bool: """ Check if a model configuration exists. + Check if a model configuration exists. Args: model_name: Name of the model. diff --git a/src/cli/utils.py b/src/cli/utils.py index 22951ef..03cd272 100644 --- a/src/cli/utils.py +++ b/src/cli/utils.py @@ -50,11 +50,11 @@ def validate_model_path(model_path): def get_config_file_path(): """ Get the path to the config file, checking the OPENARC_CONFIG_FILE environment variable first, - then defaulting to openarc_config.json in the project root. + then defaulting to config.yaml in the project root. """ env_path = os.environ.get("OPENARC_CONFIG_FILE") if env_path: return Path(env_path) else: project_root = Path(__file__).parent.parent.parent - return project_root / "openarc_config.json" + return project_root / "config.yaml" diff --git a/src/server/main.py b/src/server/main.py index 7b73af9..282d5d5 100644 --- a/src/server/main.py +++ b/src/server/main.py @@ -1,7 +1,6 @@ # The first implementation of the OpenAI-like API was contributed by @gapeleon. # They are one hero among many future heroes working to make OpenArc better. -import json import logging import os import time @@ -12,11 +11,9 @@ from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse -from src.cli.utils import get_config_file_path from starlette.middleware.base import BaseHTTPMiddleware from src.server.deps import _registry -from src.server.schemas.registration import ModelLoadConfig from src.server.routes.openai import router as openai_router from src.server.routes.openarc import router as openarc_router @@ -52,39 +49,45 @@ async def dispatch(self, request: Request, call_next): @asynccontextmanager async def lifespan(app: FastAPI): + # Let OPENARC_STARTUP_MODELS (exported by `openarc serve start`) win over the + # config file's startup_models key. models = os.getenv("OPENARC_STARTUP_MODELS", "").strip() + if not models: + from src.cli.modules.server_config import ServerConfig + + configured = ServerConfig().load_config().get("startup_models") or "" + models = configured.strip() if isinstance(configured, str) else "" + if models: from pathlib import Path - config_file = get_config_file_path() - if config_file.exists(): - with open(config_file) as f: - config = json.load(f) - - for name in models.split(","): - name = name.strip() - model_config = config.get("models", {}).get(name) - if not model_config: - logger.warning(f"Startup: model '{name}' not in config, skipping") - continue - - model_path = model_config.get("model_path") - if model_path and not Path(model_path).is_absolute(): - model_config["model_path"] = str((config_file.parent / model_path).resolve()) - - cache_dir = model_config.get("cache_dir") - if cache_dir and not Path(cache_dir).is_absolute(): - cache_dir = str((config_file.parent / cache_dir).resolve()) - model_config["cache_dir"] = cache_dir - - try: - if cache_dir: - # Create the cache directory at startup if it doesn't exist. - Path(cache_dir).mkdir(parents=True, exist_ok=True) - await _registry.register_load(ModelLoadConfig(**model_config)) - logger.info(f"Startup: loaded '{name}'") - except Exception as e: - logger.error(f"Startup: failed to load '{name}': {e}") + from src.cli.modules.server_config import ServerConfig + + # All config parsing, env interpolation and relative-path resolution + # happens inside ServerConfig so startup and the CLI share one loader. + server_config = ServerConfig() + + for name in (m.strip() for m in models.split(",")): + if not name: + continue + try: + model_config = server_config.get_model_load_config(name) + except Exception as e: + logger.error(f"Startup: invalid config for '{name}': {e}") + continue + + if model_config is None: + logger.warning(f"Startup: model '{name}' not in config, skipping") + continue + + try: + if model_config.cache_dir: + # Create the cache directory at startup if it doesn't exist. + Path(model_config.cache_dir).mkdir(parents=True, exist_ok=True) + await _registry.register_load(model_config) + logger.info(f"Startup: loaded '{name}'") + except Exception as e: + logger.error(f"Startup: failed to load '{name}': {e}") yield diff --git a/src/server/model_registry.py b/src/server/model_registry.py index 8a20885..68c12a0 100644 --- a/src/server/model_registry.py +++ b/src/server/model_registry.py @@ -14,6 +14,7 @@ ModelLoadConfig, ModelStatus, ModelType, + ToolCallParser, ) logger = logging.getLogger(__name__) @@ -31,25 +32,40 @@ class ModelRecord: # Public fields model_path: str = "" model_name: str = "" - model_type: str = "" - engine: str = "" + model_type: ModelType = ModelType.LLM + engine: EngineType = EngineType.OV_GENAI device: str = "" runtime_config: Dict[str, Any] = field(default_factory=dict) - tool_call_parser: Optional[str] = None + tool_call_parser: Optional[ToolCallParser] = None + # Model-level request defaults surfaced from config.yaml. Stored as plain + # dicts (already validated by ModelLoadConfig) and used to seed per-request + # configs; an explicit request value always wins over these. + # Model-level request defaults from config.yaml, keyed by block name + # (e.g. 'sampler_config', 'kokoro_config'). Each value is a dict of only the + # keys the author wrote and is merged under the per-request config. + model_config_blocks: Dict[str, Dict[str, Any]] = field(default_factory=dict) + + def block(self, name: str) -> Dict[str, Any]: + """Return a named config block (e.g. 'kokoro_config'), or {} if unset.""" + return self.model_config_blocks.get(name) or {} def registered_models(self) -> dict: """Return only public fields as JSON-serializable dict.""" result = { "model_name": self.model_name, - "model_type": self.model_type, - "engine": self.engine, + "model_type": self.model_type.value, + "engine": self.engine.value, "device": self.device, "runtime_config": self.runtime_config, - "tool_call_parser": self.tool_call_parser, + "tool_call_parser": ( + self.tool_call_parser.value if self.tool_call_parser else None + ), "status": self.status.value, "time_loaded": self.time_loaded.isoformat(), } + if self.model_config_blocks: + result["model_config_blocks"] = self.model_config_blocks if self.error_message: result["error_message"] = self.error_message return result @@ -90,6 +106,10 @@ async def register_load(self, loader: ModelLoadConfig) -> str: logger.info(f"Load failed! model_name '{loader.model_name}' already exists") raise ValueError(f"model_name '{loader.model_name}' already registered") + # Reject config blocks that don't match this model's model_type before + # anything is loaded, so a mismatched config.yaml fails fast. + loader.validate_config_blocks() + # Create a model record with LOADING status record = ModelRecord( model_path=loader.model_path, @@ -98,9 +118,8 @@ async def register_load(self, loader: ModelLoadConfig) -> str: engine=loader.engine, device=loader.device, runtime_config=loader.runtime_config, - tool_call_parser=( - loader.tool_call_parser.value if loader.tool_call_parser else None - ), + tool_call_parser=loader.tool_call_parser, + model_config_blocks=dict(loader.model_config_blocks or {}), status=ModelStatus.LOADING, ) diff --git a/src/server/routes/openai.py b/src/server/routes/openai.py index e02dd06..833ec31 100644 --- a/src/server/routes/openai.py +++ b/src/server/routes/openai.py @@ -11,9 +11,14 @@ from fastapi.responses import JSONResponse, StreamingResponse from src.server.deps import _registry, _workers, verify_api_key -from src.server.schemas.modeling.contract_kokoro import KokoroLanguage, KokoroVoice +from src.server.schemas.modeling.contract_kokoro import ( + KokoroLanguage, + KokoroVoice, + OV_KokoroGenConfig, +) from src.server.schemas.modeling.contract_optimum_emb import PreTrainedTokenizerConfig from src.server.schemas.modeling.contract_optimum_rerank import RerankerConfig +from src.server.schemas.modeling.contract_qwen3asr import OV_Qwen3ASRGenConfig from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig from src.server.schemas.modeling.contract_qwen3tts import ( OV_Qwen3TTSCustomVoice, @@ -31,6 +36,7 @@ OpenArcASRConfig, RerankRequest, ) +from src.server.utils.merge import build_config, defaults_for_record from src.engine.ov_genai.tool_parse import gemma4, hermes, museglimmer, qwen35 logger = logging.getLogger(__name__) @@ -47,6 +53,26 @@ } +def _get_record(model_name: str): + """Return the loaded ModelRecord for a model_name, or None. + + Caller must not hold _registry._lock (this takes it). The returned record's + plain-dict config blocks are safe to read outside the lock. + """ + for record in list(_registry._models.values()): + if record.model_name == model_name: + return record + return None + + +def _record_defaults(model_name: str) -> Dict[str, Any]: + """Resolve the config.yaml defaults that apply to a loaded model.""" + record = _get_record(model_name) + if record is None: + return {} + return defaults_for_record(record) + + def _prepend_system_instruction(messages: Any, instruction: str) -> Any: if not isinstance(messages, list): return messages @@ -145,7 +171,10 @@ async def openai_chat_completions( async with _registry._lock: for record in _registry._models.values(): if record.model_name == request.model: - tool_parser_name = record.tool_call_parser + # record.tool_call_parser is a ToolCallParser enum; the + # parser registry below is keyed by its string value. + parser_enum = record.tool_call_parser + tool_parser_name = parser_enum.value if parser_enum else None break if tool_parser_name is None and request.tools: @@ -166,7 +195,6 @@ async def openai_chat_completions( chat_template_kwargs.setdefault("enable_thinking", False) config_kwargs = { - "messages": messages, "temperature": request.temperature, "max_tokens": request.max_tokens, "top_p": request.top_p, @@ -174,18 +202,23 @@ async def openai_chat_completions( "repetition_penalty": request.repetition_penalty, "do_sample": request.do_sample, "num_return_sequences": request.num_return_sequences, - "stream": request.stream, - "tools": tools, "seed": request.seed, "frequency_penalty": request.frequency_penalty, "presence_penalty": request.presence_penalty, - "chat_template_kwargs": chat_template_kwargs, } if parser_module is not None: config_kwargs["tool_call_parser"] = tool_parser_name - config_kwargs = {k: v for k, v in config_kwargs.items() if v is not None} - generation_config = OVGenAI_GenConfig(**config_kwargs) + # Layer the model's config.yaml sampler defaults under the request. + # Precedence: request-time > config.yaml > engine default. + generation_config = build_config( + OVGenAI_GenConfig, + request={**config_kwargs, "chat_template_kwargs": chat_template_kwargs}, + defaults=_record_defaults(request.model), + messages=messages, + tools=tools, + stream=request.stream, + ) model_name = request.model created_ts = int(time.time()) @@ -373,7 +406,6 @@ async def openai_completions(request: OpenAICompletionRequest, raw_request: Requ ) config_kwargs = { - "prompt": prompt, "temperature": request.temperature, "max_tokens": request.max_tokens, "top_p": request.top_p, @@ -381,11 +413,16 @@ async def openai_completions(request: OpenAICompletionRequest, raw_request: Requ "repetition_penalty": request.repetition_penalty, "do_sample": request.do_sample, "num_return_sequences": request.num_return_sequences, - "stream": request.stream, } - config_kwargs = {k: v for k, v in config_kwargs.items() if v is not None} - generation_config = OVGenAI_GenConfig(**config_kwargs) + # Layer the model's config.yaml sampler defaults under the request. + generation_config = build_config( + OVGenAI_GenConfig, + request=config_kwargs, + defaults=_record_defaults(request.model), + prompt=prompt, + stream=request.stream, + ) model_name = request.model created_ts = int(time.time()) @@ -548,14 +585,23 @@ async def openai_audio_transcriptions( if not payload.get("qwen3_asr"): # Fall back to defaults if qwen3_asr config is not provided payload["qwen3_asr"] = {} - + cfg = OpenArcASRConfig.model_validate(payload) - update = {"audio_base64": audio_base64} - if language: + # Layer the model's config.yaml qwen3_asr_config under the request. + # The original handler used model_copy(update=...), which skips + # validation; passing through build_config re-validates, so only + # real values (not unset FastAPI Form sentinels) may be injected. + request_fields = cfg.qwen3_asr.model_dump(exclude_unset=True) + request_fields["audio_base64"] = audio_base64 + if isinstance(language, str) and language: # Whisper-style top-level `language` takes precedence; otherwise # fall back to openarc_asr.qwen3_asr.language (current behavior). - update["language"] = language - gen_config = cfg.qwen3_asr.model_copy(update=update) + request_fields["language"] = language + gen_config = build_config( + OV_Qwen3ASRGenConfig, + request=request_fields, + defaults=_record_defaults(model), + ) result = await _workers.transcribe_qwen3_asr(model, gen_config) else: gen_config = OVGenAI_WhisperGenConfig(audio_base64=audio_base64) @@ -607,6 +653,7 @@ async def openai_audio_speech(request: OpenAISpeechRequest): raise ValueError(f"Model '{request.model}' is not loaded") normalized = ModelType(selected_model_type) + tts_defaults = _record_defaults(request.model) if normalized in ( ModelType.QWEN3_TTS_CUSTOM_VOICE, @@ -620,10 +667,23 @@ async def openai_audio_speech(request: OpenAISpeechRequest): ModelType.QWEN3_TTS_VOICE_DESIGN: "qwen3_tts_voice_design", ModelType.QWEN3_TTS_VOICE_CLONE: "qwen3_tts_voice_clone", }[normalized] - gen_config = getattr(request.openarc_tts, _qwen3_tts_field) - if gen_config is None: + _contract = { + ModelType.QWEN3_TTS_CUSTOM_VOICE: OV_Qwen3TTSCustomVoice, + ModelType.QWEN3_TTS_VOICE_DESIGN: OV_Qwen3TTSVoiceDesign, + ModelType.QWEN3_TTS_VOICE_CLONE: OV_Qwen3TTSVoiceClone, + }[normalized] + supplied = getattr(request.openarc_tts, _qwen3_tts_field) + if supplied is None: raise ValueError(f"openarc_tts.{_qwen3_tts_field} required for {normalized.value} models") - gen_config.input = request.input + # Seed the contract from config.yaml, letting explicitly supplied + # request fields win. Anything neither layer sets stays unset, so + # the guards below still see "not provided by the caller". + gen_config = build_config( + _contract, + request=supplied.model_dump(exclude_unset=True), + defaults=tts_defaults, + input=request.input, + ) if request.language is not None and "language" not in gen_config.model_fields_set: gen_config.language = request.language if ( @@ -649,8 +709,12 @@ async def openai_audio_speech(request: OpenAISpeechRequest): else: if not request.openarc_tts or not request.openarc_tts.kokoro: raise ValueError("openarc_tts.kokoro required for Kokoro models") - gen_config = request.openarc_tts.kokoro - gen_config.input = request.input + gen_config = build_config( + OV_KokoroGenConfig, + request=request.openarc_tts.kokoro.model_dump(exclude_unset=True), + defaults=tts_defaults, + input=request.input, + ) if request.voice is not None and "voice" not in gen_config.model_fields_set: try: gen_config.voice = KokoroVoice(request.voice) diff --git a/src/server/schemas/modeling/config_blocks.py b/src/server/schemas/modeling/config_blocks.py new file mode 100644 index 0000000..c2a1de6 --- /dev/null +++ b/src/server/schemas/modeling/config_blocks.py @@ -0,0 +1,152 @@ +"""Which request contract backs each config.yaml model-defaults block. + +There is exactly one definition per config shape: the request contract in +``contract_*.py``. This module only maps block names to those contracts, so a +block can never drift from the contract it configures and adding a field to a +contract makes it configurable in config.yaml automatically. + +Precedence, highest wins: + + request-time > config.yaml block > engine/pydantic default + +A block is stored as a plain dict of only the keys the author actually wrote +(see ``ServerConfig.get_model_load_config``), and is merged under the request by +``src/server/utils/merge.py``. Because unset keys are simply absent, the +contract's own default applies when neither layer supplies a value. +""" +from typing import Any, Dict, Optional, Type + +from pydantic import BaseModel, ValidationError + +from src.server.schemas.modeling.contract_kokoro import OV_KokoroGenConfig +from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig +from src.server.schemas.modeling.contract_qwen3asr import OV_Qwen3ASRGenConfig +from src.server.schemas.modeling.contract_qwen3tts import ( + OV_Qwen3TTSCustomVoice, + OV_Qwen3TTSGenConfig, + OV_Qwen3TTSVoiceClone, + OV_Qwen3TTSVoiceDesign, +) + +# Block name -> (contract, model_type it applies to). A None model_type means +# the block is valid for any member of that family (see SHARED_TTS_BLOCK). +BLOCK_CONTRACTS: Dict[str, tuple[Type[BaseModel], Optional[str]]] = { + "sampler_config": (OVGenAI_GenConfig, None), # llm/vlm only, see SAMPLER_MODEL_TYPES + "kokoro_config": (OV_KokoroGenConfig, "kokoro"), + "qwen3_asr_config": (OV_Qwen3ASRGenConfig, "qwen3_asr"), + "qwen3_tts_config": (OV_Qwen3TTSGenConfig, None), # shared by all qwen3_tts_* modes + "qwen3_tts_custom_voice_config": (OV_Qwen3TTSCustomVoice, "qwen3_tts_custom_voice"), + "qwen3_tts_voice_design_config": (OV_Qwen3TTSVoiceDesign, "qwen3_tts_voice_design"), + "qwen3_tts_voice_clone_config": (OV_Qwen3TTSVoiceClone, "qwen3_tts_voice_clone"), +} + +# sampler_config applies only to llm/vlm. +SAMPLER_BLOCK = "sampler_config" +SAMPLER_MODEL_TYPES = {"llm", "vlm"} + +# One block shared by every qwen3_tts_* mode. +SHARED_TTS_BLOCK = "qwen3_tts_config" +QWEN3_TTS_PREFIX = "qwen3_tts_" + +# Fields that describe one specific request rather than a reusable model +# default. They may appear in a contract but must not be authored in config.yaml. +REQUEST_ONLY_FIELDS = frozenset( + { + "messages", + "prompt", + "input_ids", + "tools", + "tool_call_parser", + "request_id", + "chat_template_kwargs", + "input", + "audio_base64", + "ref_audio_b64", + } +) + + +def contract_for(block_name: str) -> Optional[Type[BaseModel]]: + """Return the contract backing a block name, or None if unknown.""" + entry = BLOCK_CONTRACTS.get(block_name) + return entry[0] if entry else None + + +def block_applies_to(block_name: str, model_type: str) -> bool: + """Whether a config block may be used with a given model_type.""" + if block_name == SAMPLER_BLOCK: + return model_type in SAMPLER_MODEL_TYPES + if block_name == SHARED_TTS_BLOCK: + return model_type.startswith(QWEN3_TTS_PREFIX) + + entry = BLOCK_CONTRACTS.get(block_name) + if entry is None: + return False + required = entry[1] + return required is not None and model_type == required + + +def validate_block(block_name: str, payload: Any, model_type: str) -> Dict[str, Any]: + """Validate an authored block against its contract. + + Returns the block as a dict of only the authored keys. + + Raises: + ValueError: If the block is unknown, does not apply to model_type, or + contains a key the contract does not define. + """ + contract = contract_for(block_name) + if contract is None: + raise ValueError( + f"Unknown config block '{block_name}'. Valid blocks: " + f"{', '.join(sorted(BLOCK_CONTRACTS))}" + ) + if not block_applies_to(block_name, model_type): + if block_name == SAMPLER_BLOCK: + raise ValueError( + f"sampler_config is only valid for {sorted(SAMPLER_MODEL_TYPES)} models, " + f"but this model is model_type '{model_type}'" + ) + entry = BLOCK_CONTRACTS[block_name] + required = entry[1] or f"{QWEN3_TTS_PREFIX}*" + raise ValueError( + f"{block_name} requires model_type '{required}', " + f"but this model is model_type '{model_type}'" + ) + if payload is None: + return {} + if not isinstance(payload, dict): + raise ValueError(f"{block_name} must be a mapping, got {type(payload).__name__}") + + unknown = sorted(set(payload) - set(contract.model_fields)) + if unknown: + raise ValueError( + f"{block_name} has unknown key(s): {', '.join(unknown)}. " + f"Valid keys: {', '.join(sorted(contract.model_fields))}" + ) + + # Fields that describe one request rather than a reusable model default. + request_only = sorted(set(payload) & REQUEST_ONLY_FIELDS) + if request_only: + raise ValueError( + f"{block_name} cannot set request-only field(s): {', '.join(request_only)}" + ) + + # Coerce/validate values against the contract so a bad type or an invalid + # enum member is caught here rather than at model load time. + authored = {k: v for k, v in payload.items() if v is not None} + try: + validated = contract(**authored) + except ValidationError as exc: + raise ValueError(f"{block_name} has invalid values: {exc}") from exc + + # Re-read only the authored keys so unset fields stay absent. + return {k: getattr(validated, k) for k in authored} + + +def model_block_name(model_type: str) -> Optional[str]: + """Return the mode-specific block name for a model_type, if it has one.""" + for block_name, (_, required) in BLOCK_CONTRACTS.items(): + if required == model_type and block_name != SAMPLER_BLOCK: + return block_name + return None diff --git a/src/server/schemas/registration.py b/src/server/schemas/registration.py index 6fd226a..f7e0983 100644 --- a/src/server/schemas/registration.py +++ b/src/server/schemas/registration.py @@ -2,7 +2,7 @@ from enum import Enum from typing import Any, Dict, Optional -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import SchedulerConfigSchema @@ -77,6 +77,10 @@ class ToolCallParser(str, Enum): class ModelLoadConfig(BaseModel): + # Strict: an unrecognized key is a config error, not something to silently + # drop. Typos in hand-written config.yaml fail loudly at load time. + model_config = ConfigDict(extra="forbid") + model_path: str = Field( description=""" Top level path to directory containing OpenVINO IR converted model. @@ -90,10 +94,6 @@ class ModelLoadConfig(BaseModel): """ ) model_type: ModelType = Field(...) - vlm_type: Optional[str] = Field( - default=None, - description="Deprecated legacy VLM token type. VLM tokens are resolved from config.json." - ) engine: EngineType = Field(...) device: str = Field( ..., @@ -143,6 +143,47 @@ class ModelLoadConfig(BaseModel): with 400.""", ) + # --- Model-level request defaults, authored in config.yaml --- + # Each entry is a plain dict of only the keys the author wrote, keyed by + # block name (e.g. 'sampler_config', 'kokoro_config'). Validated against the + # matching request contract by validate_config_blocks(). + # These seed per-request configs; anything the client sends wins. + # Precedence: request > config.yaml block > contract default. + model_config_blocks: Dict[str, Dict[str, Any]] = Field( + default_factory=dict, + description=( + "Model-level request defaults keyed by block name. Each block is " + "validated against the request contract it configures." + ), + ) + + def validate_config_blocks(self) -> None: + """Validate every supplied config block against its contract. + + Checks that each block name is known, applies to this model's + model_type, and contains no key the contract does not define. Unknown + keys are an error so a typo fails loudly instead of being ignored. + + Also normalizes each block, dropping unset (None) values so that + "not authored" stays distinct from "authored as the contract default". + + Raises: + ValueError: If any block is unknown, mismatched, or has bad keys. + """ + from src.server.schemas.modeling.config_blocks import validate_block + + model_type = self.model_type.value + validated: Dict[str, Dict[str, Any]] = {} + + for block_name, payload in (self.model_config_blocks or {}).items(): + try: + resolved = validate_block(block_name, payload, model_type) + except ValueError as exc: + raise ValueError(f"'{self.model_name}': {exc}") from exc + if resolved: + validated[block_name] = resolved + + self.model_config_blocks = validated class ModelUnloadConfig(BaseModel): model_name: str = Field(..., description="Name of the model to unload") diff --git a/src/server/utils/merge.py b/src/server/utils/merge.py new file mode 100644 index 0000000..75abd77 --- /dev/null +++ b/src/server/utils/merge.py @@ -0,0 +1,135 @@ +"""Layered config merging for per-request model defaults. + +Precedence, highest wins: + + request-time > config.yaml block > engine/pydantic default + +The tricky part is that the request contracts use ``model_fields_set`` to ask +"did the caller explicitly provide this?", and several call sites in +``routes/openai.py`` rely on that to decide whether a top-level request field +(e.g. ``request.language``) may override a value in the contract. If we built +the contract by dumping every field and re-passing it, everything would look +explicitly set and those guards would silently stop working. + +``build_config`` therefore only ever passes keys that were actually provided by +either layer, leaving pydantic to supply defaults for the rest. +""" +from __future__ import annotations + +from typing import Any, Dict, Mapping, Optional, Type, TypeVar + +from pydantic import BaseModel + +from src.server.schemas.modeling.config_blocks import ( + QWEN3_TTS_PREFIX, + SHARED_TTS_BLOCK, + block_applies_to, +) + +T = TypeVar("T", bound=BaseModel) + + +def deep_merge(base: Mapping[str, Any], overlay: Mapping[str, Any]) -> Dict[str, Any]: + """Merge ``overlay`` onto ``base``. + + Nested mappings are merged recursively. Scalars and lists are replaced + wholesale. ``None`` values in ``overlay`` are skipped so that an omitted + request field never clobbers a configured default. + """ + result: Dict[str, Any] = dict(base) + for key, value in overlay.items(): + if value is None: + continue + existing = result.get(key) + if isinstance(existing, Mapping) and isinstance(value, Mapping): + result[key] = deep_merge(existing, value) + else: + result[key] = value + return result + + +def resolve_fields( + request: Optional[Mapping[str, Any]] = None, + defaults: Optional[Mapping[str, Any]] = None, +) -> Dict[str, Any]: + """Resolve raw mappings into the field dict to construct a contract with. + + ``None`` values in ``request`` are dropped rather than treated as explicit + nulls, which is what makes "client omitted the field" fall through to the + ``defaults`` layer. + """ + clean_request = {k: v for k, v in (request or {}).items() if v is not None} + clean_defaults = {k: v for k, v in (defaults or {}).items() if v is not None} + return deep_merge(clean_defaults, clean_request) + + +def build_config( + contract: Type[T], + request: Optional[Mapping[str, Any]] = None, + defaults: Optional[Mapping[str, Any]] = None, + **extra: Any, +) -> T: + """Construct ``contract`` from a request layer over a defaults layer. + + Only keys supplied by one of the layers are passed to the constructor, so + ``model_fields_set`` on the result contains exactly the fields that were + genuinely provided. Pydantic fills in the contract's own defaults for + everything else. + + Args: + contract: The pydantic request-contract class to build. + request: Values supplied by the caller (highest precedence). ``None`` + values are ignored. + defaults: Values from the model's config.yaml block. ``None`` values are + ignored. + **extra: Keys to force, applied last (e.g. transport/mode fields the + route derives rather than the client supplying). + + Returns: + An instance of ``contract``. + """ + fields = resolve_fields(request=request, defaults=defaults) + fields.update(extra) + # Restrict to real contract fields so a stray config key can't blow up the + # constructor; unknown keys in a block are caught earlier by ModelLoadConfig. + allowed = {k: v for k, v in fields.items() if k in contract.model_fields} + return contract(**allowed) + + +def defaults_for_record(record: Any) -> Dict[str, Any]: + """Resolve the config.yaml defaults that apply to a loaded ModelRecord. + + Merges every block on the record that applies to its model_type. For + llm/vlm that is sampler_config; for a qwen3_tts_* mode it is the shared + qwen3_tts_config plus the mode's own block (mode-specific values win). + """ + blocks = getattr(record, "model_config_blocks", None) or {} + if not blocks: + return {} + + model_type = _model_type_value(record) + + applicable: Dict[str, Any] = {} + for block_name, payload in blocks.items(): + if not isinstance(payload, Mapping): + continue + if not block_applies_to(block_name, model_type): + continue + # Applied in dict order, so a later block overrides an earlier one. + # Merge the shared TTS block first so the mode block wins. + if block_name == SHARED_TTS_BLOCK: + continue + applicable = deep_merge(applicable, payload) + + if model_type.startswith(QWEN3_TTS_PREFIX): + shared = blocks.get(SHARED_TTS_BLOCK) + if isinstance(shared, Mapping): + applicable = deep_merge(shared, applicable) + + return applicable + + +def _model_type_value(record: Any) -> str: + """Return a ModelRecord's model_type as a plain string.""" + model_type = getattr(record, "model_type", None) + return model_type.value if hasattr(model_type, "value") else str(model_type or "") diff --git a/tests/unit/test_cli_add_unit.py b/tests/unit/test_cli_add_unit.py index 79c795b..9011691 100644 --- a/tests/unit/test_cli_add_unit.py +++ b/tests/unit/test_cli_add_unit.py @@ -1,4 +1,4 @@ -import json +import yaml from click.testing import CliRunner @@ -22,7 +22,7 @@ def test_add_help_omits_vlm_type_option() -> None: def test_add_does_not_save_vlm_type(tmp_path) -> None: - config_file = tmp_path / "openarc_config.json" + config_file = tmp_path / "config.yaml" model_dir = _model_dir(tmp_path) result = CliRunner().invoke( @@ -44,6 +44,69 @@ def test_add_does_not_save_vlm_type(tmp_path) -> None: ) assert result.exit_code == 0 - config = json.loads(config_file.read_text(encoding="utf-8")) - model_config = config["models"]["test-vlm"] - assert "vlm_type" not in model_config + config = yaml.safe_load(config_file.read_text(encoding="utf-8")) + entry = config["models"]["test-vlm"] + assert "load_config" in entry + assert "vlm_type" not in entry["load_config"] + + +def test_add_writes_nested_shape_with_sampler_config(tmp_path) -> None: + config_file = tmp_path / "config.yaml" + model_dir = _model_dir(tmp_path) + + result = CliRunner().invoke( + cli, + [ + "add", + "--model-name", + "test-llm", + "--model-path", + str(model_dir), + "--engine", + "ovgenai", + "--model-type", + "llm", + "--device", + "CPU", + "--sampler-config", + '{"temperature": 0.7, "top_k": 40}', + "--tool-call-parser", + "hermes", + ], + env={"OPENARC_CONFIG_FILE": str(config_file)}, + ) + + assert result.exit_code == 0 + entry = yaml.safe_load(config_file.read_text(encoding="utf-8"))["models"]["test-llm"] + # load fields nest under load_config; request defaults are siblings. + assert entry["load_config"]["tool_call_parser"] == "hermes" + assert entry["load_config"]["model_type"] == "llm" + assert entry["sampler_config"] == {"temperature": 0.7, "top_k": 40} + + +def test_add_rejects_invalid_sampler_config(tmp_path) -> None: + config_file = tmp_path / "config.yaml" + model_dir = _model_dir(tmp_path) + + result = CliRunner().invoke( + cli, + [ + "add", + "--model-name", + "bad", + "--model-path", + str(model_dir), + "--engine", + "ovgenai", + "--model-type", + "llm", + "--device", + "CPU", + "--sampler-config", + '{"temperature": "not-a-number"}', + ], + env={"OPENARC_CONFIG_FILE": str(config_file)}, + ) + + assert result.exit_code != 0 + assert not config_file.exists() diff --git a/tests/unit/test_config_merge_unit.py b/tests/unit/test_config_merge_unit.py new file mode 100644 index 0000000..247141d --- /dev/null +++ b/tests/unit/test_config_merge_unit.py @@ -0,0 +1,229 @@ +"""Precedence tests for the layered config merge. + + request-time > config.yaml block > engine/pydantic default + +Config blocks are plain dicts on ModelRecord, keyed by block name, validated +against the request contract they configure (see config_blocks.BLOCK_CONTRACTS). +""" +from typing import Any, Dict + +import pytest # type: ignore[import] + +from src.server.model_registry import ModelRecord +from src.server.schemas.modeling.config_blocks import validate_block +from src.server.schemas.modeling.contract_kokoro import OV_KokoroGenConfig +from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig +from src.server.schemas.modeling.contract_qwen3asr import OV_Qwen3ASRGenConfig +from src.server.schemas.registration import EngineType, ModelType +from src.server.utils.merge import build_config, deep_merge, defaults_for_record, resolve_fields + + +def _record( + model_type: ModelType, blocks: Dict[str, Dict[str, Any]] | None = None +) -> ModelRecord: + return ModelRecord( + model_name="m", + model_type=model_type, + engine=EngineType.OV_GENAI, + model_config_blocks=blocks or {}, + ) + + +# ---- deep_merge / resolve_fields ---- + + +def test_deep_merge_recurses_into_nested_dicts() -> None: + assert deep_merge({"a": {"x": 1, "y": 2}}, {"a": {"y": 9, "z": 3}}) == { + "a": {"x": 1, "y": 9, "z": 3} + } + + +def test_deep_merge_replaces_lists_wholesale() -> None: + assert deep_merge({"a": [1, 2, 3]}, {"a": [9]}) == {"a": [9]} + + +def test_resolve_fields_request_none_does_not_clobber_default() -> None: + assert resolve_fields(request={"temperature": None}, defaults={"temperature": 0.7}) == { + "temperature": 0.7 + } + + +def test_resolve_fields_request_wins_over_default() -> None: + assert resolve_fields(request={"temperature": 0.1}, defaults={"temperature": 0.7}) == { + "temperature": 0.1 + } + + +def test_resolve_fields_drops_none_defaults() -> None: + assert resolve_fields(request={}, defaults={"top_k": None, "top_p": 0.9}) == {"top_p": 0.9} + + +# ---- three-layer precedence ---- + + +def test_yaml_default_applies_when_request_omits_field() -> None: + config = build_config( + OVGenAI_GenConfig, + request={"temperature": 0.1}, + defaults={"temperature": 0.7, "top_k": 40}, + messages=[], + ) + assert config.temperature == 0.1 # request wins + assert config.top_k == 40 # yaml fills the rest + + +def test_engine_default_applies_when_neither_layer_sets_field() -> None: + config = build_config(OVGenAI_GenConfig, request={}, defaults={}, messages=[]) + assert config.top_k == 50 # OVGenAI_GenConfig default + + +def test_fields_set_contains_only_provided_fields() -> None: + """Purely-defaulted fields must stay out of model_fields_set. + + routes/openai.py gates its top-level overrides on this, so a field no layer + provided must not look explicitly set. + """ + config = build_config( + OVGenAI_GenConfig, request={"temperature": 0.3}, defaults={}, messages=[] + ) + assert "temperature" in config.model_fields_set + assert "seed" not in config.model_fields_set + + +def test_build_config_ignores_keys_not_on_contract() -> None: + config = build_config( + OVGenAI_GenConfig, + request={"not_a_field": 1}, + defaults={"also_not_a_field": 2}, + messages=[], + ) + assert not hasattr(config, "not_a_field") + + +# ---- block validation against contracts ---- + + +def test_validate_block_accepts_known_keys() -> None: + assert validate_block("sampler_config", {"temperature": 0.7}, "llm") == {"temperature": 0.7} + + +def test_validate_block_rejects_unknown_key() -> None: + with pytest.raises(ValueError, match="unknown key"): + validate_block("sampler_config", {"temperatur": 0.7}, "llm") + + +def test_validate_block_drops_none_values() -> None: + assert validate_block("sampler_config", {"temperature": None, "top_k": 5}, "llm") == { + "top_k": 5 + } + + +def test_validate_block_rejects_model_type_mismatch() -> None: + with pytest.raises(ValueError, match="requires model_type"): + validate_block("kokoro_config", {"voice": "af_sarah"}, "llm") + + +def test_validate_block_rejects_sampler_for_non_llm() -> None: + with pytest.raises(ValueError, match="sampler_config is only valid"): + validate_block("sampler_config", {"temperature": 0.7}, "kokoro") + + +def test_validate_block_rejects_unknown_block() -> None: + with pytest.raises(ValueError, match="Unknown config block"): + validate_block("bogus_config", {}, "llm") + + +def test_validate_block_shared_tts_applies_to_every_mode() -> None: + for mode in ( + "qwen3_tts_custom_voice", + "qwen3_tts_voice_design", + "qwen3_tts_voice_clone", + ): + assert validate_block("qwen3_tts_config", {"top_k": 5}, mode) == {"top_k": 5} + + +def test_validate_block_mode_block_rejected_for_other_mode() -> None: + with pytest.raises(ValueError, match="requires model_type"): + validate_block("qwen3_tts_voice_clone_config", {"ref_text": "x"}, "qwen3_tts_voice_design") + + +def test_validate_block_rejects_request_only_field() -> None: + # `messages` is not a model-level default; the contract defines it but a + # config author should not be able to set it. + with pytest.raises(ValueError, match="request-only field"): + validate_block("sampler_config", {"messages": [{"role": "user"}]}, "llm") + + +# ---- defaults_for_record ---- + + +def test_defaults_for_record_reads_sampler_for_llm() -> None: + record = _record(ModelType.LLM, blocks={"sampler_config": {"temperature": 0.7}}) + assert defaults_for_record(record) == {"temperature": 0.7} + + +def test_defaults_for_record_reads_block_for_kokoro() -> None: + record = _record(ModelType.KOKORO, blocks={"kokoro_config": {"voice": "af_sarah"}}) + assert defaults_for_record(record) == {"voice": "af_sarah"} + + +def test_defaults_for_record_ignores_mismatched_block() -> None: + record = _record(ModelType.LLM, blocks={"kokoro_config": {"voice": "af_sarah"}}) + assert defaults_for_record(record) == {} + + +def test_defaults_for_record_merges_shared_and_mode_tts_blocks() -> None: + record = _record( + ModelType.QWEN3_TTS_VOICE_CLONE, + blocks={ + "qwen3_tts_config": {"top_k": 5, "temperature": 0.9}, + "qwen3_tts_voice_clone_config": {"ref_text": "hi"}, + }, + ) + assert defaults_for_record(record) == {"top_k": 5, "temperature": 0.9, "ref_text": "hi"} + + +def test_mode_block_wins_over_shared_tts_block() -> None: + record = _record( + ModelType.QWEN3_TTS_VOICE_CLONE, + blocks={ + "qwen3_tts_config": {"top_k": 5}, + "qwen3_tts_voice_clone_config": {"top_k": 99}, + }, + ) + assert defaults_for_record(record)["top_k"] == 99 + + +def test_defaults_for_record_handles_empty_record() -> None: + assert defaults_for_record(_record(ModelType.LLM)) == {} + + +# ---- end-to-end per-contract precedence ---- + + +def test_kokoro_yaml_default_yields_to_request() -> None: + record = _record(ModelType.KOKORO, blocks={"kokoro_config": {"voice": "af_sarah", "speed": 1.0}}) + config = build_config( + OV_KokoroGenConfig, + request={"speed": 2.0}, + defaults=defaults_for_record(record), + input="hi", + ) + assert config.speed == 2.0 # request wins + assert config.voice.value == "af_sarah" # yaml default survives + assert config.character_count_chunk == 400 # engine default + + +def test_asr_yaml_default_yields_to_request() -> None: + record = _record( + ModelType.QWEN3_ASR, + blocks={"qwen3_asr_config": {"language": "English", "max_chunk_sec": 25.0}}, + ) + config = build_config( + OV_Qwen3ASRGenConfig, + request={"audio_base64": "AAA=", "language": "Chinese"}, + defaults=defaults_for_record(record), + ) + assert config.language == "Chinese" # request wins + assert config.max_chunk_sec == 25.0 # yaml default + assert config.max_tokens == 1024 # engine default diff --git a/tests/unit/test_openai_audio_transcriptions_unit.py b/tests/unit/test_openai_audio_transcriptions_unit.py index 5df32c3..883759f 100644 --- a/tests/unit/test_openai_audio_transcriptions_unit.py +++ b/tests/unit/test_openai_audio_transcriptions_unit.py @@ -29,7 +29,11 @@ def _call(monkeypatch: pytest.MonkeyPatch, response_format: str, result=None, op fake_registry = SimpleNamespace( _lock=asyncio.Lock(), _models={ - "qwen3": SimpleNamespace(model_name="qwen3-asr", model_type=ModelType.QWEN3_ASR), + "qwen3": SimpleNamespace( + model_name="qwen3-asr", + model_type=ModelType.QWEN3_ASR, + model_config_blocks={}, + ), }, ) transcribe_mock = AsyncMock(return_value=result) diff --git a/tests/unit/test_tool_call_parser_unit.py b/tests/unit/test_tool_call_parser_unit.py index cf861f6..5155654 100644 --- a/tests/unit/test_tool_call_parser_unit.py +++ b/tests/unit/test_tool_call_parser_unit.py @@ -8,6 +8,7 @@ import src.server.routes.openai as openai_routes from src.engine.ov_genai.tool_parse import gemma4, hermes, museglimmer, qwen35 +from src.server.schemas.registration import ModelType, ToolCallParser from src.server.schemas.requests_openai import OpenAIChatCompletionRequest from src.server.utils.chat import flatten_messages, normalize_tool_calls_for_template @@ -67,7 +68,12 @@ async def is_disconnected(self) -> bool: class _FakeRegistry: """Minimal registry stand-in: one record with a configurable parser.""" - def __init__(self, tool_call_parser: Optional[str]) -> None: + def __init__( + self, + tool_call_parser: Optional[str], + model_config_blocks: Optional[Dict[str, Dict[str, Any]]] = None, + model_type: str = "llm", + ) -> None: class _Lock: async def __aenter__(self) -> "_Lock": return self @@ -76,10 +82,16 @@ async def __aexit__(self, *exc: Any) -> bool: return False self._lock = _Lock() + # ModelRecord stores tool_call_parser as a ToolCallParser enum (that is + # what streamers.ensure_tool_call_parser expects); the fake mirrors it. self._models = { "fake-id": SimpleNamespace( model_name="demo-model", - tool_call_parser=tool_call_parser, + model_type=ModelType(model_type), + tool_call_parser=( + ToolCallParser(tool_call_parser) if tool_call_parser else None + ), + model_config_blocks=model_config_blocks or {}, ) } diff --git a/tests/unit/test_yaml_config_unit.py b/tests/unit/test_yaml_config_unit.py new file mode 100644 index 0000000..578d611 --- /dev/null +++ b/tests/unit/test_yaml_config_unit.py @@ -0,0 +1,221 @@ +"""YAML config loading, env interpolation, and strict validation.""" +from pathlib import Path + +import pytest # type: ignore[import] +import yaml +from pydantic import ValidationError + +from src.cli.modules.server_config import ServerConfig +from src.server.schemas.registration import ModelLoadConfig + + +def _cfg(tmp_path: Path) -> ServerConfig: + return ServerConfig(config_file=tmp_path / "config.yaml") + + +def _write(cfg: ServerConfig, payload: dict) -> None: + cfg.config_file.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + + +# ---- load / save ---- + + +def test_load_returns_empty_when_file_missing(tmp_path: Path) -> None: + assert _cfg(tmp_path).load_config() == {} + + +def test_load_parses_yaml(tmp_path: Path) -> None: + cfg = _cfg(tmp_path) + _write(cfg, {"server": {"port": 8000}, "models": {}}) + assert cfg.load_config()["server"]["port"] == 8000 + + +def test_load_returns_empty_on_invalid_yaml(tmp_path: Path) -> None: + cfg = _cfg(tmp_path) + cfg.config_file.write_text("models: [unclosed\n", encoding="utf-8") + assert cfg.load_config() == {} + + +def test_load_rejects_non_mapping_top_level(tmp_path: Path) -> None: + cfg = _cfg(tmp_path) + cfg.config_file.write_text("- just\n- a\n- list\n", encoding="utf-8") + assert cfg.load_config() == {} + + +def test_save_then_load_round_trips(tmp_path: Path) -> None: + cfg = _cfg(tmp_path) + payload = {"models": {"a": {"load_config": {"model_type": "llm", "device": "CPU"}}}} + cfg.save_config(payload) + assert cfg.load_config() == payload + + +def test_save_is_noop_when_unchanged(tmp_path: Path) -> None: + cfg = _cfg(tmp_path) + payload = {"models": {"a": {"load_config": {"device": "CPU"}}}} + cfg.save_config(payload) + first_mtime = cfg.config_file.stat().st_mtime_ns + cfg.save_config(payload) + assert cfg.config_file.stat().st_mtime_ns == first_mtime + + +# ---- env interpolation ---- + + +def test_interpolation_expands_set_variable(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setenv("MY_MODELS", "a,b") + cfg = _cfg(tmp_path) + _write(cfg, {"startup_models": "${MY_MODELS}"}) + assert cfg.load_config()["startup_models"] == "a,b" + + +def test_interpolation_uses_default_when_unset(tmp_path: Path, monkeypatch) -> None: + monkeypatch.delenv("MISSING_VAR", raising=False) + cfg = _cfg(tmp_path) + _write(cfg, {"startup_models": "${MISSING_VAR:-fallback}"}) + assert cfg.load_config()["startup_models"] == "fallback" + + +def test_interpolation_unset_without_default_becomes_empty(tmp_path: Path, monkeypatch) -> None: + monkeypatch.delenv("MISSING_VAR", raising=False) + cfg = _cfg(tmp_path) + _write(cfg, {"startup_models": "${MISSING_VAR}"}) + assert cfg.load_config()["startup_models"] == "" + + +def test_interpolation_recurses_into_nested_values(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setenv("DEV", "GPU.0") + cfg = _cfg(tmp_path) + _write(cfg, {"models": {"a": {"load_config": {"device": "${DEV}"}}}}) + assert cfg.load_config()["models"]["a"]["load_config"]["device"] == "GPU.0" + + +def test_interpolation_does_not_expand_keys(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setenv("KEY", "expanded") + cfg = _cfg(tmp_path) + _write(cfg, {"${KEY}": "value"}) + assert cfg.load_config() == {"${KEY}": "value"} + + +# ---- nested entry access ---- + + +def test_get_model_config_flattens_nested_entry(tmp_path: Path) -> None: + cfg = _cfg(tmp_path) + _write( + cfg, + { + "models": { + "a": { + "load_config": {"model_type": "llm", "device": "CPU"}, + "sampler_config": {"temperature": 0.7}, + } + } + }, + ) + flat = cfg.get_model_config("a") + assert flat["model_type"] == "llm" + assert flat["device"] == "CPU" + + +def test_get_model_config_still_accepts_flat_entry(tmp_path: Path) -> None: + cfg = _cfg(tmp_path) + _write(cfg, {"models": {"a": {"model_type": "llm", "device": "CPU"}}}) + assert cfg.get_model_config("a")["model_type"] == "llm" + + +def test_get_model_load_config_injects_name_from_key(tmp_path: Path) -> None: + cfg = _cfg(tmp_path) + _write(cfg, {"models": {"real-name": {"load_config": {"model_type": "llm", "engine": "ovgenai", "device": "CPU", "model_path": "/p"}}}}) + assert cfg.get_model_load_config("real-name").model_name == "real-name" + + +def test_get_model_load_config_returns_none_for_unknown(tmp_path: Path) -> None: + _write(_cfg(tmp_path), {"models": {}}) + assert _cfg(tmp_path).get_model_load_config("nope") is None + + +# ---- strict validation ---- + + +def test_unknown_key_is_rejected(tmp_path: Path) -> None: + cfg = _cfg(tmp_path) + _write( + cfg, + { + "models": { + "a": { + "load_config": { + "model_type": "llm", + "engine": "ovgenai", + "device": "CPU", + "model_path": "/p", + "bogus_key": 1, + } + } + } + }, + ) + with pytest.raises(ValueError, match="Invalid configuration"): + cfg.get_model_load_config("a") + + +def test_block_model_type_mismatch_is_rejected(tmp_path: Path) -> None: + cfg = _cfg(tmp_path) + _write( + cfg, + { + "models": { + "a": { + "load_config": { + "model_type": "qwen3_tts_voice_clone", + "engine": "openvino", + "device": "CPU", + "model_path": "/p", + }, + "qwen3_tts_custom_voice_config": {"speaker": "x"}, + } + } + }, + ) + with pytest.raises(ValueError, match="qwen3_tts_custom_voice_config requires"): + cfg.get_model_load_config("a") + + +def test_sampler_config_rejected_for_non_llm(tmp_path: Path) -> None: + cfg = _cfg(tmp_path) + _write( + cfg, + { + "models": { + "a": { + "load_config": { + "model_type": "kokoro", + "engine": "openvino", + "device": "CPU", + "model_path": "/p", + }, + "sampler_config": {"temperature": 0.7}, + } + } + }, + ) + with pytest.raises(ValueError, match="sampler_config is only valid"): + cfg.get_model_load_config("a") + + +def test_save_model_config_writes_nested_shape(tmp_path: Path) -> None: + cfg = _cfg(tmp_path) + cfg.save_model_config("a", {"model_type": "llm", "device": "CPU"}) + assert yaml.safe_load(cfg.config_file.read_text())["models"]["a"] == { + "load_config": {"model_type": "llm", "device": "CPU"} + } + + +def test_save_model_entry_preserves_sibling_blocks(tmp_path: Path) -> None: + cfg = _cfg(tmp_path) + cfg.save_model_entry( + "a", + {"load_config": {"model_type": "llm"}, "sampler_config": {"temperature": 0.7}}, + ) + entry = yaml.safe_load(cfg.config_file.read_text())["models"]["a"] + assert entry["sampler_config"] == {"temperature": 0.7} From 5d08e45228be4186420ab4e4738801c4e11d133f Mon Sep 17 00:00:00 2001 From: SearchSavior Date: Fri, 18 Sep 2026 01:17:14 -0400 Subject: [PATCH 2/5] rewrite openarc add to support the new yaml config keys --- src/cli/groups/add.py | 112 ++++++----- src/cli/modules/config_options.py | 308 ++++++++++++++++++++++++++++++ 2 files changed, 370 insertions(+), 50 deletions(-) create mode 100644 src/cli/modules/config_options.py diff --git a/src/cli/groups/add.py b/src/cli/groups/add.py index 42b9b55..34d7aa5 100644 --- a/src/cli/groups/add.py +++ b/src/cli/groups/add.py @@ -4,12 +4,19 @@ import json import click -from pydantic import ValidationError -from src.server.schemas.modeling.config_blocks import validate_block -from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import SchedulerConfigSchema +# rich_click's package namespace falls through to click for unknown names, so +# the globals module is imported directly. Mutating its dict in place is what +# RichHelpConfiguration.load_from_globals() reads. +from rich_click.rich_click import OPTION_GROUPS as RICH_CLICK_OPTION_GROUPS from ..main import cli, console +from ..modules.config_options import ( + ConfigOptionError, + config_options, + option_groups, + resolve_config_values, +) from ..utils import validate_model_path @@ -38,12 +45,6 @@ @click.option("--runtime-config", "--rtc", default=None, help='OpenVINO runtime configuration as JSON string (e.g., \'{"MODEL_DISTRIBUTION_POLICY": "PIPELINE_PARALLEL"}\').') -@click.option("--scheduler-config", "-sc", - default=None, - help='OpenVINO runtime scheduler configuration as JSON string (e.g., \'{"use_sparse_attention": true}\').') -@click.option("--sampler-config", "--smc", - default=None, - help='Default sampler settings for llm/vlm models as JSON string (e.g., \'{"temperature": 0.7, "top_k": 40}\'). Overridden per request.') @click.option('--cache-dir', '--cd', required=False, default=None, @@ -71,9 +72,20 @@ required=False, default=None, help='Tool-call output format for this model (qwen35 XML, hermes JSON, gemma4 call syntax, or museglimmer Harmony atem). llm/vlm only; required for tool calling.') +@config_options @click.pass_context -def add(ctx, model_path, model_name, engine, model_type, device, runtime_config, scheduler_config, sampler_config, cache_dir, draft_model_path, draft_device, num_assistant_tokens, assistant_confidence_threshold, tool_call_parser): - """- Add a model configuration to the config file.""" +def add(ctx, model_path, model_name, engine, model_type, device, runtime_config, cache_dir, draft_model_path, draft_device, num_assistant_tokens, assistant_confidence_threshold, tool_call_parser, **config_values): + """- Add a model configuration to the config file. + + \b + Model defaults (--temperature, --max-tokens, --max-num-seqs, ...) are + generated from the pydantic contracts in src/server/schemas/modeling, so + they never drift from the request schema. Each help panel below is one + config.yaml key. A flag is written to the key that backs it for the chosen + --model-type, and a flag that does not apply to that model type is + rejected. Only flags you actually pass are written, so everything else + keeps using the contract's own default. + """ # Validate model path if not validate_model_path(model_path): @@ -93,43 +105,16 @@ def add(ctx, model_path, model_name, engine, model_type, device, runtime_config, console.print(f"[red]Error parsing runtime_config JSON:[/red] {e}") console.print('[yellow]Example format: \'{"MODEL_DISTRIBUTION_POLICY": "PIPELINE_PARALLEL"}\'[/yellow]') ctx.exit(1) - parsed_scheduler_config = {} - if scheduler_config: - # Let the model validate the JSON itself. If it validates, assume we can safely load the JSON. - try: - parsed_scheduler_config = json.loads(scheduler_config) - if not isinstance(parsed_scheduler_config, dict): - console.print(f"[red]Error: scheduler_config must be a JSON object (dictionary), got {type(scheduler_config).__name__}[/red]") - console.print('[yellow]Example format: \'{"max_num_batched_tokens": 256, "enable_prefix_caching": true}\'[/yellow]') - SchedulerConfigSchema.model_validate_json(scheduler_config) - except ValidationError as e: - console.print("[red]Error: Failed validating scheduler_config:[/red]") - console.print('[yellow]Example format: \'{"max_num_batched_tokens": 256, "enable_prefix_caching": true}\'[/yellow]') - console.print('') - console.print('[yellow]Error:[/yellow]') - console.print(e) - ctx.exit(1) - - # Validate against the contract that backs the block, so an invalid default - # is caught here rather than at model load time. - parsed_sampler_config = {} - if sampler_config: - try: - parsed_sampler_config = json.loads(sampler_config) - if not isinstance(parsed_sampler_config, dict): - console.print(f"[red]Error: sampler_config must be a JSON object (dictionary), got {type(parsed_sampler_config).__name__}[/red]") - console.print('[yellow]Example format: \'{"temperature": 0.7, "top_k": 40}\'[/yellow]') - ctx.exit(1) - validate_block("sampler_config", parsed_sampler_config, model_type) - except (json.JSONDecodeError, ValidationError) as e: - console.print("[red]Error: Failed validating sampler_config:[/red]") - console.print('[yellow]Example format: \'{"temperature": 0.7, "top_k": 40}\'[/yellow]') - console.print('') - console.print('[yellow]Error:[/yellow]') - console.print(e) - ctx.exit(1) + # Route the contract-flagged values into their blocks. Validation happens + # against the contracts themselves, so a bad value is caught here rather + # than at model load time. + try: + scheduler_config, blocks = resolve_config_values(model_type, config_values) + except ConfigOptionError as e: + console.print("[red]Error: invalid model configuration options:[/red]") + console.print(e) + ctx.exit(1) - # Model-level request defaults are stored as siblings of load_config. entry = { "load_config": { "model_name": model_name, @@ -138,11 +123,14 @@ def add(ctx, model_path, model_name, engine, model_type, device, runtime_config, "engine": engine, "device": device, "runtime_config": parsed_runtime_config, - "scheduler_config": parsed_scheduler_config, } } - if parsed_sampler_config: - entry["sampler_config"] = parsed_sampler_config + if scheduler_config: + entry["load_config"]["scheduler_config"] = scheduler_config + + # Model-level request defaults are stored as siblings of load_config. + for block_name, payload in blocks.items(): + entry[block_name] = payload # Store the cache directory (resolved relative to the config file at load time) if cache_dir: @@ -166,3 +154,27 @@ def add(ctx, model_path, model_name, engine, model_type, device, runtime_config, ctx.obj.server_config.save_model_entry(model_name, entry) console.print(f"[green]Model configuration saved:[/green] {model_name}") console.print(f"[dim]Use 'openarc load {model_name}' to load this model.[/dim]") + + +# Hand-declared load_config flags, plus click's own --help. Everything else in +# the help comes from the contracts via option_groups(). +_LOAD_OPTIONS = [ + "help", + "model_name", + "model_path", + "engine", + "model_type", + "device", + "runtime_config", + "cache_dir", + "draft_model_path", + "draft_device", + "num_assistant_tokens", + "assistant_confidence_threshold", + "tool_call_parser", +] + +# One help panel per config.yaml key, keyed by the command path rich_click +# matches against. This mutates rich_click's own groups, so there is no custom +# panel code to maintain. +RICH_CLICK_OPTION_GROUPS["*add"] = option_groups(_LOAD_OPTIONS) diff --git a/src/cli/modules/config_options.py b/src/cli/modules/config_options.py new file mode 100644 index 0000000..797b319 --- /dev/null +++ b/src/cli/modules/config_options.py @@ -0,0 +1,308 @@ +"""CLI flags generated from the pydantic request contracts. + +``openarc add`` must not keep its own copy of the model-default fields: the +contracts under ``src/server/schemas/modeling/`` are the single definition of +what a request - and therefore a model default - may contain. This module +introspects those contracts and builds one ``click`` option per field, so +adding a field to a contract makes it available on ``openarc add`` with no +separate edit. + +The same field name can appear in more than one contract (``temperature`` backs +both ``sampler_config`` for llm/vlm and ``qwen3_tts_config`` for Qwen3-TTS). +Because ``--model-type`` is not known until after parsing, one flag is declared +per field name and the destination is resolved after parse: the flag lands in +whichever block the chosen model_type uses. A flag with no home for that +model_type is rejected rather than silently dropped. + +``runtime_config`` is deliberately absent: it is a free-form dict of OpenVINO +properties with no contract to generate flags from, and stays a JSON string. + +For --help, ``option_groups()`` produces rich_click's own OPTION_GROUPS layout, +so each config.yaml key becomes one help panel. +""" +from __future__ import annotations + +import enum +import re +import types +from typing import Any, Dict, List, Optional, Tuple, Type, Union, get_args, get_origin + +import click +from pydantic import BaseModel, ValidationError + +from src.server.schemas.modeling.config_blocks import ( + BLOCK_CONTRACTS, + QWEN3_TTS_PREFIX, + REQUEST_ONLY_FIELDS, + SAMPLER_BLOCK, + SAMPLER_MODEL_TYPES, + SHARED_TTS_BLOCK, + model_block_name, + validate_block, +) +from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import ( + SchedulerConfigSchema, +) + +# scheduler_config is a load-time field (a sibling of the config blocks in +# config.yaml, and a field on ModelLoadConfig), not a request-default block. +SCHEDULER_DEST = "scheduler_config" +LOAD_CONFIG_DEST = "load_config" + +# Contract fields `openarc add` already exposes as dedicated load_config flags. +# They exist in OVGenAI_GenConfig as request-time overrides, but at add time +# speculative decoding is configured on load_config (ModelLoadConfig), so they +# are not re-declared as block flags and cannot land in sampler_config. +LOAD_CONFIG_FIELDS = frozenset( + {"num_assistant_tokens", "assistant_confidence_threshold"} +) + +# Destination -> contract. +_DEST_CONTRACTS: Dict[str, Type[BaseModel]] = { + name: contract for name, (contract, _) in BLOCK_CONTRACTS.items() +} +_DEST_CONTRACTS[SCHEDULER_DEST] = SchedulerConfigSchema + +# --help/registry order: the request-default blocks grouped by model family, +# then scheduler. +_GROUP_ORDER: List[str] = [ + SAMPLER_BLOCK, + SHARED_TTS_BLOCK, + "qwen3_tts_custom_voice_config", + "qwen3_tts_voice_design_config", + "qwen3_tts_voice_clone_config", + "kokoro_config", + "qwen3_asr_config", + SCHEDULER_DEST, +] + +_WHITESPACE = re.compile(r"\s+") + + +class ConfigOptionError(ValueError): + """A provided flag is invalid, or does not apply to the chosen model_type.""" + + +def kebab(field: str) -> str: + return field.replace("_", "-") + + +def _scalar(annotation: Any) -> Optional[Any]: + """Reduce an annotation to the bool/int/float/str/Enum it carries. + + ``Optional[X]`` and ``X | None`` unwrap to ``X``. Returns None when the + field is not expressible as a single CLI flag. + """ + origin = get_origin(annotation) + if origin is Union or origin is types.UnionType: + args = [arg for arg in get_args(annotation) if arg is not type(None)] + return _scalar(args[0]) if len(args) == 1 else None + if isinstance(annotation, type) and issubclass(annotation, enum.Enum): + return annotation + if annotation in (bool, int, float, str): + return annotation + return None + + +def _help_text(field: str, info: Any) -> str: + description = _WHITESPACE.sub(" ", info.description or "").strip() + return description or field + + +def exposed_fields(dest: str) -> List[str]: + """Fields of a destination that become CLI flags, in contract order.""" + return [ + name + for name in _DEST_CONTRACTS[dest].model_fields + if name not in REQUEST_ONLY_FIELDS and name not in LOAD_CONFIG_FIELDS + ] + + +def field_registry() -> Dict[str, Tuple[Any, str]]: + """Map every CLI-exposed field to (scalar type, help text). + + Fields are deduped by name in group order, so a name shared by two + contracts takes the first contract's type and description. + """ + registry: Dict[str, Tuple[Any, str]] = {} + for dest in _GROUP_ORDER: + contract = _DEST_CONTRACTS[dest] + for name in exposed_fields(dest): + if name in registry: + continue + scalar = _scalar(contract.model_fields[name].annotation) + if scalar is None: + raise ConfigOptionError( + f"Cannot expose contract field '{dest}.{name}' " + f"({contract.model_fields[name].annotation!r}) as a CLI flag. " + f"Teach config_options.py to render it or exclude it via " + f"REQUEST_ONLY_FIELDS." + ) + registry[name] = (scalar, _help_text(name, contract.model_fields[name])) + return registry + + +def build_config_options() -> List[click.Option]: + """Build one click option per field defined by the contracts.""" + options: List[click.Option] = [] + for name, (scalar, help_text) in field_registry().items(): + kebab_name = kebab(name) + if scalar is bool: + params = [f"--{kebab_name}/--no-{kebab_name}"] + attrs: Dict[str, Any] = {"default": None, "help": help_text} + else: + params = [f"--{kebab_name}"] + attrs = {"default": None, "help": help_text} + if isinstance(scalar, type) and issubclass(scalar, enum.Enum): + attrs["type"] = click.Choice([member.value for member in scalar]) + else: + attrs["type"] = scalar + options.append(click.Option(params, **attrs)) + return options + + +def config_options(command): + """Decorator attaching every contract-derived flag to a command. + + Params are appended in reverse: click reverses ``__click_params__`` when it + builds the Command, so this is what lands them in registry order. + """ + params = getattr(command, "__click_params__", None) + if params is None: + params = [] + command.__click_params__ = params + params.extend(reversed(build_config_options())) + return command + + +def _group_label(dest: str) -> str: + """Human label for a help group, naming the model types the key serves.""" + if dest == SAMPLER_BLOCK: + return f"{dest} · {', '.join(sorted(SAMPLER_MODEL_TYPES))}" + if dest == SHARED_TTS_BLOCK: + return f"{dest} · {QWEN3_TTS_PREFIX}*" + if dest == SCHEDULER_DEST: + return f"{dest} · all model types, written to load_config" + required = BLOCK_CONTRACTS[dest][1] + return f"{dest} · {required}" if required else dest + + +def option_groups(load_options: List[str]) -> List[Dict[str, Any]]: + """Build rich_click's OPTION_GROUPS value for `openarc add`. + + One group per config.yaml key, each listing every setting the key accepts, + so a field shared by two keys appears in both. ``deduplicate: False`` is + what lets rich_click render the same option in more than one panel. + + Args: + load_options: Parameter names of the hand-declared load_config flags, + plus click's own ``help``. + """ + groups: List[Dict[str, Any]] = [ + {"name": LOAD_CONFIG_DEST, "options": list(load_options)} + ] + shared_tts = set(exposed_fields(SHARED_TTS_BLOCK)) + for dest in _GROUP_ORDER: + fields = exposed_fields(dest) + if dest != SHARED_TTS_BLOCK and dest.startswith(QWEN3_TTS_PREFIX): + # A Qwen3-TTS mode contract subclasses the shared TTS contract, so + # it carries every shared field. The shared panel already lists + # them; this panel shows only the mode's extras. + fields = [name for name in fields if name not in shared_tts] + groups.append( + { + "name": _group_label(dest), + "options": fields, + "deduplicate": False, + } + ) + return groups + + +def allowed_destinations(model_type: str) -> List[str]: + """Destination blocks a model_type may be configured with, in claim order. + + A Qwen3-TTS mode contract subclasses the shared TTS contract, so the shared + block claims the fields they have in common and the mode block keeps only + its extras. + """ + destinations: List[str] = [] + if model_type in SAMPLER_MODEL_TYPES: + destinations.append(SAMPLER_BLOCK) + if model_type.startswith(QWEN3_TTS_PREFIX): + destinations.append(SHARED_TTS_BLOCK) + mode_block = model_block_name(model_type) + if mode_block: + destinations.append(mode_block) + destinations.append(SCHEDULER_DEST) + return destinations + + +def _destinations_by_field(model_type: str) -> Dict[str, str]: + mapping: Dict[str, str] = {} + for dest in allowed_destinations(model_type): + for name in exposed_fields(dest): + mapping.setdefault(name, dest) + return mapping + + +def _plain(value: Any) -> Any: + """Reduce enum members to their serializable values.""" + return value.value if isinstance(value, enum.Enum) else value + + +def _validated_scheduler(payload: Dict[str, Any]) -> Dict[str, Any]: + if not payload: + return {} + try: + validated = SchedulerConfigSchema(**payload) + except ValidationError as exc: + raise ConfigOptionError(f"scheduler_config has invalid values: {exc}") from exc + return {key: _plain(getattr(validated, key)) for key in payload} + + +def resolve_config_values( + model_type: str, values: Dict[str, Any] +) -> Tuple[Dict[str, Any], Dict[str, Dict[str, Any]]]: + """Route provided flag values to the config blocks they belong to. + + Args: + model_type: The ``--model-type`` chosen by the operator. + values: Raw values keyed by field name; None means "not provided". + + Returns: + ``(scheduler_config, blocks)`` where scheduler_config is written under + ``load_config`` and blocks map block name -> validated authored keys. + + Raises: + ConfigOptionError: If a flag does not apply to model_type, or a value + fails contract validation. + """ + mapping = _destinations_by_field(model_type) + provided = {name: value for name, value in values.items() if value is not None} + + unknown = sorted(set(provided) - set(mapping)) + if unknown: + flags = ", ".join(f"--{kebab(name)}" for name in unknown) + valid = ", ".join(f"--{kebab(name)}" for name in sorted(mapping)) + raise ConfigOptionError( + f"Option(s) {flags} do not apply to model_type '{model_type}'. " + f"Valid config options for this model type: {valid or 'none'}." + ) + + payloads: Dict[str, Dict[str, Any]] = {} + for name, value in provided.items(): + payloads.setdefault(mapping[name], {})[name] = value + + scheduler = _validated_scheduler(payloads.pop(SCHEDULER_DEST, {})) + + blocks: Dict[str, Dict[str, Any]] = {} + for dest, payload in payloads.items(): + try: + resolved = validate_block(dest, payload, model_type) + except ValueError as exc: + raise ConfigOptionError(str(exc)) from exc + if resolved: + blocks[dest] = {key: _plain(value) for key, value in resolved.items()} + + return scheduler, blocks \ No newline at end of file From 6386f75f32327810f912079a4e74821778e815f3 Mon Sep 17 00:00:00 2001 From: SearchSavior Date: Fri, 18 Sep 2026 20:11:39 -0400 Subject: [PATCH 3/5] deprecate openarc add --- src/cli/groups/add.py | 180 ----------------- src/cli/main.py | 2 +- src/cli/modules/config_options.py | 308 ------------------------------ tests/unit/test_cli_add_unit.py | 112 ----------- 4 files changed, 1 insertion(+), 601 deletions(-) delete mode 100644 src/cli/groups/add.py delete mode 100644 src/cli/modules/config_options.py delete mode 100644 tests/unit/test_cli_add_unit.py diff --git a/src/cli/groups/add.py b/src/cli/groups/add.py deleted file mode 100644 index 34d7aa5..0000000 --- a/src/cli/groups/add.py +++ /dev/null @@ -1,180 +0,0 @@ -""" -Add command - Add a model configuration to the config file. -""" -import json - -import click - -# rich_click's package namespace falls through to click for unknown names, so -# the globals module is imported directly. Mutating its dict in place is what -# RichHelpConfiguration.load_from_globals() reads. -from rich_click.rich_click import OPTION_GROUPS as RICH_CLICK_OPTION_GROUPS - -from ..main import cli, console -from ..modules.config_options import ( - ConfigOptionError, - config_options, - option_groups, - resolve_config_values, -) -from ..utils import validate_model_path - - -@cli.command() -@click.option('--model-name', '--mn', - required=True, - help='Public facing name of the model.') -@click.option('--model-path', '--m', - required=True, - help='Path to OpenVINO IR converted model.') -@click.option('--engine', '--en', - type=click.Choice(['ovgenai', 'openvino', 'optimum']), - required=True, - help='Engine used to load the model (ovgenai, openvino, optimum)') -@click.option('--model-type', '--mt', - type=click.Choice([ - 'llm', 'vlm', 'whisper', 'qwen3_asr', 'kokoro', - 'qwen3_tts_custom_voice', 'qwen3_tts_voice_design', 'qwen3_tts_voice_clone', - 'emb', 'rerank', - ]), - required=True, - help='Model type (llm, vlm, whisper, qwen3_asr, kokoro, qwen3_tts_custom_voice, qwen3_tts_voice_design, qwen3_tts_voice_clone, emb, rerank)') -@click.option('--device', '--d', - required=True, - help='Device(s) to load the model on.') -@click.option("--runtime-config", "--rtc", - default=None, - help='OpenVINO runtime configuration as JSON string (e.g., \'{"MODEL_DISTRIBUTION_POLICY": "PIPELINE_PARALLEL"}\').') -@click.option('--cache-dir', '--cd', - required=False, - default=None, - help='Directory for the OpenVINO model cache. Caching compiled model blobs here speeds up subsequent loads of this model. Relative paths are resolved against the config file, like --model-path.') -@click.option('--draft-model-path', '--dmp', - required=False, - default=None, - help='Path to draft model for speculative decoding.') -@click.option('--draft-device', '--dd', - required=False, - default=None, - help='Draft model device.') -@click.option('--num-assistant-tokens', '--nat', - required=False, - default=None, - type=int, - help='Number of tokens draft model generates per step.') -@click.option('--assistant-confidence-threshold', '--act', - required=False, - default=None, - type=float, - help='Confidence threshold for accepting draft tokens.') -@click.option('--tool-call-parser', - type=click.Choice(['qwen35', 'hermes', 'gemma4', 'museglimmer']), - required=False, - default=None, - help='Tool-call output format for this model (qwen35 XML, hermes JSON, gemma4 call syntax, or museglimmer Harmony atem). llm/vlm only; required for tool calling.') -@config_options -@click.pass_context -def add(ctx, model_path, model_name, engine, model_type, device, runtime_config, cache_dir, draft_model_path, draft_device, num_assistant_tokens, assistant_confidence_threshold, tool_call_parser, **config_values): - """- Add a model configuration to the config file. - - \b - Model defaults (--temperature, --max-tokens, --max-num-seqs, ...) are - generated from the pydantic contracts in src/server/schemas/modeling, so - they never drift from the request schema. Each help panel below is one - config.yaml key. A flag is written to the key that backs it for the chosen - --model-type, and a flag that does not apply to that model type is - rejected. Only flags you actually pass are written, so everything else - keeps using the contract's own default. - """ - - # Validate model path - if not validate_model_path(model_path): - console.print(f"[red]Model file check failed! {model_path} does not contain openvino model files OR your chosen path is malformed. Verify chosen path is correct and acquired model files match source on the hub, or the destination of converted model.[/red]") - ctx.exit(1) - - # Parse runtime_config if provided - parsed_runtime_config = {} - if runtime_config: - try: - parsed_runtime_config = json.loads(runtime_config) - if not isinstance(parsed_runtime_config, dict): - console.print(f"[red]Error: runtime_config must be a JSON object (dictionary), got {type(parsed_runtime_config).__name__}[/red]") - console.print('[yellow]Example format: \'{"MODEL_DISTRIBUTION_POLICY": "PIPELINE_PARALLEL"}\'[/yellow]') - ctx.exit(1) - except json.JSONDecodeError as e: - console.print(f"[red]Error parsing runtime_config JSON:[/red] {e}") - console.print('[yellow]Example format: \'{"MODEL_DISTRIBUTION_POLICY": "PIPELINE_PARALLEL"}\'[/yellow]') - ctx.exit(1) - # Route the contract-flagged values into their blocks. Validation happens - # against the contracts themselves, so a bad value is caught here rather - # than at model load time. - try: - scheduler_config, blocks = resolve_config_values(model_type, config_values) - except ConfigOptionError as e: - console.print("[red]Error: invalid model configuration options:[/red]") - console.print(e) - ctx.exit(1) - - entry = { - "load_config": { - "model_name": model_name, - "model_path": model_path, - "model_type": model_type, - "engine": engine, - "device": device, - "runtime_config": parsed_runtime_config, - } - } - if scheduler_config: - entry["load_config"]["scheduler_config"] = scheduler_config - - # Model-level request defaults are stored as siblings of load_config. - for block_name, payload in blocks.items(): - entry[block_name] = payload - - # Store the cache directory (resolved relative to the config file at load time) - if cache_dir: - entry["load_config"]["cache_dir"] = cache_dir - - # Add speculative decoding options if provided - if draft_model_path: - if not validate_model_path(draft_model_path): - console.print(f"[red]Model file check failed! {draft_model_path} does not contain openvino model files OR your chosen path is malformed. Verify chosen path is correct and acquired model files match source on the hub, or the destination of converted model.[/red]") - ctx.exit(1) - entry["load_config"]["draft_model_path"] = draft_model_path - if draft_device: - entry["load_config"]["draft_device"] = draft_device - if num_assistant_tokens is not None: - entry["load_config"]["num_assistant_tokens"] = num_assistant_tokens - if assistant_confidence_threshold is not None: - entry["load_config"]["assistant_confidence_threshold"] = assistant_confidence_threshold - if tool_call_parser: - entry["load_config"]["tool_call_parser"] = tool_call_parser - - ctx.obj.server_config.save_model_entry(model_name, entry) - console.print(f"[green]Model configuration saved:[/green] {model_name}") - console.print(f"[dim]Use 'openarc load {model_name}' to load this model.[/dim]") - - -# Hand-declared load_config flags, plus click's own --help. Everything else in -# the help comes from the contracts via option_groups(). -_LOAD_OPTIONS = [ - "help", - "model_name", - "model_path", - "engine", - "model_type", - "device", - "runtime_config", - "cache_dir", - "draft_model_path", - "draft_device", - "num_assistant_tokens", - "assistant_confidence_threshold", - "tool_call_parser", -] - -# One help panel per config.yaml key, keyed by the command path rich_click -# matches against. This mutates rich_click's own groups, so there is no custom -# panel code to maintain. -RICH_CLICK_OPTION_GROUPS["*add"] = option_groups(_LOAD_OPTIONS) diff --git a/src/cli/main.py b/src/cli/main.py index d4afc01..e9a4a78 100644 --- a/src/cli/main.py +++ b/src/cli/main.py @@ -126,7 +126,7 @@ def cli(ctx): # Import command groups to register them with the CLI -from .groups import add, bench, list, load, serve, status, tool, unload # noqa: E402, F401 +from .groups import bench, list, load, serve, status, tool, unload # noqa: E402, F401 if __name__ == "__main__": diff --git a/src/cli/modules/config_options.py b/src/cli/modules/config_options.py deleted file mode 100644 index 797b319..0000000 --- a/src/cli/modules/config_options.py +++ /dev/null @@ -1,308 +0,0 @@ -"""CLI flags generated from the pydantic request contracts. - -``openarc add`` must not keep its own copy of the model-default fields: the -contracts under ``src/server/schemas/modeling/`` are the single definition of -what a request - and therefore a model default - may contain. This module -introspects those contracts and builds one ``click`` option per field, so -adding a field to a contract makes it available on ``openarc add`` with no -separate edit. - -The same field name can appear in more than one contract (``temperature`` backs -both ``sampler_config`` for llm/vlm and ``qwen3_tts_config`` for Qwen3-TTS). -Because ``--model-type`` is not known until after parsing, one flag is declared -per field name and the destination is resolved after parse: the flag lands in -whichever block the chosen model_type uses. A flag with no home for that -model_type is rejected rather than silently dropped. - -``runtime_config`` is deliberately absent: it is a free-form dict of OpenVINO -properties with no contract to generate flags from, and stays a JSON string. - -For --help, ``option_groups()`` produces rich_click's own OPTION_GROUPS layout, -so each config.yaml key becomes one help panel. -""" -from __future__ import annotations - -import enum -import re -import types -from typing import Any, Dict, List, Optional, Tuple, Type, Union, get_args, get_origin - -import click -from pydantic import BaseModel, ValidationError - -from src.server.schemas.modeling.config_blocks import ( - BLOCK_CONTRACTS, - QWEN3_TTS_PREFIX, - REQUEST_ONLY_FIELDS, - SAMPLER_BLOCK, - SAMPLER_MODEL_TYPES, - SHARED_TTS_BLOCK, - model_block_name, - validate_block, -) -from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import ( - SchedulerConfigSchema, -) - -# scheduler_config is a load-time field (a sibling of the config blocks in -# config.yaml, and a field on ModelLoadConfig), not a request-default block. -SCHEDULER_DEST = "scheduler_config" -LOAD_CONFIG_DEST = "load_config" - -# Contract fields `openarc add` already exposes as dedicated load_config flags. -# They exist in OVGenAI_GenConfig as request-time overrides, but at add time -# speculative decoding is configured on load_config (ModelLoadConfig), so they -# are not re-declared as block flags and cannot land in sampler_config. -LOAD_CONFIG_FIELDS = frozenset( - {"num_assistant_tokens", "assistant_confidence_threshold"} -) - -# Destination -> contract. -_DEST_CONTRACTS: Dict[str, Type[BaseModel]] = { - name: contract for name, (contract, _) in BLOCK_CONTRACTS.items() -} -_DEST_CONTRACTS[SCHEDULER_DEST] = SchedulerConfigSchema - -# --help/registry order: the request-default blocks grouped by model family, -# then scheduler. -_GROUP_ORDER: List[str] = [ - SAMPLER_BLOCK, - SHARED_TTS_BLOCK, - "qwen3_tts_custom_voice_config", - "qwen3_tts_voice_design_config", - "qwen3_tts_voice_clone_config", - "kokoro_config", - "qwen3_asr_config", - SCHEDULER_DEST, -] - -_WHITESPACE = re.compile(r"\s+") - - -class ConfigOptionError(ValueError): - """A provided flag is invalid, or does not apply to the chosen model_type.""" - - -def kebab(field: str) -> str: - return field.replace("_", "-") - - -def _scalar(annotation: Any) -> Optional[Any]: - """Reduce an annotation to the bool/int/float/str/Enum it carries. - - ``Optional[X]`` and ``X | None`` unwrap to ``X``. Returns None when the - field is not expressible as a single CLI flag. - """ - origin = get_origin(annotation) - if origin is Union or origin is types.UnionType: - args = [arg for arg in get_args(annotation) if arg is not type(None)] - return _scalar(args[0]) if len(args) == 1 else None - if isinstance(annotation, type) and issubclass(annotation, enum.Enum): - return annotation - if annotation in (bool, int, float, str): - return annotation - return None - - -def _help_text(field: str, info: Any) -> str: - description = _WHITESPACE.sub(" ", info.description or "").strip() - return description or field - - -def exposed_fields(dest: str) -> List[str]: - """Fields of a destination that become CLI flags, in contract order.""" - return [ - name - for name in _DEST_CONTRACTS[dest].model_fields - if name not in REQUEST_ONLY_FIELDS and name not in LOAD_CONFIG_FIELDS - ] - - -def field_registry() -> Dict[str, Tuple[Any, str]]: - """Map every CLI-exposed field to (scalar type, help text). - - Fields are deduped by name in group order, so a name shared by two - contracts takes the first contract's type and description. - """ - registry: Dict[str, Tuple[Any, str]] = {} - for dest in _GROUP_ORDER: - contract = _DEST_CONTRACTS[dest] - for name in exposed_fields(dest): - if name in registry: - continue - scalar = _scalar(contract.model_fields[name].annotation) - if scalar is None: - raise ConfigOptionError( - f"Cannot expose contract field '{dest}.{name}' " - f"({contract.model_fields[name].annotation!r}) as a CLI flag. " - f"Teach config_options.py to render it or exclude it via " - f"REQUEST_ONLY_FIELDS." - ) - registry[name] = (scalar, _help_text(name, contract.model_fields[name])) - return registry - - -def build_config_options() -> List[click.Option]: - """Build one click option per field defined by the contracts.""" - options: List[click.Option] = [] - for name, (scalar, help_text) in field_registry().items(): - kebab_name = kebab(name) - if scalar is bool: - params = [f"--{kebab_name}/--no-{kebab_name}"] - attrs: Dict[str, Any] = {"default": None, "help": help_text} - else: - params = [f"--{kebab_name}"] - attrs = {"default": None, "help": help_text} - if isinstance(scalar, type) and issubclass(scalar, enum.Enum): - attrs["type"] = click.Choice([member.value for member in scalar]) - else: - attrs["type"] = scalar - options.append(click.Option(params, **attrs)) - return options - - -def config_options(command): - """Decorator attaching every contract-derived flag to a command. - - Params are appended in reverse: click reverses ``__click_params__`` when it - builds the Command, so this is what lands them in registry order. - """ - params = getattr(command, "__click_params__", None) - if params is None: - params = [] - command.__click_params__ = params - params.extend(reversed(build_config_options())) - return command - - -def _group_label(dest: str) -> str: - """Human label for a help group, naming the model types the key serves.""" - if dest == SAMPLER_BLOCK: - return f"{dest} · {', '.join(sorted(SAMPLER_MODEL_TYPES))}" - if dest == SHARED_TTS_BLOCK: - return f"{dest} · {QWEN3_TTS_PREFIX}*" - if dest == SCHEDULER_DEST: - return f"{dest} · all model types, written to load_config" - required = BLOCK_CONTRACTS[dest][1] - return f"{dest} · {required}" if required else dest - - -def option_groups(load_options: List[str]) -> List[Dict[str, Any]]: - """Build rich_click's OPTION_GROUPS value for `openarc add`. - - One group per config.yaml key, each listing every setting the key accepts, - so a field shared by two keys appears in both. ``deduplicate: False`` is - what lets rich_click render the same option in more than one panel. - - Args: - load_options: Parameter names of the hand-declared load_config flags, - plus click's own ``help``. - """ - groups: List[Dict[str, Any]] = [ - {"name": LOAD_CONFIG_DEST, "options": list(load_options)} - ] - shared_tts = set(exposed_fields(SHARED_TTS_BLOCK)) - for dest in _GROUP_ORDER: - fields = exposed_fields(dest) - if dest != SHARED_TTS_BLOCK and dest.startswith(QWEN3_TTS_PREFIX): - # A Qwen3-TTS mode contract subclasses the shared TTS contract, so - # it carries every shared field. The shared panel already lists - # them; this panel shows only the mode's extras. - fields = [name for name in fields if name not in shared_tts] - groups.append( - { - "name": _group_label(dest), - "options": fields, - "deduplicate": False, - } - ) - return groups - - -def allowed_destinations(model_type: str) -> List[str]: - """Destination blocks a model_type may be configured with, in claim order. - - A Qwen3-TTS mode contract subclasses the shared TTS contract, so the shared - block claims the fields they have in common and the mode block keeps only - its extras. - """ - destinations: List[str] = [] - if model_type in SAMPLER_MODEL_TYPES: - destinations.append(SAMPLER_BLOCK) - if model_type.startswith(QWEN3_TTS_PREFIX): - destinations.append(SHARED_TTS_BLOCK) - mode_block = model_block_name(model_type) - if mode_block: - destinations.append(mode_block) - destinations.append(SCHEDULER_DEST) - return destinations - - -def _destinations_by_field(model_type: str) -> Dict[str, str]: - mapping: Dict[str, str] = {} - for dest in allowed_destinations(model_type): - for name in exposed_fields(dest): - mapping.setdefault(name, dest) - return mapping - - -def _plain(value: Any) -> Any: - """Reduce enum members to their serializable values.""" - return value.value if isinstance(value, enum.Enum) else value - - -def _validated_scheduler(payload: Dict[str, Any]) -> Dict[str, Any]: - if not payload: - return {} - try: - validated = SchedulerConfigSchema(**payload) - except ValidationError as exc: - raise ConfigOptionError(f"scheduler_config has invalid values: {exc}") from exc - return {key: _plain(getattr(validated, key)) for key in payload} - - -def resolve_config_values( - model_type: str, values: Dict[str, Any] -) -> Tuple[Dict[str, Any], Dict[str, Dict[str, Any]]]: - """Route provided flag values to the config blocks they belong to. - - Args: - model_type: The ``--model-type`` chosen by the operator. - values: Raw values keyed by field name; None means "not provided". - - Returns: - ``(scheduler_config, blocks)`` where scheduler_config is written under - ``load_config`` and blocks map block name -> validated authored keys. - - Raises: - ConfigOptionError: If a flag does not apply to model_type, or a value - fails contract validation. - """ - mapping = _destinations_by_field(model_type) - provided = {name: value for name, value in values.items() if value is not None} - - unknown = sorted(set(provided) - set(mapping)) - if unknown: - flags = ", ".join(f"--{kebab(name)}" for name in unknown) - valid = ", ".join(f"--{kebab(name)}" for name in sorted(mapping)) - raise ConfigOptionError( - f"Option(s) {flags} do not apply to model_type '{model_type}'. " - f"Valid config options for this model type: {valid or 'none'}." - ) - - payloads: Dict[str, Dict[str, Any]] = {} - for name, value in provided.items(): - payloads.setdefault(mapping[name], {})[name] = value - - scheduler = _validated_scheduler(payloads.pop(SCHEDULER_DEST, {})) - - blocks: Dict[str, Dict[str, Any]] = {} - for dest, payload in payloads.items(): - try: - resolved = validate_block(dest, payload, model_type) - except ValueError as exc: - raise ConfigOptionError(str(exc)) from exc - if resolved: - blocks[dest] = {key: _plain(value) for key, value in resolved.items()} - - return scheduler, blocks \ No newline at end of file diff --git a/tests/unit/test_cli_add_unit.py b/tests/unit/test_cli_add_unit.py deleted file mode 100644 index 9011691..0000000 --- a/tests/unit/test_cli_add_unit.py +++ /dev/null @@ -1,112 +0,0 @@ -import yaml - -from click.testing import CliRunner - -from src.cli import cli - - -def _model_dir(tmp_path): - model_dir = tmp_path / "model" - model_dir.mkdir() - (model_dir / "openvino_model.xml").write_text("", encoding="utf-8") - (model_dir / "openvino_model.bin").write_bytes(b"bin") - return model_dir - - -def test_add_help_omits_vlm_type_option() -> None: - result = CliRunner().invoke(cli, ["add", "--help"]) - - assert result.exit_code == 0 - assert "--vlm-type" not in result.output - assert "--vt" not in result.output - - -def test_add_does_not_save_vlm_type(tmp_path) -> None: - config_file = tmp_path / "config.yaml" - model_dir = _model_dir(tmp_path) - - result = CliRunner().invoke( - cli, - [ - "add", - "--model-name", - "test-vlm", - "--model-path", - str(model_dir), - "--engine", - "ovgenai", - "--model-type", - "vlm", - "--device", - "CPU", - ], - env={"OPENARC_CONFIG_FILE": str(config_file)}, - ) - - assert result.exit_code == 0 - config = yaml.safe_load(config_file.read_text(encoding="utf-8")) - entry = config["models"]["test-vlm"] - assert "load_config" in entry - assert "vlm_type" not in entry["load_config"] - - -def test_add_writes_nested_shape_with_sampler_config(tmp_path) -> None: - config_file = tmp_path / "config.yaml" - model_dir = _model_dir(tmp_path) - - result = CliRunner().invoke( - cli, - [ - "add", - "--model-name", - "test-llm", - "--model-path", - str(model_dir), - "--engine", - "ovgenai", - "--model-type", - "llm", - "--device", - "CPU", - "--sampler-config", - '{"temperature": 0.7, "top_k": 40}', - "--tool-call-parser", - "hermes", - ], - env={"OPENARC_CONFIG_FILE": str(config_file)}, - ) - - assert result.exit_code == 0 - entry = yaml.safe_load(config_file.read_text(encoding="utf-8"))["models"]["test-llm"] - # load fields nest under load_config; request defaults are siblings. - assert entry["load_config"]["tool_call_parser"] == "hermes" - assert entry["load_config"]["model_type"] == "llm" - assert entry["sampler_config"] == {"temperature": 0.7, "top_k": 40} - - -def test_add_rejects_invalid_sampler_config(tmp_path) -> None: - config_file = tmp_path / "config.yaml" - model_dir = _model_dir(tmp_path) - - result = CliRunner().invoke( - cli, - [ - "add", - "--model-name", - "bad", - "--model-path", - str(model_dir), - "--engine", - "ovgenai", - "--model-type", - "llm", - "--device", - "CPU", - "--sampler-config", - '{"temperature": "not-a-number"}', - ], - env={"OPENARC_CONFIG_FILE": str(config_file)}, - ) - - assert result.exit_code != 0 - assert not config_file.exists() From bd8645c8caa1799f55266f46c46dae0a54866fbd Mon Sep 17 00:00:00 2001 From: SearchSavior Date: Fri, 18 Sep 2026 22:55:11 -0400 Subject: [PATCH 4/5] updates to the config docs --- config.yaml | 19 ++--- docs/configure.md | 195 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+), 14 deletions(-) diff --git a/config.yaml b/config.yaml index e479751..6501295 100644 --- a/config.yaml +++ b/config.yaml @@ -32,7 +32,7 @@ models: use_cache_eviction: use_sparse_attention: sampler_config: - temperature: 0.7 + tempeature: 0.7 top_k: 40 top_p: 0.95 repetition_penalty: 1.05 @@ -41,7 +41,7 @@ models: qwen3-8b-spec: load_config: engine: ovgenai - model_type: vlm + model_type: llm model_path: /mnt/Ironwolf-4TB/Models/OpenVINO/Qwen/Qwen3-8B-ShiningValiant3-int4-asym-ov device: GPU.0 tool_call_parser: hermes @@ -50,16 +50,6 @@ models: num_assistant_tokens: 5 runtime_config: PERFORMANCE_HINT: LATENCY - scheduler_config: - max_num_batched_tokens: - num_kv_blocks: - cache_size: - num_linear_attention_blocks: - cache_interval_multiplier: - dynamic_split_fuse: - enable_prefix_caching: - use_cache_eviction: - use_sparse_attention: sampler_config: temperature: 0.7 top_k: 40 @@ -91,8 +81,9 @@ models: runtime_config: ATTENTION_BACKEND: PA CACHE_DIR: /mnt/Ironwolf-4TB/Models/OpenVINO/Qwen/Ornith-1.5-9B/Ornith-1.5-9B-int4_asym-awq-se256-ov/model_cache -####################################################################################################### -####################################################################################################### +# ==== +# LLM +# qwen38-27b-davidau ####################################################################################################### qwen38-27b-davidau: load_config: diff --git a/docs/configure.md b/docs/configure.md index ca3b21a..f5888ed 100644 --- a/docs/configure.md +++ b/docs/configure.md @@ -1,3 +1,198 @@ +OpenArc now uses a YAML based configutation system! Before we did things with a CLI tool- but now, you are free to configure defaults to your hearts content. + + + + +# Examples + +## LLM + +```yaml +models: + qwen35-08b: + load_config: + engine: ovgenai + model_type: llm + model_path: /mnt/models/Qwen3.5-0.8B-int8-asym-ov + device: CPU + tool_call_parser: qwen35 + runtime_config: + PERFORMANCE_HINT: LATENCY + scheduler_config: + max_num_batched_tokens: + num_kv_blocks: + cache_size: + num_linear_attention_blocks: + cache_interval_multiplier: + dynamic_split_fuse: + enable_prefix_caching: + use_cache_eviction: + use_sparse_attention: + sampler_config: + temperature: 0.7 + top_k: 40 + top_p: 0.95 + repetition_penalty: 1.05 + max_tokens: 1024 +``` + +## VLM + +```yaml +models: + qwen35-08b: + load_config: + engine: ovgenai + model_type: vlm + model_path: /mnt/models/Qwen3.5-0.8B-int8-asym-ov + device: CPU + tool_call_parser: qwen35 + runtime_config: + PERFORMANCE_HINT: LATENCY + scheduler_config: + max_num_batched_tokens: + num_kv_blocks: + cache_size: + num_linear_attention_blocks: + cache_interval_multiplier: + dynamic_split_fuse: + enable_prefix_caching: + use_cache_eviction: + use_sparse_attention: + sampler_config: + temperature: 0.7 + top_k: 40 + top_p: 0.95 + repetition_penalty: 1.05 + max_tokens: 1024 +``` + +## Kokoro + +OpenArc supports kokoro! Support in openvino has improved since I first implemented this model, so its possible we should look into review. + +```yaml +models: + kokoro-82m: + load_config: + engine: ovgenai + model_type: kokoro + model_path: /mnt/models/Kokoro-82M-ov + device: CPU + kokoro_config: + voice: af_sarah + voice_blend: af_heart:0.7,af_nicole:0.3 + lang_code: a + speed: 1.0 + character_count_chunk: 400 + response_format: wav +``` + +## Qwen3-ASR + +Another great model for ASR task. THe below settings around chunking have model-specific audio chunking logic which adapt and improve on what Qwen-Team released at launch- we study the energy of audio to choose when to slice up to `max_chunk_sec`, so you rarely hit `30` seconds. Therefore `max_chunk_sec` is a unit of inference, and performance should be judged by how long it takes to process a window. + +Reference Implementation lives at [SearchSavior/Qwen3-ASR-OpenVINO](https://github.com/SearchSavior/Qwen3-ASR-OpenVINO) + + +```yaml +models: + qwen3-asr: + load_config: + engine: ovgenai + model_type: qwen3_asr + model_path: /mnt/models/Qwen3-ASR-ov + device: GPU.0 + qwen3_asr_config: + language: # None = auto-detect + max_tokens: 1024 # must be > 0 + max_chunk_sec: 30.0 # chunk upper bound, seconds + search_expand_sec: 5.0 # boundary search expansion, seconds + min_window_ms: 100.0 # energy window, ms +``` + + +## Qwen3-TTS + +All 3 flavors of qwen3-tts are supported, with a rich set of configuration options. PRs to improve performance or other enhancements are welcome + +Qwen3-TTS family models process audio and text in an interleaved way, leveraging the reasoning of the Qwen3-0.6b language model backbone to augment the latents produced up to that point; then, Qwen team trains a streaming audio codec decoder with multi-token prediction heads that spit audio into a tokenizer. + +If you want to learn more or contribute improvements, check out my reference implementation [SearchSavior/Qwen3-TTS-OpenVINO](https://github.com/SearchSavior/Qwen3-TTS-OpenVINO). + + + +```yaml +models: + qwen3-tts-custom: + load_config: + engine: ovgenai + model_type: qwen3_tts_custom_voice + model_path: /mnt/models/Qwen3-TTS-CustomVoice-ov + device: GPU.0 + # Shared by all three modes. + qwen3_tts_config: + language: # None = auto-detect + max_new_tokens: 2048 + do_sample: true + top_k: 50 + top_p: 1.0 + temperature: 0.9 + repetition_penalty: 1.05 + non_streaming_mode: true # false = drip-feed text during decode + subtalker_do_sample: true + subtalker_top_k: 50 + subtalker_top_p: 1.0 + subtalker_temperature: 0.9 + stream: true # chunked audio/L16 response + stream_chunk_frames: 300 + stream_left_context: 25 +``` + +### Voice Clone + +```yaml + qwen3_tts_voice_clone_config: + ref_text: "Transcript of the reference clip." # enables ICL + x_vector_only: false # true = skip ICL even when ref_text is set + instruct: "Speak slowly." +``` + +### Voice Design +```yaml + qwen3_tts_voice_design_config: + voice_description: "A red furry muppet with an orange nose." +``` +### Custom Voice + +```yaml + qwen3_tts_custom_voice_config: + speaker: chelsie + instruct: "Sound cheerful." +``` + + + + + + + + +## scheduler_config + + + + + + + + + + + + + + ## runtime_config From 2750a0147163d1c4281cfbd5bfc52d18a651c1c4 Mon Sep 17 00:00:00 2001 From: SearchSavior Date: Sat, 19 Sep 2026 01:09:40 -0400 Subject: [PATCH 5/5] update docs for openarc configuration --- docs/api_usage.md | 208 ++++++++++ docs/commands.md | 403 +------------------ docs/configure.md | 61 +-- docs/index.md | 26 +- docs/models.md | 29 -- docs/performance.md | 132 ++++++ src/cli/groups/list.py | 2 +- src/server/routes/openai.py | 2 +- src/server/schemas/modeling/config_blocks.py | 14 +- tests/unit/test_config_merge_unit.py | 17 + zensical.toml | 4 + 11 files changed, 423 insertions(+), 475 deletions(-) create mode 100644 docs/api_usage.md create mode 100644 docs/performance.md diff --git a/docs/api_usage.md b/docs/api_usage.md new file mode 100644 index 0000000..7664176 --- /dev/null +++ b/docs/api_usage.md @@ -0,0 +1,208 @@ +--- +icon: lucide/code +--- + +# API Usage + +Request-time parameters are supplied per request via `extra_body` on the OpenAI-compatible API. Values set here override the model's config block in `config.yaml` (see [Configuration](configure.md)); anything omitted falls back to the config block, then the built-in defaults. + +## Qwen3-TTS + +Qwen3-TTS has three modes, selected by `model_type` in `config.yaml` (`qwen3_tts_custom_voice`, `qwen3_tts_voice_design`, `qwen3_tts_voice_clone`). Inference parameters (speaker, voice description, reference audio, sampling settings) are supplied per-request via the API. + +CPU and GPU device are supported. + +When GPU is selected as device, part of the model still runs on CPU. + +Supported languages: `english`, `chinese`, `japanese`, `korean`, `german`, `french`, `spanish`, `italian`, `portuguese`, `russian`, `beijing_dialect`, `sichuan_dialect`. Pass `None` to auto-detect. See `demos/qwen3_tts_example.py` for a full request example. + +### Custom Voice + +Pick a predefined speaker at inference time (`serena`, `vivian`, `uncle_fu`, `ryan`, `aiden`, `ono_anna`, `sohee`, `eric`, `dylan`): + +```python +import os +from openai import OpenAI +from pathlib import Path + +client = OpenAI( + base_url="http://localhost:8000/v1", + api_key=os.environ["OPENARC_API_KEY"], +) + +response = client.audio.speech.create( + model="", + input="Hello, this is a test.", + extra_body={ + "openarc_tts": { + "qwen3_tts": { + # --- content --- + "input": "Hello, this is a test.", + "speaker": "uncle_fu", # serena, vivian, uncle_fu, ryan, aiden, ono_anna, sohee, eric, dylan + "instruct": None, # optional style instruction e.g. "Speak slowly and clearly." + "language": "english", # None to auto-detect + # --- sampling --- + "max_new_tokens": 2048, + "do_sample": True, + "top_k": 50, + "top_p": 1.0, + "temperature": 0.9, + "repetition_penalty": 1.05, + "non_streaming_mode": True, + "subtalker_do_sample": True, + "subtalker_top_k": 50, + "subtalker_top_p": 1.0, + "subtalker_temperature": 0.9, + # --- streaming --- + "stream": True, + "stream_chunk_frames": 50, + "stream_left_context": 25, + } + } + }, +) + +Path("speech.wav").write_bytes(response.content) +``` + +### Voice Design + +Describe the voice in free-form text at inference time: + +```python +import os +from openai import OpenAI +from pathlib import Path + +client = OpenAI( + base_url="http://localhost:8000/v1", + api_key=os.environ["OPENARC_API_KEY"], +) + +response = client.audio.speech.create( + model="", + input="Hello, this is a test.", + voice="alloy", + extra_body={ + "openarc_tts": { + "qwen3_tts": { + # --- content --- + "input": "Hello, this is a test.", + "voice_description": "A calm, deep male voice with a slight British accent.", + "language": "english", # None to auto-detect + # --- sampling --- + "max_new_tokens": 2048, + "do_sample": True, + "top_k": 50, + "top_p": 1.0, + "temperature": 0.9, + "repetition_penalty": 1.05, + "subtalker_do_sample": True, + "subtalker_top_k": 50, + "subtalker_top_p": 1.0, + "subtalker_temperature": 0.9, + # --- streaming --- + "stream": True, + "stream_chunk_frames": 300, + "stream_left_context": 25, + } + } + }, +) + +Path("speech.wav").write_bytes(response.content) +``` + +### Voice Clone + +Provide a reference WAV at inference time to clone a speaker: + +```python +import base64 +import os +from openai import OpenAI +from pathlib import Path + +client = OpenAI( + base_url="http://localhost:8000/v1", + api_key=os.environ["OPENARC_API_KEY"], +) + +ref_audio_b64 = base64.b64encode(Path("reference.wav").read_bytes()).decode() + +response = client.audio.speech.create( + model="", + input="Hello, this is a test.", + voice="alloy", + extra_body={ + "openarc_tts": { + "qwen3_tts": { + # --- content --- + "ref_audio_b64": ref_audio_b64, + "ref_text": "Transcript of the reference audio.", # optional, enables ICL + "x_vector_only": False, # True = x-vector only, skips ICL even if ref_text is set + "instruct": None, # optional style instruction + "language": "english", # None to auto-detect + # --- sampling --- + "max_new_tokens": 2048, + "do_sample": True, + "top_k": 50, + "top_p": 1.0, + "temperature": 0.9, + "repetition_penalty": 1.05, + "subtalker_do_sample": True, + "subtalker_top_k": 50, + "subtalker_top_p": 1.0, + "subtalker_temperature": 0.9, + # --- streaming --- + "stream": True, + "stream_chunk_frames": 300, + "stream_left_context": 25, + } + } + }, +) + +Path("speech.wav").write_bytes(response.content) +``` + +## Qwen3-ASR + +Qwen3-ASR long-form transcription — supports Qwen3-ASR-0.6B. Audio is chunked automatically at silence boundaries up to `max_chunk_sec` (default `30s`). This is not a hard limit; chunking happens dynamically based on the energy of the audio. + +Chunking can be configured on a per-request basis via `openarc_asr` in the request body for `/v1/audio/transcriptions`. Anything not set falls back to the model's `qwen3_asr_config` block in `config.yaml`, then the built-in defaults. + +These `extra_body` options are OpenArc-specific, so third-party tools will not expose them; pass them yourself. Per-request tinkering works on both CPU and GPU. At this time NPU device is unsupported. + +```python +import json +import os +from pathlib import Path +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:8000/v1", + api_key=os.environ["OPENARC_API_KEY"], +) + +with Path("audio.wav").open("rb") as f: + response = client.audio.transcriptions.create( + model="", + file=f, + response_format="verbose_json", + # Optional. Values below are used as defaults if `openarc_asr` is not provided. + extra_body={ + "openarc_asr": json.dumps({ + "qwen3_asr": { + "language": None, # auto-detect, or e.g. "english" + "max_tokens": 1024, # max tokens per chunk + "max_chunk_sec": 30.0, # max audio chunk length in seconds + "search_expand_sec": 5.0, # silence-search window expansion + "min_window_ms": 100.0, # minimum silence window in ms + } + }) + }, + ) + +print(response.text) +``` diff --git a/docs/commands.md b/docs/commands.md index 60f9422..b78102e 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -9,404 +9,11 @@ After installation run ```openarc --help``` to see focused usage documentation i This page contains example commands to help you choose models and configure OpenArc. -=== "add" - - Add a model to `openarc_config.json` for easy loading with `openarc load`. - - === "Required" - - ``` - openarc add \ - --model-name \ - --model-path \ - --engine \ - --model-type \ - --device - --tool-call-parser - ``` - - To see what options you have for `--device`, use `openarc tool device-detect`. - - - === "LLM" - - ``` - openarc add \ - --model-name \ - --model-path \ - --engine \ - --model-type llm \ - --device - --tool-call-parser # text only models currently supported use hermes style in most cases - ``` - - === "VLM" - - ``` - openarc add \ - --model-name \ - --model-path \ - --engine \ - --model-type vlm \ - --device - --tool-call-parser - ``` - - === "Whisper" - - ``` - openarc add \ - --model-name \ - --model-path \ - --engine ovgenai \ - --model-type whisper \ - --device - ``` - - === "Kokoro" - - ``` - openarc add \ - --model-name \ - --model-path \ - --engine openvino \ - --model-type kokoro \ - --device CPU - ``` - - === "Qwen3-TTS" - - Qwen3-TTS has three modes, each selected by `--model-type` at add time. Inference parameters (speaker, voice description, reference audio, sampling settings) are supplied per-request via the API, not here. - - CPU and GPU device are supported. - - When GPU is selected as device, part of the model still runs on CPU. - - Supported languages: `english`, `chinese`, `japanese`, `korean`, `german`, `french`, `spanish`, `italian`, `portuguese`, `russian`, `beijing_dialect`, `sichuan_dialect`. Pass `None` to auto-detect. See `demos/qwen3_tts_example.py` for a full request example. - - === "Custom voice" - - Pick a predefined speaker at inference time (`serena`, `vivian`, `uncle_fu`, `ryan`, `aiden`, `ono_anna`, `sohee`, `eric`, `dylan`): - - ``` - openarc add \ - --model-name \ - --model-path \ - --engine openvino \ - --model-type qwen3_tts_custom_voice \ - --device CPU - ``` - - ```python - import os - from openai import OpenAI - from pathlib import Path - - client = OpenAI( - base_url="http://localhost:8000/v1", - api_key=os.environ["OPENARC_API_KEY"], - ) - - response = client.audio.speech.create( - model="", - input="Hello, this is a test.", - extra_body={ - "openarc_tts": { - "qwen3_tts": { - # --- content --- - "input": "Hello, this is a test.", - "speaker": "uncle_fu", # serena, vivian, uncle_fu, ryan, aiden, ono_anna, sohee, eric, dylan - "instruct": None, # optional style instruction e.g. "Speak slowly and clearly." - "language": "english", # None to auto-detect - # --- sampling --- - "max_new_tokens": 2048, - "do_sample": True, - "top_k": 50, - "top_p": 1.0, - "temperature": 0.9, - "repetition_penalty": 1.05, - "non_streaming_mode": True, - "subtalker_do_sample": True, - "subtalker_top_k": 50, - "subtalker_top_p": 1.0, - "subtalker_temperature": 0.9, - # --- streaming --- - "stream": True, - "stream_chunk_frames": 50, - "stream_left_context": 25, - } - } - }, - ) - - Path("speech.wav").write_bytes(response.content) - ``` - - === "Voice design" - - Describe the voice in free-form text at inference time: - - ``` - openarc add \ - --model-name \ - --model-path \ - --engine openvino \ - --model-type qwen3_tts_voice_design \ - --device CPU - ``` - - ```python - import os - from openai import OpenAI - from pathlib import Path - - client = OpenAI( - base_url="http://localhost:8000/v1", - api_key=os.environ["OPENARC_API_KEY"], - ) - - response = client.audio.speech.create( - model="", - input="Hello, this is a test.", - voice="alloy", - extra_body={ - "openarc_tts": { - "qwen3_tts": { - # --- content --- - "input": "Hello, this is a test.", - "voice_description": "A calm, deep male voice with a slight British accent.", - "language": "english", # None to auto-detect - # --- sampling --- - "max_new_tokens": 2048, - "do_sample": True, - "top_k": 50, - "top_p": 1.0, - "temperature": 0.9, - "repetition_penalty": 1.05, - "subtalker_do_sample": True, - "subtalker_top_k": 50, - "subtalker_top_p": 1.0, - "subtalker_temperature": 0.9, - # --- streaming --- - "stream": True, - "stream_chunk_frames": 300, - "stream_left_context": 25, - } - } - }, - ) - - Path("speech.wav").write_bytes(response.content) - ``` - - === "Voice clone" - - Provide a reference WAV at inference time to clone a speaker: - - ``` - openarc add \ - --model-name \ - --model-path \ - --engine openvino \ - --model-type qwen3_tts_voice_clone \ - --device CPU - ``` - - ```python - import base64 - import os - from openai import OpenAI - from pathlib import Path - - client = OpenAI( - base_url="http://localhost:8000/v1", - api_key=os.environ["OPENARC_API_KEY"], - ) - - ref_audio_b64 = base64.b64encode(Path("reference.wav").read_bytes()).decode() - - response = client.audio.speech.create( - model="", - input="Hello, this is a test.", - voice="alloy", - extra_body={ - "openarc_tts": { - "qwen3_tts": { - # --- content --- - "ref_audio_b64": ref_audio_b64, - "ref_text": "Transcript of the reference audio.", # optional, enables ICL - "x_vector_only": False, # True = x-vector only, skips ICL even if ref_text is set - "instruct": None, # optional style instruction - "language": "english", # None to auto-detect - # --- sampling --- - "max_new_tokens": 2048, - "do_sample": True, - "top_k": 50, - "top_p": 1.0, - "temperature": 0.9, - "repetition_penalty": 1.05, - "subtalker_do_sample": True, - "subtalker_top_k": 50, - "subtalker_top_p": 1.0, - "subtalker_temperature": 0.9, - # --- streaming --- - "stream": True, - "stream_chunk_frames": 300, - "stream_left_context": 25, - } - } - }, - ) - - Path("speech.wav").write_bytes(response.content) - ``` - - === "Qwen3-ASR" - - Qwen3-ASR long-form transcription — supports Qwen3-ASR-0.6B. Audio is chunked automatically at silence boundaries up to `max_chunk_sec` (default `30s`). This is not a hard limit and happens dynamically based - - ``` - openarc add \ - --model-name \ - --model-path \ - --engine openvino \ - --model-type qwen3_asr \ - --device CPU - ``` - - Chunking can be configured on a per-request basis via the `openarc_asr` in the request body for the `/v1/audio/transcriptions`. If not set, the below defaults will be used. Defaults values cannot currently be configured. - - For the options in `extra_body`, they will likely not have support in any third party tool you don't build from scratch. I'm working on improving how these can be configured. Currently, the behavior is modified per request, so you can tinker with performance on CPU and GPU. At this time NPU device is unsupported. - - ```python - import json - import os - from pathlib import Path - from openai import OpenAI - - client = OpenAI( - base_url="http://localhost:8000/v1", - api_key=os.environ["OPENARC_API_KEY"], - ) - - with Path("audio.wav").open("rb") as f: - response = client.audio.transcriptions.create( - model="", - file=f, - response_format="verbose_json", - # Optional. The below values will be used as defaults if `openarc_asr` is not provided. - extra_body={ - "openarc_asr": json.dumps({ - "qwen3_asr": { - "language": None, # auto-detect, or e.g. "english" - "max_tokens": 1024, # max tokens per chunk - "max_chunk_sec": 30.0, # max audio chunk length in seconds - "search_expand_sec": 5.0, # silence-search window expansion - "min_window_ms": 100.0, # minimum silence window in ms - } - }) - }, - ) - - print(response.text) - ``` - - === "Advanced" - - `runtime-config` accepts many options to modify `openvino` runtime behavior for different inference scenarios. OpenArc reports C++ errors to the server when these fail, making experimentation easy. - - See OpenVINO documentation on [Inference Optimization](https://docs.openvino.ai/2025/openvino-workflow/running-inference/optimize-inference.html) to learn more about what can be customized. - - Not all options are designed for transformers, so `runtime-config` was implemented in a way where you get immediate feedback from the OpenVINO runtime after loading a model. Add an argument, load that model, get feedback from the server, run `openarc bench`. This makes iterating faster in an area where the documentation is sparse. The options listed here have been validated. - - Review the [pipeline-parallelism preview](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/hetero-execution.html#pipeline-parallelism-preview) to learn how you can customize multi-device inference using the HETERO device plugin. - - === "Multi-GPU Pipeline Parallel" - - ``` - openarc add \ - --model-name \ - --model-path \ - --engine ovgenai \ - --model-type llm \ - --device HETERO:GPU.0,GPU.1 \ - --runtime-config "{"MODEL_DISTRIBUTION_POLICY": "PIPELINE_PARALLEL"}" - ``` - - === "Tensor Parallel" - - Requires more than one CPU socket in a single node. - - ``` - openarc add \ - --model-name \ - --model-path \ - --engine ovgenai \ - --model-type llm \ - --device CPU \ - --runtime-config "{"MODEL_DISTRIBUTION_POLICY": "TENSOR_PARALLEL"}" - ``` - - === "Hybrid / CPU Offload" - - ``` - openarc add \ - --model-name \ - --model-path \ - --engine ovgenai \ - --model-type llm \ - --device HETERO:GPU.0,CPU \ - --runtime-config "{"MODEL_DISTRIBUTION_POLICY": "PIPELINE_PARALLEL"}" - ``` - - === "Speculative Decoding" - - ``` - openarc add \ - --model-name \ - --model-path \ - --engine ovgenai \ - --model-type llm \ - --device GPU.0 \ - --draft-model-path \ - --draft-device CPU \ - --num-assistant-tokens 5 \ - --assistant-confidence-threshold 0.5 - ``` - - === "Model caching" - - The `--cache-dir` parameter can be specified to cache compiled models upon first start. This can greatly reduce startup memory cost and time for subsequent process starts. - On some setups this can reduce peak memory utilization on subsequent restarts by 3x or more, and start time by 7x. For additional details, see - [here](https://docs.openvino.ai/2026/model-server/ovms_docs_model_cache.html). - - The cache can be shared by multiple processes (and on shared network filesystems such as NFS or CephFS) provided that only one process updates it at a time. - - The cache will be fully or partially invalidated when doing any of the below: - * Changing the utilized device(s) (swapping GPU models, adding or removing a GPU, adding or removing a CPU, etc.) - * Changing `runtime_config` that impacts the model itself (e.g. `PERFORMANCE_HINT: THROUGHPUT` to `PERFORMANCE_HINT: LATENCY` but not `NUM_STREAMS: 1 to `NUM_STREAMS: 2`) - * Changing any part of the software stack from the firmware up - GPU firmware, OS kernel, kernel modules/drivers, dependency libraries, OpenARC, model versions. - - > [!WARNING] - > Due to OpenVINO limitations, unused cache files are never cleaned up and will persist until an operator removes them. The cache can grow large over time. It is recommended - > that operators monitor the cache size and manually clean it up as needed to reduce disk usage. - - - ``` - openarc add \ - --model-name \ - --model-path \ - --engine ovgenai \ - --model-type llm \ - --device GPU \ - --cache-dir - ``` - === "list" - Reads added configurations from `openarc_config.json`. + Reads model entries from `config.yaml`. - Display all added models: + Display all model entries: ``` openarc list ``` @@ -418,7 +25,7 @@ This page contains example commands to help you choose models and configure Open -v ``` - Remove a configuration: + Remove a model entry: ``` openarc list \ --remove @@ -463,9 +70,9 @@ This page contains example commands to help you choose models and configure Open === "load" - After using `openarc add` you can use `openarc load` to read the added configuration and load models onto the OpenArc server. + `openarc load` reads a model's entry from `config.yaml` and loads it onto the OpenArc server. - OpenArc uses arguments from `openarc add` as metadata to make routing decisions internally; you are querying for correct inference code. + OpenArc uses the entry's metadata (engine, model_type, device) to make routing decisions internally; you are querying for correct inference code. ``` openarc load diff --git a/docs/configure.md b/docs/configure.md index f5888ed..4d956a1 100644 --- a/docs/configure.md +++ b/docs/configure.md @@ -34,6 +34,8 @@ models: top_p: 0.95 repetition_penalty: 1.05 max_tokens: 1024 + chat_template_kwargs: + enable_thinking: true ``` ## VLM @@ -111,6 +113,22 @@ models: min_window_ms: 100.0 # energy window, ms ``` +## Whisper + +Whisper has no config block; audio arrives per request, so only load-time options apply. + +```yaml +models: + whisper-large-v3: + load_config: + engine: ovgenai + model_type: whisper + model_path: /mnt/models/whisper-large-v3-int8-ov + device: GPU.0 + runtime_config: + PERFORMANCE_HINT: LATENCY +``` + ## Qwen3-TTS @@ -171,49 +189,10 @@ models: instruct: "Sound cheerful." ``` - - - - - - - ## scheduler_config - - - - - - - - - - - - +Scheduler properties (KV cache, prefix caching, batching) live on the [Performance](performance.md#scheduler_config) page. ## runtime_config - -runtime_config is an OpenArc entrypoint to the *properties* way of configuring openvino runtime. These settings allow users to tune the behavior of openivno runtime without needing to change application logic and are meant to be "portable", requring no code changes. Since OpenArc - -OpenArc does not validate these, and OpenVINO upstream does not provide a way to check the behvaior of these settings in all cases. They can help you access hardware features not available to all devices like `SCHEDULING_CORE_TYPE` for more recent Intel CPUs, debug numeircal precision issues with `INFERENCE_PRECISION_HINT` or control `KV_CACHE_PRECISION`. - -*properties* have the worst documentation in all of OpenVINO ecosystem, yet they are used everywhere in the openvino_notebooks, PRs and sometimes are even hardcoded depending on the needs of OpenVINO team. In that way, poking at these settings can drastically change performance but have less knobs than users of projects like `llama.cpp`, `vllm`, `sglang` are familiar with making. - - -Even though we can learn from the source code what these settings do knowing when they are useful comes with practice - - -ATTENTION_BACKEND "SDPA", "PA" -KV_CACHE_PRECISION "u4", "u8", "f16", "f32" -PERFORMANCE_HINT "LATNENCY", "THROUGHPUT" -EXECUTION_MODE_HINT "ACCURACY", "PERFORMANCE" -INFERENCE_PRECISION_HINT "f16", "f32" -MODEL_DISTRIBUTION_POLICY "TENSOR_PARALLEL", "PIPELINE_PARALLEL" -ACTIVATIONS_SCALING_FACTOR: -DYNAMIC_QUANTIZATION_GROUP_SIZE: integer -ENABLE_HYPER_THREADING: bool, defaults to true -SCHEDULING_CORE_TYPE: "ANY_CORE", "ECORE_ONLY", "PCORE_ONLY" -LOG_LEVEL: "ERR", "WARN", "INFO", "DEBUG", "TRACE" # might require building openvino \ No newline at end of file +OpenVINO runtime *properties* and multi-device / speculative / caching recipes live on the [Performance](performance.md#runtime_config) page. \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index f2e7890..eec47af 100644 --- a/docs/index.md +++ b/docs/index.md @@ -23,7 +23,6 @@ Welcome to the OpenArc documentation! OpenArc includes a command line tool for controlling the server. -- [openarc add](commands.md#add) — Add a model to the config. - [openarc list](commands.md#list) — List models added to the config. - [openarc serve](commands.md#serve) — Start the OpenArc server. - [openarc load](commands.md#load) — Load a model from the config. @@ -33,9 +32,30 @@ OpenArc includes a command line tool for controlling the server. ## Configuration ->Under construction! +OpenArc uses a YAML based config system. See [Examples](configure.md#examples) for a config block for each model type. -- [Advanced openvino properties](configure.md#runtime_config) +- [LLM](configure.md#llm) +- [VLM](configure.md#vlm) +- [Kokoro](configure.md#kokoro) +- [Qwen3-ASR](configure.md#qwen3-asr) +- [Qwen3-TTS](configure.md#qwen3-tts) + +## Performance + +Scheduler, runtime properties, and multi-device recipes. + +- [scheduler_config](performance.md#scheduler_config) +- [runtime_config](performance.md#runtime_config) +- [Multi-Device Inference](performance.md#multi-device-inference) +- [Speculative Decoding](performance.md#speculative-decoding) +- [Model Caching](performance.md#model-caching) + +## API Usage + +Request-time parameters passed via `extra_body` on the OpenAI-compatible API. + +- [Qwen3-TTS](api_usage.md#qwen3-tts) +- [Qwen3-ASR](api_usage.md#qwen3-asr) ## Concepts diff --git a/docs/models.md b/docs/models.md index be57dfc..d8427e2 100644 --- a/docs/models.md +++ b/docs/models.md @@ -98,18 +98,6 @@ If you need help converting a particular model join Discord and we can help you! | [Qwen3-Reranker-0.6B-fp16-ov](https://huggingface.co/OpenVINO/Qwen3-Reranker-0.6B-fp16-ov) | -## Model-Specific Instructions - -### Tool and Reasoning Parsing - - -OpenArc now supports tool and reasoning parsing for several architectures. Our approach ensures correct openai compatible parsing using openvino genai by leveraging some useful facts about tokens. - -Since Autoregressive language models emit tokens in a continuous stream one by one, we can visualize it this way -- streaming decoded tokens to a buff - - - **How do I control thinking?** Qwen3.5 utilizes chat instructions for thinking control. You can enable thinking by using the parameter `chat_template_kwargs` with a value of `{"enable_thinking": true}` and disable it by setting the value to `{"enable_thinking": false}`. @@ -118,20 +106,3 @@ Previous reasoning is also retained within a conversation. You can enable or dis For example, to enable thinking and disable previous reasoning, you would pass `chat_template_kwargs` with a value of `{"enable_thinking": true, "preserve_previous_think": false}`. -### Qwen3 MoE - -**GPU loading fails with "FuseMOE3GemmCompressed transformation did not match the routing subgraph"?** - -Qwen3 MoE models (e.g., `Qwen3-Coder-30B-A3B-Instruct`) require a recent OpenVINO build on GPU. Older builds fail during pipeline creation with: - -``` -[GPU] MOECompressed (GEMM3_SWIGLU) reached the GPU backend without being fused: FuseMOE3GemmCompressed transformation did not match the routing subgraph -``` - -This was a bug in the OpenVINO GPU plugin's MoE fusion pass and is fixed upstream in nightly `2026.4.0.dev20260730` and newer. Upgrade with: - -``` -uv pip install --pre -U openvino openvino-genai openvino-tokenizers --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly -``` - -After upgrading, clear the model cache directory (if one is configured) so the model recompiles, and restart the server if it was running before the upgrade. diff --git a/docs/performance.md b/docs/performance.md new file mode 100644 index 0000000..f324f95 --- /dev/null +++ b/docs/performance.md @@ -0,0 +1,132 @@ +--- +icon: lucide/gauge +--- + +# Performance + +Tuning knobs which live in a model's entry in `config.yaml`, alongside `load_config`. See [Configuration](configure.md) for the full config layout and per-model-type blocks. + +## scheduler_config + +OpenVINO GenAI scheduler properties for `llm` and `vlm` models. All fields are optional; unset fields use engine defaults. + +```yaml +models: + qwen35-08b: + load_config: + engine: ovgenai + model_type: llm + model_path: /mnt/models/Qwen3.5-0.8B-int8-asym-ov + device: CPU + scheduler_config: + max_num_batched_tokens: # max tokens per batch (across all sequences) + max_num_seqs: # max scheduled sequences ("max batch size") + num_kv_blocks: # total KV blocks available to the scheduler + cache_size: # total cache size in GB + num_linear_attention_blocks: # linear attention models only + cache_interval_multiplier: # linear-attn checkpoint interval multiplier (default 8) + dynamic_split_fuse: # split prompt / generate into separate scheduling phases + enable_prefix_caching: # keep KV blocks for reuse across sequences + use_cache_eviction: # evict cache during generation + use_sparse_attention: # sparse attention during prefill +``` + +Notes from the schema: + +- `num_linear_attention_blocks` and `cache_interval_multiplier` only apply to models with linear attention cache inputs. `cache_interval_multiplier: 0` is valid only when prefix caching is disabled. +- With `enable_prefix_caching` on, all previously calculated KV caches are kept in memory and blocks are not released; maximum RAM usage is bounded by `cache_size` or `num_kv_blocks`. With it off, only the KV cache required for the current batch is kept and released when a sequence finishes. + +## runtime_config + +`runtime_config` is an OpenArc entrypoint to the *properties* way of configuring the OpenVINO runtime. These settings tune runtime behavior without changing application logic and are meant to be portable, requiring no code changes. + +OpenArc does not validate these, and OpenVINO upstream does not provide a way to check the behavior of these settings in all cases. They can help you access hardware features not available on all devices like `SCHEDULING_CORE_TYPE` for more recent Intel CPUs, debug numerical precision issues with `INFERENCE_PRECISION_HINT`, or control `KV_CACHE_PRECISION`. + +*properties* have the worst documentation in the OpenVINO ecosystem, yet they are used everywhere in the openvino_notebooks, PRs, and are sometimes hardcoded depending on the needs of the OpenVINO team. Poking at these settings can drastically change performance but offers fewer knobs than users of projects like `llama.cpp`, `vllm`, or `sglang` are familiar with. + +Because invalid values surface as C++ errors from the runtime when the model loads, experimentation is cheap: edit the entry, reload the model, read the server error, run `openarc bench`. See the OpenVINO documentation on [Inference Optimization](https://docs.openvino.ai/2025/openvino-workflow/running-inference/optimize-inference.html) for what can be customized. Even though we can learn from the source code what these settings do, knowing when they are useful comes with practice. + +| Property | Values | +| --- | --- | +| `ATTENTION_BACKEND` | `"SDPA"`, `"PA"` | +| `KV_CACHE_PRECISION` | `"u4"`, `"u8"`, `"f16"`, `"f32"` | +| `PERFORMANCE_HINT` | `"LATENCY"`, `"THROUGHPUT"` | +| `EXECUTION_MODE_HINT` | `"ACCURACY"`, `"PERFORMANCE"` | +| `INFERENCE_PRECISION_HINT` | `"f16"`, `"f32"` | +| `MODEL_DISTRIBUTION_POLICY` | `"TENSOR_PARALLEL"`, `"PIPELINE_PARALLEL"` | +| `ACTIVATIONS_SCALING_FACTOR` | number | +| `DYNAMIC_QUANTIZATION_GROUP_SIZE` | integer | +| `ENABLE_HYPER_THREADING` | bool, defaults to true | +| `SCHEDULING_CORE_TYPE` | `"ANY_CORE"`, `"ECORE_ONLY"`, `"PCORE_ONLY"` | +| `LOG_LEVEL` | `"ERR"`, `"WARN"`, `"INFO"`, `"DEBUG"`, `"TRACE"` — may require building openvino | + +## Multi-Device Inference + +Review the [pipeline-parallelism preview](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/hetero-execution.html#pipeline-parallelism-preview) to learn how to customize multi-device inference using the HETERO device plugin. + +### Multi-GPU Pipeline Parallel + +```yaml + load_config: + device: HETERO:GPU.0,GPU.1 + runtime_config: + MODEL_DISTRIBUTION_POLICY: PIPELINE_PARALLEL +``` + +### Tensor Parallel + +Requires more than one CPU socket in a single node. + +```yaml + load_config: + device: CPU + runtime_config: + MODEL_DISTRIBUTION_POLICY: TENSOR_PARALLEL +``` + +### Hybrid / CPU Offload + +```yaml + load_config: + device: HETERO:GPU.0,CPU + runtime_config: + MODEL_DISTRIBUTION_POLICY: PIPELINE_PARALLEL +``` + +## Speculative Decoding + +Draft-model speculative decoding, enabled per model entry. Enables a 1.3-1.4x speedup. + +```yaml +models: + qwen35-08b: + load_config: + engine: ovgenai + model_type: llm + model_path: /mnt/models/Qwen3.5-0.8B-int8-asym-ov + device: GPU.0 + draft_model_path: /mnt/models/Qwen3.5-0.5B-int8-asym-ov + draft_device: CPU + num_assistant_tokens: 5 + assistant_confidence_threshold: 0.5 +``` + +## Model Caching + +`cache_dir` caches compiled model blobs on first load, greatly reducing startup time and memory cost for subsequent process starts. On some setups this can reduce peak memory utilization on restarts by 3x or more, and start time by 7x. For additional details, see the [OpenVINO model cache docs](https://docs.openvino.ai/2026/model-server/ovms_docs_model_cache.html). + +```yaml + load_config: + device: GPU + cache_dir: /mnt/models/cache/qwen35-08b +``` + +The cache can be shared by multiple processes (and on shared network filesystems such as NFS or CephFS) provided that only one process updates it at a time. + +The cache will be fully or partially invalidated when doing any of the below: + +- Changing the utilized device(s) (swapping GPU models, adding or removing a GPU, adding or removing a CPU, etc.) +- Changing `runtime_config` that impacts the model itself (e.g. `PERFORMANCE_HINT: THROUGHPUT` to `PERFORMANCE_HINT: LATENCY`, but not `NUM_STREAMS: 1` to `NUM_STREAMS: 2`) +- Changing any part of the software stack from the firmware up — GPU firmware, OS kernel, kernel modules/drivers, dependency libraries, OpenArc, model versions + +> **Warning:** Due to OpenVINO limitations, unused cache files are never cleaned up and will persist until an operator removes them. The cache can grow large over time. It is recommended that operators monitor the cache size and manually clean it up as needed to reduce disk usage. diff --git a/src/cli/groups/list.py b/src/cli/groups/list.py index 7bc9b6a..82a922e 100644 --- a/src/cli/groups/list.py +++ b/src/cli/groups/list.py @@ -70,7 +70,7 @@ def list(ctx, model_name, verbose, remove): if not models: console.print("[yellow]No model configurations found.[/yellow]") - console.print("[dim]Use 'openarc add --help' to see how to save configurations.[/dim]") + console.print("[dim]Add a model entry to config.yaml to get started.[/dim]") return # Reserved command mode: openarc list prune diff --git a/src/server/routes/openai.py b/src/server/routes/openai.py index 833ec31..3401473 100644 --- a/src/server/routes/openai.py +++ b/src/server/routes/openai.py @@ -180,7 +180,7 @@ async def openai_chat_completions( if tool_parser_name is None and request.tools: raise ValueError( f"Model '{request.model}' has no tool_call_parser configured; " - "set one in the model config (e.g. 'openarc add --tool-call-parser qwen35|hermes|gemma4|museglimmer')" + "set one under load_config in config.yaml (tool_call_parser: qwen35|hermes|gemma4|museglimmer')" ) parser_module = _TOOL_PARSERS.get(tool_parser_name) if tool_parser_name else None diff --git a/src/server/schemas/modeling/config_blocks.py b/src/server/schemas/modeling/config_blocks.py index c2a1de6..6f6db7c 100644 --- a/src/server/schemas/modeling/config_blocks.py +++ b/src/server/schemas/modeling/config_blocks.py @@ -56,15 +56,18 @@ "prompt", "input_ids", "tools", - "tool_call_parser", "request_id", - "chat_template_kwargs", "input", "audio_base64", "ref_audio_b64", } ) +# Fields that belong under load_config, not in a request-defaults block. They +# exist in the request contract as transport/dispatch plumbing, so they pass +# the unknown-key check; rejecting them here points the author at load_config. +LOAD_CONFIG_FIELDS = frozenset({"tool_call_parser"}) + def contract_for(block_name: str) -> Optional[Type[BaseModel]]: """Return the contract backing a block name, or None if unknown.""" @@ -132,6 +135,13 @@ def validate_block(block_name: str, payload: Any, model_type: str) -> Dict[str, f"{block_name} cannot set request-only field(s): {', '.join(request_only)}" ) + load_config = sorted(set(payload) & LOAD_CONFIG_FIELDS) + if load_config: + raise ValueError( + f"{block_name} cannot set load-time field(s): {', '.join(load_config)}. " + "Move them under load_config" + ) + # Coerce/validate values against the contract so a bad type or an invalid # enum member is caught here rather than at model load time. authored = {k: v for k, v in payload.items() if v is not None} diff --git a/tests/unit/test_config_merge_unit.py b/tests/unit/test_config_merge_unit.py index 247141d..ac545ca 100644 --- a/tests/unit/test_config_merge_unit.py +++ b/tests/unit/test_config_merge_unit.py @@ -154,6 +154,23 @@ def test_validate_block_rejects_request_only_field() -> None: validate_block("sampler_config", {"messages": [{"role": "user"}]}, "llm") +def test_validate_block_accepts_chat_template_kwargs_default() -> None: + # Thinking behavior is a reusable model default, so chat_template_kwargs + # is a legitimate sampler_config block key. + assert validate_block( + "sampler_config", + {"chat_template_kwargs": {"enable_thinking": False}}, + "llm", + ) == {"chat_template_kwargs": {"enable_thinking": False}} + + +def test_validate_block_rejects_load_config_field() -> None: + # tool_call_parser is selected at load time; authoring it in a block should + # point the author at load_config. + with pytest.raises(ValueError, match="load-time field"): + validate_block("sampler_config", {"tool_call_parser": "qwen35"}, "llm") + + # ---- defaults_for_record ---- diff --git a/zensical.toml b/zensical.toml index 478b138..4f94014 100644 --- a/zensical.toml +++ b/zensical.toml @@ -49,6 +49,10 @@ nav = [ { "Install" = "install.md" }, { "Models" = "models.md" }, { "Commands" = "commands.md" }, + { "Configuration" = "configure.md" }, + { "Performance" = "performance.md" }, + { "API Usage" = "api_usage.md" }, + { "Tool Use" = "tool_use.md" }, ] # With the "extra_css" option you can add your own CSS styling to customize